Merge pull request #286 from danielsclint/ft_overflow

Fix Integer Overflow

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
     if MAX_BLOCK_BYTES:
         max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
-        max_skims_per_block = max_block_items // np.prod(omx_shape)
+        max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
     else:
         max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
     skim_dtype = skim_info['dtype']
-    omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+    omx_shape = skim_info['omx_shape']
     blocks = skim_info['blocks']
 
     skim_buffers = {}
     for block_name, block_size in iteritems(blocks):
 
         # buffer_size must be int (or p2.7 long), not np.int64
-        buffer_size = int(np.prod(omx_shape) * block_size)
+        buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
         csz = buffer_size * np.dtype(skim_dtype).itemsize
         logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
     for block_name, block_size in iteritems(blocks):
         skims_shape = omx_shape + (block_size,)
         block_buffer = skim_buffers[block_name]
-        assert len(block_buffer) == int(np.prod(skims_shape))
+        assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
         block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
         skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
     return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+    return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+    return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+    return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+    time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+    omx_keys = OrderedDict()
+    omx_key1_block_offsets = OrderedDict()
+    omx_block_offsets = OrderedDict()
+    omx_blocks = OrderedDict()
+    omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+    for i in range(0, num_of_matrices + 1):
+        key1_name = 'm{}'.format(i // len(time_periods) + 1)
+        time_period = time_periods[i % len(time_periods)]
+
+        omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+        omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+        if 0 == i % len(time_periods):
+            omx_key1_block_offsets[key1_name] = (0, i)
+
+    skim_info = {
+        'omx_name': 'arc_skims',
+        'omx_shape': (matrix_dimension, matrix_dimension),
+        'num_skims': num_of_matrices,
+        'dtype': np.float32,
+        'omx_keys': omx_keys,
+        'key1_block_offsets': omx_key1_block_offsets,
+        'block_offsets': omx_block_offsets,
+        'blocks': omx_blocks
+    }
+
+    return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+    omx_shape = skim_info['omx_shape']
+    blocks = skim_info['blocks']
+
+    for block_name, block_size in iteritems(blocks):
+        # If overflow, this number will go negative
+        assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+               num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+    calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+    actual_value = 483200518316.9472
+    assert abs(calculated_value - actual_value) < 0.0001