added df comparison integration tests

.circleci/config.yml

@@ -11,7 +11,9 @@ jobs:
       - python/load-cache
       - python/install-deps
       - python/save-cache
-      - run: pip install cadcad
+      - run: pip install -r requirements.txt
+      - run: python setup.py sdist bdist_wheel
+      - run: pip install dist/*.whl
       - run:
           command: python testing/tests/param_sweep.py
           name: Parameter sweep

.gitignore

@@ -1,34 +1,40 @@
-.idea
-.ipynb_checkpoints
-.DS_Store
-.idea
-.eggs
-.pytest_cache
-.mypy_cache
+*.idea
+*.vscode
+*.ipynb_checkpoints
+*.DS_Store
+*.eggs
+*.pytest_cache
+*.mypy_cache
 *.csv
 *.egg-info
-*.ipynb_checkpoints
 *.sqlite3
 *.pyc
-venv
+*venv
+*external_data
+*sys_exec.py
+*/__pycache__
+*.DS_Store
+*notes.*
+*announcement.md
+*dist.py
+*/poc
 
 __pycache__
 Pipfile
 Pipfile.lock
-
 build
 results
 result
 notebooks
 client_work
 monkeytype
+venv
+notes.*
+announcement.md
+
 
 simulations/tickets
 simulations/validation
+simulations/poc/local/
 simulations/regression_tests/poc
 simulations/regression_tests/poc_configs
-
-*/poc
-
-notes.*
-announcement.md

CHANGELOG.md

@@ -7,6 +7,7 @@
         * Multi - System Model simulation results will no longer return truncated results (exclude the results of the last 
         `cadCAD.configuration.Configuration` appended to `cadCAD.configs`).
         * Issue: [#195](https://github.com/cadCAD-org/cadCAD/issues/195)
+    * Parameter Sweep value `M` (Params) requires up to a maximum of 2 distinct lengths
 
 
 ### August 5, 2020

testing/generic_test.py

@@ -1,10 +1,7 @@
 import unittest
-from pprint import pprint
-
-from parameterized import parameterized
 from functools import reduce
-
 from tabulate import tabulate
+from parameterized import parameterized
 
 
 def generate_assertions_df(df, expected_results, target_cols, evaluations):
@@ -31,8 +28,6 @@ def wrapped_eval(a, b):
 
 def make_generic_test(params):
     class TestSequence(unittest.TestCase):
-        # pprint(params)
-
         def generic_test(self, tested_df, expected_reults, test_name):
             erroneous = tested_df[(tested_df[test_name] == False)]
             print(tabulate(tested_df, headers='keys', tablefmt='psql'))

testing/models/param_sweep.py

@@ -1,9 +1,8 @@
 import pprint
 from typing import Dict, List
 
-# from cadCAD.configuration import append_configs
+from cadCAD.configuration import Experiment
 from cadCAD.configuration.utils import env_trigger, var_substep_trigger, config_sim, psub_list
-from testing.experiments import exp_param_sweep
 
 pp = pprint.PrettyPrinter(indent=4)
 
@@ -54,14 +53,14 @@ def policies(_g, step, sL, s, _input, **kwargs):
 def sweeped(_g, step, sL, s, _input, **kwargs):
     return 'sweeped', {'beta': _g['beta'], 'gamma': _g['gamma']}
 
-psu_block = {k: {"policies": {}, "variables": {}} for k in psu_steps}
+psu_block = {k: {"policies": {}, "states": {}} for k in psu_steps}
 for m in psu_steps:
     psu_block[m]['policies']['gamma'] = gamma
     psu_block[m]['policies']['omega'] = omega
-    psu_block[m]["variables"]['alpha'] = alpha
-    psu_block[m]["variables"]['beta'] = beta
-    psu_block[m]['variables']['policies'] = policies
-    psu_block[m]["variables"]['sweeped'] = var_timestep_trigger(y='sweeped', f=sweeped)
+    psu_block[m]["states"]['alpha'] = alpha
+    psu_block[m]["states"]['beta'] = beta
+    psu_block[m]['states']['policies'] = policies
+    psu_block[m]["states"]['sweeped'] = var_timestep_trigger(y='sweeped', f=sweeped)
 
 
 # Genesis States
@@ -86,7 +85,9 @@ def sweeped(_g, step, sL, s, _input, **kwargs):
 
 # New Convention
 partial_state_update_blocks = psub_list(psu_block, psu_steps)
-exp_param_sweep.append_configs(
+
+exp = Experiment()
+exp.append_model(
     sim_configs=sim_config,
     initial_state=genesis_states,
     env_processes=env_process,

testing/models/policy_aggregation.py

@@ -1,10 +1,7 @@
+from cadCAD.configuration import Experiment
 from cadCAD.configuration.utils import config_sim
 
-
 # Policies per Mechanism
-from testing.experiments import exp_policy_agg
-
-
 def p1m1(_g, step, sL, s, **kwargs):
     return {'policy1': 1}
 def p2m1(_g, step, sL, s, **kwargs):
@@ -74,8 +71,8 @@ def policies(_g, step, sH, s, _input, **kwargs):
     }
 )
 
-
-exp_policy_agg.append_configs(
+exp = Experiment()
+exp.append_model(
     sim_configs=sim_config,
     initial_state=genesis_states,
     partial_state_update_blocks=partial_state_update_block,

testing/results_comparison.py

@@ -0,0 +1,95 @@
+import unittest
+import pandas as pd
+from tabulate import tabulate
+from pandas._testing import assert_frame_equal
+
+
+def compare_results(result_diff):
+    class CompareResults(unittest.TestCase):
+        def setUp(self):
+            result_diff.columns
+            self.val_error_status_ind = False not in result_diff.val_error_status.tolist()
+            self.type_error_status_ind = False not in result_diff.type_error_status.tolist()
+            self.df_out = None
+            self.erroneous_indexes = None
+            self.select_result_diff = None
+
+            if (self.val_error_status_ind is False) or (self.type_error_status_ind is False):
+                self.erroneous_indexes = list(result_diff.index[result_diff["val_error_status"] == False])
+                self.select_result_diff = result_diff.iloc[self.erroneous_indexes]
+                self.df_out = tabulate(self.select_result_diff, headers='keys', tablefmt='psql')
+
+        def test_val_error_status(self):
+            print(self.df_out)
+            self.assertEqual(self.val_error_status_ind, True, "Value Error")
+
+        def test_type_error_status(self):
+            print(self.df_out)
+            self.assertEqual(self.type_error_status_ind, True, "Type Error")
+
+    return CompareResults
+
+
+def dataframe_difference(df1, df2):
+    def discrepancies(row):
+        return {
+            col: {'values': list(vals), 'type': [type(v) for v in vals]} for col, vals in row.items()
+            if vals[0] != vals[1]
+        }
+
+    def val_error_status(val):
+        if type(val) is dict and len(val) != 0:
+            return False
+        else:
+            return True
+
+    def type_error_status(val):
+        if type(val) is dict and len(val) != 0:
+            return list(set([v['type'][0] == v['type'][1] for k, v in val.items()]))[0]
+        else:
+            return True
+
+    df1_cols, df2_cols = list(df1.columns), list(df2.columns)
+    if set(df1_cols) == set(df2_cols) and df1.shape == df2.shape:
+        df_equal_ind = True
+        try:
+            assert_frame_equal(df1, df2)
+        except:
+            df_equal_ind = False
+
+        if df_equal_ind is False:
+            data = [list(zip(df1[col], df2[col])) for col in df1_cols]
+
+            df = pd.DataFrame(data).T
+            df.columns = df1_cols
+            df['discrepancies'] = df.apply(discrepancies, axis=1)
+
+            result_df1 = df1.reset_index()
+            discrepancies_df = df[['discrepancies']].reset_index()
+            result_diff = result_df1.merge(discrepancies_df, how='left', on='index')
+            result_diff['val_error_status'] = result_diff['discrepancies'].apply(val_error_status)
+            result_diff['type_error_status'] = result_diff['discrepancies'].apply(type_error_status)
+
+            return result_diff
+        else:
+            df1['discrepancies'] = None
+            df1['val_error_status'] = True
+            df1['type_error_status'] = True
+            return df1
+    else:
+        df1_row_count, df1_col_count = df1.shape
+        df2_row_count, df2_col_count = df2.shape
+        raise Exception(f"""
+        DataFrames have mismatched dimensions or columns:
+
+            Columns:
+                * df1: {df1_cols}
+                * df2: {df2_cols}     
+            Shapes:
+                * df1: 
+                    *    Row Count: {df1_row_count}
+                    * Column Count: {df1_col_count}
+                * df2:
+                    *    Row Count: {df2_row_count}
+                    * Column Count: {df2_col_count}
+        """)

testing/tests/dev/compare_results_dev.py

@@ -0,0 +1,40 @@
+import unittest
+import pandas as pd
+from copy import deepcopy
+
+from testing.models import param_sweep
+from testing.results_comparison import dataframe_difference, compare_results
+from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
+
+exec_mode = ExecutionMode()
+exec_ctx = ExecutionContext(context=exec_mode.local_mode)
+run = Executor(exec_context=exec_ctx, configs=param_sweep.exp.configs)
+
+raw_result, tensor_fields, sessions = run.execute()
+result = pd.DataFrame(raw_result)
+# print(tabulate(tensor_fields[0], headers='keys', tablefmt='psql'))
+# pprint(sessions)
+# print(tabulate(result, headers='keys', tablefmt='psql'))
+
+
+result_1 = result
+result_2 = deepcopy(result)
+result_df1 = pd.DataFrame({'a': [1, 2], 'b': [3, 5]})
+result_df2 = pd.DataFrame({'a': ['hi', 2], 'b': [3.0, 4.0]})
+
+# print(result_df1.shape)
+# exit()
+
+
+
+equivalent_result_diff = dataframe_difference(result_1, result_2)
+different_result_diff = dataframe_difference(result_df1, result_df2)
+
+class dfCompareTest(compare_results(different_result_diff)):
+    pass
+
+class EquivalentTest(compare_results(equivalent_result_diff)):
+    pass
+
+if __name__ == '__main__':
+    unittest.main()

testing/tests/dev/compare_results_old.py

@@ -0,0 +1,79 @@
+from copy import deepcopy
+
+import pandas as pd
+import numpy as np
+# import pandasql
+# from tabulate import tabulate
+from tabulate import tabulate
+
+from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
+from testing.models import param_sweep
+
+exec_mode = ExecutionMode()
+exec_ctx = ExecutionContext(context=exec_mode.local_mode)
+run = Executor(exec_context=exec_ctx, configs=param_sweep.exp.configs)
+
+raw_result, tensor_fields, sessions = run.execute()
+result = pd.DataFrame(raw_result)
+# print(tabulate(tensor_fields[0], headers='keys', tablefmt='psql'))
+# pprint(sessions)
+# print(tabulate(result, headers='keys', tablefmt='psql'))
+
+# result_1 = result
+# result_2 = deepcopy(result)
+
+# test_df1 = pd.testing.assert_frame_equal(result_1, result_2)
+# print(tabulate(test_df1, headers='keys', tablefmt='psql'))
+
+result_df = pd.DataFrame({'a': [1, 2], 'b': [3, 5]}).reset_index()
+df2 = pd.DataFrame({'a': [3.1, 2], 'b': [3.0, 4.0]}).reset_index()
+
+# test_df2 = pd.testing.assert_frame_equal(df1, df2)
+# print(tabulate(test_df2, headers='keys', tablefmt='psql'))
+
+def dataframe_difference(df1: pd.DataFrame, df2: pd.DataFrame, which=None):
+    """
+        Find rows which are different between two DataFrames.
+        https://hackersandslackers.com/compare-rows-pandas-dataframes/
+    """
+    comparison_df = df1.merge(
+        df2,
+        indicator=True,
+        how='outer'
+    )
+    if which is None:
+        diff_df = comparison_df[comparison_df['_merge'] != 'both']
+    else:
+        diff_df = comparison_df[comparison_df['_merge'] == which]
+    # diff_df.to_csv('data/diff.csv')
+    return diff_df
+
+
+merge_df = dataframe_difference(result_df, df2)
+cols_no__merge = list(filter(lambda col: '_merge' not in col, merge_df.columns.tolist()))
+cols_no_index = list(filter(lambda col: 'index' not in col, cols_no__merge))
+aggregation = dict((k, 'unique') for k in cols_no_index)
+diff_df = merge_df[cols_no__merge].groupby('index').agg(aggregation)
+# print(tabulate(diff_df, headers='keys', tablefmt='psql'))
+
+
+def discrepancies(row):
+    return dict([
+        (col, list(vals)) for col, vals in row.items()
+            if type(vals) is np.ndarray and len(vals) > 1
+    ])
+
+
+def val_error(val):
+    if type(val) is dict:
+        return False
+    else:
+        return True
+
+
+diff_df['discrepancies'] = diff_df.apply(discrepancies, axis=1)
+discrepancies_df = diff_df[['discrepancies']]
+result_diff = result_df.merge(discrepancies_df, how='left', on='index')
+result_diff['val_error'] = result_diff['discrepancies'].apply(val_error)
+print(tabulate(result_diff, headers='keys', tablefmt='psql'))
+