Added date to generate filename (#538)

* Minor correction to tests (#537)

* Added datetime option on generate filenames

* Fixed missing exclude_dirs in bids_file_group bug

* Corrected the hed type summary format

hed/tools/analysis/hed_type_factors.py

@@ -74,8 +74,8 @@ def get_summary(self, full=True):
         number_events, number_multiple, max_multiple = self.count_events(count_list)
         summary = {'name': self.variable_value, 'variable_type': self.variable_type, 'levels': len(self.levels.keys()),
                    'direct_references': len(self.direct_indices.keys()),
-                   'total_events': self.number_elements, 'number_type_events': number_events,
-                   'number_multiple_events': number_multiple, 'multiple_event_maximum': max_multiple}
+                   'total_events': self.number_elements, 'events': number_events,
+                   'multiple_events': number_multiple, 'multiple_event_maximum': max_multiple}
         if full:
             summary['level_counts'] = self._get_level_counts()
         return summary

hed/tools/analysis/hed_variable_summary.py

@@ -18,8 +18,8 @@ def __init__(self, variable_value, variable_type):
         self.variable_type = variable_type.lower()
         self.direct_references = 0
         self.total_events = 0
-        self.number_type_events = 0
-        self.number_multiple_events = 0
+        self.events = 0
+        self.multiple_events = 0
         self.multiple_event_maximum = 0
         self.level_counts = {}
 
@@ -33,10 +33,10 @@ def update(self, variable_info):
         var_sum = variable_info.get_summary(full=True)
         self.direct_references += var_sum['direct_references']
         self.total_events += var_sum['total_events']
-        self.number_type_events += var_sum['number_type_events']
-        self.number_multiple_events += var_sum['number_multiple_events']
+        self.events += var_sum['events']
+        self.multiple_events += var_sum['multiple_events']
         self.multiple_event_maximum = max(self.multiple_event_maximum, var_sum['multiple_event_maximum'])
-        self._update_levels(var_sum['level_counts'])
+        self._update_levels(var_sum.get('level_counts', {}))
 
     def _update_levels(self, level_dict):
         for key, item in level_dict.items():
@@ -51,10 +51,11 @@ def get_summary(self, as_json=False):
                    'levels': len(self.level_counts.keys()),
                    'direct_references': self.direct_references,
                    'total_events': self.total_events,
-                   'number_type_events': self.number_type_events,
-                   'number_multiple_events': self.number_multiple_events,
-                   'multiple_event_maximum': self.multiple_event_maximum,
-                   'level_counts': self.level_counts}
+                   'events': self.events,
+                   'multiple_events': self.multiple_events,
+                   'multiple_event_maximum': self.multiple_event_maximum}
+        if self.level_counts:
+            summary['level_counts'] = self.level_counts
         if as_json:
             return json.dumps(summary, indent=4)
         return summary

hed/tools/bids/bids_file_group.py

@@ -207,7 +207,8 @@ def _make_sidecar_dir_dict(self):
             dict: A dictionary of lists of sidecar BidsSidecarFiles
 
         """
-        dir_dict = get_dir_dictionary(self.root_path, name_suffix=self.suffix, extensions=['.json'])
+        dir_dict = get_dir_dictionary(self.root_path, name_suffix=self.suffix, extensions=['.json'],
+                                      exclude_dirs=self.exclude_dirs)
         sidecar_dir_dict = {}
         for this_dir, dir_list in dir_dict.items():
             new_dir_list = []

hed/tools/remodeling/cli/run_remodel_backup.py

@@ -11,7 +11,7 @@ def get_parser():
                         help="Directories names to exclude from search for files." +
                              "Note data_dir/remodel/backup will always be excluded.")
     parser.add_argument("-f", "--file-suffix", dest="file_suffix", default='events',
-                        help="Filename suffix including file type.")
+                        help="Filename suffix of files to be backed up.")
     parser.add_argument("-e", "--extensions", nargs="*", default=['.tsv'], dest="extensions",
                         help="File extensions to allow in locating files.")
     parser.add_argument("-w", "--write-over-backup", action='store_true',

hed/tools/remodeling/operations/base_context.py

@@ -1,7 +1,7 @@
 import os
 from abc import ABC
-from datetime import datetime
 import json
+from hed.tools.util.io_util import generate_filename
 
 
 class BaseContext(ABC):
@@ -55,8 +55,7 @@ def get_text_summary(self, title='', verbose=True):
     def save(self, save_dir, file_formats, verbose=True):
         if not file_formats:
             return
-        now = datetime.now()
-        file_base = os.path.join(save_dir, self.context_filename) + '_' + now.strftime('%Y_%m_%d_T_%H_%M_%S_%f')
+        file_base = os.path.join(save_dir, generate_filename(self.context_filename, append_datetime=True))
         for file_format in file_formats:
             if file_format == '.txt':
                 summary = self.get_text_summary(verbose=verbose)

hed/tools/remodeling/operations/split_event_op.py

@@ -82,5 +82,6 @@ def do_op(self, dispatcher, df, name, sidecar=None):
             df_list.append(add_events)
 
         df_new = pd.concat(df_list, axis=0, ignore_index=True)
+        df_new["onset"] = df_new["onset"].apply(pd.to_numeric)
         df_new = df_new.sort_values('onset').reset_index(drop=True)
         return df_new

hed/tools/util/io_util.py

@@ -2,6 +2,7 @@
 
 import os
 import shutil
+from datetime import datetime
 from pathlib import Path
 from werkzeug.utils import secure_filename
 from hed.errors import HedFileError
@@ -71,7 +72,7 @@ def find_task_files(bids_dir, task=None, suffix='*_events.tsv'):
         return [path for path in Path(bids_dir).rglob(suffix) if task in str(path)]
 
 
-def generate_filename(base_name, name_prefix=None, name_suffix=None, extension=None):
+def generate_filename(base_name, name_prefix=None, name_suffix=None, extension=None, append_datetime=False):
     """ Generate a filename for the attachment.
 
     Args:
@@ -96,8 +97,12 @@ def generate_filename(base_name, name_prefix=None, name_suffix=None, extension=N
     if name_suffix:
         pieces = pieces + [name_suffix]
     filename = "".join(pieces)
+    if append_datetime:
+        now = datetime.now()
+        filename = filename + '_' + now.strftime('%Y_%m_%d_T_%H_%M_%S_%f')
     if filename and extension:
         filename = filename + extension
+
     return secure_filename(filename)
 
 

tests/data/remodeling/only_splitevents_example.json

@@ -0,0 +1,16 @@
+[
+   {
+       "command": "split_event",
+       "description": "Create separate response event from response time column.",
+       "parameters": {
+           "anchor_column": "event_type",
+           "new_events": {"response": {"onset_source": ["response_time"],
+                                       "duration": [0],
+                                       "copy_columns": ["response_accuracy",
+                                                        "response_hand",
+                                                        "trial_type"]}},
+           "remove_parent_event": false
+       }
+   }
+]
+

tests/data/remodeling/sub-0013_task-stopsignal_acq-seq_events.tsv

@@ -0,0 +1,101 @@
+onset	duration	trial_type	stop_signal_delay	response_time	response_accuracy	response_hand	sex
+0.0776	0.5083	go	n/a	0.565	correct	right	female
+5.5774	0.5083	unsuccesful_stop	0.2	0.49	correct	right	female
+9.5856	0.5084	go	n/a	0.45	correct	right	female
+13.5939	0.5083	succesful_stop	0.2	n/a	n/a	right	female
+17.1021	0.5083	unsuccesful_stop	0.25	0.633	correct	left	male
+21.6103	0.5083	go	n/a	0.443	correct	left	male
+24.6186	0.5083	go	n/a	0.439	correct	left	male
+28.6268	0.5083	go	n/a	0.667	correct	left	male
+32.1434	0.5083	go	n/a	0.55	correct	right	female
+36.1516	0.5083	succesful_stop	0.25	n/a	n/a	right	female
+41.6514	0.5084	go	n/a	0.59	correct	right	female
+44.6597	0.5083	unsuccesful_stop	0.3	0.511	correct	right	female
+49.6679	0.5083	go	n/a	0.604	correct	right	female
+52.1845	0.5083	go	n/a	0.743	correct	left	male
+56.1927	0.5084	succesful_stop	0.3	n/a	n/a	right	female
+60.6926	0.5083	unsuccesful_stop	0.35	0.555	correct	left	male
+65.7008	0.5083	go	n/a	0.584	correct	right	female
+73.7173	0.5083	succesful_stop	0.35	n/a	n/a	right	female
+76.7255	0.5083	succesful_stop	0.4	n/a	n/a	right	male
+81.2337	0.5084	go	n/a	0.615	correct	left	male
+84.742	0.5083	go	n/a	0.754	correct	left	male
+89.2502	0.5083	go	n/a	0.777	correct	right	female
+92.2668	0.5083	go	n/a	0.644	correct	right	female
+97.2666	0.5084	unsuccesful_stop	0.45	0.629	correct	right	female
+100.2832	0.5083	go	n/a	0.714	correct	right	female
+104.7831	0.5083	go	n/a	0.627	correct	left	male
+108.2997	0.5083	go	n/a	0.668	correct	left	male
+113.2995	0.5084	go	n/a	0.558	correct	left	male
+117.3078	0.5083	go	n/a	1.038	incorrect	left	female
+120.816	0.5083	go	n/a	0.764	correct	left	male
+125.8242	0.5083	go	n/a	0.782	correct	right	female
+129.3325	0.5083	unsuccesful_stop	0.5	0.722	correct	left	male
+132.8407	0.5083	go	n/a	0.716	correct	right	female
+137.8489	0.5083	go	n/a	0.741	correct	right	female
+141.3571	0.5084	succesful_stop	0.5	n/a	n/a	right	male
+145.8653	0.5084	go	n/a	1.027	correct	right	female
+149.3736	0.5083	go	n/a	0.881	correct	left	male
+153.3818	0.5083	go	n/a	0.801	correct	right	female
+157.89	0.5084	go	n/a	0.803	correct	left	male
+160.8983	0.5083	go	n/a	0.771	correct	right	female
+164.4149	0.5083	succesful_stop	0.55	n/a	n/a	right	female
+169.4147	0.5083	go	n/a	0.899	correct	left	male
+172.923	0.5083	unsuccesful_stop	0.6	0.754	correct	left	male
+176.9312	0.5083	go	n/a	1.11	correct	left	male
+180.4478	0.5083	succesful_stop	0.65	n/a	n/a	right	male
+188.9559	0.5083	unsuccesful_stop	0.7	0.867	correct	right	female
+193.4641	0.5083	unsuccesful_stop	0.75	0.814	correct	left	male
+197.4723	0.5083	go	n/a	1.21	correct	right	female
+201.4805	0.5084	go	n/a	0.859	correct	left	male
+204.9888	0.5083	unsuccesful_stop	0.75	0.973	correct	right	female
+212.5136	0.5083	go	n/a	1.02	correct	left	male
+221.5217	0.5083	go	n/a	0.817	correct	left	male
+225.5299	0.5083	go	n/a	1.038	correct	right	female
+228.5465	0.5083	go	n/a	1.049	correct	right	female
+234.0463	0.5084	go	n/a	0.92	correct	left	male
+237.0546	0.5083	succesful_stop	0.7	n/a	n/a	right	female
+241.0628	0.5083	go	n/a	1.266	correct	right	female
+245.071	0.5084	unsuccesful_stop	0.7	0.854	correct	right	female
+248.5876	0.5083	go	n/a	0.985	correct	left	male
+254.0875	0.5083	go	n/a	0.789	correct	right	female
+260.6123	0.5083	go	n/a	0.928	correct	right	female
+266.1122	0.5083	go	n/a	0.807	correct	left	male
+269.6204	0.5083	go	n/a	0.735	correct	left	male
+273.6286	0.5083	succesful_stop	0.65	n/a	n/a	right	male
+277.6368	0.5084	go	n/a	0.896	correct	right	female
+281.6451	0.5083	succesful_stop	0.65	n/a	n/a	right	female
+289.6615	0.5083	unsuccesful_stop	0.7	0.831	correct	right	female
+293.1698	0.5083	go	n/a	0.876	correct	left	male
+296.6863	0.5084	go	n/a	1.021	correct	right	female
+302.1862	0.5083	unsuccesful_stop	0.7	1.085	correct	left	male
+306.1944	0.5083	succesful_stop	0.65	n/a	n/a	right	female
+309.2027	0.5083	go	n/a	0.814	correct	right	female
+313.2109	0.5083	go	n/a	1.053	correct	left	male
+318.2191	0.5083	go	n/a	1.002	correct	left	male
+322.2273	0.5083	go	n/a	1.057	correct	right	female
+326.2355	0.5084	succesful_stop	0.65	n/a	n/a	right	male
+330.2438	0.5083	succesful_stop	0.7	n/a	n/a	right	male
+334.252	0.5083	go	n/a	0.962	correct	left	male
+341.2685	0.5083	go	n/a	0.817	correct	right	female
+346.2767	0.5083	unsuccesful_stop	0.75	0.822	correct	left	male
+350.2849	0.5083	go	n/a	0.889	correct	left	male
+353.2932	0.5083	go	n/a	0.946	correct	right	female
+358.3014	0.5083	go	n/a	0.911	correct	right	female
+360.818	0.5083	unsuccesful_stop	0.8	1.054	correct	left	male
+364.8262	0.5083	go	n/a	0.966	correct	right	female
+368.8344	0.5083	unsuccesful_stop	0.8	0.99	correct	right	female
+373.8343	0.5083	go	n/a	1.004	correct	right	female
+377.8425	0.5083	unsuccesful_stop	0.75	0.909	correct	left	male
+381.8507	0.5084	go	n/a	0.859	correct	left	male
+385.859	0.5083	go	n/a	1.186	correct	right	female
+389.3672	0.5083	go	n/a	1.288	correct	right	female
+393.3754	0.5083	go	n/a	0.979	correct	left	male
+398.3836	0.5084	go	n/a	1.067	correct	left	male
+400.9002	0.5083	succesful_stop	0.7	n/a	n/a	right	male
+409.4083	0.5084	go	n/a	0.901	correct	left	male
+414.4165	0.5084	unsuccesful_stop	0.65	0.879	correct	left	male
+418.4248	0.5083	go	n/a	1.003	correct	left	male
+422.433	0.5083	succesful_stop	0.6	n/a	n/a	right	female
+429.9495	0.5083	succesful_stop	0.55	n/a	n/a	right	female
+437.9659	0.5083	go	n/a	0.866	correct	left	male

tests/data/remodeling/summarize_hed_types_remdl.json

@@ -0,0 +1,11 @@
+[
+    {
+        "command": "summarize_hed_type",
+        "description": "Summarize a hed type tag such as condition-variable",
+        "parameters": {
+            "summary_name": "Hed type summary",
+            "summary_filename": "hed_type_summary",
+            "type_tag": "condition-variable"
+        }
+    }
+]

tests/tools/analysis/test_hed_type_factors.py

@@ -128,13 +128,13 @@ def test_get_summary(self):
         var_manager = HedTypeVariable(HedContextManager(hed_strings, self.schema), self.schema, definitions)
         var_key = var_manager.get_variable('key-assignment')
         sum_key = var_key.get_summary()
-        self.assertEqual(sum_key['number_type_events'], 200, "get_summary has right number of events")
+        self.assertEqual(sum_key['events'], 200, "get_summary has right number of events")
         self.assertEqual(sum_key['multiple_event_maximum'], 1, "Get_summary has right multiple maximum")
         self.assertIsInstance(sum_key['level_counts'], dict, "get_summary level counts is a dictionary")
         self.assertEqual(sum_key['level_counts']['right-sym-cond'], 200, "get_summary level counts value correct.")
         var_face = var_manager.get_variable('face-type')
         sum_key = var_face.get_summary()
-        self.assertEqual(sum_key['number_type_events'], 52, "get_summary has right number of events")
+        self.assertEqual(sum_key['events'], 52, "get_summary has right number of events")
         self.assertEqual(sum_key['multiple_event_maximum'], 1, "Get_summary has right multiple maximum")
         self.assertIsInstance(sum_key['level_counts'], dict, "get_summary level counts is a dictionary")
         self.assertEqual(sum_key['level_counts']['unfamiliar-face-cond'], 20, "get_summary level counts value correct.")

tests/tools/analysis/test_hed_variable_counts.py

@@ -54,14 +54,14 @@ def test_get_summary_multiple_levels(self):
         self.assertEqual(3, len(count1.level_counts))
         self.assertIn('unfamiliar-face-cond', count1.level_counts)
         self.assertEqual(200, count1.total_events)
-        self.assertEqual(52, count1.number_type_events)
+        self.assertEqual(52, count1.events)
         self.assertEqual(1, count1.level_counts['unfamiliar-face-cond']['files'])
         count1.update(var1)
         self.assertEqual(0, count1.direct_references, "get_summary")
         self.assertEqual(3, len(count1.level_counts))
         self.assertIn('unfamiliar-face-cond', count1.level_counts)
         self.assertEqual(400, count1.total_events)
-        self.assertEqual(104, count1.number_type_events)
+        self.assertEqual(104, count1.events)
         self.assertEqual(2, count1.level_counts['unfamiliar-face-cond']['files'])
 
 

tests/tools/analysis/test_hed_variable_summary.py

@@ -38,11 +38,11 @@ def test_get_summary(self):
         var_summary1.update_summary(self.var_type1)
         summary2 = var_summary1.get_summary(as_json=False)
         self.assertEqual(len(summary2), 3)
-        self.assertEqual(summary2['repetition-type']['number_type_events'], 52)
+        self.assertEqual(summary2['repetition-type']['events'], 52)
         var_summary1.update_summary(self.var_type1)
         summary3 = var_summary1.get_summary(as_json=False)
         self.assertEqual(len(summary3), 3)
-        self.assertEqual(summary3['repetition-type']['number_type_events'], 104)
+        self.assertEqual(summary3['repetition-type']['events'], 104)
 
 
 if __name__ == '__main__':

tests/tools/bids/test_bids_file_group.py

@@ -5,6 +5,7 @@
 from hed.tools.bids.bids_file_group import BidsFileGroup
 from hed.validator.hed_validator import HedValidator
 
+# TODO: Add test when exclude directories have files of the type needed (such as JSON in code directory).
 
 class Test(unittest.TestCase):
 

tests/tools/remodeling/operations/test_split_event_op.py

@@ -1,3 +1,4 @@
+import os
 import json
 import pandas as pd
 import numpy as np
@@ -18,6 +19,10 @@ class Test(unittest.TestCase):
     """
     @classmethod
     def setUpClass(cls):
+        cls.events_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                           '../../../data/remodeling/sub-0013_task-stopsignal_acq-seq_events.tsv'))
+        cls.model1_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                           '../../../data/remodeling/only_splitevents_example.json'))
         cls.sample_data = [[0.0776, 0.5083, 'go', 'n/a', 0.565, 'correct', 'right', 'female'],
                            [5.5774, 0.5083, 'unsuccesful_stop', 0.2, 0.49, 'correct', 'right', 'female'],
                            [9.5856, 0.5084, 'go', 'n/a', 0.45, 'correct', 'right', 'female'],
@@ -98,6 +103,33 @@ def test_split_event_valid_existing_column(self):
         self.assertTrue(np.array_equal(df.to_numpy(), df_test.to_numpy()),
                         "split_event should not change the input df values when existing column anchor")
 
+    def test_split_event_from_files(self):
+        # Test when existing column is used as anchor event
+        df = pd.read_csv(self.events_path, delimiter='\t', header=0, dtype=str, keep_default_na=False, na_values=None)
+        with open(self.model1_path) as fp:
+            command_list = json.load(fp)
+        commands, errors = Dispatcher.parse_commands(command_list)
+        self.assertFalse(errors, 'split_event should not give errors if command is correct')
+        dispatch = Dispatcher(command_list)
+        df = dispatch.prep_events(df)
+        for operation in dispatch.parsed_ops:
+            df_new = operation.do_op(dispatch, df, "Name")
+        df_new = df_new.fillna('n/a')
+        df_check = pd.read_csv(self.events_path, delimiter='\t', header=0, dtype=str,
+                               keep_default_na=False, na_values=None)
+        # parms = json.loads(self.json_parms)
+        # op = SplitEventOp(parms)
+        # df = pd.DataFrame(self.sample_data, columns=self.sample_columns)
+        # df_check = pd.DataFrame(self.split, columns=self.split_columns)
+        # df_test = pd.DataFrame(self.sample_data, columns=self.sample_columns)
+        # df_new = op.do_op(self.dispatch, df_test, name="sample_data")
+        # df_new = df_new.fillna('n/a')
+        #
+        # Test that this
+        self.assertEqual(len(df_check), len(df),
+                         "split_event should not change the length of the original dataframe")
+        self.assertEqual(len(df_check.columns), len(df.columns),
+                         "split_event should change the number of columns of the original dataframe")
 
 if __name__ == '__main__':
     unittest.main()