From 4bc96bd3868393153ec251aeb59597aaf4fe4e18 Mon Sep 17 00:00:00 2001 From: Will Powelson Date: Tue, 27 Oct 2020 12:43:38 -0700 Subject: [PATCH 1/5] add failing tests --- beep/tests/test_protocol.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/beep/tests/test_protocol.py b/beep/tests/test_protocol.py index 77c6e516..3944e148 100644 --- a/beep/tests/test_protocol.py +++ b/beep/tests/test_protocol.py @@ -530,7 +530,7 @@ def test_from_csv(self): with ScratchDir(".") as scratch_dir: makedirs_p(os.path.join(scratch_dir, "procedures")) makedirs_p(os.path.join(scratch_dir, "names")) - new_files, result, message = generate_protocol_files_from_csv( + _new_files, _generation_failures, result, message = generate_protocol_files_from_csv( csv_file, output_directory=scratch_dir ) self.assertEqual( @@ -543,7 +543,7 @@ def test_from_csv(self): makedirs_p(os.path.join(scratch_dir, "procedures")) makedirs_p(os.path.join(scratch_dir, "names")) dumpfn({"hello": "world"}, os.path.join("procedures", "name_000007.000")) - new_files, result, message = generate_protocol_files_from_csv( + _new_files, generation_failures, result, message = generate_protocol_files_from_csv( csv_file, output_directory=scratch_dir ) post_file = loadfn(os.path.join("procedures", "name_000007.000")) @@ -555,8 +555,15 @@ def test_from_csv(self): self.assertEqual( message, { - "comment": "Unable to find template: EXP-D3.000", - "error": "Not Found", + "comment": "Generated 2 of 32 protocols", + "error": "Failed to generate 30 of 32 protocols", + }, + ) + self.assertEqual( + generation_failures[0], + { + "comment": "Unable to find template: EXP-SOH.000", + "error": "Not Found" }, ) @@ -567,11 +574,11 @@ def test_from_csv_2(self): with ScratchDir(".") as scratch_dir: makedirs_p(os.path.join(scratch_dir, "procedures")) makedirs_p(os.path.join(scratch_dir, "names")) - new_files, result, message = generate_protocol_files_from_csv( + _new_files, _generation_failures, result, message = generate_protocol_files_from_csv( csv_file, output_directory=scratch_dir ) self.assertEqual(result, "success") - self.assertEqual(message, {"comment": "Generated 2 protocols", "error": ""}) + self.assertEqual(message, {"comment": "Generated 2 of 2 protocols", "error": ""}) self.assertEqual( len(os.listdir(os.path.join(scratch_dir, "procedures"))), 2 ) @@ -644,7 +651,7 @@ def test_console_script(self): csv_file = os.path.join(TEST_FILE_DIR, "parameter_test.csv") # Test script functionality - with ScratchDir(".") as scratch_dir: + with ScratchDir(".") as _scratch_dir: # Set BEEP_PROCESSING_DIR directory to scratch_dir os.environ["BEEP_PROCESSING_DIR"] = os.getcwd() procedures_path = os.path.join("data-share", "protocols", "procedures") From bd217ade2c206986e82947f007cfcab97cf308a4 Mon Sep 17 00:00:00 2001 From: Will Powelson Date: Tue, 27 Oct 2020 12:55:36 -0700 Subject: [PATCH 2/5] return failures and successes for each conversion --- beep/generate_protocol.py | 48 ++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/beep/generate_protocol.py b/beep/generate_protocol.py index a3cba7f8..81a01dc4 100644 --- a/beep/generate_protocol.py +++ b/beep/generate_protocol.py @@ -82,7 +82,8 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None): # Read csv file protocol_params_df = pd.read_csv(csv_filename) - new_files = [] + successfully_generated_files = [] + file_generation_failures = [] names = [] result = "" message = {"comment": "", "error": ""} @@ -167,27 +168,29 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None): filename = os.path.join(output_directory, "settings", filename) else: - warnings.warn("Unsupported file template {}, skipping.".format(template)) - result = "error" - message = { + failure = { "comment": "Unable to find template: " + template, "error": "Not Found", } + file_generation_failures.append(failure) + warnings.warn("Unsupported file template {}, skipping.".format(template)) + result = "error" continue logger.info(filename, extra=s) if not os.path.isfile(filename): protocol.to_file(filename) - new_files.append(filename) + successfully_generated_files.append(filename) names.append(filename_prefix + "_") elif ".sdu" in template: + failure = { + "comment": "Unable to find template: " + template, + "error": "Unsupported Template", + } + file_generation_failures.append(failure) logger.warning("Schedule file generation not yet implemented", extra=s) result = "error" - message = { - "comment": "Schedule file generation is not yet implemented", - "error": "Not Implemented", - } # This block of code produces the file containing all of the run file # names produced in this function call. This is to make starting tests easier @@ -202,14 +205,22 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None): wr.writerow([name]) outputfile.close() + num_generated_files = len(successfully_generated_files) + num_generation_failures = len(file_generation_failures) + num_files = num_generated_files + num_generation_failures + + message = { + "comment": "Generated {} of {} protocols".format(num_generated_files, num_files), + "error": "" + } if not result: result = "success" - message = { - "comment": "Generated {} protocols".format(str(len(new_files))), - "error": "", - } + else: + message["error"] = "Failed to generate {} of {} protocols".format(num_generation_failures, num_files) + + - return new_files, result, message + return successfully_generated_files, file_generation_failures, result, message def process_csv_file_list_from_json( @@ -240,12 +251,17 @@ def process_csv_file_list_from_json( os.environ.get("BEEP_PROCESSING_DIR", "/"), processed_dir ) for filename in file_list: - output_files, result, message = generate_protocol_files_from_csv( + output_files, file_generation_failures, result, message = generate_protocol_files_from_csv( filename, output_directory=protocol_dir ) all_output_files.extend(output_files) - output_data = {"file_list": all_output_files, "result": result, "message": message} + output_data = { + "file_list": all_output_files, + "failures": file_generation_failures, + "result": result, + "message": message + } # Workflow outputs outputs.put_generate_outputs_list(output_data, "complete") From e5d98991ea140a4b29dfab1b973fc362506cc56f Mon Sep 17 00:00:00 2001 From: Will Powelson Date: Wed, 28 Oct 2020 09:34:57 -0700 Subject: [PATCH 3/5] whitespace --- beep/generate_protocol.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/beep/generate_protocol.py b/beep/generate_protocol.py index 81a01dc4..208f84cb 100644 --- a/beep/generate_protocol.py +++ b/beep/generate_protocol.py @@ -218,8 +218,6 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None): else: message["error"] = "Failed to generate {} of {} protocols".format(num_generation_failures, num_files) - - return successfully_generated_files, file_generation_failures, result, message From d5cc11bf43bbcdc6da72c59b9ae741ef8cbf46f6 Mon Sep 17 00:00:00 2001 From: Will Powelson Date: Thu, 29 Oct 2020 08:33:47 -0700 Subject: [PATCH 4/5] correctly log unsupported file --- beep/generate_protocol.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beep/generate_protocol.py b/beep/generate_protocol.py index 208f84cb..23685243 100644 --- a/beep/generate_protocol.py +++ b/beep/generate_protocol.py @@ -184,9 +184,9 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None): names.append(filename_prefix + "_") elif ".sdu" in template: - failure = { - "comment": "Unable to find template: " + template, - "error": "Unsupported Template", + failure = { + "comment": "Schedule file generation is not yet implemented", + "error": "Not Implemented" } file_generation_failures.append(failure) logger.warning("Schedule file generation not yet implemented", extra=s) From 207f222ea379eac6d83fe6f1bc364fabcd98d85a Mon Sep 17 00:00:00 2001 From: Will Powelson Date: Tue, 3 Nov 2020 11:30:40 -0800 Subject: [PATCH 5/5] add error logging --- beep/generate_protocol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beep/generate_protocol.py b/beep/generate_protocol.py index 23685243..5d4e3356 100644 --- a/beep/generate_protocol.py +++ b/beep/generate_protocol.py @@ -217,6 +217,7 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None): result = "success" else: message["error"] = "Failed to generate {} of {} protocols".format(num_generation_failures, num_files) + logger.error(message["error"]) return successfully_generated_files, file_generation_failures, result, message