Merge pull request #140 from willpowelson/MAT-2014

MAT-2014 Handles partial success poorly

beep/generate_protocol.py

@@ -82,7 +82,8 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None):
     # Read csv file
     protocol_params_df = pd.read_csv(csv_filename)
 
-    new_files = []
+    successfully_generated_files = []
+    file_generation_failures = []
     names = []
     result = ""
     message = {"comment": "", "error": ""}
@@ -167,27 +168,29 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None):
             filename = os.path.join(output_directory, "settings", filename)
 
         else:
-            warnings.warn("Unsupported file template {}, skipping.".format(template))
-            result = "error"
-            message = {
+            failure = {
                 "comment": "Unable to find template: " + template,
                 "error": "Not Found",
             }
+            file_generation_failures.append(failure)
+            warnings.warn("Unsupported file template {}, skipping.".format(template))
+            result = "error"
             continue
 
         logger.info(filename, extra=s)
         if not os.path.isfile(filename):
             protocol.to_file(filename)
-            new_files.append(filename)
+            successfully_generated_files.append(filename)
             names.append(filename_prefix + "_")
 
         elif ".sdu" in template:
+            failure = {	
+                "comment": "Schedule file generation is not yet implemented",	
+                "error": "Not Implemented"
+            }
+            file_generation_failures.append(failure)
             logger.warning("Schedule file generation not yet implemented", extra=s)
             result = "error"
-            message = {
-                "comment": "Schedule file generation is not yet implemented",
-                "error": "Not Implemented",
-            }
 
     # This block of code produces the file containing all of the run file
     # names produced in this function call. This is to make starting tests easier
@@ -202,14 +205,21 @@ def generate_protocol_files_from_csv(csv_filename, output_directory=None):
             wr.writerow([name])
     outputfile.close()
 
+    num_generated_files = len(successfully_generated_files)
+    num_generation_failures = len(file_generation_failures)
+    num_files = num_generated_files + num_generation_failures
+
+    message = {
+        "comment": "Generated {} of {} protocols".format(num_generated_files, num_files),
+        "error": ""
+    }
     if not result:
         result = "success"
-        message = {
-            "comment": "Generated {} protocols".format(str(len(new_files))),
-            "error": "",
-        }
+    else:
+        message["error"] = "Failed to generate {} of {} protocols".format(num_generation_failures, num_files)
+        logger.error(message["error"])
 
-    return new_files, result, message
+    return successfully_generated_files, file_generation_failures, result, message
 
 
 def process_csv_file_list_from_json(
@@ -240,12 +250,17 @@ def process_csv_file_list_from_json(
         os.environ.get("BEEP_PROCESSING_DIR", "/"), processed_dir
     )
     for filename in file_list:
-        output_files, result, message = generate_protocol_files_from_csv(
+        output_files, file_generation_failures, result, message = generate_protocol_files_from_csv(
             filename, output_directory=protocol_dir
         )
         all_output_files.extend(output_files)
 
-    output_data = {"file_list": all_output_files, "result": result, "message": message}
+    output_data = {
+        "file_list": all_output_files,
+        "failures": file_generation_failures,
+        "result": result,
+        "message": message
+    }
 
     # Workflow outputs
     outputs.put_generate_outputs_list(output_data, "complete")

beep/tests/test_protocol.py

@@ -530,7 +530,7 @@ def test_from_csv(self):
         with ScratchDir(".") as scratch_dir:
             makedirs_p(os.path.join(scratch_dir, "procedures"))
             makedirs_p(os.path.join(scratch_dir, "names"))
-            new_files, result, message = generate_protocol_files_from_csv(
+            _new_files, _generation_failures, result, message = generate_protocol_files_from_csv(
                 csv_file, output_directory=scratch_dir
             )
             self.assertEqual(
@@ -543,7 +543,7 @@ def test_from_csv(self):
             makedirs_p(os.path.join(scratch_dir, "procedures"))
             makedirs_p(os.path.join(scratch_dir, "names"))
             dumpfn({"hello": "world"}, os.path.join("procedures", "name_000007.000"))
-            new_files, result, message = generate_protocol_files_from_csv(
+            _new_files, generation_failures, result, message = generate_protocol_files_from_csv(
                 csv_file, output_directory=scratch_dir
             )
             post_file = loadfn(os.path.join("procedures", "name_000007.000"))
@@ -555,8 +555,15 @@ def test_from_csv(self):
             self.assertEqual(
                 message,
                 {
-                    "comment": "Unable to find template: EXP-D3.000",
-                    "error": "Not Found",
+                    "comment": "Generated 2 of 32 protocols",
+                    "error": "Failed to generate 30 of 32 protocols",
+                },
+            )
+            self.assertEqual(
+                generation_failures[0],
+                {
+                    "comment": "Unable to find template: EXP-SOH.000",
+                    "error": "Not Found"
                 },
             )
 
@@ -567,11 +574,11 @@ def test_from_csv_2(self):
         with ScratchDir(".") as scratch_dir:
             makedirs_p(os.path.join(scratch_dir, "procedures"))
             makedirs_p(os.path.join(scratch_dir, "names"))
-            new_files, result, message = generate_protocol_files_from_csv(
+            _new_files, _generation_failures, result, message = generate_protocol_files_from_csv(
                 csv_file, output_directory=scratch_dir
             )
             self.assertEqual(result, "success")
-            self.assertEqual(message, {"comment": "Generated 2 protocols", "error": ""})
+            self.assertEqual(message, {"comment": "Generated 2 of 2 protocols", "error": ""})
             self.assertEqual(
                 len(os.listdir(os.path.join(scratch_dir, "procedures"))), 2
             )
@@ -644,7 +651,7 @@ def test_console_script(self):
         csv_file = os.path.join(TEST_FILE_DIR, "parameter_test.csv")
 
         # Test script functionality
-        with ScratchDir(".") as scratch_dir:
+        with ScratchDir(".") as _scratch_dir:
             # Set BEEP_PROCESSING_DIR directory to scratch_dir
             os.environ["BEEP_PROCESSING_DIR"] = os.getcwd()
             procedures_path = os.path.join("data-share", "protocols", "procedures")