Merge pull request #15 from usdot-its-jpo-data-portal/ode-1264

ODE-1264 Show serialId and errors in slack message

.gitignore

@@ -1,3 +1,4 @@
 /package
 *.ini
 *.zip
+__pycache__/

main.py

@@ -69,6 +69,7 @@ def validate(local_test, context):
     total_validations_failed = 0
     records_analyzed = 0
     msg_queue = queue.Queue()
+    error_list = []
     for filename in s3_file_list:
         logger.info("============================================================================")
         logger.info("Analyzing file '%s'" % filename)
@@ -95,19 +96,16 @@ def validate(local_test, context):
             for validation in result.field_validations:
                 if validation.valid == False:
                     num_errors += 1
-                    validation_message = validation.details
-                    if validation_message in error_dict:
-                        error_dict[validation_message] += 1
-                    else:
-                        error_dict[validation_message] = 1
+                    error_dict[json.dumps(validation.serial_id)] = validation.details
 
         total_validation_count += num_validations
         total_validations_failed += num_errors
         if num_errors > 0:
             logger.error("Validation has FAILED for file '%s'. Detected %d errors out of %d total validation checks." % (filename, num_errors, num_validations))
             if VERBOSE_OUTPUT:
                 for error in error_dict:
-                    logger.error("[Error: '%s', Occurrences: '%d'" % (error, error_dict[error]))
+                    error_list.append("SerialID: '%s', Error: '%s'" % (error, error_dict[error]))
+                logger.info("\n".join(error_list))
             logger.error("============================================================================")
         else:
             logger.info("Validation has PASSED for file '%s'. Detected no errors out and performed %d total validation checks." % (filename, num_validations))
@@ -118,10 +116,11 @@ def validate(local_test, context):
         slack_message = SlackMessage(
             success = total_validations_failed == 0,
             prefixes = prefix_strings,
-            files = s3_file_list,
+            filecount = len(s3_file_list),
             recordcount = records_analyzed,
             validationcount = total_validation_count,
             errorcount = total_validations_failed,
+            errorstring = "\n\n".join(error_list),
             starttime = function_start_time,
             endtime = datetime.now(),
             function_name = context.function_name,

slacker.py

@@ -2,18 +2,17 @@
 import json
 
 class SlackMessage():
-    def __init__(self, success, files, prefixes, recordcount, validationcount, errorcount, starttime, endtime, function_name, aws_request_id, log_group_name, log_stream_name):
+    def __init__(self, success, prefixes, filecount, recordcount, validationcount, errorcount, errorstring, starttime, endtime, function_name, aws_request_id, log_group_name, log_stream_name):
         if success and validationcount > 0:
             self.validation = "PASSED"
         elif success and validationcount == 0:
             self.validation = "N/A"
         else:
             self.validation = "FAILED"
-        self.file_count = len(files)
-        self.files_string = "\n".join(files)
-        if len(self.files_string) > 2971:
-            self.files_string = self.files_string[:2948] # 3000 character maximum for Slack blocks (31 characters for wrap, 21 characters for truncated message)
-            self.files_string += " ... [TRUNCATED LIST]"
+        self.filecount = filecount
+        self.errorstring = "*Failed Validations:* ```%s```" % errorstring
+        if len(self.errorstring) > 2950:
+            self.errorstring = self.errorstring[:2950] + " ... [TRUNCATED LIST]"
         self.prefixes = prefixes
         self.recordcount = recordcount
         self.validationcount = validationcount
@@ -49,7 +48,7 @@ def send(self, logger, dest_url):
             		"type": "section",
             		"text": {
             			"type": "mrkdwn",
-            			"text": "*Files Analyzed (%d):* ```%s```" % (self.file_count, self.files_string)
+            			"text": "*Files Analyzed:* %d" % self.filecount
             		}
             	},
                 {
@@ -73,6 +72,13 @@ def send(self, logger, dest_url):
             			"text": "*Validations Failed:* %d" % self.errorcount
             		}
             	},
+                {
+            		"type": "section",
+            		"text": {
+            			"type": "mrkdwn",
+            			"text": self.errorstring
+            		}
+            	},
                 {
             		"type": "section",
             		"text": {