From ecf190667a5be0aed8c623344c4c469660597afb Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Thu, 18 Jul 2024 09:51:42 -0500 Subject: [PATCH 1/3] fixing bugs when some but not all custom API URLs are specified --- lightbeam/lightbeam.py | 2 +- lightbeam/send.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lightbeam/lightbeam.py b/lightbeam/lightbeam.py index 908f8d8..6290b50 100644 --- a/lightbeam/lightbeam.py +++ b/lightbeam/lightbeam.py @@ -174,7 +174,7 @@ def get_data_files_for_endpoint(self, endpoint): return file_list # Prunes the list of endpoints down to those for which .jsonl files exist in the config.data_dir - def get_endpoints_with_data(self): + def get_endpoints_with_data(self, filter_endpoints=None): self.logger.debug("discovering data...") endpoints_with_data = [] data_dir_list = os.listdir(self.config["data_dir"]) diff --git a/lightbeam/send.py b/lightbeam/send.py index 7577c4c..59e4c2f 100644 --- a/lightbeam/send.py +++ b/lightbeam/send.py @@ -163,6 +163,13 @@ async def do_send(self, endpoint): hashlog.save(hashlog_file, self.hashlog_data) # update metadata counts for this endpoint + statuses = self.lightbeam.status_counts.keys() + successes = [] + for status in statuses: + if status>=200 and status<300: + successes.append({"status_code": status, "count": self.lightbeam.status_counts[status]}) + if len(successes)>0: + self.metadata["resources"][endpoint].update({"successes": successes}) self.metadata["resources"][endpoint].update({ "records_processed": total_counter, "records_skipped": self.lightbeam.num_skipped, From f1f8f38c02617e44f841a836ee9814739ac27590 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Thu, 18 Jul 2024 09:55:06 -0500 Subject: [PATCH 2/3] this adds a successes section to the structured log results file to report how many payloads resulted in a 200, 201, etc --- lightbeam/lightbeam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightbeam/lightbeam.py b/lightbeam/lightbeam.py index 6290b50..908f8d8 100644 --- a/lightbeam/lightbeam.py +++ b/lightbeam/lightbeam.py @@ -174,7 +174,7 @@ def get_data_files_for_endpoint(self, endpoint): return file_list # Prunes the list of endpoints down to those for which .jsonl files exist in the config.data_dir - def get_endpoints_with_data(self, filter_endpoints=None): + def get_endpoints_with_data(self): self.logger.debug("discovering data...") endpoints_with_data = [] data_dir_list = os.listdir(self.config["data_dir"]) From af042f1bd12556f52293d2f2bb5fc086c81be7eb Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Thu, 18 Jul 2024 09:58:47 -0500 Subject: [PATCH 3/3] update readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f026c3b..3f2a087 100644 --- a/README.md +++ b/README.md @@ -336,6 +336,12 @@ A sample results file could be: "count": 14 } ], + "successes": [ + { + "status_code": 201, + "count": 28 + } + ], "records_processed": 50, "records_skipped": 0, "records_failed": 22