From d7f63eabb22c0836105677fa19639046ed4920c8 Mon Sep 17 00:00:00 2001 From: harshilgajera-crest <69803385+harshilgajera-crest@users.noreply.github.com> Date: Wed, 29 May 2024 15:54:45 +0530 Subject: [PATCH] fix: do not log .conf parser warnings from all workers (#845) When tests are ran with multiple workers, duplicate logging is observed because every workers logs the warning. This PR fixes that. --------- Co-authored-by: Artem Rys --- pytest_splunk_addon/splunk.py | 17 ++++-------- .../standard_lib/addon_parser/props_parser.py | 4 ++- .../sample_xdist_generator.py | 8 +++--- pytest_splunk_addon/standard_lib/utils.py | 26 +++++++++++++++++++ 4 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 pytest_splunk_addon/standard_lib/utils.py diff --git a/pytest_splunk_addon/splunk.py b/pytest_splunk_addon/splunk.py index c92eaa085..271c998fe 100644 --- a/pytest_splunk_addon/splunk.py +++ b/pytest_splunk_addon/splunk.py @@ -32,6 +32,8 @@ import configparser from filelock import FileLock +from pytest_splunk_addon.standard_lib import utils + RESPONSIVE_SPLUNK_TIMEOUT = 300 # seconds LOGGER = logging.getLogger("pytest-splunk-addon") @@ -732,10 +734,7 @@ def splunk_ingest_data(request, splunk_hec_uri, sc4s, uf, splunk_events_cleanup) if request.config.getoption("ingest_events").lower() in ["n", "no", "false", "f"]: return global PYTEST_XDIST_TESTRUNUID - if ( - "PYTEST_XDIST_WORKER" not in os.environ - or os.environ.get("PYTEST_XDIST_WORKER") == "gw0" - ): + if utils.check_first_worker(): addon_path = request.config.getoption("splunk_app") config_path = request.config.getoption("splunk_data_generator") ingest_meta_data = { @@ -783,10 +782,7 @@ def splunk_events_cleanup(request, splunk_search_util): """ if request.config.getoption("splunk_cleanup"): - if ( - "PYTEST_XDIST_WORKER" not in os.environ - or os.environ.get("PYTEST_XDIST_WORKER") == "gw0" - ): + if utils.check_first_worker(): LOGGER.info("Running the old events cleanup") splunk_search_util.deleteEventsFromIndex() else: @@ -801,10 +797,7 @@ def file_system_prerequisite(): """ UF_FILE_MONTOR_DIR = "uf_files" monitor_dir = os.path.join(os.getcwd(), UF_FILE_MONTOR_DIR) - if ( - "PYTEST_XDIST_WORKER" not in os.environ - or os.environ.get("PYTEST_XDIST_WORKER") == "gw0" - ): + if utils.check_first_worker(): if os.path.exists(monitor_dir): shutil.rmtree(monitor_dir, ignore_errors=True) os.mkdir(monitor_dir) diff --git a/pytest_splunk_addon/standard_lib/addon_parser/props_parser.py b/pytest_splunk_addon/standard_lib/addon_parser/props_parser.py index e2757eb52..c85302d31 100644 --- a/pytest_splunk_addon/standard_lib/addon_parser/props_parser.py +++ b/pytest_splunk_addon/standard_lib/addon_parser/props_parser.py @@ -28,6 +28,7 @@ from .fields import convert_to_fields from .transforms_parser import TransformsParser +from pytest_splunk_addon.standard_lib import utils LOGGER = logging.getLogger("pytest-splunk-addon") @@ -110,7 +111,8 @@ def _get_props_method(self, class_name: str): LOGGER.info(f"Matched method of type={each_type}") return method_mapping[each_type] else: - LOGGER.warning(f"No parser available for {class_name}. Skipping...") + if utils.check_first_worker(): + LOGGER.warning(f"No parser available for {class_name}. Skipping...") def _get_props_stanzas(self) -> Optional[Generator]: """ diff --git a/pytest_splunk_addon/standard_lib/sample_generation/sample_xdist_generator.py b/pytest_splunk_addon/standard_lib/sample_generation/sample_xdist_generator.py index 6a69642a2..b17427b67 100644 --- a/pytest_splunk_addon/standard_lib/sample_generation/sample_xdist_generator.py +++ b/pytest_splunk_addon/standard_lib/sample_generation/sample_xdist_generator.py @@ -20,6 +20,8 @@ import json import pytest +from pytest_splunk_addon.standard_lib import utils + class SampleXdistGenerator: def __init__(self, addon_path, config_path=None, process_count=4): @@ -28,14 +30,10 @@ def __init__(self, addon_path, config_path=None, process_count=4): self.config_path = config_path def get_samples(self, store_events): - if self.tokenized_event_source == "pregenerated": with open(self.event_path, "rb") as file_obj: store_sample = pickle.load(file_obj) - if store_events and ( - "PYTEST_XDIST_WORKER" not in os.environ - or os.environ.get("PYTEST_XDIST_WORKER") == "gw0" - ): + if store_events and utils.check_first_worker(): try: tokenized_events = store_sample.get("tokenized_events") self.store_events(tokenized_events) diff --git a/pytest_splunk_addon/standard_lib/utils.py b/pytest_splunk_addon/standard_lib/utils.py new file mode 100644 index 000000000..360335eb2 --- /dev/null +++ b/pytest_splunk_addon/standard_lib/utils.py @@ -0,0 +1,26 @@ +# +# Copyright 2024 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + + +def check_first_worker() -> bool: + """ + returns True if the current execution is under gw0 (first worker) + """ + return ( + "PYTEST_XDIST_WORKER" not in os.environ + or os.environ.get("PYTEST_XDIST_WORKER") == "gw0" + )