diff --git a/lab/connector_coding_lab.md b/lab/connector_coding_lab.md index fcf5f3edd..9d38f51a7 100644 --- a/lab/connector_coding_lab.md +++ b/lab/connector_coding_lab.md @@ -184,13 +184,13 @@ class APIClient(): ``` import datetime import json -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.api_client = APIClient(connection, configuration) diff --git a/stix_shifter_modules/alertflex/stix_transmission/results_connector.py b/stix_shifter_modules/alertflex/stix_transmission/results_connector.py index 0aed3ab37..0332ed797 100644 --- a/stix_shifter_modules/alertflex/stix_transmission/results_connector.py +++ b/stix_shifter_modules/alertflex/stix_transmission/results_connector.py @@ -1,10 +1,10 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger import json -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/arcsight/stix_transmission/results_connector.py b/stix_shifter_modules/arcsight/stix_transmission/results_connector.py index 7de13bc87..68a1f42b1 100644 --- a/stix_shifter_modules/arcsight/stix_transmission/results_connector.py +++ b/stix_shifter_modules/arcsight/stix_transmission/results_connector.py @@ -1,6 +1,6 @@ import json import re -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder MAX_LIMIT = 10000 @@ -10,7 +10,7 @@ FILE_HASHES = ["MD5", "SHA256", "SHA1"] -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.connector = __name__.split('.')[1] diff --git a/stix_shifter_modules/async_template/stix_transmission/results_connector.py b/stix_shifter_modules/async_template/stix_transmission/results_connector.py index 555aee76d..3abe38ee3 100644 --- a/stix_shifter_modules/async_template/stix_transmission/results_connector.py +++ b/stix_shifter_modules/async_template/stix_transmission/results_connector.py @@ -1,8 +1,8 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/aws_athena/stix_transmission/results_connector.py b/stix_shifter_modules/aws_athena/stix_transmission/results_connector.py index 603095335..e6960c275 100644 --- a/stix_shifter_modules/aws_athena/stix_transmission/results_connector.py +++ b/stix_shifter_modules/aws_athena/stix_transmission/results_connector.py @@ -1,5 +1,5 @@ import json -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger from flatten_json import flatten @@ -13,7 +13,7 @@ class AccessDeniedException(Exception): pass -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): # https://docs.aws.amazon.com/athena/latest/APIReference/API_GetQueryResults.html#API_GetQueryResults_RequestSyntax # Athena MaxResults count is 1000 when calling get_query_results ATHENA_MAX_RESULTS_SIZE = 1000 diff --git a/stix_shifter_modules/aws_cloud_watch_logs/stix_transmission/results_connector.py b/stix_shifter_modules/aws_cloud_watch_logs/stix_transmission/results_connector.py index e01b8c013..c3cf89259 100644 --- a/stix_shifter_modules/aws_cloud_watch_logs/stix_transmission/results_connector.py +++ b/stix_shifter_modules/aws_cloud_watch_logs/stix_transmission/results_connector.py @@ -1,13 +1,13 @@ import json from flatten_json import flatten import copy -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger from stix_shifter_utils.utils.file_helper import read_json -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, client, options): self.client = client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/aws_security_hub/stix_transmission/connector.py b/stix_shifter_modules/aws_security_hub/stix_transmission/connector.py index 786474f7b..f5026b60e 100644 --- a/stix_shifter_modules/aws_security_hub/stix_transmission/connector.py +++ b/stix_shifter_modules/aws_security_hub/stix_transmission/connector.py @@ -1,10 +1,10 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector import aioboto3 from json import loads -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.connection = connection self.configuration = configuration diff --git a/stix_shifter_modules/azure_sentinel/stix_transmission/connector.py b/stix_shifter_modules/azure_sentinel/stix_transmission/connector.py index a4b929174..68144faea 100644 --- a/stix_shifter_modules/azure_sentinel/stix_transmission/connector.py +++ b/stix_shifter_modules/azure_sentinel/stix_transmission/connector.py @@ -2,13 +2,13 @@ import adal import re from flatten_json import flatten -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): init_error = None max_limit = 1000 diff --git a/stix_shifter_modules/bigfix/stix_transmission/results_connector.py b/stix_shifter_modules/bigfix/stix_transmission/results_connector.py index 9e0235020..c7f0d1390 100644 --- a/stix_shifter_modules/bigfix/stix_transmission/results_connector.py +++ b/stix_shifter_modules/bigfix/stix_transmission/results_connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector import json import time from stix_shifter_utils.utils.error_response import ErrorResponder @@ -10,7 +10,7 @@ class UnexpectedResponseException(Exception): pass -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/carbonblack/stix_transmission/connector.py b/stix_shifter_modules/carbonblack/stix_transmission/connector.py index 07f6711e6..1d9195c3b 100644 --- a/stix_shifter_modules/carbonblack/stix_transmission/connector.py +++ b/stix_shifter_modules/carbonblack/stix_transmission/connector.py @@ -1,5 +1,5 @@ import json -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger @@ -11,7 +11,7 @@ class UnexpectedResponseException(Exception): pass -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.api_client = APIClient(connection, configuration) self.show_events = Connector.get_show_events_mode(connection) diff --git a/stix_shifter_modules/cbcloud/stix_transmission/results_connector.py b/stix_shifter_modules/cbcloud/stix_transmission/results_connector.py index dc478480f..538837f7e 100644 --- a/stix_shifter_modules/cbcloud/stix_transmission/results_connector.py +++ b/stix_shifter_modules/cbcloud/stix_transmission/results_connector.py @@ -1,11 +1,11 @@ import json -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/cloudsql/stix_transmission/results_connector.py b/stix_shifter_modules/cloudsql/stix_transmission/results_connector.py index 03d2b7acb..41aef0556 100644 --- a/stix_shifter_modules/cloudsql/stix_transmission/results_connector.py +++ b/stix_shifter_modules/cloudsql/stix_transmission/results_connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils import logger import json import re @@ -6,7 +6,7 @@ import pandas as pd -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/crowdstrike/stix_transmission/connector.py b/stix_shifter_modules/crowdstrike/stix_transmission/connector.py index 5269a29b1..71347b596 100644 --- a/stix_shifter_modules/crowdstrike/stix_transmission/connector.py +++ b/stix_shifter_modules/crowdstrike/stix_transmission/connector.py @@ -1,11 +1,11 @@ import json -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): init_error = None logger = logger.set_logger(__name__) PROVIDER = 'CrowdStrike' diff --git a/stix_shifter_modules/cybereason/stix_transmission/connector.py b/stix_shifter_modules/cybereason/stix_transmission/connector.py index f4b7ad716..530742b02 100644 --- a/stix_shifter_modules/cybereason/stix_transmission/connector.py +++ b/stix_shifter_modules/cybereason/stix_transmission/connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger from .api_client import APIClient @@ -18,7 +18,7 @@ class InvalidAuthenticationException(Exception): pass -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.api_client = APIClient(connection, configuration) diff --git a/stix_shifter_modules/darktrace/stix_transmission/connector.py b/stix_shifter_modules/darktrace/stix_transmission/connector.py index 7b62078f8..99450d82c 100644 --- a/stix_shifter_modules/darktrace/stix_transmission/connector.py +++ b/stix_shifter_modules/darktrace/stix_transmission/connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger from .api_client import APIClient @@ -25,7 +25,7 @@ class InvalidAuthenticationException(Exception): pass -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.api_client = APIClient(connection, configuration) diff --git a/stix_shifter_modules/datadog/stix_transmission/connector.py b/stix_shifter_modules/datadog/stix_transmission/connector.py index 9c19ffa14..c0a7aa7c8 100644 --- a/stix_shifter_modules/datadog/stix_transmission/connector.py +++ b/stix_shifter_modules/datadog/stix_transmission/connector.py @@ -1,11 +1,11 @@ import json from stix_shifter_modules.datadog.stix_transmission.api_client import APIClient -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.api_client = APIClient(connection, configuration) self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/demo_template/stix_transmission/connector.py b/stix_shifter_modules/demo_template/stix_transmission/connector.py index a1b9479f3..bca35559d 100644 --- a/stix_shifter_modules/demo_template/stix_transmission/connector.py +++ b/stix_shifter_modules/demo_template/stix_transmission/connector.py @@ -1,12 +1,12 @@ import datetime import json -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.api_client = APIClient(connection, configuration) diff --git a/stix_shifter_modules/elastic_ecs/stix_transmission/connector.py b/stix_shifter_modules/elastic_ecs/stix_transmission/connector.py index 966915765..ef9c13c77 100644 --- a/stix_shifter_modules/elastic_ecs/stix_transmission/connector.py +++ b/stix_shifter_modules/elastic_ecs/stix_transmission/connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient import json from stix_shifter_utils.utils.error_response import ErrorResponder @@ -9,7 +9,7 @@ class UnexpectedResponseException(Exception): pass -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.api_client = APIClient(connection, configuration) self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/error_test/stix_transmission/connector.py b/stix_shifter_modules/error_test/stix_transmission/connector.py index 87ec7f73e..818628579 100644 --- a/stix_shifter_modules/error_test/stix_transmission/connector.py +++ b/stix_shifter_modules/error_test/stix_transmission/connector.py @@ -5,7 +5,7 @@ from stix2matcher.matcher import MatchListener from stix2validator import validate_instance -from stix_shifter_utils.modules.base.stix_transmission.base_connector import BaseConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync from stix_shifter_utils.utils.error_response import ErrorResponder @@ -18,7 +18,7 @@ class UnexpectedResponseException(Exception): pass -class Connector(BaseConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.connector = __name__.split('.')[1] self.connection = connection diff --git a/stix_shifter_modules/gcp_chronicle/stix_transmission/results_connector.py b/stix_shifter_modules/gcp_chronicle/stix_transmission/results_connector.py index 9a95c9676..9e828772b 100644 --- a/stix_shifter_modules/gcp_chronicle/stix_transmission/results_connector.py +++ b/stix_shifter_modules/gcp_chronicle/stix_transmission/results_connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger import json @@ -8,7 +8,7 @@ import re -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): EMAIL_PATTERN = re.compile(r'(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)') GCP_MAX_PAGE_SIZE = 1000 diff --git a/stix_shifter_modules/guardium/stix_transmission/results_connector.py b/stix_shifter_modules/guardium/stix_transmission/results_connector.py index d07c3f8b9..067bfe0f9 100644 --- a/stix_shifter_modules/guardium/stix_transmission/results_connector.py +++ b/stix_shifter_modules/guardium/stix_transmission/results_connector.py @@ -1,10 +1,10 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger import json -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/ibm_security_verify/stix_transmission/results_connector.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/results_connector.py index 03aae405b..7c10c88d8 100644 --- a/stix_shifter_modules/ibm_security_verify/stix_transmission/results_connector.py +++ b/stix_shifter_modules/ibm_security_verify/stix_transmission/results_connector.py @@ -1,13 +1,13 @@ import json import traceback -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import \ - BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import \ + BaseJsonResultsConnector from stix_shifter_utils.utils import logger from stix_shifter_utils.utils.error_response import ErrorResponder -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client diff --git a/stix_shifter_modules/infoblox/stix_transmission/results_connector.py b/stix_shifter_modules/infoblox/stix_transmission/results_connector.py index d20d09d85..dbb8165c8 100644 --- a/stix_shifter_modules/infoblox/stix_transmission/results_connector.py +++ b/stix_shifter_modules/infoblox/stix_transmission/results_connector.py @@ -3,11 +3,11 @@ See: https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/develop-transmission-module.md """ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils import logger from stix_shifter_utils.utils.error_response import ErrorResponder -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): """ Class that handles results connector integration. diff --git a/stix_shifter_modules/msatp/stix_transmission/connector.py b/stix_shifter_modules/msatp/stix_transmission/connector.py index 692f8ab5c..73ba87406 100644 --- a/stix_shifter_modules/msatp/stix_transmission/connector.py +++ b/stix_shifter_modules/msatp/stix_transmission/connector.py @@ -1,13 +1,13 @@ import json import adal -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger import copy -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): init_error = None logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/mysql/stix_transmission/connector.py b/stix_shifter_modules/mysql/stix_transmission/connector.py index 675d983ca..71b0506cc 100644 --- a/stix_shifter_modules/mysql/stix_transmission/connector.py +++ b/stix_shifter_modules/mysql/stix_transmission/connector.py @@ -1,12 +1,12 @@ import datetime import json -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.api_client = APIClient(connection, configuration) diff --git a/stix_shifter_modules/onelogin/stix_transmission/connector.py b/stix_shifter_modules/onelogin/stix_transmission/connector.py index 6c8e2b46c..acbf15737 100644 --- a/stix_shifter_modules/onelogin/stix_transmission/connector.py +++ b/stix_shifter_modules/onelogin/stix_transmission/connector.py @@ -1,12 +1,12 @@ import datetime import json -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): max_limit = 50 def __init__(self, connection, configuration): diff --git a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py index 4f6c1f97e..cdb8fcd8b 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py +++ b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger import json @@ -18,7 +18,7 @@ class InvalidQueryException(Exception): pass -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index c26b0d476..724e0f44d 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -1,10 +1,10 @@ import json -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/proxy/stix_transmission/connector.py b/stix_shifter_modules/proxy/stix_transmission/connector.py index 12d84c6af..cc6560c5b 100644 --- a/stix_shifter_modules/proxy/stix_transmission/connector.py +++ b/stix_shifter_modules/proxy/stix_transmission/connector.py @@ -1,10 +1,10 @@ import copy import json -from stix_shifter_utils.modules.base.stix_transmission.base_connector import BaseConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync -class Connector(BaseConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.request_http_path = "https://{}:{}".format(connection['options']['proxy_host'], connection['options']['proxy_port']) self.timeout = connection['options']['timeout'] diff --git a/stix_shifter_modules/qradar/stix_transmission/results_connector.py b/stix_shifter_modules/qradar/stix_transmission/results_connector.py index eecb85d25..2d21dda81 100644 --- a/stix_shifter_modules/qradar/stix_transmission/results_connector.py +++ b/stix_shifter_modules/qradar/stix_transmission/results_connector.py @@ -1,10 +1,10 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger import json -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/qradar_perf_test/stix_transmission/connector.py b/stix_shifter_modules/qradar_perf_test/stix_transmission/connector.py index a99e455d5..2aaa85979 100644 --- a/stix_shifter_modules/qradar_perf_test/stix_transmission/connector.py +++ b/stix_shifter_modules/qradar_perf_test/stix_transmission/connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync import json from stix_shifter_utils.utils.error_response import ErrorResponder @@ -8,7 +8,7 @@ class UnexpectedResponseException(Exception): pass -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.connector = __name__.split('.')[1] self.connection = connection diff --git a/stix_shifter_modules/reaqta/stix_transmission/connector.py b/stix_shifter_modules/reaqta/stix_transmission/connector.py index 581c0526d..714547227 100644 --- a/stix_shifter_modules/reaqta/stix_transmission/connector.py +++ b/stix_shifter_modules/reaqta/stix_transmission/connector.py @@ -1,11 +1,11 @@ import json -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): # LOOKS LIKE MAX COUNT is 500. response doesn't show why it fails MAX_LIMIT = 500 diff --git a/stix_shifter_modules/reversinglabs/stix_transmission/results_connector.py b/stix_shifter_modules/reversinglabs/stix_transmission/results_connector.py index ddcfbb39c..9af3f7d72 100644 --- a/stix_shifter_modules/reversinglabs/stix_transmission/results_connector.py +++ b/stix_shifter_modules/reversinglabs/stix_transmission/results_connector.py @@ -1,9 +1,9 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger import json -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/rhacs/stix_transmission/connector.py b/stix_shifter_modules/rhacs/stix_transmission/connector.py index d0179640c..ad52e20f5 100644 --- a/stix_shifter_modules/rhacs/stix_transmission/connector.py +++ b/stix_shifter_modules/rhacs/stix_transmission/connector.py @@ -1,6 +1,6 @@ import json from requests.exceptions import ConnectionError, RetryError -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger from .api_client import APIClient @@ -10,7 +10,7 @@ class InvalidAuthenticationException(Exception): pass -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): """connector class""" def __init__(self, connection, configuration): diff --git a/stix_shifter_modules/secretserver/stix_transmission/results_connector.py b/stix_shifter_modules/secretserver/stix_transmission/results_connector.py index 7be5b3dba..dfc8f7010 100644 --- a/stix_shifter_modules/secretserver/stix_transmission/results_connector.py +++ b/stix_shifter_modules/secretserver/stix_transmission/results_connector.py @@ -1,10 +1,10 @@ import json -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/security_advisor/stix_transmission/results_connector.py b/stix_shifter_modules/security_advisor/stix_transmission/results_connector.py index 642ffc284..8fe1f4a3f 100644 --- a/stix_shifter_modules/security_advisor/stix_transmission/results_connector.py +++ b/stix_shifter_modules/security_advisor/stix_transmission/results_connector.py @@ -1,10 +1,10 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from .utils.stix_pattern_processor import StixPatternProcessor from stix_shifter_utils.utils.error_response import ErrorResponder from .auth import Auth -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, host, auth ): self.connector = __name__.split('.')[1] self.auth = auth diff --git a/stix_shifter_modules/sentinelone/stix_transmission/results_connector.py b/stix_shifter_modules/sentinelone/stix_transmission/results_connector.py index 8e04227ff..72a3ce0d1 100644 --- a/stix_shifter_modules/sentinelone/stix_transmission/results_connector.py +++ b/stix_shifter_modules/sentinelone/stix_transmission/results_connector.py @@ -1,6 +1,6 @@ import json -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector\ - import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector\ + import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger from aiohttp.client_exceptions import ClientConnectionError @@ -12,7 +12,7 @@ class LimitOutOfRangeError(Exception): class QueryIdNotFoundError(Exception): pass -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): """ResultsConnector class""" def __init__(self, api_client): self.api_client = api_client diff --git a/stix_shifter_modules/splunk/stix_transmission/results_connector.py b/stix_shifter_modules/splunk/stix_transmission/results_connector.py index d90417f0f..32efb70fa 100644 --- a/stix_shifter_modules/splunk/stix_transmission/results_connector.py +++ b/stix_shifter_modules/splunk/stix_transmission/results_connector.py @@ -1,10 +1,10 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from .api_client import APIClient import json from stix_shifter_utils.utils.error_response import ErrorResponder -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.connector = __name__.split('.')[1] diff --git a/stix_shifter_modules/stix_bundle/stix_transmission/connector.py b/stix_shifter_modules/stix_bundle/stix_transmission/connector.py index 1b6ddccea..f8bd87a51 100644 --- a/stix_shifter_modules/stix_bundle/stix_transmission/connector.py +++ b/stix_shifter_modules/stix_bundle/stix_transmission/connector.py @@ -1,4 +1,4 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync from stix2matcher.matcher import Pattern from stix2matcher.matcher import MatchListener @@ -12,7 +12,7 @@ class UnexpectedResponseException(Exception): pass -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): def __init__(self, connection, configuration): self.connector = __name__.split('.')[1] self.connection = connection diff --git a/stix_shifter_modules/sumologic/stix_transmission/results_connector.py b/stix_shifter_modules/sumologic/stix_transmission/results_connector.py index 9cf345cef..26190ccd1 100644 --- a/stix_shifter_modules/sumologic/stix_transmission/results_connector.py +++ b/stix_shifter_modules/sumologic/stix_transmission/results_connector.py @@ -1,9 +1,9 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/synchronous_template/stix_transmission/results_connector.py b/stix_shifter_modules/synchronous_template/stix_transmission/results_connector.py index 65c8bbacb..3618a3878 100644 --- a/stix_shifter_modules/synchronous_template/stix_transmission/results_connector.py +++ b/stix_shifter_modules/synchronous_template/stix_transmission/results_connector.py @@ -1,8 +1,8 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/trendmicro_vision_one/stix_transmission/results_connector.py b/stix_shifter_modules/trendmicro_vision_one/stix_transmission/results_connector.py index ffc7f9f12..0ccd4acb3 100644 --- a/stix_shifter_modules/trendmicro_vision_one/stix_transmission/results_connector.py +++ b/stix_shifter_modules/trendmicro_vision_one/stix_transmission/results_connector.py @@ -1,9 +1,9 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector from stix_shifter_utils.utils import logger from stix_shifter_utils.utils.error_response import ErrorResponder -class ResultsConnector(BaseResultsConnector): +class ResultsConnector(BaseJsonResultsConnector): def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) diff --git a/stix_shifter_utils/modules/base/stix_transmission/base_connector.py b/stix_shifter_utils/modules/base/stix_transmission/base_connector.py index ac261880d..2d8207e88 100644 --- a/stix_shifter_utils/modules/base/stix_transmission/base_connector.py +++ b/stix_shifter_utils/modules/base/stix_transmission/base_connector.py @@ -3,6 +3,7 @@ from .base_status_connector import BaseStatusConnector from .base_delete_connector import BaseDeleteConnector from .base_results_connector import BaseResultsConnector +import json import time @@ -102,7 +103,7 @@ async def create_results_stix_connection(self, entry_point, search_id, offset, l if result.get('success'): data = result['data'] data = data[:length] - result = await entry_point.translate_results(data_source, data) + result = await entry_point.translate_results(json.dumps(data_source), json.dumps(data)) stats.append({'action': 'translation', 'time': int(time.time()*1000)}) result['stats'] = stats if metadata: diff --git a/stix_shifter_utils/modules/base/stix_transmission/base_json_connector.py b/stix_shifter_utils/modules/base/stix_transmission/base_json_connector.py new file mode 100644 index 000000000..d2ee2d603 --- /dev/null +++ b/stix_shifter_utils/modules/base/stix_transmission/base_json_connector.py @@ -0,0 +1,26 @@ +from .base_connector import BaseConnector +import time + + +class BaseJsonConnector(BaseConnector): + + async def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source, metadata=None): + stats = [] + if metadata: + result = await entry_point.create_results_connection(search_id, offset, length, metadata) + else: + result = await entry_point.create_results_connection(search_id, offset, length) + stats.append({'action': 'transmission', 'time': int(time.time()*1000)}) + metadata = None + if 'metadata' in result: + metadata = result['metadata'] + del result['metadata'] + if result.get('success'): + data = result['data'] + data = data[:length] + result = await entry_point.translate_results(data_source, data) + stats.append({'action': 'translation', 'time': int(time.time()*1000)}) + result['stats'] = stats + if metadata: + result['metadata'] = metadata + return result diff --git a/stix_shifter_utils/modules/base/stix_transmission/base_json_results_connector.py b/stix_shifter_utils/modules/base/stix_transmission/base_json_results_connector.py new file mode 100644 index 000000000..d4a974047 --- /dev/null +++ b/stix_shifter_utils/modules/base/stix_transmission/base_json_results_connector.py @@ -0,0 +1,24 @@ +import time +from .base_results_connector import BaseResultsConnector + +class BaseJsonResultsConnector(BaseResultsConnector): + async def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source, metadata=None): + stats = [] + if metadata: + result = await entry_point.create_results_connection(search_id, offset, length, metadata) + else: + result = await entry_point.create_results_connection(search_id, offset, length) + metadata = None + if 'metadata' in result: + metadata = result['metadata'] + del result['metadata'] + stats.append({'action': 'transmission', 'time': int(time.time()*1000)}) + if result.get('success'): + data = result['data'] + data = data[:int(length)] + result = await entry_point.translate_results(data_source, data) + stats.append({'action': 'translation', 'time': int(time.time()*1000)}) + result['stats'] = stats + if metadata: + result['metadata'] = metadata + return result diff --git a/stix_shifter_utils/modules/base/stix_transmission/base_json_sync_connector.py b/stix_shifter_utils/modules/base/stix_transmission/base_json_sync_connector.py new file mode 100644 index 000000000..cacc38c23 --- /dev/null +++ b/stix_shifter_utils/modules/base/stix_transmission/base_json_sync_connector.py @@ -0,0 +1,13 @@ +from .base_json_connector import BaseJsonConnector + + +class BaseJsonSyncConnector(BaseJsonConnector): + async def create_query_connection(self, query): + return {"success": True, "search_id": query} + + async def create_status_connection(self, search_id): + return {"success": True, "status": "COMPLETED", "progress": 100} + + async def delete_query_connection(self, search_id): + return {"success": True} + diff --git a/stix_shifter_utils/modules/base/stix_transmission/base_results_connector.py b/stix_shifter_utils/modules/base/stix_transmission/base_results_connector.py index c9456eb47..3b65cb885 100644 --- a/stix_shifter_utils/modules/base/stix_transmission/base_results_connector.py +++ b/stix_shifter_utils/modules/base/stix_transmission/base_results_connector.py @@ -1,26 +1,9 @@ from abc import ABCMeta, abstractmethod +import json import time class BaseResultsConnector(object, metaclass=ABCMeta): - @abstractmethod - async def create_results_connection(self, search_id, offset, length, metadata=None): - """ - Creates a connection to the specified datasource to retrieve query results - - Args: - search_id (str): The datasource query ID. - offset: data offset to start fetch from. - length: data length to fetch - - Returns: - dict: The return value. - keys: - success (bool): True or False - data (str): The query result data - error (str): error message (when success=False) - """ - raise NotImplementedError() async def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source, metadata=None): stats = [] @@ -36,7 +19,7 @@ async def create_results_stix_connection(self, entry_point, search_id, offset, l if result.get('success'): data = result['data'] data = data[:int(length)] - result = await entry_point.translate_results(data_source, data) + result = await entry_point.translate_results(data_source, json.dumps(data)) stats.append({'action': 'translation', 'time': int(time.time()*1000)}) result['stats'] = stats if metadata: diff --git a/stix_shifter_utils/modules/base/stix_transmission/base_sync_connector.py b/stix_shifter_utils/modules/base/stix_transmission/base_sync_connector.py index c527c9969..e606dcc78 100644 --- a/stix_shifter_utils/modules/base/stix_transmission/base_sync_connector.py +++ b/stix_shifter_utils/modules/base/stix_transmission/base_sync_connector.py @@ -2,7 +2,7 @@ from .base_query_connector import BaseQueryConnector from .base_status_connector import BaseStatusConnector from .base_delete_connector import BaseDeleteConnector -from .base_results_connector import BaseResultsConnector +from .base_json_results_connector import BaseResultsConnector from .base_connector import BaseConnector diff --git a/stix_shifter_utils/utils/base_entry_point.py b/stix_shifter_utils/utils/base_entry_point.py index ad70686ab..36cd1fd00 100644 --- a/stix_shifter_utils/utils/base_entry_point.py +++ b/stix_shifter_utils/utils/base_entry_point.py @@ -14,7 +14,7 @@ from stix_shifter_utils.modules.base.stix_transmission.base_query_connector import BaseQueryConnector from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import BaseStatusConnector from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseResultsConnector from stix_shifter_utils.utils.param_validator import param_validator, modernize_objects from stix_shifter_utils.stix_translation.src.utils.exceptions import UnsupportedDialectException from stix_shifter_utils.utils.error_response import ErrorResponder