From 9236070d4562b23a52a2f0478f32814d680fa366 Mon Sep 17 00:00:00 2001 From: Danny Elliott Date: Wed, 9 Aug 2023 15:58:03 -0300 Subject: [PATCH] CLI lab updates and STIX validator removal (#1555) --- bundle_validator/README.md | 5 +- docs/OVERVIEW.md | 12 -- .../develop-translation-module.md | 6 +- docs/lab/STIX-shifter CLI Quick Lab.ipynb | 103 ++++++------------ docs/lab/requirements-lab.txt | 6 + stix_shifter/requirements.txt | 3 +- stix_shifter/scripts/stix_shifter.py | 2 - .../test_azure_log_analytics.py | 2 +- .../error_test/stix_transmission/connector.py | 8 -- .../test_reversinglabs_json_to_stix.py | 3 +- .../stix_transmission/connector.py | 7 -- .../normalization/BaseNormalization.py | 40 ------- .../json_to_stix/json_to_stix_translator.py | 11 -- 13 files changed, 51 insertions(+), 157 deletions(-) create mode 100644 docs/lab/requirements-lab.txt diff --git a/bundle_validator/README.md b/bundle_validator/README.md index c8d744bb3..5ea1ea28a 100644 --- a/bundle_validator/README.md +++ b/bundle_validator/README.md @@ -6,8 +6,9 @@ You can easily validate your stix bundle file by following the below steps: The following needs to be installed on your local machine: -* Python 3 -* GIT +* Python 3.8 or greater +* git +* [`stix2-validator`](https://github.com/oasis-open/cti-stix-validator) python library If you have not already cloned the [stix-shifter github project](https://github.com/opencybersecurityalliance/stix-shifter): ``` diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md index 89a2cac01..da8feaea6 100644 --- a/docs/OVERVIEW.md +++ b/docs/OVERVIEW.md @@ -154,7 +154,6 @@ These are general translation options defined in [`config.json`](../stix_shifter | time_range | query | A default time range, in minutes, applied to the translated query when no `START STOP` qualifier is present in the STIX pattern. As an example, this would be the `last x minutes` in a SQL query. The default is `5` | A number between `1` and `10000` | | dialects | query | Dialects to be used for pattern translation. This will determine what `from_stix_map.json` files will be used. | A list of one or more dialect strings supported by the connector | | validate_pattern | query | Specifies if pattern validation is run during the query translation call. This can catch errors in the submitted STIX pattern that would otherwise raise exceptions during translation. | `true` or `false` | -| stix_validator | results | Specifies if validation is run on the bundle of STIX data returned with results translation. This is performance intensive and should be used on a small result set. The default if `false`. | `true` or `false` | | unmapped_fallback | results | If set to `true`, any results data returned, that is not specifired in the to-STIX mapping, will be included in the results in the following STIX object:property format `x-:`. The default is `false` | `true` or `false` | | stix_2.1 | results | Results are returned as STIX 2.0 objects by default. Setting this option will return results in STIX 2.1 format. The default is `false` | `true` or `false` | @@ -297,16 +296,6 @@ python main.py translate qradar results \ ``` -### Validating translated STIX 2.1 bundle from the CLI - -You can validate translated STIX results from the CLI provided they conform to the 2.1 standard. The `--stix-validator` flag at the end will run validation on the returned STIX objects to ensure they conform to the STIX 2.1 standard. Alternatively, `'{ "stix_validator": true }'` can be passed in at the end as an options dictionary. - -``` -python main.py translate qradar results \ -'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ -'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' '{"stix_2.1": true, "stix_validator: true}' -``` - ### Validating STIX 2.0 and 2.1 bundles with the validator script Refer to the [STIX validator](../bundle_validator/) @@ -594,7 +583,6 @@ These are general options defined in [`config.json`](../stix_shifter_modules/con | time_range | query translation | A default time range, in minutes, applied to the translated query when no `START STOP` qualifier is present in the STIX pattern. As an example, this would be the `last x minutes` in a SQL query. The default is `5` | A number between `1` and `10000` | | dialects | query translation | Dialects to be used for pattern translation. This will determine what `from_stix_map.json` files will be used. | A list of one or more dialect strings supported by the connector | | validate_pattern | query translation | Specifies if pattern validation is run during the query translation call. This can catch errors in the submitted STIX pattern that would otherwise raise exceptions during translation. | `true` or `false` | -| stix_validator | results translation | Specifies if validation is run on the bundle of STIX data returned with results translation. This is performance intensive and should be used on a small result set. The default if `false`. | `true` or `false` | | unmapped_fallback | results translation | If set to `true`, any results data returned, that is not specifired in the to-STIX mapping, will be included in the results in the following STIX object:property format `x-:`. The default is `false` | `true` or `false` | | stix_2.1 | results translation | Results are returned as STIX 2.0 objects by default. Setting this option will return results in STIX 2.1 format. The default is `false` | `true` or `false` | | timeout | transmission | The max amount of time in seconds before the query times out. The default is `30`. | A number between `1` and `60` | diff --git a/docs/adapter-guide/develop-translation-module.md b/docs/adapter-guide/develop-translation-module.md index 5bf974eac..e70ef9b56 100644 --- a/docs/adapter-guide/develop-translation-module.md +++ b/docs/adapter-guide/develop-translation-module.md @@ -476,14 +476,14 @@ python main.py translate abc_security_monitor results '{"type": "identity","id": 6a4751cae5ff", "name": "abc_security_monitor", "identity_class": "events"}' '[{"Url": "www.example.com", "SourcePort": 3000, "DestinationPort": 1000, "SourceIpV4": "192.0.2.0", "DestinationIpV4": "198.51.100.0", "NetworkProtocol": "TCP"}]' ``` -By default, STIX 2.0 results will be returned. Adding the `{"stix_2.1": true}` option to the end of the CLI command will return STIX 2.1 objects. STIX 2.1 results can be validated against the 2.1 standard by including the `{"stix_validator": true }` option at the end. Only STIX 2.1 can be validated this way. For example: +By default, STIX 2.0 results will be returned. Adding the `{"stix_2.1": true}` option to the end of the CLI command will return STIX 2.1 objects. For example: ``` python main.py translate abc_security_monitor results '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c- -6a4751cae5ff", "name": "abc_security_monitor", "identity_class": "events"}' '[{"Url": "www.example.com", "SourcePort": 3000, "DestinationPort": 1000, "SourceIpV4": "192.0.2.0", "DestinationIpV4": "198.51.100.0", "NetworkProtocol": "TCP"}]' '{"stix_2.1": true, "stix_validator": true }' +6a4751cae5ff", "name": "abc_security_monitor", "identity_class": "events"}' '[{"Url": "www.example.com", "SourcePort": 3000, "DestinationPort": 1000, "SourceIpV4": "192.0.2.0", "DestinationIpV4": "198.51.100.0", "NetworkProtocol": "TCP"}]' '{"stix_2.1": true}' ``` -An alternative way to validate both STIX 2.0 and 2.1 results is to use the [Bundle validator](../../bundle_validator/README.md). +You may validate both STIX 2.0 and 2.1 results with the [Bundle validator](../../bundle_validator/README.md) script. 2. Visually verify that all expected data is in the returned STIX bundle. If a data source field in your sample results is mapped in `to_stix_map.json`, the value must be in the STIX bundle under the mapped STIX property. diff --git a/docs/lab/STIX-shifter CLI Quick Lab.ipynb b/docs/lab/STIX-shifter CLI Quick Lab.ipynb index 57d759878..c2c9e1813 100644 --- a/docs/lab/STIX-shifter CLI Quick Lab.ipynb +++ b/docs/lab/STIX-shifter CLI Quick Lab.ipynb @@ -166,7 +166,8 @@ "\n", "* Python 3.8 or greater\n", "* pip\n", - "* venv\n", + "* git\n", + "* `virtualevn` python library\n", "* Ability to run bash commands\n", "\n", "### Run the following steps in a terminal\n", @@ -175,7 +176,6 @@ "\n", "Call the folder **connector_lab**\n", "\n", - "\n", "```bash\n", "mkdir connector_lab\n", "cd connector_lab\n", @@ -183,7 +183,6 @@ "\n", "### 2. Clone the stix-shifter project\n", "\n", - "\n", "```bash\n", "git clone https://github.com/opencybersecurityalliance/stix-shifter.git\n", "```\n", @@ -191,52 +190,46 @@ "### 3. Install and activate a Python virtual environment and upgrade pip\n", "\n", "```bash\n", - "python3 -m venv labenv\n", + "python3.9 -m venv labenv\n", "source labenv/bin/activate\n", "python3 -m pip install --upgrade pip\n", "```\n", "\n", - "### 4. Install jupyter notebook and the ipython kernal to use the virtual environment\n", + "### 4. CD into the STIX-shifter lab directory\n", "\n", "```bash\n", - "python3 -m pip install notebook\n", - "ipython kernel install --user --name=labenv\n", + "cd stix-shifter/docs/lab\n", "```\n", "\n", - "### 5. CD into the STIX-shifter lab directory\n", + "### 5. Install the required libraries used in this lab\n", + "\n", + "This installs the core stix-shifter and stix-shifter-utils libraries, the STIX-bundle and MySQL connectors, and the Jupyter Notebook library.\n", "\n", "```bash\n", - "cd stix-shifter/lab\n", + "python3 -m pip install -r requirements-lab.txt\n", "```\n", "\n", - "### 6. Run jupyter notebook\n", + "### 6. Install the ipython kernal to use the virtual environment\n", "\n", "```bash\n", - "jupyter notebook\n", + "ipython kernel install --user --name=labenv\n", "```\n", "\n", - "### All remaining steps take place in the jupyter notebook\n", + "### 7. Run jupyter notebook\n", "\n", - "### 7. Confirm the Jupyter Kernel is using the labenv virtual environment\n", + "```bash\n", + "jupyter notebook\n", + "```\n", "\n", - "If the kernel is something other than `labenv`, change it. This will ensure that every notebook cell runs in the virtual environment.\n", + "----------------------------------------------------------------------------------------------------------------------\n", "\n", - "![set_virtual_env.png](attachment:set_virtual_env.png)\n", + "## All remaining steps take place directly in the jupyter notebook\n", "\n", - "### 8. Install the shared STIX-shifter libraries used in this lab\n", + "### 8. Confirm the Jupyter Kernel is using the labenv virtual environment\n", "\n", - "This installs the core stix-shifter and utils library along with the STIX-bundle and QRadar connectors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4d2049c2", - "metadata": {}, - "outputs": [], - "source": [ - "%%bash\n", - "python3 -m pip install stix-shifter stix-shifter-utils" + "If the kernel is something other than `labenv`, change it. This will ensure that every notebook cell runs in the virtual environment.\n", + "\n", + "![set_virtual_env.png](attachment:set_virtual_env.png)" ] }, { @@ -266,31 +259,12 @@ "Note the bundle of observed-data objects that are returned. Each of these objects contains a numbered set of cyber observable objects (`url`, `network-traffic`, `ipv4-addr`…) which contain the data from the target data source." ] }, - { - "cell_type": "markdown", - "id": "1204ef99", - "metadata": {}, - "source": [ - "## Step 2: Install the STIX Bundle connector library" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ceff7d8", - "metadata": {}, - "outputs": [], - "source": [ - "%%bash\n", - "python3 -m pip install stix-shifter-modules-stix_bundle" - ] - }, { "cell_type": "markdown", "id": "0257bd78", "metadata": {}, "source": [ - "## Step 3: Set environment variables to be used in the CLI\n", + "## Step 2: Set environment variables to be used in the CLI\n", "\n", "### STIX Bundle URL\n", "This points to a publicly aviablable, static JSON file of STIX data. " @@ -367,7 +341,7 @@ "id": "e407e416", "metadata": {}, "source": [ - "## Step 4: Run the ping command\n", + "## Step 3: Run the ping command\n", "The `ping` command checks that the data source can be reached by the stix-shifter connector." ] }, @@ -387,7 +361,7 @@ "id": "533c714d", "metadata": {}, "source": [ - "## Step 5: Run the query command\n", + "## Step 4: Run the query command\n", "This command sends the native query to the data source." ] }, @@ -399,7 +373,8 @@ "outputs": [], "source": [ "%%bash\n", - "stix-shifter transmit stix_bundle '{\"url\": \"'\"$BUNDLE_URL\"'\"}' \"$BUNDLE_AUTH\" query \"[ipv4-addr:value = '192.168.0.8']\"\n" + "stix-shifter transmit stix_bundle '{\"url\": \"'\"$BUNDLE_URL\"'\"}' \"$BUNDLE_AUTH\" query \\\n", + "\"[ipv4-addr:value = '192.168.6.242'] START t'2021-06-01T11:36:21Z' STOP t'2021-06-20T11:36:21Z'\"\n" ] }, { @@ -407,7 +382,7 @@ "id": "a5496cbf", "metadata": {}, "source": [ - "## Step 6: Run the status command\n", + "## Step 5: Run the status command\n", "This command checks the status of the query." ] }, @@ -419,7 +394,8 @@ "outputs": [], "source": [ "%%bash\n", - "stix-shifter transmit stix_bundle '{\"url\": \"'\"$BUNDLE_URL\"'\"}' \"$BUNDLE_AUTH\" status \"[ipv4-addr:value = '192.168.0.8']\"\n" + "stix-shifter transmit stix_bundle '{\"url\": \"'\"$BUNDLE_URL\"'\"}' \"$BUNDLE_AUTH\" status \\\n", + "\"[ipv4-addr:value = '192.168.6.242'] START t'2021-06-01T11:36:21Z' STOP t'2021-06-20T11:36:21Z'\"\n" ] }, { @@ -427,7 +403,7 @@ "id": "a94dde00", "metadata": {}, "source": [ - "## Step 7: Run the results command\n", + "## Step 6: Run the results command\n", "This command fetches the query results" ] }, @@ -439,7 +415,8 @@ "outputs": [], "source": [ "%%bash\n", - "stix-shifter transmit stix_bundle '{\"url\": \"'\"$BUNDLE_URL\"'\"}' \"$BUNDLE_AUTH\" results \"[ipv4-addr:value = '192.168.6.242']\" 0 2\n" + "stix-shifter transmit stix_bundle '{\"url\": \"'\"$BUNDLE_URL\"'\"}' \"$BUNDLE_AUTH\" results \\\n", + "\"[ipv4-addr:value = '192.168.6.242'] START t'2021-06-01T11:36:21Z' STOP t'2021-06-20T11:36:21Z'\" 0 2\n" ] }, { @@ -447,7 +424,7 @@ "id": "49f1244e", "metadata": {}, "source": [ - "## Step 8: Run the execute command\n", + "## Step 7: Run the execute command\n", "Notice how the identity object, bundle URL and authentication, and STIX pattern are passed in. The result is a subset of observed-data objects from the original STIX bundle matching the pattern." ] }, @@ -460,7 +437,7 @@ "source": [ "%%bash\n", "stix-shifter execute stix_bundle stix_bundle \"$IDENTITY_OBJECT\" '{\"url\": \"'\"$BUNDLE_URL\"'\"}' \\\n", - "\"$BUNDLE_AUTH\" \"[ipv4-addr:value = '12.111.222.0']\"" + "\"$BUNDLE_AUTH\" \"[ipv4-addr:value = '192.168.6.242'] START t'2021-06-01T11:36:21Z' STOP t'2021-06-20T11:36:21Z'\"" ] }, { @@ -1177,7 +1154,7 @@ "outputs": [], "source": [ "%%bash\n", - "stix-shifter translate mysql results \"$MYSQL_IDENTITY_OBJECT\" \"$MYSQL_RESULTS\" \"$MYSQL_TABLE_OPTIONS\"\n" + "stix-shifter translate mysql results \"$MYSQL_IDENTITY_OBJECT\" \"$MYSQL_RESULTS\" \"$MYSQL_TABLE_OPTIONS\"" ] }, { @@ -1213,16 +1190,8 @@ "outputs": [], "source": [ "%%bash\n", - "stix-shifter execute mysql mysql \"$MYSQL_IDENTITY_OBJECT\" \"$MYSQL_CONNECTION_OBJECT\" \"$MYSQL_AUTH_OBJECT\" \"$STIX_PATTERN\"\n" + "stix-shifter execute mysql mysql \"$MYSQL_IDENTITY_OBJECT\" \"$MYSQL_CONNECTION_OBJECT\" \"$MYSQL_AUTH_OBJECT\" \"$STIX_PATTERN\"" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3a1c8f2f", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/docs/lab/requirements-lab.txt b/docs/lab/requirements-lab.txt new file mode 100644 index 000000000..be3a230d8 --- /dev/null +++ b/docs/lab/requirements-lab.txt @@ -0,0 +1,6 @@ +stix-shifter +stix-shifter-utils +stix-shifter-modules-stix-bundle +stix-shifter-modules-mysql +notebook +mysql-connector-python \ No newline at end of file diff --git a/stix_shifter/requirements.txt b/stix_shifter/requirements.txt index aa5e6bdcf..a05d95cc0 100644 --- a/stix_shifter/requirements.txt +++ b/stix_shifter/requirements.txt @@ -11,11 +11,10 @@ flask==2.3.2 flatten_json==0.1.13 json-fix==0.5.2 jsonmerge==1.9.0 -pyOpenSSL==23.1.1 +pyOpenSSL==23.2.0 python-dateutil==2.8.2 requests_toolbelt==0.10.1 stix2-matcher==3.0.0 stix2-patterns==1.3.2 -stix2-validator==3.1.4 xmltodict==0.13.0 urllib3==1.26.15 diff --git a/stix_shifter/scripts/stix_shifter.py b/stix_shifter/scripts/stix_shifter.py index fa836ac51..8720fe071 100644 --- a/stix_shifter/scripts/stix_shifter.py +++ b/stix_shifter/scripts/stix_shifter.py @@ -350,8 +350,6 @@ def is_async(): for line in sys.stdin: data_lines.append(line) data = '\n'.join(data_lines) - if args.stix_validator: - options['stix_validator'] = args.stix_validator recursion_limit = args.recursion_limit if args.recursion_limit else 1000 translation = stix_translation.StixTranslation() result = translation.translate( diff --git a/stix_shifter_modules/azure_log_analytics/tests/stix_transmission/test_azure_log_analytics.py b/stix_shifter_modules/azure_log_analytics/tests/stix_transmission/test_azure_log_analytics.py index 644f43784..c5a589505 100644 --- a/stix_shifter_modules/azure_log_analytics/tests/stix_transmission/test_azure_log_analytics.py +++ b/stix_shifter_modules/azure_log_analytics/tests/stix_transmission/test_azure_log_analytics.py @@ -15,7 +15,7 @@ class MockToken: class ClientSecretMockResponse: @staticmethod - async def get_token(scope): + async def get_token(scope, **kwargs): return MockToken @staticmethod diff --git a/stix_shifter_modules/error_test/stix_transmission/connector.py b/stix_shifter_modules/error_test/stix_transmission/connector.py index 1104545f2..b18b48e25 100644 --- a/stix_shifter_modules/error_test/stix_transmission/connector.py +++ b/stix_shifter_modules/error_test/stix_transmission/connector.py @@ -6,7 +6,6 @@ from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync from stix2matcher.matcher import Pattern from stix2matcher.matcher import MatchListener -from stix2validator import validate_instance, ValidationOptions from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status @@ -116,13 +115,6 @@ async def create_results_connection(self, search_id, offset, length): response_txt = response.read().decode('utf-8') bundle = json.loads(response_txt) - if "stix_validator" in self.connection['options'] and self.connection['options'].get("stix_validator") is True: - results = validate_instance(bundle, ValidationOptions(version=stix_version)) - - if results.is_valid is not True: - ErrorResponder.fill_error(return_obj, message='Invalid Objects in STIX Bundle.', connector=self.connector) - return return_obj - for obj in bundle["objects"]: if obj["type"] == "observed-data": observations.append(obj) diff --git a/stix_shifter_modules/reversinglabs/tests/stix_translation/test_reversinglabs_json_to_stix.py b/stix_shifter_modules/reversinglabs/tests/stix_translation/test_reversinglabs_json_to_stix.py index a1d002a15..d3f73e7c0 100644 --- a/stix_shifter_modules/reversinglabs/tests/stix_translation/test_reversinglabs_json_to_stix.py +++ b/stix_shifter_modules/reversinglabs/tests/stix_translation/test_reversinglabs_json_to_stix.py @@ -6,8 +6,7 @@ MODULE = "reversinglabs" DATA_SOURCE = {"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "ReversingLabs_Connector", "identity_class": "system"} -options = {'stix_validator':True} -entry_point = EntryPoint(options=options) +entry_point = EntryPoint() translation_options = {} domain_name = "google.com" extension_types = ['toplevel-property-extension'] diff --git a/stix_shifter_modules/stix_bundle/stix_transmission/connector.py b/stix_shifter_modules/stix_bundle/stix_transmission/connector.py index 144654807..fe2800ff1 100644 --- a/stix_shifter_modules/stix_bundle/stix_transmission/connector.py +++ b/stix_shifter_modules/stix_bundle/stix_transmission/connector.py @@ -6,7 +6,6 @@ from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync from stix2matcher.matcher import Pattern from stix2matcher.matcher import MatchListener -from stix2validator import validate_instance, ValidationOptions from stix_shifter_utils.utils.error_response import ErrorResponder @@ -89,12 +88,6 @@ async def create_results_connection(self, search_id, offset, length): response_txt = response.read().decode('utf-8') bundle = json.loads(response_txt) - if "stix_validator" in self.connection['options'] and self.connection['options'].get("stix_validator") is True: - results = validate_instance(bundle, ValidationOptions(version=stix_version)) - - if results.is_valid is not True: - ErrorResponder.fill_error(return_obj, message='Invalid Objects in STIX Bundle.', connector=self.connector) - return return_obj if is_stix_21: observations = [bundle] diff --git a/stix_shifter_utils/normalization/BaseNormalization.py b/stix_shifter_utils/normalization/BaseNormalization.py index 1da9e40a5..b5ecaedab 100644 --- a/stix_shifter_utils/normalization/BaseNormalization.py +++ b/stix_shifter_utils/normalization/BaseNormalization.py @@ -3,7 +3,6 @@ from stix_shifter_utils.utils import logger from stix_shifter_utils.utils.file_helper import read_json as helper_read_json from datetime import datetime -from stix2validator import validate_instance, print_results, ValidationOptions import uuid @@ -16,7 +15,6 @@ class BaseNormalization(object,metaclass=ABCMeta): def __init__(self, options): self.logger = logger.set_logger(__name__) - self.stix_validator = options.get('stix_validator') def create_stix_bundle(self, version="2.1"): @@ -60,12 +58,6 @@ def create_identity_sdo(self, data_source, namespace): if data_source.get('sectors'): stix_identity_sdo['sectors'] = data_source['sectors'] if data_source.get('contact_information'): stix_identity_sdo['contact_information'] = data_source['contact_information'] - if self.stix_validator: - options = ValidationOptions(version="2.1") - results = validate_instance(stix_identity_sdo, options) - if results.is_valid is False: - print_results(results) - raise Exception(f'Invalid parameter set in identity SDO. Please follow STIX 2.1 spec for properties') return [stix_identity_sdo] @@ -113,12 +105,6 @@ def create_extension_sdo(self, identity_object, namespace, nested_properties=[], if (not len(extension_object['extension_properties']) > 0): del extension_object['extension_properties'] - if self.stix_validator: - options = ValidationOptions(version="2.1") - results = validate_instance(extension_object, options) - if results.is_valid is False: - print_results(results) - raise Exception(f'Invalid parameter set in extension_object SDO. Please follow STIX 2.1 spec for properties') stix_extension_sdo = [extension_object] return stix_extension_sdo @@ -249,12 +235,6 @@ def create_sighting_sdo(self, sighting_object, indicator_id): 'modified': now } - if self.stix_validator: - options = ValidationOptions(version="2.1") - results = validate_instance(sighting, options) - if results.is_valid is False: - print_results(results) - raise Exception(f'Invalid parameter set in sighting SDO. Please follow STIX 2.1 spec for properties') return [sighting] except Exception as err: @@ -284,12 +264,6 @@ def create_infrastructure_object_sdo(self, infrastructure_object, enriched_ioc, infrastructure_types = self.normalized_infra_type(infrastructure['infrastructure_types']) infrastructure['infrastructure_types'] = infrastructure_types - if self.stix_validator: - options = ValidationOptions(version="2.1") - results = validate_instance(infrastructure, options) - if results.is_valid is False: - print_results(results) - raise Exception(f'Invalid parameter set in infrastructure SDO. Please follow STIX 2.1 spec for properties') infrastructure_array = [infrastructure] relationship = self.createRelationship(infrastructure_array, indicator_id) @@ -355,13 +329,6 @@ def create_malware_sdo(self,malware_object, indicator_id, enriched_ioc): malware_types = self.normalized_malware_type(malware['malware_types']) malware['malware_types'] = malware_types - # malware SDO properties validation - if self.stix_validator: - options = ValidationOptions(version="2.1") - results = validate_instance(malware, options) - if results.is_valid is False: - print_results(results) - raise Exception(f'Invalid parameter set in malware SDO. Please follow STIX 2.1 spec for properties') # if name is not present then compare only malware_types to remove duplicate else check malware types and name. if (len([i for i in malware_array if (i['malware_types'] == malware ['malware_types'] and i['name'] == malware ['name'])]) == 0): @@ -417,13 +384,6 @@ def create_indicator_sdo(self, indicator_object: dict, identity_id: str, extensi if (extension_id): indicator = self.add_extension(indicator, extension_id, nested_properties, top_properties) - # indicator SDO properties validation - if self.stix_validator: - options = ValidationOptions(version="2.1") - results = validate_instance(indicator, options) - if results.is_valid is False: - print_results(results) - raise Exception(f'Invalid parameter set in indicator SDO. Please follow STIX 2.1 spec for properties') return [indicator] diff --git a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py index 4ede2efc7..6b6909c30 100644 --- a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py +++ b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py @@ -5,7 +5,6 @@ from stix_shifter_utils.utils.helpers import dict_merge from stix_shifter_utils.stix_translation.src.json_to_stix import observable, id_contributing_properties -from stix2validator import validate_instance, print_results, ValidationOptions from datetime import datetime from stix_shifter_utils.utils import logger from stix_shifter_utils.utils.helpers import StixObjectId @@ -34,16 +33,6 @@ def convert_to_stix(data_source, map_data, data, transformers, options, callback for _, value in ds2stix.unique_cybox_objects.items(): ds2stix.bundle["objects"].append(value) - if options.get('stix_validator'): - if ds2stix.spec_version == "2.1": - # Serialize and Deserialize bundle to covert StixObjectIds to strings - bundle_obj = json.dumps(ds2stix.bundle, sort_keys=False) - bundle_obj = json.loads(bundle_obj) - else: - bundle_obj = ds2stix.bundle - validated_result = validate_instance(bundle_obj, ValidationOptions(version=ds2stix.spec_version)) - print_results(validated_result) - return ds2stix.bundle except Exception as e: