From 2bec8b1e054f69a2f8e3c87b79973507c74bfc6a Mon Sep 17 00:00:00 2001 From: RMS Date: Fri, 4 Feb 2022 14:25:47 +0530 Subject: [PATCH 01/19] adding verify for new pull request --- stix_shifter_modules/verify/README.md | 63 +++++ stix_shifter_modules/verify/__init__.py | 0 .../verify/configuration/config.json | 36 +++ .../verify/configuration/lang_en.json | 77 ++++++ stix_shifter_modules/verify/entry_point.py | 49 ++++ .../verify/stix_translation/__init__.py | 0 .../stix_translation/json/from_stix_map.json | 70 ++++++ .../json/from_to_stix_1.8.json | 58 +++++ .../stix_translation/json/operators.json | 5 + .../stix_translation/json/to_stix_map.json | 224 +++++++++++++++++ .../stix_translation/query_constructor.py | 234 ++++++++++++++++++ .../stix_translation/query_translator.py | 26 ++ .../stix_translation/results_translator.py | 5 + .../verify/stix_transmission/__init__.py | 0 .../verify/stix_transmission/api_client.py | 184 ++++++++++++++ .../stix_transmission/delete_connector.py | 12 + .../verify/stix_transmission/error_mapper.py | 36 +++ .../stix_transmission/ping_connector.py | 29 +++ .../stix_transmission/query_connector.py | 30 +++ .../stix_transmission/results_connector.py | 38 +++ .../stix_transmission/status_connector.py | 16 ++ .../verify/test/stix_translation/___init__.py | 0 .../test/stix_translation/test_transform.py | 90 +++++++ .../test_verify_stix_to_query.py | 73 ++++++ .../test/stix_transmission/___init__.py | 0 .../test/stix_transmission/test_verify.py | 158 ++++++++++++ 26 files changed, 1513 insertions(+) create mode 100644 stix_shifter_modules/verify/README.md create mode 100644 stix_shifter_modules/verify/__init__.py create mode 100644 stix_shifter_modules/verify/configuration/config.json create mode 100644 stix_shifter_modules/verify/configuration/lang_en.json create mode 100644 stix_shifter_modules/verify/entry_point.py create mode 100644 stix_shifter_modules/verify/stix_translation/__init__.py create mode 100644 stix_shifter_modules/verify/stix_translation/json/from_stix_map.json create mode 100644 stix_shifter_modules/verify/stix_translation/json/from_to_stix_1.8.json create mode 100644 stix_shifter_modules/verify/stix_translation/json/operators.json create mode 100644 stix_shifter_modules/verify/stix_translation/json/to_stix_map.json create mode 100644 stix_shifter_modules/verify/stix_translation/query_constructor.py create mode 100644 stix_shifter_modules/verify/stix_translation/query_translator.py create mode 100644 stix_shifter_modules/verify/stix_translation/results_translator.py create mode 100644 stix_shifter_modules/verify/stix_transmission/__init__.py create mode 100644 stix_shifter_modules/verify/stix_transmission/api_client.py create mode 100644 stix_shifter_modules/verify/stix_transmission/delete_connector.py create mode 100644 stix_shifter_modules/verify/stix_transmission/error_mapper.py create mode 100644 stix_shifter_modules/verify/stix_transmission/ping_connector.py create mode 100644 stix_shifter_modules/verify/stix_transmission/query_connector.py create mode 100644 stix_shifter_modules/verify/stix_transmission/results_connector.py create mode 100644 stix_shifter_modules/verify/stix_transmission/status_connector.py create mode 100644 stix_shifter_modules/verify/test/stix_translation/___init__.py create mode 100644 stix_shifter_modules/verify/test/stix_translation/test_transform.py create mode 100644 stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py create mode 100644 stix_shifter_modules/verify/test/stix_transmission/___init__.py create mode 100644 stix_shifter_modules/verify/test/stix_transmission/test_verify.py diff --git a/stix_shifter_modules/verify/README.md b/stix_shifter_modules/verify/README.md new file mode 100644 index 000000000..369881ab3 --- /dev/null +++ b/stix_shifter_modules/verify/README.md @@ -0,0 +1,63 @@ +# Verify + +This is a connector for searching IBM Verify events. + +### Format for making STIX translation calls via the CLI + +`python main.py ` + +Note the STIX identity object is only used when translating data source results into STIX, so it can be passed in as an empty object for query translation calls. + +### Converting from STIX patterns to verify_event queries + +This example input pattern for TRANSLATE: + +python main.py translate verify query "{}" "[x-oca-event:category='sso']"' +Returns the following search query: + +``` +{ + "queries": [ + "event_type=\"sso\"&limit=10000" + ] +} +``` + +## Transmit + +### Transmit functions + +Transmit offers several functions: ping, query, results and is_async. + +### Ping + +Uses the data source API to ping the connection + +CLI command example: +``` +python main.py transmit verify '{ "host": "" }' '{ "auth": { "clientId": "" }' '{ "auth": { "clientId": ""}}' results "event_type=\"authentication\"&limit=10000" 0 10 +``` +Returns JSON response data from verify +``` +### Execute +python main.py execute verify verify '{"type": "identity","id": "","name":"verify","identity_class":"events"}' '{ }' '{ "host": "" }' '{ "auth": { "clientId": ""}}' "[x-oca-event:category = 'sso' AND x-oca-event:category = 'authentication']" + +''' + Return stix format data +''' + diff --git a/stix_shifter_modules/verify/__init__.py b/stix_shifter_modules/verify/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/verify/configuration/config.json b/stix_shifter_modules/verify/configuration/config.json new file mode 100644 index 000000000..b5f63c0bb --- /dev/null +++ b/stix_shifter_modules/verify/configuration/config.json @@ -0,0 +1,36 @@ +{ + "connection": { + "type": { + "displayName": "IBM Security Verify", + "type": "connectorType" + }, + "host": { + "type": "text", + "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_:/\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_:/\\-]*[A-Za-z0-9])$" + }, + "region": { + "type": "text", + "default": "us" + }, + "help": { + "type": "link", + "default": "data-sources.html" + }, + "options": { + "unmapped_fallback": { + "default": false + } + } + }, + "configuration": { + "auth": { + "type": "fields", + "clientId": { + "type": "password" + }, + "clientSecret": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/verify/configuration/lang_en.json b/stix_shifter_modules/verify/configuration/lang_en.json new file mode 100644 index 000000000..777559d3c --- /dev/null +++ b/stix_shifter_modules/verify/configuration/lang_en.json @@ -0,0 +1,77 @@ +{ + "connection": { + "help": { + "label": "Need additional help?", + "description": "More details on the data source setting can be found in the specified link" + }, + "type": { + "description": "Data source type" + }, + "name": { + "label": "Data source name", + "description": "Assign a name to identify the connection (minimum of 5 characters)" + }, + "description": { + "label": "Data source description", + "description": "Assign a description to indicate the purpose of the connection" + }, + "options": { + "result_limit": { + "label": "Result Size Limit", + "description": "The maximum number of entries or objects that are returned by search query. Valid input range is {{min}} to {{max}}." + }, + "time_range": { + "label": "Query Time Range", + "description": "Time range for the search, in minutes, represented as last x minutes. Valid input range is {{min}} to {{max}}." + }, + "timeout": { + "label": "Query Response Timeout Limit", + "description": "The limit on how long to wait for the data source response, in seconds. Valid input range is {{min}} to {{max}}." + }, + "search_timeout": { + "label": "Query Search Timeout Limit", + "description": "The limit on how long the query will run, in minutes, on the data source." + }, + "dialects": { + "label": "Dialect", + "description": "The dialect to use if the data source needs to search multiple schema or database tables." + }, + "validate_pattern": { + "label": "STIX Pattern Validator", + "description": "Validate STIX patterns that needs to be translated into native data source query" + }, + "stix_validator": { + "label": "STIX Object Validator", + "description": "Validate translated STIX Objects" + }, + "mapping": { + "label": "Custom Mapping", + "description": "Custom stix mapping if default mapping needs to be replaced" + }, + "concurrent": { + "label": "Concurrent Search Limit", + "description": "The number of simultaneous connections that can be made between IBM Cloud Pak\u2122 for Security and the data source. Valid input range is {{min}} to {{max}}." + } + } + }, + "configuration": { + "auth": { + "clientId": { + "label": "Client Id", + "description": "Client ID of Verify" + }, + "clientSecret": { + "label": "Client Secret", + "description": "Client secret of Verify" + } + }, + "name": { + "label": "Configuration Name", + "placeholder": "Add a configuration name" + }, + "description": { + "label": "Configuration Description", + "placeholder": "Add a configuration description" + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/verify/entry_point.py b/stix_shifter_modules/verify/entry_point.py new file mode 100644 index 000000000..d43aa4baa --- /dev/null +++ b/stix_shifter_modules/verify/entry_point.py @@ -0,0 +1,49 @@ +from stix_shifter_utils.utils import logger +from stix_shifter.stix_transmission.stix_transmission import IS_ASYNC +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint +from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from .stix_translation.query_translator import QueryTranslator +from .stix_translation.results_translator import ResultsTranslator +from .stix_transmission.ping_connector import PingConnector +from .stix_transmission.query_connector import QueryConnector +from .stix_transmission.status_connector import StatusConnector +from .stix_transmission.results_connector import ResultsConnector +from .stix_transmission.delete_connector import DeleteConnector +from .stix_transmission.api_client import APIClient + +import os +class EntryPoint(BaseEntryPoint): + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + # self.logger = logger.set_logger(__name__) + self.set_async(False) + + if connection: + auth = configuration.get("auth") + host = connection.get("host") + api_client = APIClient(connection, configuration) + base_sync_connector = BaseSyncConnector() + ping_connector = PingConnector(api_client) + query_connector = base_sync_connector + status_connector = StatusConnector(api_client) + results_connector = ResultsConnector(api_client) + delete_connector = DeleteConnector(api_client) + + + + self.set_ping_connector(ping_connector) + self.set_query_connector(query_connector) + self.set_status_connector(status_connector) + self.set_results_connector(results_connector) + self.set_delete_connector(delete_connector) + + basepath = os.path.dirname(__file__) + filepath = os.path.abspath( + os.path.join(basepath, "stix_translation")) + # self.setup_translation_simple(dialect_default='default') + # self.setup_translation_simple(dialect_default='flows') + dialect = 'default' + query_translator = QueryTranslator(options, dialect, filepath) + results_translator = ResultsTranslator(options, dialect, filepath) + self.add_dialect(dialect, query_translator=query_translator, results_translator=results_translator, default=True) diff --git a/stix_shifter_modules/verify/stix_translation/__init__.py b/stix_shifter_modules/verify/stix_translation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json new file mode 100644 index 000000000..dd6211919 --- /dev/null +++ b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json @@ -0,0 +1,70 @@ + +{ + "user-account":{ + "fields": { + "user_id": ["data.username"], + "account_type":["data.sourcetype"], + "performedby_username":["data.performedby_username"] + } + }, + "ipv4-addr":{ + "fields": { + "value": ["data.origin"] + } + }, + "domain-name":{ + "fields": { + "type": ["domain-name"], + "value": ["tenantname"] + } + }, + "x-oca-event": { + "fields":{ + "action": ["data.action"], + "category":["event_type"], + "module":["servicename"], + "outcome":["data.result"], + "agent" :["data.sourcetype"], + "ip_refs":["ip"], + "domain_ref.value": ["tenantname"], + "user_ref": ["username"], + "provider":"'IBM Security Verify Event'", + "extensions.'x-iam-ext'.subject":["data.subject"], + "extensions.'x-iam-ext'.subcategory":["data.subtype"], + "extensions.'x-iam-ext'.realm":["data.realm"], + "extensions.'x-iam-ext'.browser_agent":["data.devicetype"], + "extensions.'x-iam-ext'.application_id":["data.applicationid"], + "extensions.'x-iam-ext'.user_id":["data.userid"], + "extensions.'x-iam-ext'.application_type":["data.applicationtype"], + "extensions.'x-iam-ext'.application_name":["data.applicationname"], + "extensions.'x-iam-ext'.cause":["data.cause"], + "extensions.'x-iam-ext'.target":["data.target"], + "extensions.'x-iam-ext'.performedby_username":["data.performedby_username"], + "extensions.'x-iam-ext'.message_id":["data.messageid"], + "extensions.'x-iam-ext'.deleted":["data.deleted"], + "extensions.'x-iam-ext'.performedby_clientname":["data.performedby_clientname"], + "extensions.'x-iam-ext'.performedby_realm":["data.performedby_realm"], + "extensions.'x-iam-ext'.resource":["data.resource"], + "extensions.'x-iam-ext'.targetid":["data.targetid"], + "extensions.'x-iam-ext'.targetid_realm" :["data.targetid_realm"], + "extensions.'x-iam-ext'.targetid_username" :["data.targetid_username"], + "extensions.'x-iam-ext'.userinfo_lookup":["data.userinfo_lookup"], + "extensions.'x-iam-ext'.continent_name":["geoip.continent_name"], + "extensions.'x-iam-ext'.city_name": ["geoip.city_name"], + "extensions.'x-iam-ext'.country_iso_code": ["geoip.country_iso_code"], + "extensions.'x-iam-ext'.country_name": ["geoip.country_name"], + "extensions.'x-iam-ext'.region_name":["geoip.region_name"], + "extensions.'x-iam-ext'.location_lon":["geoip.location.lon"], + "extensions.'x-iam-ext'.location_lat":["geoip.location_lat"], + "extensions.'x-iam-ext'.provider_id":["data.providerid"], + "extensions.'x-iam-ext'.'decision_decisionCode'":["data.decision_decisionCode"], + "extensions.'x-iam-ext'.'rule_name'":["data.rule_name"], + "extensions.'x-iam-ext'.'policy_name'":["data.policy_name"], + "extensions.'x-iam-ext'.'decision_reason'":["data.decision_reason"], + "extensions.'x-iam-ext'.'policy_action'":["data.policy_action"], + "extensions.'x-iam-ext'.'risk_level'":["data.risk_level"], + "extensions.'x-iam-ext'.'risk_score'":["data.risk_score"] + + } + } + } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/from_to_stix_1.8.json b/stix_shifter_modules/verify/stix_translation/json/from_to_stix_1.8.json new file mode 100644 index 000000000..3fc89044d --- /dev/null +++ b/stix_shifter_modules/verify/stix_translation/json/from_to_stix_1.8.json @@ -0,0 +1,58 @@ + +{ + + "user-account":{ + "fields": { + "'user_id'": ["data.username"], + "account_type":["data.sourcetype"] + } + }, + + "ipv4-addr":{ + "fields": { + "value": ["data.origin"] + } + }, + "domain-name":{ + "fields": { + "type": ["domain-name"], + "value": ["tenantname"] + } + }, + "x-oca-event": { + "fields":{ + "action": ["data.action"], + "category":["event_type"], + "module":["servicename"], + "agent" :["data.sourcetype"], + "outcome":["data.result"], + "extensions.x-iam-ext.result":["data.result"], + "extensions.x-iam-ext.subject":["data.subject"], + "extensions.x-iam-ext.subcategory":["data.subtype"], + "extensions.x-iam-ext.realm":["data.realm"], + "extensions.x-iam-ext.browser_agent":["data.devicetype"], + "extensions.x-iam-ext.application_id":["data.applicationid"], + "extensions.x-iam-ext.user_id":["data.userid"], + "extensions.x-iam-ext.application_type":["data.applicationtype"], + "extensions.x-iam-ext.application_name":["data.applicationname"], + "extensions.x-iam-ext.cause":["data.cause"], + "extensions.x-iam-ext.target":["data.target"], + "extensions.x-iam-ext.performedby_username":["data.performedby_username"], + "extensions.x-iam-ext.message_id":["data.messageid"], + "extensions.x-iam-ext.deleted":["data.deleted"], + "extensions.x-iam-ext.performedby_clientname":["data.performedby_clientname"], + "extensions.x-iam-ext.performedby_realm":["data.performedby_realm"], + "extensions.x-iam-ext.resource":["data.resource"], + "extensions.x-iam-ext.targetid":["data.targetid"], + "extensions.x-iam-ext.targetid_realm" :["data.targetid_realm"], + "extensions.x-iam-ext.targetid_username" :["data.targetid_username"], + "extensions.x-iam-ext.userinfo_lookup":["data.userinfo_lookup"], + "extensions.x-iam-ext.continent_name":["geoip.continent_name"], + "extensions.x-iam-ext.city_name": ["geoip.city_name"], + "extensions.x-iam-ext.country_iso_code": ["geoip.country_iso_code"], + "extensions.x-iam-ext.country_name": ["geoip.country_name"], + "extensions.x-iam-ext.region_name":["geoip.region_name"], + "extensions.x-iam-ext.provider_id":["data.providerid"] + } + } + } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/operators.json b/stix_shifter_modules/verify/stix_translation/json/operators.json new file mode 100644 index 000000000..71dd51c7e --- /dev/null +++ b/stix_shifter_modules/verify/stix_translation/json/operators.json @@ -0,0 +1,5 @@ +{ + "ComparisonExpressionOperators.And": "&", + "ComparisonComparators.Equal": "=", + "ObservationOperators.And": "or" +} \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json new file mode 100644 index 000000000..a452547d7 --- /dev/null +++ b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json @@ -0,0 +1,224 @@ +{ +"username": [ + { + "key": "user-account.user_id", + "object": "useraccount" + }, + { + "key": "x-oca-event.user_ref", + "object": "ocaevent", + "references": "useraccount" + }, + { + "key": "x-oca-event.extensions.x-iam-ext.performedby_username", + "object": "ocaevent", + "references": "useraccount" + } +], +"servicename" :{ + "key":"x-oca-event.module", + "object":"ocaevent" +}, +"sourcetype":{ + "key": "x-oca-event.agent", + "object": "ocaevent" +}, +"ip":[{ + "key": "ipv4-addr.value", + "object": "ipdata" +}, +{ + "key": "x-oca-event.ip_refs", + "object": "ocaevent", + "references": "ipdata" +} +], +"tenantname" :[ +{ + "key": "domain-name.value", + "object": "domaindata", + "transformer":"ToDomainName" +}, +{ + "key": "x-oca-event.domain_ref", + "object": "ocaevent", + "references" :"domaindata" +} +], +"result": [ + + { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.outcome" + } +], + +"subtype": + { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.subcategory" +}, +"subject": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.subject" +}, + +"cause": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.cause" +}, + +"action": [ +{ + "key" :"x-oca-event.action", + "object":"ocaevent" +} +], +"realm": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.realm" +}, +"devicetype": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.browser_agent" +}, +"applicationid" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.application_id" +}, +"applicationtype" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.application_type" +}, +"applicationname":{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.applicationname" +}, + +"target": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.target" +}, +"event_type" :[ +{ + "key":"x-oca-event.category", + "object":"ocaevent" +} +], + +"time" : [ + { + "key":"x-oca-event.created", + "transformer":"EpochToTimestamp", + "object":"ocaevent" + } +], +"performedby_username" : + { + "key": "x-oca-event.extensions.x-iam-ext.performedby_username", + "object": "ocaevent" + }, +"messageid" :{ + "key":"x-oca-event.extensions.x-iam-ext.message_id", + "object":"ocaevent" +}, +"deleted" :{ + "key":"x-oca-event.extensions.x-iam-ext.deleted", + "object":"ocaevent" +}, +"performedby_clientname" :{ + "key":"x-oca-event.extensions.x-iam-ext.performedby_clientname", + "object":"ocaevent" +}, +"performedby_realm" :{ + "key":"x-oca-event.extensions.x-iam-ext.performedby_realm", + "object" :"ocaevent" +}, +"resource" :{ + "key":"x-oca-event.extensions.x-iam-ext.resource", + "object" :"ocaevent" +}, +"targetid" :{ + "key":"x-oca-event.extensions.x-iam-ext.targetid", + "object":"ocaevent" +}, +"targetid_realm" :{ + "key":"x-oca-event.extensions.x-iam-ext.targetid_realm", + "object":"ocaevent" +}, +"targetid_username" :{ + "key":"x-oca-event.extensions.x-iam-ext.taregetid_username", + "object":"ocaevent" +}, +"userinfo_lookup" :{ + "key" :"x-oca-event.extensions.x-iam-ext.userinfo_lookup", + "object":"ocaevent" +}, +"userid" :[ + { + "key":"x-oca-event.extensions.x-iam-ext.user_id", + "object":"ocaevent" +}, + { + "key": "user-account.user_id", + "object": "useraccount" + } +], +"continent_name" : { + "key" :"x-oca-event.extensions.x-iam-ext.continent_name", + "object":"ocaevent" + }, + "city_name" : { + "key" :"x-oca-event.extensions.x-iam-ext.city_name", + "object":"ocaevent" + }, + "country_iso_code" : { + "key" :"x-oca-event.extensions.x-iam-ext.country_iso_code", + "object":"ocaevent" + }, + "country_name" : { + "key" :"x-oca-event.extensions.x-iam-ext.country_name", + "object":"ocaevent" + }, + "region_name" : { + "key" :"x-oca-event.extensions.x-iam-ext.region_name", + "object":"ocaevent" + }, + "provider" : { + "key" :"x-oca-event.provider", + "object":"ocaevent" + }, + "providerid" :{ + "key":"x-oca-event.extensions.x-iam-ext.provider_id", + "object":"ocaevent" + }, + "decision_decisionCode" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.decision_decisionCode" + }, + "rule_name" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.rule_name" + }, + "policy_name" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.policy_name" + }, + "decision_reason" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.decision_reason" + }, + "policy_action" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.policy_action" + }, + "risk_level" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.data.risk_level" + }, + "risk_score" :{ + "object":"ocaevent", + "key":"x-oca-event.extensions.x-iam-ext.data.risk_score" + } + +} \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/query_constructor.py b/stix_shifter_modules/verify/stix_translation/query_constructor.py new file mode 100644 index 000000000..87ec16bee --- /dev/null +++ b/stix_shifter_modules/verify/stix_translation/query_constructor.py @@ -0,0 +1,234 @@ +from os import stat +from typing import Union +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \ + ComparisonExpressionOperators, ComparisonComparators, Pattern, \ + CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators +from stix_shifter_utils.stix_translation.src.json_to_stix import observable +from stix_shifter_utils.stix_translation.src.utils.transformers import TimestampToMilliseconds +import logging +import re + +# Source and destination reference mapping for ip and mac addresses. +# Change the keys to match the data source fields. The value array indicates the possible data type that can come into from field. +REFERENCE_DATA_TYPES = {"ipaddr": ["ipv4", "ipv4_cidr", "ipv6", "ipv6_cidr"], + "proxy_ip": ["ipv4", "ipv4_cidr"], + } +REFERENCE_FILTER_TYPE = ["user-account","ipv4-addr","domain-name","x-oca-event"] +REFERENCE_EXCLUDE_FILTER_TYPE = ["category"] +logger = logging.getLogger(__name__) + + +class QueryStringPatternTranslator: + QUERIES = [] + # Change comparator values to match with supported data source operators + comparator_lookup = { + ComparisonExpressionOperators.And: "&", + ComparisonComparators.Equal: "=", + ObservationOperators.And: 'or', + # Treat AND's as OR's -- Unsure how two ObsExps wouldn't cancel each other out. + ObservationOperators.Or: 'or' + } + + def __init__(self, pattern: Pattern, data_model_mapper): + self.dmm = data_model_mapper + self.pattern = pattern + self.translated = self.parse_expression(pattern) + + @staticmethod + def _escape_value(value, comparator=None) -> str: + if isinstance(value, str): + return '{}'.format(value.replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)').replace("$","\"")) + else: + return value + + @staticmethod + def _negate_comparison(comparison_string): + return "NOT ({})".format(comparison_string) + + @staticmethod + def _check_value_type(value): + value = str(value) + for key, pattern in observable.REGEX.items(): + if key != 'date' and bool(re.search(pattern, value)): + return key + return None + + # TODO remove self reference from static methods + @staticmethod + def _parse_reference(self, stix_field, value_type, mapped_field, value, comparator): + if value_type not in REFERENCE_DATA_TYPES["{}".format(mapped_field)]: + return None + else: + return "{mapped_field}{comparator}{value}".format( + mapped_field=mapped_field, comparator=comparator, value=value) + + @staticmethod + def _parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array,stix_object): + comparison_string = "" + is_reference_value = self._is_reference_value(stix_field) + is_object_filter_typeValue = self._is_object_filterType(stix_object,stix_field) + # Need to use expression.value to match against regex since the passed-in value has already been formated. + value_type = self._check_value_type(expression.value) if is_reference_value else None + mapped_fields_count = 1 if is_reference_value else len(mapped_fields_array) + + for mapped_field in mapped_fields_array: + if is_reference_value: + parsed_reference = self._parse_reference(self, stix_field, value_type, mapped_field, value, comparator) + if not parsed_reference: + continue + comparison_string += parsed_reference + elif is_object_filter_typeValue : + comparison_string += 'filter_key={mapped_field}&filter_value="{value}"'.format(mapped_field=mapped_field, comparator=comparator, value=value) + else: + comparison_string += '{mapped_field}{comparator}"{value}"'.format(mapped_field=mapped_field, comparator=comparator, value=value) + + if mapped_fields_count > 1: + comparison_string += "&" + mapped_fields_count -= 1 + return comparison_string + + @staticmethod + def _is_reference_value(stix_field): + return stix_field == 'src_ref.value' or stix_field == 'dst_ref.value' + + @staticmethod + def _is_object_filterType(stix_object,stix_field): + if (stix_object in REFERENCE_FILTER_TYPE) and (stix_field not in REFERENCE_EXCLUDE_FILTER_TYPE ): + return True + else : + return False + + @staticmethod + def _check_filter_value_type(value, stix_object): + """ + Function returning value type of event type object in double quotes + :param value: str + :return: string + """ + event_object = ['event_type'] + if stix_object in event_object: + return '"{}"'.format(value) + else: + return value + + + @staticmethod + def _check_value_type(value): + value = str(value) + for key, pattern in observable.REGEX.items(): + if key != 'date' and bool(re.search(pattern, value)): + return key + return None + + @staticmethod + def _lookup_comparison_operator(self, expression_operator): + if expression_operator not in self.comparator_lookup: + raise NotImplementedError("Comparison operator {} unsupported for verify connector".format(expression_operator.name)) + return self.comparator_lookup[expression_operator] + + @classmethod + def _format_start_stop_qualifier(self, expression, qualifier) -> str: + """ + Convert a STIX start stop qualifier into a query string. + """ + transformer = TimestampToMilliseconds() + qualifier_split = qualifier.split("'") + start = qualifier_split[1] + stop = qualifier_split[3] + # convert timepestamp to millisecond which will be passed to rest service + start_epoach = transformer.transform(start); + stop_epoach = transformer.transform(stop) + + qualified_query = "%s&from=%s&to=%s" % (expression, start_epoach, stop_epoach) + return qualified_query + + def _parse_expression(self, expression, qualifier=None) -> Union[str, list]: + if isinstance(expression, ComparisonExpression): # Base Case + # Resolve STIX Object Path to a field in the target Data Model + stix_object, stix_field = expression.object_path.split(':') + # Multiple data source fields may map to the same STIX Object + mapped_fields_array = self.dmm.map_field(stix_object, stix_field) + # Resolve the comparison symbol to use in the query string (usually just ':') + comparator = self._lookup_comparison_operator(self, expression.comparator) + + # Some values are formatted differently based on how they're being compared + if expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: + # Should be in single-quotes + value = self._escape_value(expression.value) + #check if belongs to event object type. This require sepecial treatment. + value = self._check_filter_value_type(value,stix_object) + else: + value = self._escape_value(expression.value) + + comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array,stix_object) + if(len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): + # More than one data source field maps to the STIX attribute, so group comparisons together. + grouped_comparison_string = comparison_string + comparison_string = grouped_comparison_string + + if expression.negated: + comparison_string = self._negate_comparison(comparison_string) + if qualifier is not None: + return self._format_start_stop_qualifier(comparison_string, qualifier) + else: + return "{}".format(comparison_string) + + elif isinstance(expression, CombinedComparisonExpression): + operator = self._lookup_comparison_operator(self, expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + + if not expression_01 or not expression_02: + return '' + if isinstance(expression.expr1, CombinedComparisonExpression): + expression_01 = "{}".format(expression_01) + if isinstance(expression.expr2, CombinedComparisonExpression): + expression_02 = "{}".format(expression_02) + + query_string = "{}{}{}".format(expression_01, operator, expression_02) + if qualifier is not None: + return self._format_start_stop_qualifier(query_string, qualifier) + else: + return "{}".format(query_string) + elif isinstance(expression, ObservationExpression): + return self._parse_expression(expression.comparison_expression, qualifier) + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + if isinstance(expression.observation_expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(self, expression.observation_expression.operator) + expression_01 = self._parse_expression(expression.observation_expression.expr1) + # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions + expression_02 = self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + return "{} {} {}".format(expression_01, operator, expression_02) + else: + return self._parse_expression(expression.observation_expression.comparison_expression, expression.qualifier) + elif isinstance(expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(self, expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + if not isinstance(expression_01, list): + QueryStringPatternTranslator.QUERIES.extend([expression_01]) + if not isinstance(expression_02, list): + QueryStringPatternTranslator.QUERIES.extend([expression_02]) + return QueryStringPatternTranslator.QUERIES + elif isinstance(expression, Pattern): + return self._parse_expression(expression.expression) + else: + raise RuntimeError("Unknown Recursion Case for expression={}, type(expression)={}".format( + expression, type(expression))) + + def parse_expression(self, pattern: Pattern): + return self._parse_expression(pattern) + + +def translate_pattern(pattern: Pattern, data_model_mapping, options): + # Query result limit and time range can be passed into the QueryStringPatternTranslator if supported by the data source. + result_limit = options['result_limit'] + list_final_query = [] + # time_range = options['time_range'] + query = QueryStringPatternTranslator(pattern, data_model_mapping).translated + query = query if isinstance(query, list) else [query] + for each_query in query: + base_query = f"{each_query}&size={result_limit}" + list_final_query.append(base_query) + + return list_final_query diff --git a/stix_shifter_modules/verify/stix_translation/query_translator.py b/stix_shifter_modules/verify/stix_translation/query_translator.py new file mode 100644 index 000000000..575372d07 --- /dev/null +++ b/stix_shifter_modules/verify/stix_translation/query_translator.py @@ -0,0 +1,26 @@ +import logging + +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator +from . import query_constructor + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file + :param antlr_parsing_object: Antlr parsing objects for the STIX pattern + :type antlr_parsing_object: object + :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into another format. This should default to something if one isn't passed in + :type mapping: str (filepath) + :return: transformed query string + :rtype: str + """ + + logger.info("Converting STIX2 Pattern to data source query") + + query_string = query_constructor.translate_pattern( + antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/verify/stix_translation/results_translator.py b/stix_shifter_modules/verify/stix_translation/results_translator.py new file mode 100644 index 000000000..ac0a65a33 --- /dev/null +++ b/stix_shifter_modules/verify/stix_translation/results_translator.py @@ -0,0 +1,5 @@ +from stix_shifter_utils.stix_translation.src.json_to_stix.json_to_stix import JSONToStix + + +class ResultsTranslator(JSONToStix): + pass \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_transmission/__init__.py b/stix_shifter_modules/verify/stix_transmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/verify/stix_transmission/api_client.py b/stix_shifter_modules/verify/stix_transmission/api_client.py new file mode 100644 index 000000000..ff65a5071 --- /dev/null +++ b/stix_shifter_modules/verify/stix_transmission/api_client.py @@ -0,0 +1,184 @@ +import json + +from requests.adapters import Response +from stix_shifter_utils.stix_transmission.utils.RestApiClient import RestApiClient +from datetime import datetime, timedelta +import requests +from urllib.parse import urlencode +from stix_shifter_utils.utils import logger + + +class APIClient: + + endpoint_start = '/v1.0/events' + toeken_endpoint = '/v1.0/endpoint/default/token' + + def __init__(self, connection, configuration): + self.logger = logger.set_logger(__name__) + + headers = dict() + url_modifier_function = None + auth = configuration.get('auth') + # self.endpoint_start = 'incidents/' + self.host = connection.get('host') + self.client = RestApiClient(connection.get('host'), connection.get('port', None), + headers,url_modifier_function=url_modifier_function, + cert_verify=connection.get('selfSignedCert', False), + sni=connection.get('sni', None) + ) + self.timeout = connection['options'].get('timeout') + self._client_id = auth['clientId'] + self._client_secret = auth['clientSecret'] + self._token = None + self._token_time = None + + def get_token(self): + """get the token and if expired re-generate and store in token variable""" + tokenResponse = self.generate_token() + return tokenResponse.json().get('access_token') + + def generate_token(self): + """To generate the Token""" + if self.token_expired(): + resp = requests.request( + 'POST', + 'https://'+self.host+self.toeken_endpoint, + headers={ + 'accept': 'application/json', + 'Content-Type': 'application/x-www-form-urlencoded' + }, + data=( + f'client_id={self._client_id}' + f'&client_secret={self._client_secret}' + f'&grant_type=client_credentials' + f'&scope=openid' + ) + ) + token = resp.json().get('access_token') + self._token = token + self._token_time = datetime.now() + self.resp =resp + return self.resp + + def token_expired(self) -> bool: + """Check if the verify token is expired. + :return: True if token is expired, False if not expired + :return type: bool + """ + expired = True + if self._token: + expired = (datetime.now() - self._token_time) >= timedelta(minutes=30) + return expired + + + def run_search(self, query_expr, range_end=None): + """get the response from verify endpoints + :param quary_expr: dict, filter parameters + :param range_end: int,length value + :return: response, json object""" + events = [] + # if self._token : + events = self.get_events(query_expr) + return self.response_handler(events,query_expr) + + + def get_events(self,query_expr): + self.headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer {0}'.format(self.get_token())} + if query_expr is None: + data=None + return self.client.call_api(self.endpoint_start,'GET',self.headers,urldata= query_expr) + + def response_handler(self, data=None,query_expr=None): + if data is None: + data = [] + response = dict() + response['data'] =json.loads(data.read()) + response['error'] = data.code + response['code'] =data.code + response['error_msg'] = data.response.reason + response['success'] =data.code + if response['code'] ==200: + + response['search_after'] = response.get("data")['response']['events']['search_after'] + + try: + response['event_data'] = self.parseJson(response.get("data")['response']['events']['events']) + except KeyError: + self.logger.debug('events data not found in respose object',response) + response['event_data'] = [] + + elif response['error'] == 500 and "true" in response['error_msg']: + response.update({"code": 200, "data": []}) + else: + response["message"] = data.response.reason + + return response + + + def parseJson(self, response): + ''' + Iterate through the response and read the nested data like geoip and data object. + ''' + jsonObj = response + finalJson = dict() + parsedJson = [] + for obj in jsonObj: + dictC = obj + if "geoip" in dictC: + dictA= json.loads(json.dumps(obj["geoip"])) + del dictC["geoip"] + dictB= json.loads(json.dumps(obj["data"])) + del dictC["data"] + finalJson = {**dictA,**dictB} + else: + dictB= json.loads(json.dumps(obj["data"])) + del dictC["data"] + finalJson = dictB + remainingJson = json.loads(json.dumps(dictC)) + finalJson = {**finalJson,**remainingJson} + parsedJson.append(finalJson) + return parsedJson + + + def key_exist(self,data_element, *keysarr) : + ''' + Check if *keys (nested) exists in `element` (dict). + ''' + if not isinstance(data_element, dict): + raise AttributeError('keys_exists() expects dict as first argument.') + if len(keysarr) == 0: + raise AttributeError('keys_exists() expects at least two arguments, one given.') + + _element = data_element + for key in list(keysarr): + try: + _element = _element[key] + except KeyError: + self.logger.debug('key not found ',key ) + return False + return True + + def get_search_results(self, search_id, response_type, range_start=None, range_end=None): + # Sends a GET request to + # https://// + # response object body should contain information pertaining to search. + #https://isrras.ice.ibmcloud.com/v1.0/events?event_type="sso"&size=10&after="1640104162523","eeb40fd5-6b84-4dc9-9251-3f7a4cfd91c0" + headers = dict() + headers['Accept'] = response_type + size = 1000 + if ((range_start is None) and (range_end is None)): + size = range_end - range_start + + request_param = search_id+"& size="+str(size) + endpoint = self.endpoint_start+ request_param + + return self.run_search(search_id) + + def get_search(self, search_id): + # Sends a GET request to + # https:///api/ariel/searches/ + response = self.run_search(search_id) + return response + + def delete_search(self,search_id): + return self.run_search(search_id) diff --git a/stix_shifter_modules/verify/stix_transmission/delete_connector.py b/stix_shifter_modules/verify/stix_transmission/delete_connector.py new file mode 100644 index 000000000..2c3b63bc9 --- /dev/null +++ b/stix_shifter_modules/verify/stix_transmission/delete_connector.py @@ -0,0 +1,12 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +import json + + +class DeleteConnector(BaseDeleteConnector): + def __init__(self, api_client): + self.api_client = api_client + + def delete_query_connection(self, search_id): + return {"success": True} + \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_transmission/error_mapper.py b/stix_shifter_modules/verify/stix_transmission/error_mapper.py new file mode 100644 index 000000000..bb8771950 --- /dev/null +++ b/stix_shifter_modules/verify/stix_transmission/error_mapper.py @@ -0,0 +1,36 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + # No Route Exists + 404: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + # Authentication Failure + 401: ErrorCode.TRANSMISSION_AUTH_CREDENTIALS, + # A request parameter is not valid + 400: ErrorCode.TRANSMISSION_INVALID_PARAMETER, +} + + +class ErrorMapper(): + logger = logger.set_logger(__name__) + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + + + @staticmethod + def set_error_code(json_data, return_obj): + code = None + try: + code = json_data['code'] + except Exception: + pass + + error_code = ErrorMapper.DEFAULT_ERROR + + if code in error_mapping: + error_code = error_mapping[code] + + if error_code == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: " + str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_code) diff --git a/stix_shifter_modules/verify/stix_transmission/ping_connector.py b/stix_shifter_modules/verify/stix_transmission/ping_connector.py new file mode 100644 index 000000000..954a26715 --- /dev/null +++ b/stix_shifter_modules/verify/stix_transmission/ping_connector.py @@ -0,0 +1,29 @@ +import datetime +import json +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector +from stix_shifter_modules.verify.stix_transmission.api_client import APIClient +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + +class PingConnector(BasePingConnector): + + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def ping_connection(self): + ''' + ping the connection and return status details. + ''' + try: + response = self.api_client.generate_token() + # Construct a response object + return_obj = dict() + if response.status_code== 200: + return_obj['success'] = True + else: + ErrorResponder.fill_error(return_obj, response, ['message']) + return return_obj + except Exception as err: + self.logger.error('error when pinging datasource {}:'.format(err)) + raise diff --git a/stix_shifter_modules/verify/stix_transmission/query_connector.py b/stix_shifter_modules/verify/stix_transmission/query_connector.py new file mode 100644 index 000000000..4fa316d1f --- /dev/null +++ b/stix_shifter_modules/verify/stix_transmission/query_connector.py @@ -0,0 +1,30 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_connector import BaseQueryConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + + +class QueryConnector(BaseQueryConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def create_query_connection(self, query): + + error = None + try : + api_response = self.api_client.run_search(query) + response_code = api_response['success'] + except ValueError as ex: + self.logger.debug(ex) + error = Exception(f'Can not parse response: {ex} ') + + + return_obj = dict() + if 200 <= response_code <= 299 and error is None: + return_obj['success'] = True + return_obj['search_id'] = query + else: + ErrorResponder.fill_error(return_obj, api_response, ['message'], error=error) + + return return_obj diff --git a/stix_shifter_modules/verify/stix_transmission/results_connector.py b/stix_shifter_modules/verify/stix_transmission/results_connector.py new file mode 100644 index 000000000..b09606c59 --- /dev/null +++ b/stix_shifter_modules/verify/stix_transmission/results_connector.py @@ -0,0 +1,38 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + + +class ResultsConnector(BaseResultsConnector): + + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def create_results_connection(self, search_id, offset, length): + offset = int(offset) + length =int(length) + + try: + response = self.api_client.run_search(search_id, length) + response_code = response['code'] + # Construct a response object + return_obj = dict() + if response_code == 200: + return_obj['success'] = True + return_obj['data'] = response.get("event_data", []) + return_obj['search_after'] =response.get("search_after",[]) + # filter data based on filter_attr + # slice the records as per the provided offset and length(limit) + return_obj['data'] = return_obj['data'][offset:length] + else: + ErrorResponder.fill_error(return_obj, response, ['message']) + + except Exception as err: + self.logger.error('error when getting search results: {}'.format(err)) + import traceback + self.logger.error(traceback.print_stack()) + raise + return return_obj + \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_transmission/status_connector.py b/stix_shifter_modules/verify/stix_transmission/status_connector.py new file mode 100644 index 000000000..834e0c8c8 --- /dev/null +++ b/stix_shifter_modules/verify/stix_transmission/status_connector.py @@ -0,0 +1,16 @@ + +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import BaseStatusConnector +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger as utils_logger +from enum import Enum +import json + + +class StatusConnector(BaseStatusConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = utils_logger.set_logger(__name__) + + def create_status_connection(self, search_id): + return {"success": True, "status": "COMPLETED", "progress": 100} diff --git a/stix_shifter_modules/verify/test/stix_translation/___init__.py b/stix_shifter_modules/verify/test/stix_translation/___init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/verify/test/stix_translation/test_transform.py b/stix_shifter_modules/verify/test/stix_translation/test_transform.py new file mode 100644 index 000000000..9e052e978 --- /dev/null +++ b/stix_shifter_modules/verify/test/stix_translation/test_transform.py @@ -0,0 +1,90 @@ +from unicodedata import category +import unittest +import json +import logging +from stix_shifter.stix_translation import stix_translation +from stix_shifter_modules.verify.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers +translation = stix_translation.StixTranslation() +# config_file = open('stix_shifter_modules/verify_event/configuration/config.json').read() +# from_stix_file = open('stix_shifter_modules/verify_event/stix_translation/json/from_stix_map.json').read() +# to_stix_file = open('stix_shifter_modules/verify_event/stix_translation/json/to_stix_map.json').read() +# OPTIONS = json.loads(from_stix_file) + +# logging.basicConfig(level=logging.DEBUG) +# logger = logging.getLogger() +MODULE = 'verify' +RESULTS = 'results' +TRANSFORMERS = get_module_transformers(MODULE) +epoch_to_timestamp_class = TRANSFORMERS.get('EpochToTimestamp') +entry_point = EntryPoint() +MAP_DATA = entry_point.get_results_translator().map_data + + +DATA_SOURCE ={ + "type": "identity", + "id": "32a23267-52fb-4e82-859b-0a15d6a2d334", + "name":"verify", + "identity_class":"events" + } +OPTION = json.dumps(DATA_SOURCE) +class TestTransformQuery(unittest.TestCase,object): + + @staticmethod + def get_first(itr, constraint): + return next( + (obj for obj in itr if constraint(obj)), + None + ) + + @staticmethod + def get_first_of_type(itr, typ): + return TestTransformQuery.get_first(itr, lambda o: type(o) == dict and o.get('type') == typ) + + @staticmethod + def get_object_keys(objects): + for k, v in objects.items(): + if k == 'type': + yield v + elif isinstance(v, dict): + for id_val in TestTransformQuery.get_object_keys(v): + yield id_val + + + def test_oca_event(self): + data = [{"continent_name": "Asia", "city_name": "Kolkata", "country_iso_code": "IN", + "ip": "47.15.98.56", "country_name": "India", "region_name": "West Bengal", + "location": {"lon": "88.3697", "lat": "22.5697"}, "result": "success", + "subtype": "saml", "providerid": "https://portal.baneandox.org:443/SAML20/SP", + "origin": "47.15.98.56", "realm": "www.ibm.com", + "applicationid": "6773634223410562472", + "userid": "652001LT0R", "applicationtype": "Custom Application", "devicetype": + "PAN GlobalProtect/5.2.4-21 (Microsoft Windows 10 Enterprise , 64-bit) Mozilla/5.0 (Windows NT 6.2; Win64; x64; Trident/7.0; rv:11.0) like Gecko", + "username": "dinepal1@in.ibm.com", "applicationname": "Bane & Ox VPN", "year": 2022, + "@metadata": {"group_id": "event-transform-prod-eu01a-prod-eu01a-01", "source_dc": "prod-eu01a"}, + "event_type": "sso", "month": 1, "indexed_at": 1642413142906, "@processing_time": 1012, + "tenantid": "c92ce528-293f-4e84-8307-c4fe188b9461", "tenantname": "isrras.ice.ibmcloud.com", + "correlationid": "CORR_ID-a134f569-8d73-45ac-8d44-2457448c9101", "servicename": "saml_runtime", + "id": "dc4523e6-6260-4349-83f8-3320365a5f25", "time": 1642413141894, "day": 17}, ] + + user_ref ="2" + category ="sso" + domain_ref = "3" + module="saml_runtime" + extensions_user_id = "652001LT0R" + result_bundle = entry_point.translate_results(json.dumps(DATA_SOURCE), json.dumps(data)) + observed_data = result_bundle['objects'][1] + objects = observed_data['objects'] + + event = TestTransformQuery.get_first_of_type(objects.values(), 'x-oca-event') + + assert(event['type']) == "x-oca-event" + assert(event['user_ref'] == user_ref) + assert(event['category'] == category) + assert(event['domain_ref'] == domain_ref) + assert(event['module'] == module) + assert(event['category'] == category) + assert(event['extensions']['x-iam-ext']['user_id']== extensions_user_id) + assert(event['created'] == '2022-01-17T09:52:21.894Z') + + \ No newline at end of file diff --git a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py new file mode 100644 index 000000000..14a0aeea5 --- /dev/null +++ b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py @@ -0,0 +1,73 @@ +from unicodedata import category +import unittest +import json +import logging +from stix_shifter.stix_translation import stix_translation +from stix_shifter_modules.verify.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers +translation = stix_translation.StixTranslation() +# config_file = open('stix_shifter_modules/verify_event/configuration/config.json').read() +# from_stix_file = open('stix_shifter_modules/verify_event/stix_translation/json/from_stix_map.json').read() +# to_stix_file = open('stix_shifter_modules/verify_event/stix_translation/json/to_stix_map.json').read() +# OPTIONS = json.loads(from_stix_file) + +# logging.basicConfig(level=logging.DEBUG) +# logger = logging.getLogger() +MODULE = 'verify' +RESULTS = 'results' +TRANSFORMERS = get_module_transformers(MODULE) +epoch_to_timestamp_class = TRANSFORMERS.get('EpochToTimestamp') +EPOCH_START = 1531169112 +EPOCH_END = 1531169254 +START_TIMESTAMP = epoch_to_timestamp_class.transform(EPOCH_START) +END_TIMESTAMP = epoch_to_timestamp_class.transform(EPOCH_END) +entry_point = EntryPoint() +MAP_DATA = entry_point.get_results_translator().map_data + + +DATA_SOURCE ={ + "type": "identity", + "id": "identity--32a23267-52fb-4e82-859b-0a15d6a2d334", + "name":"verify", + "identity_class":"events" + } +OPTION = json.dumps(DATA_SOURCE) +def _test_query_assertions(query, queries): + assert query['queries'] == [queries] + +def _translate_query(stix_pattern): + return translation.translate('verify', 'query', '{}', stix_pattern) + +class TestStixToQuery(unittest.TestCase, object): + + def test_event_type(self): + stix_pattern = "[x-oca-event:category='authentication']" + query = _translate_query(stix_pattern) + queries = "event_type=\"authentication\"&size=10000" + _test_query_assertions(query, queries) + + def test_IPV4_query(self): + stix_pattern = "[ipv4-addr:value='27.58.174.31']" + query = _translate_query(stix_pattern) + expected_queries = 'filter_key=data.origin&filter_value="27.58.174.31"&size=10000' + _test_query_assertions(query, expected_queries) + + def test_oca_event_extension(self): + stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.user_id='652001LT0R']" + query = _translate_query(stix_pattern) + expected_queries = 'filter_key=data.userid&filter_value="652001LT0R"&size=10000' + _test_query_assertions(query,expected_queries) + + def test_event_type_and_fiter(self): + stix_pattern = "[x-oca-event:category = 'sso' AND x-oca-event:domain_ref.value = '77.169.74.152']START t'2022-01-17T18:24:00.000Z' STOP t'2022-01-20T18:24:00.000Z' " + query = _translate_query(stix_pattern) + expected_queries = 'filter_key=tenantname&filter_value="77.169.74.152"&event_type="sso"&from=1642443840000&to=1642703040000&size=10000' + _test_query_assertions(query, expected_queries) + + def test_domain_name_query(self): + stix_pattern ="[domain-name:value = 'isrras.ice.ibmcloud.com']" + query = _translate_query(stix_pattern) + expected_queries = 'filter_key=tenantname&filter_value="isrras.ice.ibmcloud.com"&size=10000' + _test_query_assertions(query,expected_queries) + + \ No newline at end of file diff --git a/stix_shifter_modules/verify/test/stix_transmission/___init__.py b/stix_shifter_modules/verify/test/stix_transmission/___init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/verify/test/stix_transmission/test_verify.py b/stix_shifter_modules/verify/test/stix_transmission/test_verify.py new file mode 100644 index 000000000..9321e5210 --- /dev/null +++ b/stix_shifter_modules/verify/test/stix_transmission/test_verify.py @@ -0,0 +1,158 @@ + +import json +from sre_constants import ASSERT_NOT +import unittest +from unittest.mock import ANY +from unittest.mock import patch +from stix_shifter.stix_transmission import stix_transmission +from stix_shifter_modules.verify.entry_point import EntryPoint +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from stix_shifter_utils.stix_transmission.utils.RestApiClient import ResponseWrapper +from unittest.mock import patch + + +class VerifyMockResponse: + def __init__(self, response_code, object): + self.code = response_code + self.object = object + + def read(self): + return self.object + +class VerifyMockPingResponse: + def __init__(self, response_code, status_code): + self.code = response_code + self.status_code =status_code + + def read(self): + return self.object + + +@patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.__init__', autospec=True) +class TestVerifyConnection(unittest.TestCase, object): + + def test_is_async(self, mock_api_client): + mock_api_client.return_value = None + entry_point = EntryPoint() + + config = { + "auth": { + "sec": "bla" + } + } + connection = { + "host": "hostbla", + "port": 8080, + "selfSignedCert": "cert" + } + check_async = entry_point.is_async() + assert check_async is not True + + @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.get_search', autospec=True) + def test_status_response(self, mock_status_response, mock_api_client): + mock_api_client.return_value = None + mocked_return_value = '{"search_id": "108cb8b0-0744-4dd9-8e35-ea8311cd6211", "status": "COMPLETED", "progress": "100"}' + mock_status_response.return_value = VerifyMockResponse(200, mocked_return_value) + + config = { + "host": "connection.com" + } + connection = { + "auth" :{ + "clientId": "clientId", + "clientSecret": "clientscred" + } + } + + search_id = "108cb8b0-0744-4dd9-8e35-ea8311cd6211" + transmission = stix_transmission.StixTransmission('verify', config, connection) + status_response = transmission.status(search_id) + + assert status_response['success'] + assert status_response is not None + assert 'status' in status_response + assert status_response['status'] == Status.COMPLETED.value + + @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.run_search') + def test_query_response(self, mock_query_response, mock_api_client): + mock_api_client.return_value = None + mock_query_response.return_value = {"success":200} + + config = { + "host": "cloudsecurity.com" + } + connection = { + "auth" :{ + "clientId": "clientid", + "clientSecret": "secret" + } + } + + query = '{"query":"event_type=\"sso\""}' + transmission = stix_transmission.StixTransmission('verify', config, connection) + query_response = transmission.query(query) + + assert query_response is not None + assert 'search_id' in query_response + assert query_response['search_id'] == query + + + @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.generate_token') + def test_ping(self, mock_generate_token, mock_api_client): + + config = { + "host": "cloudsecurity.com" + } + connection = { + "auth" :{ + "clientId": "clientid", + "clientSecret": "secret" + } + } + mocked_return_value = VerifyMockPingResponse(200,200) + mock_generate_token.return_value = mocked_return_value + mock_api_client.return_value = None + entry_point = EntryPoint(config, connection) + ping_result = entry_point.ping_connection() + assert ping_result["success"] is True + + + @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.generate_token') + @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.run_search', + autospec=True) + def test_results_all_response(self, mock_results_response, mock_generate_token, mock_api_client): + mock_api_client.return_value = None + mocked_return_value = {"code": 200} + mock_generate_token.return_value = mocked_return_value + config = { + "host": "ibmcloud.com" + } + connection = { + "auth" :{ + "clientId": "clientId", + "clientSecret": "secret" + } + } + + mocked_return_value = { + "code": 200, + "success":200, + "data": [ + {"id":123, "created_at":"2022-01-16T16:45:16.112Z", "account_id":123, "ipaddr":"12.22.33.44"} + ]} + + mock_results_response.return_value = mocked_return_value + + query = "event_type=\"sso\"&limit=10000" + + offset = 0 + length = 101 + entry_point = EntryPoint(config, connection) + results_response = entry_point.create_results_connection(query, offset, length) + + assert results_response is not None + assert results_response['success'] + assert 'data' in results_response + assert results_response['data'] is not None + + \ No newline at end of file From 133a91c3c0c299bd19b249489e341be1ebd343d5 Mon Sep 17 00:00:00 2001 From: RMS Date: Wed, 16 Feb 2022 12:41:33 +0530 Subject: [PATCH 02/19] updated stix mapping reference and fixed escpae character problem in query param --- requirements.txt | 18 ++++++++ .../verify/configuration/config.json | 15 ++++++ .../stix_translation/json/from_stix_map.json | 25 ++++------ .../stix_translation/json/operators.json | 4 +- .../stix_translation/json/to_stix_map.json | 46 +++++-------------- .../stix_translation/query_constructor.py | 20 ++++---- .../verify/stix_translation/transformers.py | 8 ++++ .../test_verify_stix_to_query.py | 8 +++- 8 files changed, 82 insertions(+), 62 deletions(-) create mode 100644 requirements.txt create mode 100644 stix_shifter_modules/verify/stix_translation/transformers.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..82bb96df2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +adal==1.2.7 +antlr4-python3-runtime==4.8 +boto3==1.20.33 +colorlog==6.6.0 +datadog_api_client==1.2.0 +flask==2.0.2 +flatten_json==0.1.13 +jsonmerge==1.8.0 +mysql-connector-python==8.0.25 +onelogin==2.0.1 +pyOpenSSL==21.0.0 +python-dateutil==2.8.2 +requests_toolbelt==0.9.1 +stix2-matcher==2.0.1 +stix2-patterns==1.3.2 +stix2-validator==1.1.2 +sumologic-sdk==0.1.13 +xmltodict==0.12.0 diff --git a/stix_shifter_modules/verify/configuration/config.json b/stix_shifter_modules/verify/configuration/config.json index b5f63c0bb..55d7a18fc 100644 --- a/stix_shifter_modules/verify/configuration/config.json +++ b/stix_shifter_modules/verify/configuration/config.json @@ -8,6 +8,21 @@ "type": "text", "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_:/\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_:/\\-]*[A-Za-z0-9])$" }, + "port": { + "type": "number", + "default": 443, + "min": 1, + "max": 65535 + }, + + "sni": { + "type": "text", + "optional": true + }, + "selfSignedCert": { + "type": "password", + "optional": true + }, "region": { "type": "text", "default": "us" diff --git a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json index dd6211919..3c2639ba6 100644 --- a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json @@ -3,8 +3,9 @@ "user-account":{ "fields": { "user_id": ["data.username"], - "account_type":["data.sourcetype"], - "performedby_username":["data.performedby_username"] + "account_login": ["data.username"], + "account_type":["data.realm"] + } }, "ipv4-addr":{ @@ -25,46 +26,40 @@ "module":["servicename"], "outcome":["data.result"], "agent" :["data.sourcetype"], - "ip_refs":["ip"], + "ip_refs[*].value":["ip"], "domain_ref.value": ["tenantname"], "user_ref": ["username"], "provider":"'IBM Security Verify Event'", - "extensions.'x-iam-ext'.subject":["data.subject"], "extensions.'x-iam-ext'.subcategory":["data.subtype"], "extensions.'x-iam-ext'.realm":["data.realm"], "extensions.'x-iam-ext'.browser_agent":["data.devicetype"], + "extensions.'x-iam-ext'.provider_id":["data.providerid"], "extensions.'x-iam-ext'.application_id":["data.applicationid"], "extensions.'x-iam-ext'.user_id":["data.userid"], "extensions.'x-iam-ext'.application_type":["data.applicationtype"], "extensions.'x-iam-ext'.application_name":["data.applicationname"], "extensions.'x-iam-ext'.cause":["data.cause"], "extensions.'x-iam-ext'.target":["data.target"], - "extensions.'x-iam-ext'.performedby_username":["data.performedby_username"], - "extensions.'x-iam-ext'.message_id":["data.messageid"], "extensions.'x-iam-ext'.deleted":["data.deleted"], "extensions.'x-iam-ext'.performedby_clientname":["data.performedby_clientname"], "extensions.'x-iam-ext'.performedby_realm":["data.performedby_realm"], - "extensions.'x-iam-ext'.resource":["data.resource"], + "extensions.'x-iam-ext'.performedby_username":["data.performedby_username"], "extensions.'x-iam-ext'.targetid":["data.targetid"], "extensions.'x-iam-ext'.targetid_realm" :["data.targetid_realm"], "extensions.'x-iam-ext'.targetid_username" :["data.targetid_username"], - "extensions.'x-iam-ext'.userinfo_lookup":["data.userinfo_lookup"], "extensions.'x-iam-ext'.continent_name":["geoip.continent_name"], - "extensions.'x-iam-ext'.city_name": ["geoip.city_name"], "extensions.'x-iam-ext'.country_iso_code": ["geoip.country_iso_code"], "extensions.'x-iam-ext'.country_name": ["geoip.country_name"], - "extensions.'x-iam-ext'.region_name":["geoip.region_name"], "extensions.'x-iam-ext'.location_lon":["geoip.location.lon"], "extensions.'x-iam-ext'.location_lat":["geoip.location_lat"], - "extensions.'x-iam-ext'.provider_id":["data.providerid"], - "extensions.'x-iam-ext'.'decision_decisionCode'":["data.decision_decisionCode"], - "extensions.'x-iam-ext'.'rule_name'":["data.rule_name"], + "extensions.'x-iam-ext'.city_name": ["geoip.city_name"], + "extensions.'x-iam-ext'.'policy_action'":["data.policy_action"], "extensions.'x-iam-ext'.'policy_name'":["data.policy_name"], + "extensions.'x-iam-ext'.'rule_name'":["data.rule_name"], "extensions.'x-iam-ext'.'decision_reason'":["data.decision_reason"], - "extensions.'x-iam-ext'.'policy_action'":["data.policy_action"], "extensions.'x-iam-ext'.'risk_level'":["data.risk_level"], "extensions.'x-iam-ext'.'risk_score'":["data.risk_score"] - + } } } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/operators.json b/stix_shifter_modules/verify/stix_translation/json/operators.json index 71dd51c7e..0913fd6b2 100644 --- a/stix_shifter_modules/verify/stix_translation/json/operators.json +++ b/stix_shifter_modules/verify/stix_translation/json/operators.json @@ -1,5 +1,7 @@ { "ComparisonExpressionOperators.And": "&", "ComparisonComparators.Equal": "=", - "ObservationOperators.And": "or" + "ObservationOperators.And": "or", + "ComparisonComparators.In": "=" + } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json index a452547d7..47ff8c5f7 100644 --- a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json @@ -8,12 +8,8 @@ "key": "x-oca-event.user_ref", "object": "ocaevent", "references": "useraccount" - }, - { - "key": "x-oca-event.extensions.x-iam-ext.performedby_username", - "object": "ocaevent", - "references": "useraccount" } + ], "servicename" :{ "key":"x-oca-event.module", @@ -30,7 +26,8 @@ { "key": "x-oca-event.ip_refs", "object": "ocaevent", - "references": "ipdata" + "references": ["ipdata"], + "group":true } ], "tenantname" :[ @@ -49,7 +46,7 @@ { "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.outcome" + "key": "x-oca-event.outcome" } ], @@ -58,10 +55,7 @@ "object": "ocaevent", "key": "x-oca-event.extensions.x-iam-ext.subcategory" }, -"subject": { - "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.subject" -}, + "cause": { "object": "ocaevent", @@ -103,6 +97,10 @@ { "key":"x-oca-event.category", "object":"ocaevent" +},{ + "key": "x-oca-event.provider", + "object": "ocaevent", + "transformer": "VerifyStaticTransformer" } ], @@ -118,10 +116,7 @@ "key": "x-oca-event.extensions.x-iam-ext.performedby_username", "object": "ocaevent" }, -"messageid" :{ - "key":"x-oca-event.extensions.x-iam-ext.message_id", - "object":"ocaevent" -}, + "deleted" :{ "key":"x-oca-event.extensions.x-iam-ext.deleted", "object":"ocaevent" @@ -134,10 +129,7 @@ "key":"x-oca-event.extensions.x-iam-ext.performedby_realm", "object" :"ocaevent" }, -"resource" :{ - "key":"x-oca-event.extensions.x-iam-ext.resource", - "object" :"ocaevent" -}, + "targetid" :{ "key":"x-oca-event.extensions.x-iam-ext.targetid", "object":"ocaevent" @@ -150,10 +142,6 @@ "key":"x-oca-event.extensions.x-iam-ext.taregetid_username", "object":"ocaevent" }, -"userinfo_lookup" :{ - "key" :"x-oca-event.extensions.x-iam-ext.userinfo_lookup", - "object":"ocaevent" -}, "userid" :[ { "key":"x-oca-event.extensions.x-iam-ext.user_id", @@ -180,22 +168,10 @@ "key" :"x-oca-event.extensions.x-iam-ext.country_name", "object":"ocaevent" }, - "region_name" : { - "key" :"x-oca-event.extensions.x-iam-ext.region_name", - "object":"ocaevent" - }, - "provider" : { - "key" :"x-oca-event.provider", - "object":"ocaevent" - }, "providerid" :{ "key":"x-oca-event.extensions.x-iam-ext.provider_id", "object":"ocaevent" }, - "decision_decisionCode" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.decision_decisionCode" - }, "rule_name" :{ "object":"ocaevent", "key":"x-oca-event.extensions.x-iam-ext.rule_name" diff --git a/stix_shifter_modules/verify/stix_translation/query_constructor.py b/stix_shifter_modules/verify/stix_translation/query_constructor.py index 87ec16bee..930525219 100644 --- a/stix_shifter_modules/verify/stix_translation/query_constructor.py +++ b/stix_shifter_modules/verify/stix_translation/query_constructor.py @@ -20,24 +20,26 @@ class QueryStringPatternTranslator: QUERIES = [] - # Change comparator values to match with supported data source operators + # # Change comparator values to match with supported data source operators comparator_lookup = { - ComparisonExpressionOperators.And: "&", - ComparisonComparators.Equal: "=", - ObservationOperators.And: 'or', - # Treat AND's as OR's -- Unsure how two ObsExps wouldn't cancel each other out. - ObservationOperators.Or: 'or' + "ComparisonExpressionOperators.And": "&", + "ComparisonComparators.Equal": "=", + "ObservationOperators.And": "or", + "ComparisonComparators.In": "=" + } + def __init__(self, pattern: Pattern, data_model_mapper): self.dmm = data_model_mapper + #self.comparator_lookup = self.dmm.map_comparator() #Not sure its not workig with kestral self.pattern = pattern self.translated = self.parse_expression(pattern) @staticmethod def _escape_value(value, comparator=None) -> str: if isinstance(value, str): - return '{}'.format(value.replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)').replace("$","\"")) + return '{}'.format(value.replace("\'","'").replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)').replace("$","\"").replace('&','%26')) else: return value @@ -122,9 +124,9 @@ def _check_value_type(value): @staticmethod def _lookup_comparison_operator(self, expression_operator): - if expression_operator not in self.comparator_lookup: + if str(expression_operator) not in self.comparator_lookup: raise NotImplementedError("Comparison operator {} unsupported for verify connector".format(expression_operator.name)) - return self.comparator_lookup[expression_operator] + return self.comparator_lookup[str(expression_operator)] @classmethod def _format_start_stop_qualifier(self, expression, qualifier) -> str: diff --git a/stix_shifter_modules/verify/stix_translation/transformers.py b/stix_shifter_modules/verify/stix_translation/transformers.py new file mode 100644 index 000000000..d97b1b72f --- /dev/null +++ b/stix_shifter_modules/verify/stix_translation/transformers.py @@ -0,0 +1,8 @@ +from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer + +class VerifyStaticTransformer(ValueTransformer): + """A value transformer that always returns the string 'IBM Security Verify IAM'""" + + @staticmethod + def transform(value): + return "IBM Security Verify IAM" \ No newline at end of file diff --git a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py index 14a0aeea5..9c72193ad 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py @@ -69,5 +69,9 @@ def test_domain_name_query(self): query = _translate_query(stix_pattern) expected_queries = 'filter_key=tenantname&filter_value="isrras.ice.ibmcloud.com"&size=10000' _test_query_assertions(query,expected_queries) - - \ No newline at end of file + + def test_applicationname_with_special_char_query(self): + stix_pattern ="[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" + query = _translate_query(stix_pattern) + expected_query = 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' + _test_query_assertions(query,expected_query) \ No newline at end of file From f3aebeb7fd506f9ad0199be7b065d43700327437 Mon Sep 17 00:00:00 2001 From: RMS Date: Wed, 16 Feb 2022 22:03:33 +0530 Subject: [PATCH 03/19] added IN support and corrected mapping of outcome --- .../stix_translation/json/from_stix_map.json | 3 +- .../stix_translation/json/to_stix_map.json | 7 +--- .../stix_translation/query_constructor.py | 34 ++++++++++++------- .../test/stix_translation/test_transform.py | 2 +- .../test_verify_stix_to_query.py | 29 ++++++++++++---- 5 files changed, 47 insertions(+), 28 deletions(-) diff --git a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json index 3c2639ba6..096a255ca 100644 --- a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json @@ -26,7 +26,7 @@ "module":["servicename"], "outcome":["data.result"], "agent" :["data.sourcetype"], - "ip_refs[*].value":["ip"], + "ip_refs.value":["ip"], "domain_ref.value": ["tenantname"], "user_ref": ["username"], "provider":"'IBM Security Verify Event'", @@ -35,7 +35,6 @@ "extensions.'x-iam-ext'.browser_agent":["data.devicetype"], "extensions.'x-iam-ext'.provider_id":["data.providerid"], "extensions.'x-iam-ext'.application_id":["data.applicationid"], - "extensions.'x-iam-ext'.user_id":["data.userid"], "extensions.'x-iam-ext'.application_type":["data.applicationtype"], "extensions.'x-iam-ext'.application_name":["data.applicationname"], "extensions.'x-iam-ext'.cause":["data.cause"], diff --git a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json index 47ff8c5f7..797566503 100644 --- a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json @@ -26,8 +26,7 @@ { "key": "x-oca-event.ip_refs", "object": "ocaevent", - "references": ["ipdata"], - "group":true + "references": "ipdata" } ], "tenantname" :[ @@ -143,10 +142,6 @@ "object":"ocaevent" }, "userid" :[ - { - "key":"x-oca-event.extensions.x-iam-ext.user_id", - "object":"ocaevent" -}, { "key": "user-account.user_id", "object": "useraccount" diff --git a/stix_shifter_modules/verify/stix_translation/query_constructor.py b/stix_shifter_modules/verify/stix_translation/query_constructor.py index 930525219..6889edca4 100644 --- a/stix_shifter_modules/verify/stix_translation/query_constructor.py +++ b/stix_shifter_modules/verify/stix_translation/query_constructor.py @@ -1,4 +1,5 @@ from os import stat +from sre_constants import IN from typing import Union from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \ ComparisonExpressionOperators, ComparisonComparators, Pattern, \ @@ -7,6 +8,7 @@ from stix_shifter_utils.stix_translation.src.utils.transformers import TimestampToMilliseconds import logging import re +import json # Source and destination reference mapping for ip and mac addresses. # Change the keys to match the data source fields. The value array indicates the possible data type that can come into from field. @@ -39,9 +41,19 @@ def __init__(self, pattern: Pattern, data_model_mapper): @staticmethod def _escape_value(value, comparator=None) -> str: if isinstance(value, str): - return '{}'.format(value.replace("\'","'").replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)').replace("$","\"").replace('&','%26')) + return '{}'.format(value.replace("\'","'").replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)').replace("$","\"").replace('&','%26') + .replace('""','"')) else: return value + @staticmethod + def _convert_list_string_in_condition(value)->str: + if isinstance(value,list): + contcated_string = ''.join(f'"{str}",'.format(str) for str in value ).removesuffix(',') + contcated_string= contcated_string[1:-1] + return contcated_string + + + @staticmethod def _negate_comparison(comparison_string): @@ -54,7 +66,7 @@ def _check_value_type(value): if key != 'date' and bool(re.search(pattern, value)): return key return None - + # TODO remove self reference from static methods @staticmethod def _parse_reference(self, stix_field, value_type, mapped_field, value, comparator): @@ -112,16 +124,6 @@ def _check_filter_value_type(value, stix_object): return '"{}"'.format(value) else: return value - - - @staticmethod - def _check_value_type(value): - value = str(value) - for key, pattern in observable.REGEX.items(): - if key != 'date' and bool(re.search(pattern, value)): - return key - return None - @staticmethod def _lookup_comparison_operator(self, expression_operator): if str(expression_operator) not in self.comparator_lookup: @@ -159,7 +161,13 @@ def _parse_expression(self, expression, qualifier=None) -> Union[str, list]: value = self._escape_value(expression.value) #check if belongs to event object type. This require sepecial treatment. value = self._check_filter_value_type(value,stix_object) - else: + elif expression.comparator == ComparisonComparators.In: + in_string = expression.value.values if hasattr(expression.value, 'values') else expression.value + values = self._convert_list_string_in_condition(in_string) + #apply escape value to remove unwanted char in string. + value = self._escape_value(values) + + else : value = self._escape_value(expression.value) comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array,stix_object) diff --git a/stix_shifter_modules/verify/test/stix_translation/test_transform.py b/stix_shifter_modules/verify/test/stix_translation/test_transform.py index 9e052e978..ae198a53c 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_transform.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_transform.py @@ -84,7 +84,7 @@ def test_oca_event(self): assert(event['domain_ref'] == domain_ref) assert(event['module'] == module) assert(event['category'] == category) - assert(event['extensions']['x-iam-ext']['user_id']== extensions_user_id) + assert(event['extensions']) assert(event['created'] == '2022-01-17T09:52:21.894Z') \ No newline at end of file diff --git a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py index 9c72193ad..4ec207f91 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py @@ -52,11 +52,11 @@ def test_IPV4_query(self): expected_queries = 'filter_key=data.origin&filter_value="27.58.174.31"&size=10000' _test_query_assertions(query, expected_queries) - def test_oca_event_extension(self): - stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.user_id='652001LT0R']" - query = _translate_query(stix_pattern) - expected_queries = 'filter_key=data.userid&filter_value="652001LT0R"&size=10000' - _test_query_assertions(query,expected_queries) + # def test_oca_event_extension(self): + # stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.user_id='652001LT0R']" + # query = _translate_query(stix_pattern) + # expected_queries = 'filter_key=data.userid&filter_value="652001LT0R"&size=10000' + # _test_query_assertions(query,expected_queries) def test_event_type_and_fiter(self): stix_pattern = "[x-oca-event:category = 'sso' AND x-oca-event:domain_ref.value = '77.169.74.152']START t'2022-01-17T18:24:00.000Z' STOP t'2022-01-20T18:24:00.000Z' " @@ -74,4 +74,21 @@ def test_applicationname_with_special_char_query(self): stix_pattern ="[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" query = _translate_query(stix_pattern) expected_query = 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' - _test_query_assertions(query,expected_query) \ No newline at end of file + _test_query_assertions(query,expected_query) + + def test_in_statement(self): + stix_pattern = "[ ipv4-addr:value IN ('65.154.226.165', '96.244.40.143', '37.8.230.16') ]" + query = _translate_query(stix_pattern) + expected_query = 'filter_key=data.origin&filter_value="65.154.226.165","96.244.40.143","37.8.230.16"&size=10000' + _test_query_assertions(query,expected_query) + + def test_in_statement_with_start_stop(self): + stix_pattern = "[ ipv4-addr:value IN ('65.154.226.165', '96.244.40.143', '37.8.230.16') ] START t'2022-02-06T07:19:00.000Z' STOP t'2022-02-08T07:19:00.000Z' " + query = _translate_query(stix_pattern) + expected_query = 'filter_key=data.origin&filter_value="65.154.226.165","96.244.40.143","37.8.230.16"&from=1644131940000&to=1644304740000&size=10000' + _test_query_assertions(query,expected_query) + def test_in_statement_with_event_type_start_stop(self): + stix_pattern = "[ x-oca-event:category IN ('sso', 'authentication') ] START t'2022-02-06T07:19:00.000Z' STOP t'2022-02-08T07:19:00.000Z' " + query = _translate_query(stix_pattern) + expected_query = 'event_type="sso","authentication"&from=1644131940000&to=1644304740000&size=10000' + _test_query_assertions(query,expected_query) From b989b46b8cbc5ab6cc801498f30d14b020601bfb Mon Sep 17 00:00:00 2001 From: RMS Date: Thu, 17 Feb 2022 22:26:10 +0530 Subject: [PATCH 04/19] changes are done as per reviewr feedback --- requirements.txt | 18 - stix_shifter_modules/verify/README.md | 151 ++++++-- .../verify/configuration/config.json | 22 -- .../verify/configuration/lang_en.json | 47 +-- stix_shifter_modules/verify/entry_point.py | 44 +-- .../stix_translation/json/from_stix_map.json | 5 +- .../json/from_to_stix_1.8.json | 58 --- .../stix_translation/json/operators.json | 3 +- .../stix_translation/json/to_stix_map.json | 366 +++++++++--------- .../stix_translation/query_constructor.py | 8 +- .../stix_transmission/delete_connector.py | 3 +- .../verify/stix_transmission/error_mapper.py | 2 - .../stix_transmission/query_connector.py | 19 +- .../stix_transmission/results_connector.py | 14 +- .../test_verify_stix_to_query.py | 53 +-- .../test/stix_transmission/test_verify.py | 3 +- 16 files changed, 345 insertions(+), 471 deletions(-) delete mode 100644 requirements.txt delete mode 100644 stix_shifter_modules/verify/stix_translation/json/from_to_stix_1.8.json diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 82bb96df2..000000000 --- a/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -adal==1.2.7 -antlr4-python3-runtime==4.8 -boto3==1.20.33 -colorlog==6.6.0 -datadog_api_client==1.2.0 -flask==2.0.2 -flatten_json==0.1.13 -jsonmerge==1.8.0 -mysql-connector-python==8.0.25 -onelogin==2.0.1 -pyOpenSSL==21.0.0 -python-dateutil==2.8.2 -requests_toolbelt==0.9.1 -stix2-matcher==2.0.1 -stix2-patterns==1.3.2 -stix2-validator==1.1.2 -sumologic-sdk==0.1.13 -xmltodict==0.12.0 diff --git a/stix_shifter_modules/verify/README.md b/stix_shifter_modules/verify/README.md index 369881ab3..45a8bce98 100644 --- a/stix_shifter_modules/verify/README.md +++ b/stix_shifter_modules/verify/README.md @@ -1,63 +1,152 @@ -# Verify +# IBM Security Verify -This is a connector for searching IBM Verify events. +This is a connector for searching IBM Security Verify events. Connector uses stix-patterns and IBM event verify REST API to make a convert and execute the qurey. -### Format for making STIX translation calls via the CLI +* To know more about IBM Security Verify API refer to the [API Reference](https://docs.verify.ibm.com/verify/reference/getallevents) +* Connector uses the stix schema defined as per [stix-extension/stix2.0/x-oca-event](https://github.ibm.com/IBM-Security-STIX/stix-extensions/blob/verify/STIX%202.0/x-oca-event.md) +* Connector supports ` Equal, AND, IN ` stix operations +* Possible event types are ` sso,authentication,management,risk,adaptive_risk ` + +### Some of Stix pattern examples + + * Event type `[x-oca-event:category='authentication']` + * IPv4-Addr `[ ipv4-addr:value IN ('192.168.1.1', '192.168.1.2', '192.168.1.3') ]"` + * oca-event `[x-oca-event:extensions.'x-iam-ext'.application_name='Bane']` +` +` -`python main.py ` +### Format for making STIX translation calls via the CLI -Note the STIX identity object is only used when translating data source results into STIX, so it can be passed in as an empty object for query translation calls. +`python main.py +` ### Converting from STIX patterns to verify_event queries -This example input pattern for TRANSLATE: +CLI example of stix input pattern for TRANSLATE + + +` + python main.py translate verify query "{}" "[x-oca-event:category='sso']" +` -python main.py translate verify query "{}" "[x-oca-event:category='sso']"' Returns the following search query: -``` -{ +` + { "queries": [ "event_type=\"sso\"&limit=10000" ] -} -``` - -## Transmit + } +` ### Transmit functions -Transmit offers several functions: ping, query, results and is_async. - +Transmit offers several functions: ping, query, results and execute. ### Ping +Uses the data source API to ping the connection. -Uses the data source API to ping the connection -CLI command example: -``` -python main.py transmit verify '{ "host": "" }' '{ "auth": { "clientId": "","port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' ping +` -``` -If connection establish returns the following response: -``` +If connection is established, Connector will return the following response: + +` { "success": true } -``` +` ### Results Uses the data source API to fetch the query results based on the search ID, offset, and length. -CLI Command example: +CLI Command + +` +python main.py transmit verify '{ "host": "" ,"port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' +` + +Response + +` +{ + "success": true, + "search_id": "event_type=\"sso\"&limit=10000" +} +` + +### Execute -python main.py transmit verify '{ "host": "" }' '{ "auth": { "clientId": ""}}' results "event_type=\"authentication\"&limit=10000" 0 10 ``` -Returns JSON response data from verify +python main.py execute verify verify '{"type": "identity","id": "","name":"verify","identity_class":"events"}' '{ }' '{ "host": "" ,"port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' "[x-oca-event:category = 'sso']" ``` -### Execute -python main.py execute verify verify '{"type": "identity","id": "","name":"verify","identity_class":"events"}' '{ }' '{ "host": "" }' '{ "auth": { "clientId": ""}}' "[x-oca-event:category = 'sso' AND x-oca-event:category = 'authentication']" -''' - Return stix format data -''' +Response object + +```json +{ + "type": "bundle", + "id": "bundle--65fc22ff-0063-4afc-a61e-b9b50c0b1e18", + "objects": [ + { + "type": "identity", + "id": "32a23267-52fb-4e82-859b-0a15d6a2d334", + "name": "verify", + "identity_class": "events" + }, + { + "id": "observed-data--63964544-6b66-4673-ad37-bbeab66d328d", + "type": "observed-data", + "created_by_ref": "32a23267-52fb-4e82-859b-0a15d6a2d334", + "created": "2022-02-17T07:33:21.969Z", + "modified": "2022-02-17T07:33:21.969Z", + "objects": { + "0": { + "type": "x-oca-event", + "extensions": { + "x-iam-ext": { + "continent_name": "Asia", + "city_name": "mumbai", + "country_iso_code": "IN", + "country_name": "India", + "subcategory": "saml", + "provider_id": "http://ibm.com", + "realm": "www.google.com", + "application_id": "6773634223410562472", + "application_type": "Custom Application", + "browser_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + "applicationname": "Bane" + } + }, + "ip_refs": [ + "1" + ], + "outcome": "success", + "user_ref": "2", + "category": "sso", + "provider": "IBM Security Verify IAM", + "domain_ref": "3", + "module": "saml_runtime", + "created": "2022-02-17T07:31:38.824Z" + }, + "1": { + "type": "ipv4-addr", + "value": "192.168.1.1" + }, + "2": { + "type": "user-account", + "user_id": "123456" + }, + "3": { + "type": "domain-name", + "value": "ibmcloud.com" + } + }, + "first_observed": "2022-02-17T07:33:21.969Z", + "last_observed": "2022-02-17T07:33:21.969Z", + "number_observed": 1 + } +} +``` diff --git a/stix_shifter_modules/verify/configuration/config.json b/stix_shifter_modules/verify/configuration/config.json index 55d7a18fc..f7fd1c0c9 100644 --- a/stix_shifter_modules/verify/configuration/config.json +++ b/stix_shifter_modules/verify/configuration/config.json @@ -13,28 +13,6 @@ "default": 443, "min": 1, "max": 65535 - }, - - "sni": { - "type": "text", - "optional": true - }, - "selfSignedCert": { - "type": "password", - "optional": true - }, - "region": { - "type": "text", - "default": "us" - }, - "help": { - "type": "link", - "default": "data-sources.html" - }, - "options": { - "unmapped_fallback": { - "default": false - } } }, "configuration": { diff --git a/stix_shifter_modules/verify/configuration/lang_en.json b/stix_shifter_modules/verify/configuration/lang_en.json index 777559d3c..6b25a5b6d 100644 --- a/stix_shifter_modules/verify/configuration/lang_en.json +++ b/stix_shifter_modules/verify/configuration/lang_en.json @@ -4,54 +4,9 @@ "label": "Need additional help?", "description": "More details on the data source setting can be found in the specified link" }, - "type": { - "description": "Data source type" - }, "name": { "label": "Data source name", "description": "Assign a name to identify the connection (minimum of 5 characters)" - }, - "description": { - "label": "Data source description", - "description": "Assign a description to indicate the purpose of the connection" - }, - "options": { - "result_limit": { - "label": "Result Size Limit", - "description": "The maximum number of entries or objects that are returned by search query. Valid input range is {{min}} to {{max}}." - }, - "time_range": { - "label": "Query Time Range", - "description": "Time range for the search, in minutes, represented as last x minutes. Valid input range is {{min}} to {{max}}." - }, - "timeout": { - "label": "Query Response Timeout Limit", - "description": "The limit on how long to wait for the data source response, in seconds. Valid input range is {{min}} to {{max}}." - }, - "search_timeout": { - "label": "Query Search Timeout Limit", - "description": "The limit on how long the query will run, in minutes, on the data source." - }, - "dialects": { - "label": "Dialect", - "description": "The dialect to use if the data source needs to search multiple schema or database tables." - }, - "validate_pattern": { - "label": "STIX Pattern Validator", - "description": "Validate STIX patterns that needs to be translated into native data source query" - }, - "stix_validator": { - "label": "STIX Object Validator", - "description": "Validate translated STIX Objects" - }, - "mapping": { - "label": "Custom Mapping", - "description": "Custom stix mapping if default mapping needs to be replaced" - }, - "concurrent": { - "label": "Concurrent Search Limit", - "description": "The number of simultaneous connections that can be made between IBM Cloud Pak\u2122 for Security and the data source. Valid input range is {{min}} to {{max}}." - } } }, "configuration": { @@ -73,5 +28,5 @@ "label": "Configuration Description", "placeholder": "Add a configuration description" } - } + } } \ No newline at end of file diff --git a/stix_shifter_modules/verify/entry_point.py b/stix_shifter_modules/verify/entry_point.py index d43aa4baa..be618f427 100644 --- a/stix_shifter_modules/verify/entry_point.py +++ b/stix_shifter_modules/verify/entry_point.py @@ -1,49 +1,11 @@ -from stix_shifter_utils.utils import logger -from stix_shifter.stix_transmission.stix_transmission import IS_ASYNC from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector -from .stix_translation.query_translator import QueryTranslator -from .stix_translation.results_translator import ResultsTranslator -from .stix_transmission.ping_connector import PingConnector -from .stix_transmission.query_connector import QueryConnector -from .stix_transmission.status_connector import StatusConnector -from .stix_transmission.results_connector import ResultsConnector -from .stix_transmission.delete_connector import DeleteConnector -from .stix_transmission.api_client import APIClient -import os + class EntryPoint(BaseEntryPoint): def __init__(self, connection={}, configuration={}, options={}): super().__init__(connection, configuration, options) - # self.logger = logger.set_logger(__name__) - self.set_async(False) - if connection: - auth = configuration.get("auth") - host = connection.get("host") - api_client = APIClient(connection, configuration) - base_sync_connector = BaseSyncConnector() - ping_connector = PingConnector(api_client) - query_connector = base_sync_connector - status_connector = StatusConnector(api_client) - results_connector = ResultsConnector(api_client) - delete_connector = DeleteConnector(api_client) - - - - self.set_ping_connector(ping_connector) - self.set_query_connector(query_connector) - self.set_status_connector(status_connector) - self.set_results_connector(results_connector) - self.set_delete_connector(delete_connector) + self.setup_transmission_simple(connection, configuration) - basepath = os.path.dirname(__file__) - filepath = os.path.abspath( - os.path.join(basepath, "stix_translation")) - # self.setup_translation_simple(dialect_default='default') - # self.setup_translation_simple(dialect_default='flows') - dialect = 'default' - query_translator = QueryTranslator(options, dialect, filepath) - results_translator = ResultsTranslator(options, dialect, filepath) - self.add_dialect(dialect, query_translator=query_translator, results_translator=results_translator, default=True) + self.setup_translation_simple(dialect_default='default') \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json index 096a255ca..34c4c9d1e 100644 --- a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json @@ -1,11 +1,9 @@ - { "user-account":{ "fields": { "user_id": ["data.username"], "account_login": ["data.username"], "account_type":["data.realm"] - } }, "ipv4-addr":{ @@ -26,7 +24,7 @@ "module":["servicename"], "outcome":["data.result"], "agent" :["data.sourcetype"], - "ip_refs.value":["ip"], + "ip_refs[*].value":["ip"], "domain_ref.value": ["tenantname"], "user_ref": ["username"], "provider":"'IBM Security Verify Event'", @@ -58,7 +56,6 @@ "extensions.'x-iam-ext'.'decision_reason'":["data.decision_reason"], "extensions.'x-iam-ext'.'risk_level'":["data.risk_level"], "extensions.'x-iam-ext'.'risk_score'":["data.risk_score"] - } } } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/from_to_stix_1.8.json b/stix_shifter_modules/verify/stix_translation/json/from_to_stix_1.8.json deleted file mode 100644 index 3fc89044d..000000000 --- a/stix_shifter_modules/verify/stix_translation/json/from_to_stix_1.8.json +++ /dev/null @@ -1,58 +0,0 @@ - -{ - - "user-account":{ - "fields": { - "'user_id'": ["data.username"], - "account_type":["data.sourcetype"] - } - }, - - "ipv4-addr":{ - "fields": { - "value": ["data.origin"] - } - }, - "domain-name":{ - "fields": { - "type": ["domain-name"], - "value": ["tenantname"] - } - }, - "x-oca-event": { - "fields":{ - "action": ["data.action"], - "category":["event_type"], - "module":["servicename"], - "agent" :["data.sourcetype"], - "outcome":["data.result"], - "extensions.x-iam-ext.result":["data.result"], - "extensions.x-iam-ext.subject":["data.subject"], - "extensions.x-iam-ext.subcategory":["data.subtype"], - "extensions.x-iam-ext.realm":["data.realm"], - "extensions.x-iam-ext.browser_agent":["data.devicetype"], - "extensions.x-iam-ext.application_id":["data.applicationid"], - "extensions.x-iam-ext.user_id":["data.userid"], - "extensions.x-iam-ext.application_type":["data.applicationtype"], - "extensions.x-iam-ext.application_name":["data.applicationname"], - "extensions.x-iam-ext.cause":["data.cause"], - "extensions.x-iam-ext.target":["data.target"], - "extensions.x-iam-ext.performedby_username":["data.performedby_username"], - "extensions.x-iam-ext.message_id":["data.messageid"], - "extensions.x-iam-ext.deleted":["data.deleted"], - "extensions.x-iam-ext.performedby_clientname":["data.performedby_clientname"], - "extensions.x-iam-ext.performedby_realm":["data.performedby_realm"], - "extensions.x-iam-ext.resource":["data.resource"], - "extensions.x-iam-ext.targetid":["data.targetid"], - "extensions.x-iam-ext.targetid_realm" :["data.targetid_realm"], - "extensions.x-iam-ext.targetid_username" :["data.targetid_username"], - "extensions.x-iam-ext.userinfo_lookup":["data.userinfo_lookup"], - "extensions.x-iam-ext.continent_name":["geoip.continent_name"], - "extensions.x-iam-ext.city_name": ["geoip.city_name"], - "extensions.x-iam-ext.country_iso_code": ["geoip.country_iso_code"], - "extensions.x-iam-ext.country_name": ["geoip.country_name"], - "extensions.x-iam-ext.region_name":["geoip.region_name"], - "extensions.x-iam-ext.provider_id":["data.providerid"] - } - } - } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/operators.json b/stix_shifter_modules/verify/stix_translation/json/operators.json index 0913fd6b2..154c2fae2 100644 --- a/stix_shifter_modules/verify/stix_translation/json/operators.json +++ b/stix_shifter_modules/verify/stix_translation/json/operators.json @@ -1,7 +1,6 @@ { "ComparisonExpressionOperators.And": "&", "ComparisonComparators.Equal": "=", - "ObservationOperators.And": "or", + "ObservationOperators.And": "=", "ComparisonComparators.In": "=" - } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json index 797566503..3c3ca9d32 100644 --- a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json @@ -1,195 +1,187 @@ -{ -"username": [ - { - "key": "user-account.user_id", - "object": "useraccount" - }, - { - "key": "x-oca-event.user_ref", - "object": "ocaevent", - "references": "useraccount" - } - -], -"servicename" :{ - "key":"x-oca-event.module", - "object":"ocaevent" -}, -"sourcetype":{ - "key": "x-oca-event.agent", - "object": "ocaevent" -}, -"ip":[{ - "key": "ipv4-addr.value", - "object": "ipdata" -}, -{ - "key": "x-oca-event.ip_refs", - "object": "ocaevent", - "references": "ipdata" -} -], -"tenantname" :[ -{ - "key": "domain-name.value", - "object": "domaindata", - "transformer":"ToDomainName" -}, -{ - "key": "x-oca-event.domain_ref", - "object": "ocaevent", - "references" :"domaindata" -} -], -"result": [ - - { - "object": "ocaevent", - "key": "x-oca-event.outcome" - } -], - -"subtype": - { - "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.subcategory" -}, - - -"cause": { - "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.cause" -}, - -"action": [ -{ - "key" :"x-oca-event.action", - "object":"ocaevent" -} -], -"realm": { - "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.realm" -}, -"devicetype": { - "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.browser_agent" -}, -"applicationid" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.application_id" -}, -"applicationtype" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.application_type" -}, -"applicationname":{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.applicationname" -}, - -"target": { - "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.target" -}, -"event_type" :[ { - "key":"x-oca-event.category", - "object":"ocaevent" -},{ - "key": "x-oca-event.provider", - "object": "ocaevent", - "transformer": "VerifyStaticTransformer" -} -], - -"time" : [ - { - "key":"x-oca-event.created", - "transformer":"EpochToTimestamp", - "object":"ocaevent" - } -], -"performedby_username" : - { - "key": "x-oca-event.extensions.x-iam-ext.performedby_username", + "username": [ + { + "key": "user-account.user_id", + "object": "useraccount" + }, + { + "key": "x-oca-event.user_ref", + "object": "ocaevent", + "references": "useraccount" + } + ], + "servicename": { + "key": "x-oca-event.module", "object": "ocaevent" }, - -"deleted" :{ - "key":"x-oca-event.extensions.x-iam-ext.deleted", - "object":"ocaevent" -}, -"performedby_clientname" :{ - "key":"x-oca-event.extensions.x-iam-ext.performedby_clientname", - "object":"ocaevent" -}, -"performedby_realm" :{ - "key":"x-oca-event.extensions.x-iam-ext.performedby_realm", - "object" :"ocaevent" -}, - -"targetid" :{ - "key":"x-oca-event.extensions.x-iam-ext.targetid", - "object":"ocaevent" -}, -"targetid_realm" :{ - "key":"x-oca-event.extensions.x-iam-ext.targetid_realm", - "object":"ocaevent" -}, -"targetid_username" :{ - "key":"x-oca-event.extensions.x-iam-ext.taregetid_username", - "object":"ocaevent" -}, -"userid" :[ - { - "key": "user-account.user_id", - "object": "useraccount" - } -], -"continent_name" : { - "key" :"x-oca-event.extensions.x-iam-ext.continent_name", - "object":"ocaevent" - }, - "city_name" : { - "key" :"x-oca-event.extensions.x-iam-ext.city_name", - "object":"ocaevent" - }, - "country_iso_code" : { - "key" :"x-oca-event.extensions.x-iam-ext.country_iso_code", - "object":"ocaevent" - }, - "country_name" : { - "key" :"x-oca-event.extensions.x-iam-ext.country_name", - "object":"ocaevent" - }, - "providerid" :{ - "key":"x-oca-event.extensions.x-iam-ext.provider_id", - "object":"ocaevent" - }, - "rule_name" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.rule_name" - }, - "policy_name" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.policy_name" - }, - "decision_reason" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.decision_reason" + "sourcetype": { + "key": "x-oca-event.agent", + "object": "ocaevent" + }, + "ip": [ + { + "key": "ipv4-addr.value", + "object": "ipdata" }, - "policy_action" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.policy_action" + { + "key": "x-oca-event.ip_refs", + "object": "ocaevent", + "references": [ + "ipdata" + ], + "group": true + } + ], + "tenantname": [ + { + "key": "domain-name.value", + "object": "domaindata", + "transformer": "ToDomainName" }, - "risk_level" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.data.risk_level" + { + "key": "x-oca-event.domain_ref", + "object": "ocaevent", + "references": "domaindata" + } + ], + "result": [ + { + "object": "ocaevent", + "key": "x-oca-event.outcome" + } + ], + "subtype": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.subcategory" + }, + "cause": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.cause" + }, + "action": [ + { + "key": "x-oca-event.action", + "object": "ocaevent" + } + ], + "realm": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.realm" + }, + "devicetype": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.browser_agent" + }, + "applicationid": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.application_id" + }, + "applicationtype": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.application_type" + }, + "applicationname": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.applicationname" + }, + "target": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.target" + }, + "event_type": [ + { + "key": "x-oca-event.category", + "object": "ocaevent" }, - "risk_score" :{ - "object":"ocaevent", - "key":"x-oca-event.extensions.x-iam-ext.data.risk_score" + { + "key": "x-oca-event.provider", + "object": "ocaevent", + "transformer": "VerifyStaticTransformer" + } + ], + "time": [ + { + "key": "x-oca-event.created", + "transformer": "EpochToTimestamp", + "object": "ocaevent" + } + ], + "performedby_username": { + "key": "x-oca-event.extensions.x-iam-ext.performedby_username", + "object": "ocaevent" + }, + "deleted": { + "key": "x-oca-event.extensions.x-iam-ext.deleted", + "object": "ocaevent" + }, + "performedby_clientname": { + "key": "x-oca-event.extensions.x-iam-ext.performedby_clientname", + "object": "ocaevent" + }, + "performedby_realm": { + "key": "x-oca-event.extensions.x-iam-ext.performedby_realm", + "object": "ocaevent" + }, + "targetid": { + "key": "x-oca-event.extensions.x-iam-ext.targetid", + "object": "ocaevent" + }, + "targetid_realm": { + "key": "x-oca-event.extensions.x-iam-ext.targetid_realm", + "object": "ocaevent" + }, + "targetid_username": { + "key": "x-oca-event.extensions.x-iam-ext.taregetid_username", + "object": "ocaevent" + }, + "userid": [ + { + "key": "user-account.user_id", + "object": "useraccount" } - + ], + "continent_name": { + "key": "x-oca-event.extensions.x-iam-ext.continent_name", + "object": "ocaevent" + }, + "city_name": { + "key": "x-oca-event.extensions.x-iam-ext.city_name", + "object": "ocaevent" + }, + "country_iso_code": { + "key": "x-oca-event.extensions.x-iam-ext.country_iso_code", + "object": "ocaevent" + }, + "country_name": { + "key": "x-oca-event.extensions.x-iam-ext.country_name", + "object": "ocaevent" + }, + "providerid": { + "key": "x-oca-event.extensions.x-iam-ext.provider_id", + "object": "ocaevent" + }, + "rule_name": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.rule_name" + }, + "policy_name": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.policy_name" + }, + "decision_reason": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.decision_reason" + }, + "policy_action": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.policy_action" + }, + "risk_level": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.data.risk_level" + }, + "risk_score": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.data.risk_score" + } } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/query_constructor.py b/stix_shifter_modules/verify/stix_translation/query_constructor.py index 6889edca4..562bf156b 100644 --- a/stix_shifter_modules/verify/stix_translation/query_constructor.py +++ b/stix_shifter_modules/verify/stix_translation/query_constructor.py @@ -26,12 +26,9 @@ class QueryStringPatternTranslator: comparator_lookup = { "ComparisonExpressionOperators.And": "&", "ComparisonComparators.Equal": "=", - "ObservationOperators.And": "or", + "ObservationOperators.And": "=", "ComparisonComparators.In": "=" - } - - def __init__(self, pattern: Pattern, data_model_mapper): self.dmm = data_model_mapper #self.comparator_lookup = self.dmm.map_comparator() #Not sure its not workig with kestral @@ -52,9 +49,6 @@ def _convert_list_string_in_condition(value)->str: contcated_string= contcated_string[1:-1] return contcated_string - - - @staticmethod def _negate_comparison(comparison_string): return "NOT ({})".format(comparison_string) diff --git a/stix_shifter_modules/verify/stix_transmission/delete_connector.py b/stix_shifter_modules/verify/stix_transmission/delete_connector.py index 2c3b63bc9..ea07d78c8 100644 --- a/stix_shifter_modules/verify/stix_transmission/delete_connector.py +++ b/stix_shifter_modules/verify/stix_transmission/delete_connector.py @@ -8,5 +8,4 @@ def __init__(self, api_client): self.api_client = api_client def delete_query_connection(self, search_id): - return {"success": True} - \ No newline at end of file + return {"success": True} diff --git a/stix_shifter_modules/verify/stix_transmission/error_mapper.py b/stix_shifter_modules/verify/stix_transmission/error_mapper.py index bb8771950..363bcf0ca 100644 --- a/stix_shifter_modules/verify/stix_transmission/error_mapper.py +++ b/stix_shifter_modules/verify/stix_transmission/error_mapper.py @@ -11,12 +11,10 @@ 400: ErrorCode.TRANSMISSION_INVALID_PARAMETER, } - class ErrorMapper(): logger = logger.set_logger(__name__) DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR - @staticmethod def set_error_code(json_data, return_obj): code = None diff --git a/stix_shifter_modules/verify/stix_transmission/query_connector.py b/stix_shifter_modules/verify/stix_transmission/query_connector.py index 4fa316d1f..ad48c85ab 100644 --- a/stix_shifter_modules/verify/stix_transmission/query_connector.py +++ b/stix_shifter_modules/verify/stix_transmission/query_connector.py @@ -10,21 +10,4 @@ def __init__(self, api_client): self.logger = logger.set_logger(__name__) def create_query_connection(self, query): - - error = None - try : - api_response = self.api_client.run_search(query) - response_code = api_response['success'] - except ValueError as ex: - self.logger.debug(ex) - error = Exception(f'Can not parse response: {ex} ') - - - return_obj = dict() - if 200 <= response_code <= 299 and error is None: - return_obj['success'] = True - return_obj['search_id'] = query - else: - ErrorResponder.fill_error(return_obj, api_response, ['message'], error=error) - - return return_obj + return {"success": True, "search_id": query} \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_transmission/results_connector.py b/stix_shifter_modules/verify/stix_transmission/results_connector.py index b09606c59..202974056 100644 --- a/stix_shifter_modules/verify/stix_transmission/results_connector.py +++ b/stix_shifter_modules/verify/stix_transmission/results_connector.py @@ -5,14 +5,14 @@ class ResultsConnector(BaseResultsConnector): - + def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) - + def create_results_connection(self, search_id, offset, length): offset = int(offset) - length =int(length) + length = int(length) try: response = self.api_client.run_search(search_id, length) @@ -22,17 +22,17 @@ def create_results_connection(self, search_id, offset, length): if response_code == 200: return_obj['success'] = True return_obj['data'] = response.get("event_data", []) - return_obj['search_after'] =response.get("search_after",[]) + return_obj['search_after'] = response.get("search_after", []) # filter data based on filter_attr # slice the records as per the provided offset and length(limit) return_obj['data'] = return_obj['data'][offset:length] else: ErrorResponder.fill_error(return_obj, response, ['message']) - + except Exception as err: - self.logger.error('error when getting search results: {}'.format(err)) + self.logger.error( + 'error when getting search results: {}'.format(err)) import traceback self.logger.error(traceback.print_stack()) raise return return_obj - \ No newline at end of file diff --git a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py index 4ec207f91..78b2283c9 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py @@ -25,21 +25,25 @@ MAP_DATA = entry_point.get_results_translator().map_data -DATA_SOURCE ={ +DATA_SOURCE = { "type": "identity", "id": "identity--32a23267-52fb-4e82-859b-0a15d6a2d334", - "name":"verify", - "identity_class":"events" - } + "name": "verify", + "identity_class": "events" +} OPTION = json.dumps(DATA_SOURCE) + + def _test_query_assertions(query, queries): assert query['queries'] == [queries] + def _translate_query(stix_pattern): - return translation.translate('verify', 'query', '{}', stix_pattern) + return translation.translate('verify', 'query', '{}', stix_pattern) + class TestStixToQuery(unittest.TestCase, object): - + def test_event_type(self): stix_pattern = "[x-oca-event:category='authentication']" query = _translate_query(stix_pattern) @@ -47,9 +51,9 @@ def test_event_type(self): _test_query_assertions(query, queries) def test_IPV4_query(self): - stix_pattern = "[ipv4-addr:value='27.58.174.31']" + stix_pattern = "[ipv4-addr:value='192.168.1.1']" query = _translate_query(stix_pattern) - expected_queries = 'filter_key=data.origin&filter_value="27.58.174.31"&size=10000' + expected_queries = 'filter_key=data.origin&filter_value="192.168.1.1"&size=10000' _test_query_assertions(query, expected_queries) # def test_oca_event_extension(self): @@ -59,36 +63,37 @@ def test_IPV4_query(self): # _test_query_assertions(query,expected_queries) def test_event_type_and_fiter(self): - stix_pattern = "[x-oca-event:category = 'sso' AND x-oca-event:domain_ref.value = '77.169.74.152']START t'2022-01-17T18:24:00.000Z' STOP t'2022-01-20T18:24:00.000Z' " + stix_pattern = "[x-oca-event:category = 'sso' AND x-oca-event:domain_ref.value = '192.168.1.1']START t'2022-01-17T18:24:00.000Z' STOP t'2022-01-20T18:24:00.000Z' " query = _translate_query(stix_pattern) - expected_queries = 'filter_key=tenantname&filter_value="77.169.74.152"&event_type="sso"&from=1642443840000&to=1642703040000&size=10000' + expected_queries = 'filter_key=tenantname&filter_value="192.168.1.1"&event_type="sso"&from=1642443840000&to=1642703040000&size=10000' _test_query_assertions(query, expected_queries) def test_domain_name_query(self): - stix_pattern ="[domain-name:value = 'isrras.ice.ibmcloud.com']" + stix_pattern = "[domain-name:value = 'ibmcloud.com']" query = _translate_query(stix_pattern) - expected_queries = 'filter_key=tenantname&filter_value="isrras.ice.ibmcloud.com"&size=10000' - _test_query_assertions(query,expected_queries) - + expected_queries = 'filter_key=tenantname&filter_value="ibmcloud.com"&size=10000' + _test_query_assertions(query, expected_queries) + def test_applicationname_with_special_char_query(self): - stix_pattern ="[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" + stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" query = _translate_query(stix_pattern) expected_query = 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' - _test_query_assertions(query,expected_query) - + _test_query_assertions(query, expected_query) + def test_in_statement(self): - stix_pattern = "[ ipv4-addr:value IN ('65.154.226.165', '96.244.40.143', '37.8.230.16') ]" + stix_pattern = "[ ipv4-addr:value IN ('192.168.1.1', '192.168.1.2', '192.168.1.3') ]" query = _translate_query(stix_pattern) - expected_query = 'filter_key=data.origin&filter_value="65.154.226.165","96.244.40.143","37.8.230.16"&size=10000' - _test_query_assertions(query,expected_query) + expected_query = 'filter_key=data.origin&filter_value="192.168.1.1","192.168.1.2","192.168.1.3"&size=10000' + _test_query_assertions(query, expected_query) def test_in_statement_with_start_stop(self): - stix_pattern = "[ ipv4-addr:value IN ('65.154.226.165', '96.244.40.143', '37.8.230.16') ] START t'2022-02-06T07:19:00.000Z' STOP t'2022-02-08T07:19:00.000Z' " + stix_pattern = "[ ipv4-addr:value IN ('192.168.1.1', '192.168.1.2', '192.168.1.3') ] START t'2022-02-06T07:19:00.000Z' STOP t'2022-02-08T07:19:00.000Z' " query = _translate_query(stix_pattern) - expected_query = 'filter_key=data.origin&filter_value="65.154.226.165","96.244.40.143","37.8.230.16"&from=1644131940000&to=1644304740000&size=10000' - _test_query_assertions(query,expected_query) + expected_query = 'filter_key=data.origin&filter_value="192.168.1.1", "192.168.1.2", "192.168.1.3"&from=1644131940000&to=1644304740000&size=10000' + _test_query_assertions(query, expected_query) + def test_in_statement_with_event_type_start_stop(self): stix_pattern = "[ x-oca-event:category IN ('sso', 'authentication') ] START t'2022-02-06T07:19:00.000Z' STOP t'2022-02-08T07:19:00.000Z' " query = _translate_query(stix_pattern) expected_query = 'event_type="sso","authentication"&from=1644131940000&to=1644304740000&size=10000' - _test_query_assertions(query,expected_query) + _test_query_assertions(query, expected_query) diff --git a/stix_shifter_modules/verify/test/stix_transmission/test_verify.py b/stix_shifter_modules/verify/test/stix_transmission/test_verify.py index 9321e5210..891a28d82 100644 --- a/stix_shifter_modules/verify/test/stix_transmission/test_verify.py +++ b/stix_shifter_modules/verify/test/stix_transmission/test_verify.py @@ -43,10 +43,9 @@ def test_is_async(self, mock_api_client): connection = { "host": "hostbla", "port": 8080, - "selfSignedCert": "cert" } check_async = entry_point.is_async() - assert check_async is not True + assert check_async is True @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.get_search', autospec=True) def test_status_response(self, mock_status_response, mock_api_client): From 3b69c61906210de299b77e9b8e33ac0861ecc94a Mon Sep 17 00:00:00 2001 From: RMS Date: Thu, 17 Feb 2022 23:09:11 +0530 Subject: [PATCH 05/19] fixed unit test case. removed space between character --- .../verify/test/stix_translation/test_verify_stix_to_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py index 78b2283c9..6fd93dcda 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py @@ -89,7 +89,7 @@ def test_in_statement(self): def test_in_statement_with_start_stop(self): stix_pattern = "[ ipv4-addr:value IN ('192.168.1.1', '192.168.1.2', '192.168.1.3') ] START t'2022-02-06T07:19:00.000Z' STOP t'2022-02-08T07:19:00.000Z' " query = _translate_query(stix_pattern) - expected_query = 'filter_key=data.origin&filter_value="192.168.1.1", "192.168.1.2", "192.168.1.3"&from=1644131940000&to=1644304740000&size=10000' + expected_query = 'filter_key=data.origin&filter_value="192.168.1.1","192.168.1.2","192.168.1.3"&from=1644131940000&to=1644304740000&size=10000' _test_query_assertions(query, expected_query) def test_in_statement_with_event_type_start_stop(self): From df265cf06e49477df1c85014a8548505369a96c1 Mon Sep 17 00:00:00 2001 From: RMS Date: Thu, 17 Feb 2022 23:47:11 +0530 Subject: [PATCH 06/19] removesuffice available only in 3.9 version. change the code to support --- .../verify/stix_translation/query_constructor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix_shifter_modules/verify/stix_translation/query_constructor.py b/stix_shifter_modules/verify/stix_translation/query_constructor.py index 562bf156b..ca62f6b92 100644 --- a/stix_shifter_modules/verify/stix_translation/query_constructor.py +++ b/stix_shifter_modules/verify/stix_translation/query_constructor.py @@ -45,8 +45,8 @@ def _escape_value(value, comparator=None) -> str: @staticmethod def _convert_list_string_in_condition(value)->str: if isinstance(value,list): - contcated_string = ''.join(f'"{str}",'.format(str) for str in value ).removesuffix(',') - contcated_string= contcated_string[1:-1] + contcated_string = ''.join(f'"{str}",'.format(str) for str in value ) + contcated_string= contcated_string[1:-2] return contcated_string @staticmethod From d15459f8c9fef59989a858d617aea2b2db3c76ea Mon Sep 17 00:00:00 2001 From: RMS Date: Fri, 18 Feb 2022 10:17:32 +0530 Subject: [PATCH 07/19] removed the duplicate parameters from config and lang_en files. Default params are at module level --- .../verify/configuration/config.json | 8 ++++++++ .../verify/configuration/lang_en.json | 20 +------------------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/stix_shifter_modules/verify/configuration/config.json b/stix_shifter_modules/verify/configuration/config.json index f7fd1c0c9..9a031bb4b 100644 --- a/stix_shifter_modules/verify/configuration/config.json +++ b/stix_shifter_modules/verify/configuration/config.json @@ -13,6 +13,14 @@ "default": 443, "min": 1, "max": 65535 + }, + "sni": { + "type": "text", + "optional": true + }, + "selfSignedCert": { + "type": "password", + "optional": true } }, "configuration": { diff --git a/stix_shifter_modules/verify/configuration/lang_en.json b/stix_shifter_modules/verify/configuration/lang_en.json index 6b25a5b6d..93de2ea6c 100644 --- a/stix_shifter_modules/verify/configuration/lang_en.json +++ b/stix_shifter_modules/verify/configuration/lang_en.json @@ -1,14 +1,4 @@ { - "connection": { - "help": { - "label": "Need additional help?", - "description": "More details on the data source setting can be found in the specified link" - }, - "name": { - "label": "Data source name", - "description": "Assign a name to identify the connection (minimum of 5 characters)" - } - }, "configuration": { "auth": { "clientId": { @@ -19,14 +9,6 @@ "label": "Client Secret", "description": "Client secret of Verify" } - }, - "name": { - "label": "Configuration Name", - "placeholder": "Add a configuration name" - }, - "description": { - "label": "Configuration Description", - "placeholder": "Add a configuration description" } - } + } } \ No newline at end of file From 6b7d8a4420b7a8ae8dd0255cfa924f3c98873fa6 Mon Sep 17 00:00:00 2001 From: RMS Date: Fri, 18 Feb 2022 18:42:47 +0530 Subject: [PATCH 08/19] added few more mapping --- .../stix_translation/json/from_stix_map.json | 198 +++++++++++++----- .../stix_translation/json/to_stix_map.json | 28 ++- .../verify/stix_translation/transformers.py | 4 +- .../test/stix_translation/test_transform.py | 19 +- .../test_verify_stix_to_query.py | 50 ++--- 5 files changed, 213 insertions(+), 86 deletions(-) diff --git a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json index 34c4c9d1e..5b99da589 100644 --- a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json @@ -1,61 +1,157 @@ { - "user-account":{ + "user-account": { "fields": { - "user_id": ["data.username"], - "account_login": ["data.username"], - "account_type":["data.realm"] + "user_id": [ + "data.user_id" + ], + "account_login": [ + "data.username" + ], + "account_type": [ + "data.realm" + ] } - }, - "ipv4-addr":{ + }, + "ipv4-addr": { "fields": { - "value": ["data.origin"] + "value": [ + "data.origin" + ] } }, - "domain-name":{ + "domain-name": { "fields": { - "type": ["domain-name"], - "value": ["tenantname"] + "type": [ + "domain-name" + ], + "value": [ + "tenantname" + ] } }, "x-oca-event": { - "fields":{ - "action": ["data.action"], - "category":["event_type"], - "module":["servicename"], - "outcome":["data.result"], - "agent" :["data.sourcetype"], - "ip_refs[*].value":["ip"], - "domain_ref.value": ["tenantname"], - "user_ref": ["username"], - "provider":"'IBM Security Verify Event'", - "extensions.'x-iam-ext'.subcategory":["data.subtype"], - "extensions.'x-iam-ext'.realm":["data.realm"], - "extensions.'x-iam-ext'.browser_agent":["data.devicetype"], - "extensions.'x-iam-ext'.provider_id":["data.providerid"], - "extensions.'x-iam-ext'.application_id":["data.applicationid"], - "extensions.'x-iam-ext'.application_type":["data.applicationtype"], - "extensions.'x-iam-ext'.application_name":["data.applicationname"], - "extensions.'x-iam-ext'.cause":["data.cause"], - "extensions.'x-iam-ext'.target":["data.target"], - "extensions.'x-iam-ext'.deleted":["data.deleted"], - "extensions.'x-iam-ext'.performedby_clientname":["data.performedby_clientname"], - "extensions.'x-iam-ext'.performedby_realm":["data.performedby_realm"], - "extensions.'x-iam-ext'.performedby_username":["data.performedby_username"], - "extensions.'x-iam-ext'.targetid":["data.targetid"], - "extensions.'x-iam-ext'.targetid_realm" :["data.targetid_realm"], - "extensions.'x-iam-ext'.targetid_username" :["data.targetid_username"], - "extensions.'x-iam-ext'.continent_name":["geoip.continent_name"], - "extensions.'x-iam-ext'.country_iso_code": ["geoip.country_iso_code"], - "extensions.'x-iam-ext'.country_name": ["geoip.country_name"], - "extensions.'x-iam-ext'.location_lon":["geoip.location.lon"], - "extensions.'x-iam-ext'.location_lat":["geoip.location_lat"], - "extensions.'x-iam-ext'.city_name": ["geoip.city_name"], - "extensions.'x-iam-ext'.'policy_action'":["data.policy_action"], - "extensions.'x-iam-ext'.'policy_name'":["data.policy_name"], - "extensions.'x-iam-ext'.'rule_name'":["data.rule_name"], - "extensions.'x-iam-ext'.'decision_reason'":["data.decision_reason"], - "extensions.'x-iam-ext'.'risk_level'":["data.risk_level"], - "extensions.'x-iam-ext'.'risk_score'":["data.risk_score"] - } - } - } \ No newline at end of file + "fields": { + "action": [ + "data.action" + ], + "category": [ + "event_type" + ], + "module": [ + "servicename" + ], + "outcome": [ + "data.result" + ], + "agent": [ + "data.sourcetype" + ], + "ip_refs[*].value": [ + "ip" + ], + "domain_ref.value": [ + "tenantname" + ], + "user_ref": [ + "username" + ], + "provider": "'IBM Security Verify Event'", + "extensions.'x-iam-ext'.subcategory": [ + "data.subtype" + ], + "extensions.'x-iam-ext'.realm": [ + "data.realm" + ], + "extensions.'x-iam-ext'.browser_agent": [ + "data.devicetype" + ], + "extensions.'x-iam-ext'.provider_id": [ + "data.providerid" + ], + "extensions.'x-iam-ext'.application_id": [ + "data.applicationid" + ], + "extensions.'x-iam-ext'.application_type": [ + "data.applicationtype" + ], + "extensions.'x-iam-ext'.application_name": [ + "data.applicationname" + ], + "extensions.'x-iam-ext'.cause": [ + "data.cause" + ], + "extensions.'x-iam-ext'.target": [ + "data.target" + ], + "extensions.'x-iam-ext'.deleted": [ + "data.deleted" + ], + "extensions.'x-iam-ext'.performedby_clientname": [ + "data.performedby_clientname" + ], + "extensions.'x-iam-ext'.performedby_realm": [ + "data.performedby_realm" + ], + "extensions.'x-iam-ext'.performedby_username": [ + "data.performedby_username" + ], + "extensions.'x-iam-ext'.targetid": [ + "data.targetid" + ], + "extensions.'x-iam-ext'.targetid_realm": [ + "data.targetid_realm" + ], + "extensions.'x-iam-ext'.targetid_username": [ + "data.targetid_username" + ], + "extensions.'x-iam-ext'.continent_name": [ + "geoip.continent_name" + ], + "extensions.'x-iam-ext'.country_iso_code": [ + "geoip.country_iso_code" + ], + "extensions.'x-iam-ext'.country_name": [ + "geoip.country_name" + ], + "extensions.'x-iam-ext'.location_lon": [ + "geoip.location.lon" + ], + "extensions.'x-iam-ext'.location_lat": [ + "geoip.location_lat" + ], + "extensions.'x-iam-ext'.city_name": [ + "geoip.city_name" + ], + "extensions.'x-iam-ext'.'policy_action'": [ + "data.policy_action" + ], + "extensions.'x-iam-ext'.'policy_name'": [ + "data.policy_name" + ], + "extensions.'x-iam-ext'.'rule_name'": [ + "data.rule_name" + ], + "extensions.'x-iam-ext'.'decision_reason'": [ + "data.decision_reason" + ], + "extensions.'x-iam-ext'.'risk_level'": [ + "data.risk_level" + ], + "extensions.'x-iam-ext'.'risk_score'": [ + "data.risk_score" + ], + "extensions.'x-iam-ext'.deviceid": [ + "data.deviceid" + ], + "extensions.'x-iam-ext'.is_device_compliant": [ + "data.mdmiscompliant" + ], + "extensions.'x-iam-ext'.is_device_managed": [ + "data.mdmismanaged" + ], + "extensions.'x-iam-ext'.mdm_customerid": [ + "data.billingid" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json index 3c3ca9d32..4352fbda8 100644 --- a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json @@ -4,6 +4,14 @@ "key": "user-account.user_id", "object": "useraccount" }, + { + "key": "user-account.account_login", + "object": "useraccount" + }, + { + "key": "user-account.account_type", + "object": "useraccount" + }, { "key": "x-oca-event.user_ref", "object": "ocaevent", @@ -178,10 +186,26 @@ }, "risk_level": { "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.data.risk_level" + "key": "x-oca-event.extensions.x-iam-ext.risk_level" }, "risk_score": { "object": "ocaevent", - "key": "x-oca-event.extensions.x-iam-ext.data.risk_score" + "key": "x-oca-event.extensions.x-iam-ext.risk_score" + }, + "deviceid": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.deviceid" + }, + "mdmiscompliant": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.is_device_compliant" + }, + "mdmismanaged": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.is_device_managed" + }, + "billingid": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.mdm_customerid" } } \ No newline at end of file diff --git a/stix_shifter_modules/verify/stix_translation/transformers.py b/stix_shifter_modules/verify/stix_translation/transformers.py index d97b1b72f..dde3aad43 100644 --- a/stix_shifter_modules/verify/stix_translation/transformers.py +++ b/stix_shifter_modules/verify/stix_translation/transformers.py @@ -1,8 +1,8 @@ from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer class VerifyStaticTransformer(ValueTransformer): - """A value transformer that always returns the string 'IBM Security Verify IAM'""" + """A value transformer that always returns the string 'IBM Security Verify Event'""" @staticmethod def transform(value): - return "IBM Security Verify IAM" \ No newline at end of file + return "IBM Security Verify Event" \ No newline at end of file diff --git a/stix_shifter_modules/verify/test/stix_translation/test_transform.py b/stix_shifter_modules/verify/test/stix_translation/test_transform.py index ae198a53c..b82194bed 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_transform.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_transform.py @@ -1,10 +1,13 @@ -from unicodedata import category -import unittest import json import logging +import unittest +from unicodedata import category + from stix_shifter.stix_translation import stix_translation from stix_shifter_modules.verify.entry_point import EntryPoint -from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import \ + get_module_transformers + translation = stix_translation.StixTranslation() # config_file = open('stix_shifter_modules/verify_event/configuration/config.json').read() # from_stix_file = open('stix_shifter_modules/verify_event/stix_translation/json/from_stix_map.json').read() @@ -61,11 +64,12 @@ def test_oca_event(self): "userid": "652001LT0R", "applicationtype": "Custom Application", "devicetype": "PAN GlobalProtect/5.2.4-21 (Microsoft Windows 10 Enterprise , 64-bit) Mozilla/5.0 (Windows NT 6.2; Win64; x64; Trident/7.0; rv:11.0) like Gecko", "username": "dinepal1@in.ibm.com", "applicationname": "Bane & Ox VPN", "year": 2022, + "billingid":"12345","mdmismanaged":"true","mdmiscompliant": "true","deviceid":"abc_device", "@metadata": {"group_id": "event-transform-prod-eu01a-prod-eu01a-01", "source_dc": "prod-eu01a"}, "event_type": "sso", "month": 1, "indexed_at": 1642413142906, "@processing_time": 1012, "tenantid": "c92ce528-293f-4e84-8307-c4fe188b9461", "tenantname": "isrras.ice.ibmcloud.com", "correlationid": "CORR_ID-a134f569-8d73-45ac-8d44-2457448c9101", "servicename": "saml_runtime", - "id": "dc4523e6-6260-4349-83f8-3320365a5f25", "time": 1642413141894, "day": 17}, ] + "id": "dc4523e6-6260-4349-83f8-3320365a5f25", "time": 1642413141894, "day": 17} ] user_ref ="2" category ="sso" @@ -84,7 +88,10 @@ def test_oca_event(self): assert(event['domain_ref'] == domain_ref) assert(event['module'] == module) assert(event['category'] == category) - assert(event['extensions']) - assert(event['created'] == '2022-01-17T09:52:21.894Z') + assert(event["extensions"]['x-iam-ext']['is_device_managed']=="true") + assert(event["extensions"]['x-iam-ext']['is_device_managed']=="true") + assert(event["extensions"]['x-iam-ext']['mdm_customerid']=="true") + assert(event["extensions"]['x-iam-ext']['is_device_compliant']=="true") + assert(event["extensions"]['x-iam-ext']['deviceid']=="true") \ No newline at end of file diff --git a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py index 6fd93dcda..01867cf72 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py @@ -44,17 +44,17 @@ def _translate_query(stix_pattern): class TestStixToQuery(unittest.TestCase, object): - def test_event_type(self): - stix_pattern = "[x-oca-event:category='authentication']" - query = _translate_query(stix_pattern) - queries = "event_type=\"authentication\"&size=10000" - _test_query_assertions(query, queries) + # def test_event_type(self): + # stix_pattern = "[x-oca-event:category='authentication']" + # query = _translate_query(stix_pattern) + # queries = "event_type=\"authentication\"&size=10000" + # _test_query_assertions(query, queries) - def test_IPV4_query(self): - stix_pattern = "[ipv4-addr:value='192.168.1.1']" - query = _translate_query(stix_pattern) - expected_queries = 'filter_key=data.origin&filter_value="192.168.1.1"&size=10000' - _test_query_assertions(query, expected_queries) + # def test_IPV4_query(self): + # stix_pattern = "[ipv4-addr:value='192.168.1.1']" + # query = _translate_query(stix_pattern) + # expected_queries = 'filter_key=data.origin&filter_value="192.168.1.1"&size=10000' + # _test_query_assertions(query, expected_queries) # def test_oca_event_extension(self): # stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.user_id='652001LT0R']" @@ -62,23 +62,23 @@ def test_IPV4_query(self): # expected_queries = 'filter_key=data.userid&filter_value="652001LT0R"&size=10000' # _test_query_assertions(query,expected_queries) - def test_event_type_and_fiter(self): - stix_pattern = "[x-oca-event:category = 'sso' AND x-oca-event:domain_ref.value = '192.168.1.1']START t'2022-01-17T18:24:00.000Z' STOP t'2022-01-20T18:24:00.000Z' " - query = _translate_query(stix_pattern) - expected_queries = 'filter_key=tenantname&filter_value="192.168.1.1"&event_type="sso"&from=1642443840000&to=1642703040000&size=10000' - _test_query_assertions(query, expected_queries) + # def test_event_type_and_fiter(self): + # stix_pattern = "[x-oca-event:category = 'sso' AND x-oca-event:domain_ref.value = '192.168.1.1']START t'2022-01-17T18:24:00.000Z' STOP t'2022-01-20T18:24:00.000Z' " + # query = _translate_query(stix_pattern) + # expected_queries = 'filter_key=tenantname&filter_value="192.168.1.1"&event_type="sso"&from=1642443840000&to=1642703040000&size=10000' + # _test_query_assertions(query, expected_queries) - def test_domain_name_query(self): - stix_pattern = "[domain-name:value = 'ibmcloud.com']" - query = _translate_query(stix_pattern) - expected_queries = 'filter_key=tenantname&filter_value="ibmcloud.com"&size=10000' - _test_query_assertions(query, expected_queries) + # def test_domain_name_query(self): + # stix_pattern = "[domain-name:value = 'ibmcloud.com']" + # query = _translate_query(stix_pattern) + # expected_queries = 'filter_key=tenantname&filter_value="ibmcloud.com"&size=10000' + # _test_query_assertions(query, expected_queries) - def test_applicationname_with_special_char_query(self): - stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" - query = _translate_query(stix_pattern) - expected_query = 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' - _test_query_assertions(query, expected_query) + # def test_applicationname_with_special_char_query(self): + # stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" + # query = _translate_query(stix_pattern) + # expected_query = 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' + # _test_query_assertions(query, expected_query) def test_in_statement(self): stix_pattern = "[ ipv4-addr:value IN ('192.168.1.1', '192.168.1.2', '192.168.1.3') ]" From d56015864fe04d33f626338474065539cbd5d426 Mon Sep 17 00:00:00 2001 From: RMS Date: Fri, 18 Feb 2022 20:01:59 +0530 Subject: [PATCH 09/19] fix the unit test case and apply formatting --- .../stix_translation/query_constructor.py | 127 +++++++++++------- .../test/stix_translation/test_transform.py | 5 +- .../test_verify_stix_to_query.py | 73 +++++----- 3 files changed, 112 insertions(+), 93 deletions(-) diff --git a/stix_shifter_modules/verify/stix_translation/query_constructor.py b/stix_shifter_modules/verify/stix_translation/query_constructor.py index ca62f6b92..4f86eba3d 100644 --- a/stix_shifter_modules/verify/stix_translation/query_constructor.py +++ b/stix_shifter_modules/verify/stix_translation/query_constructor.py @@ -15,7 +15,8 @@ REFERENCE_DATA_TYPES = {"ipaddr": ["ipv4", "ipv4_cidr", "ipv6", "ipv6_cidr"], "proxy_ip": ["ipv4", "ipv4_cidr"], } -REFERENCE_FILTER_TYPE = ["user-account","ipv4-addr","domain-name","x-oca-event"] +REFERENCE_FILTER_TYPE = ["user-account", + "ipv4-addr", "domain-name", "x-oca-event"] REFERENCE_EXCLUDE_FILTER_TYPE = ["category"] logger = logging.getLogger(__name__) @@ -24,30 +25,33 @@ class QueryStringPatternTranslator: QUERIES = [] # # Change comparator values to match with supported data source operators comparator_lookup = { - "ComparisonExpressionOperators.And": "&", - "ComparisonComparators.Equal": "=", - "ObservationOperators.And": "=", - "ComparisonComparators.In": "=" + "ComparisonExpressionOperators.And": "&", + "ComparisonComparators.Equal": "=", + "ObservationOperators.And": "=", + "ComparisonComparators.In": "=" } + def __init__(self, pattern: Pattern, data_model_mapper): self.dmm = data_model_mapper - #self.comparator_lookup = self.dmm.map_comparator() #Not sure its not workig with kestral + # self.comparator_lookup = self.dmm.map_comparator() #Not sure its not workig with kestral self.pattern = pattern self.translated = self.parse_expression(pattern) @staticmethod def _escape_value(value, comparator=None) -> str: if isinstance(value, str): - return '{}'.format(value.replace("\'","'").replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)').replace("$","\"").replace('&','%26') - .replace('""','"')) + return '{}'.format(value.replace("\'", "'").replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)').replace("$", "\"").replace('&', '%26') + .replace('""', '"')) else: return value + @staticmethod def _convert_list_string_in_condition(value)->str: - if isinstance(value,list): - contcated_string = ''.join(f'"{str}",'.format(str) for str in value ) - contcated_string= contcated_string[1:-2] - return contcated_string + if isinstance(value, list): + contcated_string = ''.join( + f'"{str}",'.format(str) for str in value) + contcated_string = contcated_string[1:-2] + return contcated_string @staticmethod def _negate_comparison(comparison_string): @@ -60,7 +64,7 @@ def _check_value_type(value): if key != 'date' and bool(re.search(pattern, value)): return key return None - + # TODO remove self reference from static methods @staticmethod def _parse_reference(self, stix_field, value_type, mapped_field, value, comparator): @@ -71,24 +75,30 @@ def _parse_reference(self, stix_field, value_type, mapped_field, value, comparat mapped_field=mapped_field, comparator=comparator, value=value) @staticmethod - def _parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array,stix_object): + def _parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array, stix_object): comparison_string = "" is_reference_value = self._is_reference_value(stix_field) - is_object_filter_typeValue = self._is_object_filterType(stix_object,stix_field) + is_object_filter_typeValue = self._is_object_filterType( + stix_object, stix_field) # Need to use expression.value to match against regex since the passed-in value has already been formated. - value_type = self._check_value_type(expression.value) if is_reference_value else None - mapped_fields_count = 1 if is_reference_value else len(mapped_fields_array) + value_type = self._check_value_type( + expression.value) if is_reference_value else None + mapped_fields_count = 1 if is_reference_value else len( + mapped_fields_array) for mapped_field in mapped_fields_array: if is_reference_value: - parsed_reference = self._parse_reference(self, stix_field, value_type, mapped_field, value, comparator) + parsed_reference = self._parse_reference( + self, stix_field, value_type, mapped_field, value, comparator) if not parsed_reference: continue comparison_string += parsed_reference - elif is_object_filter_typeValue : - comparison_string += 'filter_key={mapped_field}&filter_value="{value}"'.format(mapped_field=mapped_field, comparator=comparator, value=value) + elif is_object_filter_typeValue: + comparison_string += 'filter_key={mapped_field}&filter_value="{value}"'.format( + mapped_field=mapped_field, comparator=comparator, value=value) else: - comparison_string += '{mapped_field}{comparator}"{value}"'.format(mapped_field=mapped_field, comparator=comparator, value=value) + comparison_string += '{mapped_field}{comparator}"{value}"'.format( + mapped_field=mapped_field, comparator=comparator, value=value) if mapped_fields_count > 1: comparison_string += "&" @@ -98,30 +108,32 @@ def _parse_mapped_fields(self, expression, value, comparator, stix_field, mapped @staticmethod def _is_reference_value(stix_field): return stix_field == 'src_ref.value' or stix_field == 'dst_ref.value' - + @staticmethod - def _is_object_filterType(stix_object,stix_field): - if (stix_object in REFERENCE_FILTER_TYPE) and (stix_field not in REFERENCE_EXCLUDE_FILTER_TYPE ): + def _is_object_filterType(stix_object, stix_field): + if (stix_object in REFERENCE_FILTER_TYPE) and (stix_field not in REFERENCE_EXCLUDE_FILTER_TYPE): return True - else : + else: return False @staticmethod def _check_filter_value_type(value, stix_object): - """ - Function returning value type of event type object in double quotes - :param value: str - :return: string - """ - event_object = ['event_type'] - if stix_object in event_object: - return '"{}"'.format(value) - else: - return value + """ + Function returning value type of event type object in double quotes + :param value: str + :return: string + """ + event_object = ['event_type'] + if stix_object in event_object: + return '"{}"'.format(value) + else: + return value + @staticmethod def _lookup_comparison_operator(self, expression_operator): if str(expression_operator) not in self.comparator_lookup: - raise NotImplementedError("Comparison operator {} unsupported for verify connector".format(expression_operator.name)) + raise NotImplementedError( + "Comparison operator {} unsupported for verify connector".format(expression_operator.name)) return self.comparator_lookup[str(expression_operator)] @classmethod @@ -134,10 +146,11 @@ def _format_start_stop_qualifier(self, expression, qualifier) -> str: start = qualifier_split[1] stop = qualifier_split[3] # convert timepestamp to millisecond which will be passed to rest service - start_epoach = transformer.transform(start); + start_epoach = transformer.transform(start) stop_epoach = transformer.transform(stop) - qualified_query = "%s&from=%s&to=%s" % (expression, start_epoach, stop_epoach) + qualified_query = "%s&from=%s&to=%s" % ( + expression, start_epoach, stop_epoach) return qualified_query def _parse_expression(self, expression, qualifier=None) -> Union[str, list]: @@ -147,24 +160,27 @@ def _parse_expression(self, expression, qualifier=None) -> Union[str, list]: # Multiple data source fields may map to the same STIX Object mapped_fields_array = self.dmm.map_field(stix_object, stix_field) # Resolve the comparison symbol to use in the query string (usually just ':') - comparator = self._lookup_comparison_operator(self, expression.comparator) + comparator = self._lookup_comparison_operator( + self, expression.comparator) # Some values are formatted differently based on how they're being compared if expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: # Should be in single-quotes value = self._escape_value(expression.value) - #check if belongs to event object type. This require sepecial treatment. - value = self._check_filter_value_type(value,stix_object) + # check if belongs to event object type. This require sepecial treatment. + value = self._check_filter_value_type(value, stix_object) elif expression.comparator == ComparisonComparators.In: - in_string = expression.value.values if hasattr(expression.value, 'values') else expression.value + in_string = expression.value.values if hasattr( + expression.value, 'values') else expression.value values = self._convert_list_string_in_condition(in_string) - #apply escape value to remove unwanted char in string. + # apply escape value to remove unwanted char in string. value = self._escape_value(values) - else : + else: value = self._escape_value(expression.value) - comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array,stix_object) + comparison_string = self._parse_mapped_fields( + self, expression, value, comparator, stix_field, mapped_fields_array, stix_object) if(len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): # More than one data source field maps to the STIX attribute, so group comparisons together. grouped_comparison_string = comparison_string @@ -178,7 +194,8 @@ def _parse_expression(self, expression, qualifier=None) -> Union[str, list]: return "{}".format(comparison_string) elif isinstance(expression, CombinedComparisonExpression): - operator = self._lookup_comparison_operator(self, expression.operator) + operator = self._lookup_comparison_operator( + self, expression.operator) expression_01 = self._parse_expression(expression.expr1) expression_02 = self._parse_expression(expression.expr2) @@ -189,7 +206,8 @@ def _parse_expression(self, expression, qualifier=None) -> Union[str, list]: if isinstance(expression.expr2, CombinedComparisonExpression): expression_02 = "{}".format(expression_02) - query_string = "{}{}{}".format(expression_01, operator, expression_02) + query_string = "{}{}{}".format( + expression_01, operator, expression_02) if qualifier is not None: return self._format_start_stop_qualifier(query_string, qualifier) else: @@ -198,15 +216,19 @@ def _parse_expression(self, expression, qualifier=None) -> Union[str, list]: return self._parse_expression(expression.comparison_expression, qualifier) elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): if isinstance(expression.observation_expression, CombinedObservationExpression): - operator = self._lookup_comparison_operator(self, expression.observation_expression.operator) - expression_01 = self._parse_expression(expression.observation_expression.expr1) + operator = self._lookup_comparison_operator( + self, expression.observation_expression.operator) + expression_01 = self._parse_expression( + expression.observation_expression.expr1) # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions - expression_02 = self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + expression_02 = self._parse_expression( + expression.observation_expression.expr2, expression.qualifier) return "{} {} {}".format(expression_01, operator, expression_02) else: return self._parse_expression(expression.observation_expression.comparison_expression, expression.qualifier) elif isinstance(expression, CombinedObservationExpression): - operator = self._lookup_comparison_operator(self, expression.operator) + operator = self._lookup_comparison_operator( + self, expression.operator) expression_01 = self._parse_expression(expression.expr1) expression_02 = self._parse_expression(expression.expr2) if not isinstance(expression_01, list): @@ -229,7 +251,8 @@ def translate_pattern(pattern: Pattern, data_model_mapping, options): result_limit = options['result_limit'] list_final_query = [] # time_range = options['time_range'] - query = QueryStringPatternTranslator(pattern, data_model_mapping).translated + query = QueryStringPatternTranslator( + pattern, data_model_mapping).translated query = query if isinstance(query, list) else [query] for each_query in query: base_query = f"{each_query}&size={result_limit}" diff --git a/stix_shifter_modules/verify/test/stix_translation/test_transform.py b/stix_shifter_modules/verify/test/stix_translation/test_transform.py index b82194bed..2d86b93ea 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_transform.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_transform.py @@ -89,9 +89,8 @@ def test_oca_event(self): assert(event['module'] == module) assert(event['category'] == category) assert(event["extensions"]['x-iam-ext']['is_device_managed']=="true") - assert(event["extensions"]['x-iam-ext']['is_device_managed']=="true") - assert(event["extensions"]['x-iam-ext']['mdm_customerid']=="true") + assert(event["extensions"]['x-iam-ext']['mdm_customerid']=="12345") assert(event["extensions"]['x-iam-ext']['is_device_compliant']=="true") - assert(event["extensions"]['x-iam-ext']['deviceid']=="true") + assert(event["extensions"]['x-iam-ext']['deviceid']=="abc_device") \ No newline at end of file diff --git a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py index 01867cf72..38d49de22 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py @@ -1,10 +1,13 @@ -from unicodedata import category -import unittest import json import logging +import unittest +from unicodedata import category + from stix_shifter.stix_translation import stix_translation from stix_shifter_modules.verify.entry_point import EntryPoint -from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import \ + get_module_transformers + translation = stix_translation.StixTranslation() # config_file = open('stix_shifter_modules/verify_event/configuration/config.json').read() # from_stix_file = open('stix_shifter_modules/verify_event/stix_translation/json/from_stix_map.json').read() @@ -44,41 +47,35 @@ def _translate_query(stix_pattern): class TestStixToQuery(unittest.TestCase, object): - # def test_event_type(self): - # stix_pattern = "[x-oca-event:category='authentication']" - # query = _translate_query(stix_pattern) - # queries = "event_type=\"authentication\"&size=10000" - # _test_query_assertions(query, queries) - - # def test_IPV4_query(self): - # stix_pattern = "[ipv4-addr:value='192.168.1.1']" - # query = _translate_query(stix_pattern) - # expected_queries = 'filter_key=data.origin&filter_value="192.168.1.1"&size=10000' - # _test_query_assertions(query, expected_queries) - - # def test_oca_event_extension(self): - # stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.user_id='652001LT0R']" - # query = _translate_query(stix_pattern) - # expected_queries = 'filter_key=data.userid&filter_value="652001LT0R"&size=10000' - # _test_query_assertions(query,expected_queries) - - # def test_event_type_and_fiter(self): - # stix_pattern = "[x-oca-event:category = 'sso' AND x-oca-event:domain_ref.value = '192.168.1.1']START t'2022-01-17T18:24:00.000Z' STOP t'2022-01-20T18:24:00.000Z' " - # query = _translate_query(stix_pattern) - # expected_queries = 'filter_key=tenantname&filter_value="192.168.1.1"&event_type="sso"&from=1642443840000&to=1642703040000&size=10000' - # _test_query_assertions(query, expected_queries) - - # def test_domain_name_query(self): - # stix_pattern = "[domain-name:value = 'ibmcloud.com']" - # query = _translate_query(stix_pattern) - # expected_queries = 'filter_key=tenantname&filter_value="ibmcloud.com"&size=10000' - # _test_query_assertions(query, expected_queries) - - # def test_applicationname_with_special_char_query(self): - # stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" - # query = _translate_query(stix_pattern) - # expected_query = 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' - # _test_query_assertions(query, expected_query) + def test_event_type(self): + stix_pattern = "[x-oca-event:category='authentication']" + query = _translate_query(stix_pattern) + queries = "event_type=\"authentication\"&size=10000" + _test_query_assertions(query, queries) + + def test_IPV4_query(self): + stix_pattern = "[ipv4-addr:value='192.168.1.1']" + query = _translate_query(stix_pattern) + expected_queries = 'filter_key=data.origin&filter_value="192.168.1.1"&size=10000' + _test_query_assertions(query, expected_queries) + + def test_event_type_and_fiter(self): + stix_pattern = "[x-oca-event:category = 'sso' AND x-oca-event:domain_ref.value = '192.168.1.1']START t'2022-01-17T18:24:00.000Z' STOP t'2022-01-20T18:24:00.000Z' " + query = _translate_query(stix_pattern) + expected_queries = 'filter_key=tenantname&filter_value="192.168.1.1"&event_type="sso"&from=1642443840000&to=1642703040000&size=10000' + _test_query_assertions(query, expected_queries) + + def test_domain_name_query(self): + stix_pattern = "[domain-name:value = 'ibmcloud.com']" + query = _translate_query(stix_pattern) + expected_queries = 'filter_key=tenantname&filter_value="ibmcloud.com"&size=10000' + _test_query_assertions(query, expected_queries) + + def test_applicationname_with_special_char_query(self): + stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" + query = _translate_query(stix_pattern) + expected_query = 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' + _test_query_assertions(query, expected_query) def test_in_statement(self): stix_pattern = "[ ipv4-addr:value IN ('192.168.1.1', '192.168.1.2', '192.168.1.3') ]" From deb0b2f5b5f81b9f2f70e1d100eaf276e4318734 Mon Sep 17 00:00:00 2001 From: RMS Date: Mon, 21 Feb 2022 12:43:19 +0530 Subject: [PATCH 10/19] made the changes as per feedback received --- stix_shifter_modules/verify/README.md | 8 +- .../verify/configuration/lang_en.json | 28 +++- .../stix_translation/json/from_stix_map.json | 12 +- .../stix_transmission/ping_connector.py | 10 +- .../stix_transmission/results_connector.py | 10 +- ...stix_to_query.py => test_stix_to_query.py} | 44 +++-- .../test/stix_translation/test_transform.py | 131 +++++++++------ .../stix_transmission/test_transmission.py | 143 ++++++++++++++++ .../test/stix_transmission/test_verify.py | 157 ------------------ 9 files changed, 298 insertions(+), 245 deletions(-) rename stix_shifter_modules/verify/test/stix_translation/{test_verify_stix_to_query.py => test_stix_to_query.py} (76%) create mode 100644 stix_shifter_modules/verify/test/stix_transmission/test_transmission.py delete mode 100644 stix_shifter_modules/verify/test/stix_transmission/test_verify.py diff --git a/stix_shifter_modules/verify/README.md b/stix_shifter_modules/verify/README.md index 45a8bce98..c4c83808b 100644 --- a/stix_shifter_modules/verify/README.md +++ b/stix_shifter_modules/verify/README.md @@ -26,7 +26,7 @@ CLI example of stix input pattern for TRANSLATE ` - python main.py translate verify query "{}" "[x-oca-event:category='sso']" + python main.py translate ibm_security_verify query "{}" "[x-oca-event:category='sso']" ` Returns the following search query: @@ -47,7 +47,7 @@ Uses the data source API to ping the connection. ` -python main.py transmit verify '{ "host": "","port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' ping +python main.py transmit ibm_security_verify '{ "host": "","port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' ping ` If connection is established, Connector will return the following response: @@ -64,7 +64,7 @@ Uses the data source API to fetch the query results based on the search ID, offs CLI Command ` -python main.py transmit verify '{ "host": "" ,"port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' +python main.py transmit ibm_security_verify '{ "host": "" ,"port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' ` Response @@ -79,7 +79,7 @@ Response ### Execute ``` -python main.py execute verify verify '{"type": "identity","id": "","name":"verify","identity_class":"events"}' '{ }' '{ "host": "" ,"port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' "[x-oca-event:category = 'sso']" +python main.py execute ibm_security_verify ibm_security_verify '{"type": "identity","id": "","name":"verify","identity_class":"events"}' '{ }' '{ "host": "" ,"port" :}' '{ "auth": { "clientId": ", "clientSecret": ""}}' "[x-oca-event:category = 'sso']" ``` Response object diff --git a/stix_shifter_modules/verify/configuration/lang_en.json b/stix_shifter_modules/verify/configuration/lang_en.json index 93de2ea6c..9d1df265a 100644 --- a/stix_shifter_modules/verify/configuration/lang_en.json +++ b/stix_shifter_modules/verify/configuration/lang_en.json @@ -1,13 +1,37 @@ { + "connection": { + "host": { + "label": "Management IP address or Hostname", + "placeholder": "192.168.1.1", + "description": "Specify the IBM Security verify IP address or Hostname." + }, + "port": { + "label": "Host Port", + "placeholder": "443", + "description": "Specify the associated port number of the data source." + }, + "help": { + "label": "Need additional help?" + }, + "selfSignedCert": { + "label": "IBM Security Verify Certificate", + "placeholder": "Paste your certificate" + }, + "sni": { + "label": "Server name indicator", + "placeholder": "Add a server name indicator", + "description": "If your hostname or IP address does not match the common name you will need to supply a Server Name Indicator (SNI). This is used to allow a separate hostname to be provided to the TLS handshake of the resource connection." + } + }, "configuration": { "auth": { "clientId": { "label": "Client Id", - "description": "Client ID of Verify" + "description": "Client ID of IBM Seurity Verify" }, "clientSecret": { "label": "Client Secret", - "description": "Client secret of Verify" + "description": "Client secret of Client ID of IBM Seurity Verify" } } } diff --git a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json index 5b99da589..0b387d119 100644 --- a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json @@ -122,22 +122,22 @@ "extensions.'x-iam-ext'.city_name": [ "geoip.city_name" ], - "extensions.'x-iam-ext'.'policy_action'": [ + "extensions.'x-iam-ext'.policy_action": [ "data.policy_action" ], - "extensions.'x-iam-ext'.'policy_name'": [ + "extensions.'x-iam-ext'.policy_name": [ "data.policy_name" ], - "extensions.'x-iam-ext'.'rule_name'": [ + "extensions.'x-iam-ext'.rule_name": [ "data.rule_name" ], - "extensions.'x-iam-ext'.'decision_reason'": [ + "extensions.'x-iam-ext'.decision_reason": [ "data.decision_reason" ], - "extensions.'x-iam-ext'.'risk_level'": [ + "extensions.'x-iam-ext'.risk_level": [ "data.risk_level" ], - "extensions.'x-iam-ext'.'risk_score'": [ + "extensions.'x-iam-ext'.risk_score": [ "data.risk_score" ], "extensions.'x-iam-ext'.deviceid": [ diff --git a/stix_shifter_modules/verify/stix_transmission/ping_connector.py b/stix_shifter_modules/verify/stix_transmission/ping_connector.py index 954a26715..72378517b 100644 --- a/stix_shifter_modules/verify/stix_transmission/ping_connector.py +++ b/stix_shifter_modules/verify/stix_transmission/ping_connector.py @@ -1,9 +1,13 @@ import datetime import json -from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector -from stix_shifter_modules.verify.stix_transmission.api_client import APIClient -from stix_shifter_utils.utils.error_response import ErrorResponder + +from stix_shifter_modules.ibm_security_verify.stix_transmission.api_client import \ + APIClient +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import \ + BasePingConnector from stix_shifter_utils.utils import logger +from stix_shifter_utils.utils.error_response import ErrorResponder + class PingConnector(BasePingConnector): diff --git a/stix_shifter_modules/verify/stix_transmission/results_connector.py b/stix_shifter_modules/verify/stix_transmission/results_connector.py index 202974056..f1caae49a 100644 --- a/stix_shifter_modules/verify/stix_transmission/results_connector.py +++ b/stix_shifter_modules/verify/stix_transmission/results_connector.py @@ -1,7 +1,10 @@ -from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector -from stix_shifter_utils.utils.error_response import ErrorResponder -from stix_shifter_utils.utils import logger import json +import traceback + +from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import \ + BaseResultsConnector +from stix_shifter_utils.utils import logger +from stix_shifter_utils.utils.error_response import ErrorResponder class ResultsConnector(BaseResultsConnector): @@ -32,7 +35,6 @@ def create_results_connection(self, search_id, offset, length): except Exception as err: self.logger.error( 'error when getting search results: {}'.format(err)) - import traceback self.logger.error(traceback.print_stack()) raise return return_obj diff --git a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py b/stix_shifter_modules/verify/test/stix_translation/test_stix_to_query.py similarity index 76% rename from stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py rename to stix_shifter_modules/verify/test/stix_translation/test_stix_to_query.py index 38d49de22..3b1dcf0ea 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_verify_stix_to_query.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_stix_to_query.py @@ -4,9 +4,10 @@ from unicodedata import category from stix_shifter.stix_translation import stix_translation -from stix_shifter_modules.verify.entry_point import EntryPoint -from stix_shifter_utils.stix_translation.src.utils.transformer_utils import \ - get_module_transformers +from stix_shifter_modules.ibm_security_verify.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import ( + get_module_transformers, +) translation = stix_translation.StixTranslation() # config_file = open('stix_shifter_modules/verify_event/configuration/config.json').read() @@ -16,10 +17,10 @@ # logging.basicConfig(level=logging.DEBUG) # logger = logging.getLogger() -MODULE = 'verify' -RESULTS = 'results' +MODULE = "ibm_security_verify" +RESULTS = "results" TRANSFORMERS = get_module_transformers(MODULE) -epoch_to_timestamp_class = TRANSFORMERS.get('EpochToTimestamp') +epoch_to_timestamp_class = TRANSFORMERS.get("EpochToTimestamp") EPOCH_START = 1531169112 EPOCH_END = 1531169254 START_TIMESTAMP = epoch_to_timestamp_class.transform(EPOCH_START) @@ -31,32 +32,33 @@ DATA_SOURCE = { "type": "identity", "id": "identity--32a23267-52fb-4e82-859b-0a15d6a2d334", - "name": "verify", - "identity_class": "events" + "name": "ibm_security_verify", + "identity_class": "events", } OPTION = json.dumps(DATA_SOURCE) def _test_query_assertions(query, queries): - assert query['queries'] == [queries] + assert query["queries"] == [queries] def _translate_query(stix_pattern): - return translation.translate('verify', 'query', '{}', stix_pattern) + return translation.translate("ibm_security_verify", "query", "{}", stix_pattern) class TestStixToQuery(unittest.TestCase, object): - def test_event_type(self): stix_pattern = "[x-oca-event:category='authentication']" query = _translate_query(stix_pattern) - queries = "event_type=\"authentication\"&size=10000" + queries = 'event_type="authentication"&size=10000' _test_query_assertions(query, queries) def test_IPV4_query(self): stix_pattern = "[ipv4-addr:value='192.168.1.1']" query = _translate_query(stix_pattern) - expected_queries = 'filter_key=data.origin&filter_value="192.168.1.1"&size=10000' + expected_queries = ( + 'filter_key=data.origin&filter_value="192.168.1.1"&size=10000' + ) _test_query_assertions(query, expected_queries) def test_event_type_and_fiter(self): @@ -68,17 +70,25 @@ def test_event_type_and_fiter(self): def test_domain_name_query(self): stix_pattern = "[domain-name:value = 'ibmcloud.com']" query = _translate_query(stix_pattern) - expected_queries = 'filter_key=tenantname&filter_value="ibmcloud.com"&size=10000' + expected_queries = ( + 'filter_key=tenantname&filter_value="ibmcloud.com"&size=10000' + ) _test_query_assertions(query, expected_queries) def test_applicationname_with_special_char_query(self): - stix_pattern = "[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" + stix_pattern = ( + "[x-oca-event:extensions.'x-iam-ext'.application_name='Bane & Ox VPN']" + ) query = _translate_query(stix_pattern) - expected_query = 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' + expected_query = ( + 'filter_key=data.applicationname&filter_value="Bane %26 Ox VPN"&size=10000' + ) _test_query_assertions(query, expected_query) def test_in_statement(self): - stix_pattern = "[ ipv4-addr:value IN ('192.168.1.1', '192.168.1.2', '192.168.1.3') ]" + stix_pattern = ( + "[ ipv4-addr:value IN ('192.168.1.1', '192.168.1.2', '192.168.1.3') ]" + ) query = _translate_query(stix_pattern) expected_query = 'filter_key=data.origin&filter_value="192.168.1.1","192.168.1.2","192.168.1.3"&size=10000' _test_query_assertions(query, expected_query) diff --git a/stix_shifter_modules/verify/test/stix_translation/test_transform.py b/stix_shifter_modules/verify/test/stix_translation/test_transform.py index 2d86b93ea..5a275b404 100644 --- a/stix_shifter_modules/verify/test/stix_translation/test_transform.py +++ b/stix_shifter_modules/verify/test/stix_translation/test_transform.py @@ -4,9 +4,10 @@ from unicodedata import category from stix_shifter.stix_translation import stix_translation -from stix_shifter_modules.verify.entry_point import EntryPoint -from stix_shifter_utils.stix_translation.src.utils.transformer_utils import \ - get_module_transformers +from stix_shifter_modules.ibm_security_verify.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import ( + get_module_transformers, +) translation = stix_translation.StixTranslation() # config_file = open('stix_shifter_modules/verify_event/configuration/config.json').read() @@ -16,81 +17,107 @@ # logging.basicConfig(level=logging.DEBUG) # logger = logging.getLogger() -MODULE = 'verify' -RESULTS = 'results' +MODULE = "ibm_security_verify" +RESULTS = "results" TRANSFORMERS = get_module_transformers(MODULE) -epoch_to_timestamp_class = TRANSFORMERS.get('EpochToTimestamp') +epoch_to_timestamp_class = TRANSFORMERS.get("EpochToTimestamp") entry_point = EntryPoint() MAP_DATA = entry_point.get_results_translator().map_data -DATA_SOURCE ={ +DATA_SOURCE = { "type": "identity", "id": "32a23267-52fb-4e82-859b-0a15d6a2d334", - "name":"verify", - "identity_class":"events" - } + "name": "verify", + "identity_class": "events", +} OPTION = json.dumps(DATA_SOURCE) -class TestTransformQuery(unittest.TestCase,object): + +class TestTransformQuery(unittest.TestCase, object): @staticmethod def get_first(itr, constraint): - return next( - (obj for obj in itr if constraint(obj)), - None - ) + return next((obj for obj in itr if constraint(obj)), None) @staticmethod def get_first_of_type(itr, typ): - return TestTransformQuery.get_first(itr, lambda o: type(o) == dict and o.get('type') == typ) + return TestTransformQuery.get_first( + itr, lambda o: type(o) == dict and o.get("type") == typ + ) @staticmethod def get_object_keys(objects): for k, v in objects.items(): - if k == 'type': + if k == "type": yield v elif isinstance(v, dict): for id_val in TestTransformQuery.get_object_keys(v): yield id_val - def test_oca_event(self): - data = [{"continent_name": "Asia", "city_name": "Kolkata", "country_iso_code": "IN", - "ip": "47.15.98.56", "country_name": "India", "region_name": "West Bengal", - "location": {"lon": "88.3697", "lat": "22.5697"}, "result": "success", - "subtype": "saml", "providerid": "https://portal.baneandox.org:443/SAML20/SP", - "origin": "47.15.98.56", "realm": "www.ibm.com", - "applicationid": "6773634223410562472", - "userid": "652001LT0R", "applicationtype": "Custom Application", "devicetype": - "PAN GlobalProtect/5.2.4-21 (Microsoft Windows 10 Enterprise , 64-bit) Mozilla/5.0 (Windows NT 6.2; Win64; x64; Trident/7.0; rv:11.0) like Gecko", - "username": "dinepal1@in.ibm.com", "applicationname": "Bane & Ox VPN", "year": 2022, - "billingid":"12345","mdmismanaged":"true","mdmiscompliant": "true","deviceid":"abc_device", - "@metadata": {"group_id": "event-transform-prod-eu01a-prod-eu01a-01", "source_dc": "prod-eu01a"}, - "event_type": "sso", "month": 1, "indexed_at": 1642413142906, "@processing_time": 1012, - "tenantid": "c92ce528-293f-4e84-8307-c4fe188b9461", "tenantname": "isrras.ice.ibmcloud.com", - "correlationid": "CORR_ID-a134f569-8d73-45ac-8d44-2457448c9101", "servicename": "saml_runtime", - "id": "dc4523e6-6260-4349-83f8-3320365a5f25", "time": 1642413141894, "day": 17} ] + data = [ + { + "continent_name": "Asia", + "city_name": "Kolkata", + "country_iso_code": "IN", + "ip": "47.15.98.56", + "country_name": "India", + "region_name": "West Bengal", + "location": {"lon": "88.3697", "lat": "22.5697"}, + "result": "success", + "subtype": "saml", + "providerid": "https://portal.baneandox.org:443/SAML20/SP", + "origin": "47.15.98.56", + "realm": "www.ibm.com", + "applicationid": "6773634223410562472", + "userid": "652001LT0R", + "applicationtype": "Custom Application", + "devicetype": "PAN GlobalProtect/5.2.4-21 (Microsoft Windows 10 Enterprise , 64-bit) Mozilla/5.0 (Windows NT 6.2; Win64; x64; Trident/7.0; rv:11.0) like Gecko", + "username": "dinepal1@in.ibm.com", + "applicationname": "Bane & Ox VPN", + "year": 2022, + "billingid": "12345", + "mdmismanaged": "true", + "mdmiscompliant": "true", + "deviceid": "abc_device", + "@metadata": { + "group_id": "event-transform-prod-eu01a-prod-eu01a-01", + "source_dc": "prod-eu01a", + }, + "event_type": "sso", + "month": 1, + "indexed_at": 1642413142906, + "@processing_time": 1012, + "tenantid": "c92ce528-293f-4e84-8307-c4fe188b9461", + "tenantname": "isrras.ice.ibmcloud.com", + "correlationid": "CORR_ID-a134f569-8d73-45ac-8d44-2457448c9101", + "servicename": "saml_runtime", + "id": "dc4523e6-6260-4349-83f8-3320365a5f25", + "time": 1642413141894, + "day": 17, + } + ] - user_ref ="2" - category ="sso" + user_ref = "2" + category = "sso" domain_ref = "3" - module="saml_runtime" + module = "saml_runtime" extensions_user_id = "652001LT0R" - result_bundle = entry_point.translate_results(json.dumps(DATA_SOURCE), json.dumps(data)) - observed_data = result_bundle['objects'][1] - objects = observed_data['objects'] + result_bundle = entry_point.translate_results( + json.dumps(DATA_SOURCE), json.dumps(data) + ) + observed_data = result_bundle["objects"][1] + objects = observed_data["objects"] - event = TestTransformQuery.get_first_of_type(objects.values(), 'x-oca-event') + event = TestTransformQuery.get_first_of_type(objects.values(), "x-oca-event") - assert(event['type']) == "x-oca-event" - assert(event['user_ref'] == user_ref) - assert(event['category'] == category) - assert(event['domain_ref'] == domain_ref) - assert(event['module'] == module) - assert(event['category'] == category) - assert(event["extensions"]['x-iam-ext']['is_device_managed']=="true") - assert(event["extensions"]['x-iam-ext']['mdm_customerid']=="12345") - assert(event["extensions"]['x-iam-ext']['is_device_compliant']=="true") - assert(event["extensions"]['x-iam-ext']['deviceid']=="abc_device") - - \ No newline at end of file + assert (event["type"]) == "x-oca-event" + assert event["user_ref"] == user_ref + assert event["category"] == category + assert event["domain_ref"] == domain_ref + assert event["module"] == module + assert event["category"] == category + assert event["extensions"]["x-iam-ext"]["is_device_managed"] == "true" + assert event["extensions"]["x-iam-ext"]["mdm_customerid"] == "12345" + assert event["extensions"]["x-iam-ext"]["is_device_compliant"] == "true" + assert event["extensions"]["x-iam-ext"]["deviceid"] == "abc_device" diff --git a/stix_shifter_modules/verify/test/stix_transmission/test_transmission.py b/stix_shifter_modules/verify/test/stix_transmission/test_transmission.py new file mode 100644 index 000000000..1de86821a --- /dev/null +++ b/stix_shifter_modules/verify/test/stix_transmission/test_transmission.py @@ -0,0 +1,143 @@ +import json +import unittest +from sre_constants import ASSERT_NOT +from unittest.mock import ANY, patch + +from stix_shifter.stix_transmission import stix_transmission +from stix_shifter_modules.ibm_security_verify.entry_point import EntryPoint +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import \ + Status +from stix_shifter_utils.stix_transmission.utils.RestApiClient import \ + ResponseWrapper + + +class VerifyMockResponse: + def __init__(self, response_code, object): + self.code = response_code + self.object = object + + def read(self): + return self.object + + +class VerifyMockPingResponse: + def __init__(self, response_code, status_code): + self.code = response_code + self.status_code = status_code + + def read(self): + return self.object + + +@patch( + "stix_shifter_modules.ibm_security_verify.stix_transmission.api_client.APIClient.__init__", + autospec=True, +) +class TestVerifyConnection(unittest.TestCase, object): + def test_is_async(self, mock_api_client): + mock_api_client.return_value = None + entry_point = EntryPoint() + + config = {"auth": {"sec": "bla"}} + connection = { + "host": "hostbla", + "port": 8080, + } + check_async = entry_point.is_async() + assert check_async is True + + @patch( + "stix_shifter_modules.ibm_security_verify.stix_transmission.api_client.APIClient.get_search", + autospec=True, + ) + def test_status_response(self, mock_status_response, mock_api_client): + mock_api_client.return_value = None + mocked_return_value = '{"search_id": "108cb8b0-0744-4dd9-8e35-ea8311cd6211", "status": "COMPLETED", "progress": "100"}' + mock_status_response.return_value = VerifyMockResponse(200, mocked_return_value) + + config = {"host": "connection.com"} + connection = {"auth": {"clientId": "clientId", "clientSecret": "clientscred"}} + + search_id = "108cb8b0-0744-4dd9-8e35-ea8311cd6211" + transmission = stix_transmission.StixTransmission("ibm_security_verify", config, connection) + status_response = transmission.status(search_id) + + assert status_response["success"] + assert status_response is not None + assert "status" in status_response + assert status_response["status"] == Status.COMPLETED.value + + @patch( + "stix_shifter_modules.ibm_security_verify.stix_transmission.api_client.APIClient.run_search" + ) + def test_query_response(self, mock_query_response, mock_api_client): + mock_api_client.return_value = None + mock_query_response.return_value = {"success": 200} + + config = {"host": "cloudsecurity.com"} + connection = {"auth": {"clientId": "clientid", "clientSecret": "secret"}} + + query = '{"query":"event_type="sso""}' + transmission = stix_transmission.StixTransmission("ibm_security_verify", config, connection) + query_response = transmission.query(query) + + assert query_response is not None + assert "search_id" in query_response + assert query_response["search_id"] == query + + @patch( + "stix_shifter_modules.ibm_security_verify.stix_transmission.api_client.APIClient.generate_token" + ) + def test_ping(self, mock_generate_token, mock_api_client): + + config = {"host": "cloudsecurity.com"} + connection = {"auth": {"clientId": "clientid", "clientSecret": "secret"}} + mocked_return_value = VerifyMockPingResponse(200, 200) + mock_generate_token.return_value = mocked_return_value + mock_api_client.return_value = None + entry_point = EntryPoint(config, connection) + ping_result = entry_point.ping_connection() + assert ping_result["success"] is True + + @patch( + "stix_shifter_modules.ibm_security_verify.stix_transmission.api_client.APIClient.generate_token" + ) + @patch( + "stix_shifter_modules.ibm_security_verify.stix_transmission.api_client.APIClient.run_search", + autospec=True, + ) + def test_results_all_response( + self, mock_results_response, mock_generate_token, mock_api_client + ): + mock_api_client.return_value = None + mocked_return_value = {"code": 200} + mock_generate_token.return_value = mocked_return_value + config = {"host": "ibmcloud.com"} + connection = {"auth": {"clientId": "clientId", "clientSecret": "secret"}} + + mocked_return_value = { + "code": 200, + "success": 200, + "data": [ + { + "id": 123, + "created_at": "2022-01-16T16:45:16.112Z", + "account_id": 123, + "ipaddr": "12.22.33.44", + } + ], + } + + mock_results_response.return_value = mocked_return_value + + query = 'event_type="sso"&limit=10000' + + offset = 0 + length = 101 + entry_point = EntryPoint(config, connection) + results_response = entry_point.create_results_connection(query, offset, length) + + assert results_response is not None + assert results_response["success"] + assert "data" in results_response + assert results_response["data"] is not None diff --git a/stix_shifter_modules/verify/test/stix_transmission/test_verify.py b/stix_shifter_modules/verify/test/stix_transmission/test_verify.py deleted file mode 100644 index 891a28d82..000000000 --- a/stix_shifter_modules/verify/test/stix_transmission/test_verify.py +++ /dev/null @@ -1,157 +0,0 @@ - -import json -from sre_constants import ASSERT_NOT -import unittest -from unittest.mock import ANY -from unittest.mock import patch -from stix_shifter.stix_transmission import stix_transmission -from stix_shifter_modules.verify.entry_point import EntryPoint -from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status -from stix_shifter_utils.stix_transmission.utils.RestApiClient import ResponseWrapper -from unittest.mock import patch - - -class VerifyMockResponse: - def __init__(self, response_code, object): - self.code = response_code - self.object = object - - def read(self): - return self.object - -class VerifyMockPingResponse: - def __init__(self, response_code, status_code): - self.code = response_code - self.status_code =status_code - - def read(self): - return self.object - - -@patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.__init__', autospec=True) -class TestVerifyConnection(unittest.TestCase, object): - - def test_is_async(self, mock_api_client): - mock_api_client.return_value = None - entry_point = EntryPoint() - - config = { - "auth": { - "sec": "bla" - } - } - connection = { - "host": "hostbla", - "port": 8080, - } - check_async = entry_point.is_async() - assert check_async is True - - @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.get_search', autospec=True) - def test_status_response(self, mock_status_response, mock_api_client): - mock_api_client.return_value = None - mocked_return_value = '{"search_id": "108cb8b0-0744-4dd9-8e35-ea8311cd6211", "status": "COMPLETED", "progress": "100"}' - mock_status_response.return_value = VerifyMockResponse(200, mocked_return_value) - - config = { - "host": "connection.com" - } - connection = { - "auth" :{ - "clientId": "clientId", - "clientSecret": "clientscred" - } - } - - search_id = "108cb8b0-0744-4dd9-8e35-ea8311cd6211" - transmission = stix_transmission.StixTransmission('verify', config, connection) - status_response = transmission.status(search_id) - - assert status_response['success'] - assert status_response is not None - assert 'status' in status_response - assert status_response['status'] == Status.COMPLETED.value - - @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.run_search') - def test_query_response(self, mock_query_response, mock_api_client): - mock_api_client.return_value = None - mock_query_response.return_value = {"success":200} - - config = { - "host": "cloudsecurity.com" - } - connection = { - "auth" :{ - "clientId": "clientid", - "clientSecret": "secret" - } - } - - query = '{"query":"event_type=\"sso\""}' - transmission = stix_transmission.StixTransmission('verify', config, connection) - query_response = transmission.query(query) - - assert query_response is not None - assert 'search_id' in query_response - assert query_response['search_id'] == query - - - @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.generate_token') - def test_ping(self, mock_generate_token, mock_api_client): - - config = { - "host": "cloudsecurity.com" - } - connection = { - "auth" :{ - "clientId": "clientid", - "clientSecret": "secret" - } - } - mocked_return_value = VerifyMockPingResponse(200,200) - mock_generate_token.return_value = mocked_return_value - mock_api_client.return_value = None - entry_point = EntryPoint(config, connection) - ping_result = entry_point.ping_connection() - assert ping_result["success"] is True - - - @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.generate_token') - @patch('stix_shifter_modules.verify.stix_transmission.api_client.APIClient.run_search', - autospec=True) - def test_results_all_response(self, mock_results_response, mock_generate_token, mock_api_client): - mock_api_client.return_value = None - mocked_return_value = {"code": 200} - mock_generate_token.return_value = mocked_return_value - config = { - "host": "ibmcloud.com" - } - connection = { - "auth" :{ - "clientId": "clientId", - "clientSecret": "secret" - } - } - - mocked_return_value = { - "code": 200, - "success":200, - "data": [ - {"id":123, "created_at":"2022-01-16T16:45:16.112Z", "account_id":123, "ipaddr":"12.22.33.44"} - ]} - - mock_results_response.return_value = mocked_return_value - - query = "event_type=\"sso\"&limit=10000" - - offset = 0 - length = 101 - entry_point = EntryPoint(config, connection) - results_response = entry_point.create_results_connection(query, offset, length) - - assert results_response is not None - assert results_response['success'] - assert 'data' in results_response - assert results_response['data'] is not None - - \ No newline at end of file From 994096d3125a6ba76c0520316b4e5247bf9c87f6 Mon Sep 17 00:00:00 2001 From: RMS Date: Mon, 21 Feb 2022 12:51:08 +0530 Subject: [PATCH 11/19] rename module name from verify to ibm_security_verify --- stix_shifter_modules/{verify => ibm_security_verify}/README.md | 0 stix_shifter_modules/{verify => ibm_security_verify}/__init__.py | 0 .../{verify => ibm_security_verify}/configuration/config.json | 0 .../{verify => ibm_security_verify}/configuration/lang_en.json | 0 .../{verify => ibm_security_verify}/entry_point.py | 0 .../{verify => ibm_security_verify}/stix_translation/__init__.py | 0 .../stix_translation/json/from_stix_map.json | 0 .../stix_translation/json/operators.json | 0 .../stix_translation/json/to_stix_map.json | 0 .../stix_translation/query_constructor.py | 0 .../stix_translation/query_translator.py | 0 .../stix_translation/results_translator.py | 0 .../stix_translation/transformers.py | 0 .../{verify => ibm_security_verify}/stix_transmission/__init__.py | 0 .../stix_transmission/api_client.py | 0 .../stix_transmission/delete_connector.py | 0 .../stix_transmission/error_mapper.py | 0 .../stix_transmission/ping_connector.py | 0 .../stix_transmission/query_connector.py | 0 .../stix_transmission/results_connector.py | 0 .../stix_transmission/status_connector.py | 0 .../test/stix_translation/___init__.py | 0 .../test/stix_translation/test_stix_to_query.py | 0 .../test/stix_translation/test_transform.py | 0 .../test/stix_transmission/___init__.py | 0 .../test/stix_transmission/test_transmission.py | 0 26 files changed, 0 insertions(+), 0 deletions(-) rename stix_shifter_modules/{verify => ibm_security_verify}/README.md (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/__init__.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/configuration/config.json (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/configuration/lang_en.json (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/entry_point.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_translation/__init__.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_translation/json/from_stix_map.json (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_translation/json/operators.json (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_translation/json/to_stix_map.json (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_translation/query_constructor.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_translation/query_translator.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_translation/results_translator.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_translation/transformers.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_transmission/__init__.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_transmission/api_client.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_transmission/delete_connector.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_transmission/error_mapper.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_transmission/ping_connector.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_transmission/query_connector.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_transmission/results_connector.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/stix_transmission/status_connector.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/test/stix_translation/___init__.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/test/stix_translation/test_stix_to_query.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/test/stix_translation/test_transform.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/test/stix_transmission/___init__.py (100%) rename stix_shifter_modules/{verify => ibm_security_verify}/test/stix_transmission/test_transmission.py (100%) diff --git a/stix_shifter_modules/verify/README.md b/stix_shifter_modules/ibm_security_verify/README.md similarity index 100% rename from stix_shifter_modules/verify/README.md rename to stix_shifter_modules/ibm_security_verify/README.md diff --git a/stix_shifter_modules/verify/__init__.py b/stix_shifter_modules/ibm_security_verify/__init__.py similarity index 100% rename from stix_shifter_modules/verify/__init__.py rename to stix_shifter_modules/ibm_security_verify/__init__.py diff --git a/stix_shifter_modules/verify/configuration/config.json b/stix_shifter_modules/ibm_security_verify/configuration/config.json similarity index 100% rename from stix_shifter_modules/verify/configuration/config.json rename to stix_shifter_modules/ibm_security_verify/configuration/config.json diff --git a/stix_shifter_modules/verify/configuration/lang_en.json b/stix_shifter_modules/ibm_security_verify/configuration/lang_en.json similarity index 100% rename from stix_shifter_modules/verify/configuration/lang_en.json rename to stix_shifter_modules/ibm_security_verify/configuration/lang_en.json diff --git a/stix_shifter_modules/verify/entry_point.py b/stix_shifter_modules/ibm_security_verify/entry_point.py similarity index 100% rename from stix_shifter_modules/verify/entry_point.py rename to stix_shifter_modules/ibm_security_verify/entry_point.py diff --git a/stix_shifter_modules/verify/stix_translation/__init__.py b/stix_shifter_modules/ibm_security_verify/stix_translation/__init__.py similarity index 100% rename from stix_shifter_modules/verify/stix_translation/__init__.py rename to stix_shifter_modules/ibm_security_verify/stix_translation/__init__.py diff --git a/stix_shifter_modules/verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json similarity index 100% rename from stix_shifter_modules/verify/stix_translation/json/from_stix_map.json rename to stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json diff --git a/stix_shifter_modules/verify/stix_translation/json/operators.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/operators.json similarity index 100% rename from stix_shifter_modules/verify/stix_translation/json/operators.json rename to stix_shifter_modules/ibm_security_verify/stix_translation/json/operators.json diff --git a/stix_shifter_modules/verify/stix_translation/json/to_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/to_stix_map.json similarity index 100% rename from stix_shifter_modules/verify/stix_translation/json/to_stix_map.json rename to stix_shifter_modules/ibm_security_verify/stix_translation/json/to_stix_map.json diff --git a/stix_shifter_modules/verify/stix_translation/query_constructor.py b/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py similarity index 100% rename from stix_shifter_modules/verify/stix_translation/query_constructor.py rename to stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py diff --git a/stix_shifter_modules/verify/stix_translation/query_translator.py b/stix_shifter_modules/ibm_security_verify/stix_translation/query_translator.py similarity index 100% rename from stix_shifter_modules/verify/stix_translation/query_translator.py rename to stix_shifter_modules/ibm_security_verify/stix_translation/query_translator.py diff --git a/stix_shifter_modules/verify/stix_translation/results_translator.py b/stix_shifter_modules/ibm_security_verify/stix_translation/results_translator.py similarity index 100% rename from stix_shifter_modules/verify/stix_translation/results_translator.py rename to stix_shifter_modules/ibm_security_verify/stix_translation/results_translator.py diff --git a/stix_shifter_modules/verify/stix_translation/transformers.py b/stix_shifter_modules/ibm_security_verify/stix_translation/transformers.py similarity index 100% rename from stix_shifter_modules/verify/stix_translation/transformers.py rename to stix_shifter_modules/ibm_security_verify/stix_translation/transformers.py diff --git a/stix_shifter_modules/verify/stix_transmission/__init__.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/__init__.py similarity index 100% rename from stix_shifter_modules/verify/stix_transmission/__init__.py rename to stix_shifter_modules/ibm_security_verify/stix_transmission/__init__.py diff --git a/stix_shifter_modules/verify/stix_transmission/api_client.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/api_client.py similarity index 100% rename from stix_shifter_modules/verify/stix_transmission/api_client.py rename to stix_shifter_modules/ibm_security_verify/stix_transmission/api_client.py diff --git a/stix_shifter_modules/verify/stix_transmission/delete_connector.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/delete_connector.py similarity index 100% rename from stix_shifter_modules/verify/stix_transmission/delete_connector.py rename to stix_shifter_modules/ibm_security_verify/stix_transmission/delete_connector.py diff --git a/stix_shifter_modules/verify/stix_transmission/error_mapper.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/error_mapper.py similarity index 100% rename from stix_shifter_modules/verify/stix_transmission/error_mapper.py rename to stix_shifter_modules/ibm_security_verify/stix_transmission/error_mapper.py diff --git a/stix_shifter_modules/verify/stix_transmission/ping_connector.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/ping_connector.py similarity index 100% rename from stix_shifter_modules/verify/stix_transmission/ping_connector.py rename to stix_shifter_modules/ibm_security_verify/stix_transmission/ping_connector.py diff --git a/stix_shifter_modules/verify/stix_transmission/query_connector.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/query_connector.py similarity index 100% rename from stix_shifter_modules/verify/stix_transmission/query_connector.py rename to stix_shifter_modules/ibm_security_verify/stix_transmission/query_connector.py diff --git a/stix_shifter_modules/verify/stix_transmission/results_connector.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/results_connector.py similarity index 100% rename from stix_shifter_modules/verify/stix_transmission/results_connector.py rename to stix_shifter_modules/ibm_security_verify/stix_transmission/results_connector.py diff --git a/stix_shifter_modules/verify/stix_transmission/status_connector.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/status_connector.py similarity index 100% rename from stix_shifter_modules/verify/stix_transmission/status_connector.py rename to stix_shifter_modules/ibm_security_verify/stix_transmission/status_connector.py diff --git a/stix_shifter_modules/verify/test/stix_translation/___init__.py b/stix_shifter_modules/ibm_security_verify/test/stix_translation/___init__.py similarity index 100% rename from stix_shifter_modules/verify/test/stix_translation/___init__.py rename to stix_shifter_modules/ibm_security_verify/test/stix_translation/___init__.py diff --git a/stix_shifter_modules/verify/test/stix_translation/test_stix_to_query.py b/stix_shifter_modules/ibm_security_verify/test/stix_translation/test_stix_to_query.py similarity index 100% rename from stix_shifter_modules/verify/test/stix_translation/test_stix_to_query.py rename to stix_shifter_modules/ibm_security_verify/test/stix_translation/test_stix_to_query.py diff --git a/stix_shifter_modules/verify/test/stix_translation/test_transform.py b/stix_shifter_modules/ibm_security_verify/test/stix_translation/test_transform.py similarity index 100% rename from stix_shifter_modules/verify/test/stix_translation/test_transform.py rename to stix_shifter_modules/ibm_security_verify/test/stix_translation/test_transform.py diff --git a/stix_shifter_modules/verify/test/stix_transmission/___init__.py b/stix_shifter_modules/ibm_security_verify/test/stix_transmission/___init__.py similarity index 100% rename from stix_shifter_modules/verify/test/stix_transmission/___init__.py rename to stix_shifter_modules/ibm_security_verify/test/stix_transmission/___init__.py diff --git a/stix_shifter_modules/verify/test/stix_transmission/test_transmission.py b/stix_shifter_modules/ibm_security_verify/test/stix_transmission/test_transmission.py similarity index 100% rename from stix_shifter_modules/verify/test/stix_transmission/test_transmission.py rename to stix_shifter_modules/ibm_security_verify/test/stix_transmission/test_transmission.py From fa399a6c95ac542e8aeb38668a3dc9dd3e46ba5b Mon Sep 17 00:00:00 2001 From: RMS Date: Mon, 21 Feb 2022 15:55:50 +0530 Subject: [PATCH 12/19] fixed mapping for lan and lat geo location --- .../stix_translation/json/from_stix_map.json | 6 +++--- .../stix_translation/json/to_stix_map.json | 8 ++++++++ .../stix_transmission/api_client.py | 15 +++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json index 0b387d119..2bc44f539 100644 --- a/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json @@ -113,11 +113,11 @@ "extensions.'x-iam-ext'.country_name": [ "geoip.country_name" ], - "extensions.'x-iam-ext'.location_lon": [ - "geoip.location.lon" + "extensions.'x-iam-ext'.lon": [ + "geoip.lon" ], "extensions.'x-iam-ext'.location_lat": [ - "geoip.location_lat" + "geoip.lat" ], "extensions.'x-iam-ext'.city_name": [ "geoip.city_name" diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/to_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/to_stix_map.json index 4352fbda8..1d5d8365c 100644 --- a/stix_shifter_modules/ibm_security_verify/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/ibm_security_verify/stix_translation/json/to_stix_map.json @@ -207,5 +207,13 @@ "billingid": { "object": "ocaevent", "key": "x-oca-event.extensions.x-iam-ext.mdm_customerid" + }, + "lat": { + "key": "x-oca-event.extensions.x-iam-ext.location_lat", + "object": "ocaevent" + }, + "lon": { + "key": "x-oca-event.extensions.x-iam-ext.location_lon", + "object": "ocaevent" } } \ No newline at end of file diff --git a/stix_shifter_modules/ibm_security_verify/stix_transmission/api_client.py b/stix_shifter_modules/ibm_security_verify/stix_transmission/api_client.py index ff65a5071..875745f2b 100644 --- a/stix_shifter_modules/ibm_security_verify/stix_transmission/api_client.py +++ b/stix_shifter_modules/ibm_security_verify/stix_transmission/api_client.py @@ -1,10 +1,11 @@ import json - -from requests.adapters import Response -from stix_shifter_utils.stix_transmission.utils.RestApiClient import RestApiClient from datetime import datetime, timedelta -import requests from urllib.parse import urlencode + +import requests +from requests.adapters import Response +from stix_shifter_utils.stix_transmission.utils.RestApiClient import \ + RestApiClient from stix_shifter_utils.utils import logger @@ -128,8 +129,10 @@ def parseJson(self, response): dictA= json.loads(json.dumps(obj["geoip"])) del dictC["geoip"] dictB= json.loads(json.dumps(obj["data"])) - del dictC["data"] - finalJson = {**dictA,**dictB} + del dictC["data"] + dict_geo_location = json.loads(json.dumps(dictA.get('location'))) + del dictA['location'] + finalJson = {**dictA,**dictB,**dict_geo_location} else: dictB= json.loads(json.dumps(obj["data"])) del dictC["data"] From c3f9e9d9570073c0db460dfbfc89e1f6026d1ff8 Mon Sep 17 00:00:00 2001 From: RMS Date: Mon, 21 Feb 2022 17:39:15 +0530 Subject: [PATCH 13/19] mapping correction for geoloaction --- .../stix_translation/json/from_stix_map.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json index 2bc44f539..6a99ef51c 100644 --- a/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json @@ -113,7 +113,7 @@ "extensions.'x-iam-ext'.country_name": [ "geoip.country_name" ], - "extensions.'x-iam-ext'.lon": [ + "extensions.x-iam-ext.location_lon": [ "geoip.lon" ], "extensions.'x-iam-ext'.location_lat": [ From a42fe57dd517da47a4a6fb37a556f49916c0587f Mon Sep 17 00:00:00 2001 From: RMS Date: Tue, 22 Feb 2022 22:18:53 +0530 Subject: [PATCH 14/19] Rename test_stix_to_query.py to test_ibm_verify_stix_to_query.py Test case is failing because of duplicate test case name in other module. --- .../{test_stix_to_query.py => test_ibm_verify_stix_to_query.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename stix_shifter_modules/ibm_security_verify/test/stix_translation/{test_stix_to_query.py => test_ibm_verify_stix_to_query.py} (100%) diff --git a/stix_shifter_modules/ibm_security_verify/test/stix_translation/test_stix_to_query.py b/stix_shifter_modules/ibm_security_verify/test/stix_translation/test_ibm_verify_stix_to_query.py similarity index 100% rename from stix_shifter_modules/ibm_security_verify/test/stix_translation/test_stix_to_query.py rename to stix_shifter_modules/ibm_security_verify/test/stix_translation/test_ibm_verify_stix_to_query.py From e91c221ac2cc167644769b72805e580d2c1c9795 Mon Sep 17 00:00:00 2001 From: RMS Date: Tue, 22 Feb 2022 22:19:50 +0530 Subject: [PATCH 15/19] Rename test_transform.py to test_ibm_verify_transform.py Test case is failing because pytest found duplicate filename in other module. --- .../{test_transform.py => test_ibm_verify_transform.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename stix_shifter_modules/ibm_security_verify/test/stix_translation/{test_transform.py => test_ibm_verify_transform.py} (100%) diff --git a/stix_shifter_modules/ibm_security_verify/test/stix_translation/test_transform.py b/stix_shifter_modules/ibm_security_verify/test/stix_translation/test_ibm_verify_transform.py similarity index 100% rename from stix_shifter_modules/ibm_security_verify/test/stix_translation/test_transform.py rename to stix_shifter_modules/ibm_security_verify/test/stix_translation/test_ibm_verify_transform.py From 59cb6bde24a6fcddb1c5faca76caf363317c329f Mon Sep 17 00:00:00 2001 From: RMS Date: Tue, 22 Feb 2022 22:42:30 +0530 Subject: [PATCH 16/19] rename test_transmission to test_ibm_verify_transmission --- .../{test_transmission.py => test_ibm_verify_transmission.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename stix_shifter_modules/ibm_security_verify/test/stix_transmission/{test_transmission.py => test_ibm_verify_transmission.py} (100%) diff --git a/stix_shifter_modules/ibm_security_verify/test/stix_transmission/test_transmission.py b/stix_shifter_modules/ibm_security_verify/test/stix_transmission/test_ibm_verify_transmission.py similarity index 100% rename from stix_shifter_modules/ibm_security_verify/test/stix_transmission/test_transmission.py rename to stix_shifter_modules/ibm_security_verify/test/stix_transmission/test_ibm_verify_transmission.py From 667f0e2376de3ddb4781f490cae73902e7f86199 Mon Sep 17 00:00:00 2001 From: RMS Date: Tue, 1 Mar 2022 12:42:01 +0530 Subject: [PATCH 17/19] removed operator_lookup from queryConstructor class and now refering to json file. --- .../stix_translation/query_constructor.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py b/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py index 4f86eba3d..698f882b8 100644 --- a/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py +++ b/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py @@ -1,14 +1,17 @@ +import json +import logging +import re from os import stat from sre_constants import IN from typing import Union -from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \ - ComparisonExpressionOperators, ComparisonComparators, Pattern, \ - CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators + from stix_shifter_utils.stix_translation.src.json_to_stix import observable -from stix_shifter_utils.stix_translation.src.utils.transformers import TimestampToMilliseconds -import logging -import re -import json +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ( + CombinedComparisonExpression, CombinedObservationExpression, + ComparisonComparators, ComparisonExpression, ComparisonExpressionOperators, + ObservationExpression, ObservationOperators, Pattern) +from stix_shifter_utils.stix_translation.src.utils.transformers import \ + TimestampToMilliseconds # Source and destination reference mapping for ip and mac addresses. # Change the keys to match the data source fields. The value array indicates the possible data type that can come into from field. @@ -23,17 +26,10 @@ class QueryStringPatternTranslator: QUERIES = [] - # # Change comparator values to match with supported data source operators - comparator_lookup = { - "ComparisonExpressionOperators.And": "&", - "ComparisonComparators.Equal": "=", - "ObservationOperators.And": "=", - "ComparisonComparators.In": "=" - } - + def __init__(self, pattern: Pattern, data_model_mapper): self.dmm = data_model_mapper - # self.comparator_lookup = self.dmm.map_comparator() #Not sure its not workig with kestral + self.comparator_lookup = self.dmm.map_comparator() self.pattern = pattern self.translated = self.parse_expression(pattern) From d3fe306e818b648175ffaa99fcdc6828959606e0 Mon Sep 17 00:00:00 2001 From: RMS Date: Wed, 2 Mar 2022 10:20:31 +0530 Subject: [PATCH 18/19] removed unused import in queryConstructor.py --- .../ibm_security_verify/stix_translation/query_constructor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py b/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py index 698f882b8..d7b532b4b 100644 --- a/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py +++ b/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py @@ -1,8 +1,6 @@ import json import logging import re -from os import stat -from sre_constants import IN from typing import Union from stix_shifter_utils.stix_translation.src.json_to_stix import observable From 654840b09217c4b4a6006644559ee3601c6cfa84 Mon Sep 17 00:00:00 2001 From: RMS Date: Fri, 4 Mar 2022 15:32:05 +0530 Subject: [PATCH 19/19] added stix_2_1 mapping and corrected mapping of useraccount.account_type --- .../stix_translation/json/from_stix_map.json | 2 +- .../json/stix_2_1/from_stix_map.json | 157 +++++++++++++ .../json/stix_2_1/to_stix_map.json | 219 ++++++++++++++++++ 3 files changed, 377 insertions(+), 1 deletion(-) create mode 100644 stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/to_stix_map.json diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json index 6a99ef51c..eda222733 100644 --- a/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json @@ -8,7 +8,7 @@ "data.username" ], "account_type": [ - "data.realm" + "data.sourcetype" ] } }, diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..eda222733 --- /dev/null +++ b/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,157 @@ +{ + "user-account": { + "fields": { + "user_id": [ + "data.user_id" + ], + "account_login": [ + "data.username" + ], + "account_type": [ + "data.sourcetype" + ] + } + }, + "ipv4-addr": { + "fields": { + "value": [ + "data.origin" + ] + } + }, + "domain-name": { + "fields": { + "type": [ + "domain-name" + ], + "value": [ + "tenantname" + ] + } + }, + "x-oca-event": { + "fields": { + "action": [ + "data.action" + ], + "category": [ + "event_type" + ], + "module": [ + "servicename" + ], + "outcome": [ + "data.result" + ], + "agent": [ + "data.sourcetype" + ], + "ip_refs[*].value": [ + "ip" + ], + "domain_ref.value": [ + "tenantname" + ], + "user_ref": [ + "username" + ], + "provider": "'IBM Security Verify Event'", + "extensions.'x-iam-ext'.subcategory": [ + "data.subtype" + ], + "extensions.'x-iam-ext'.realm": [ + "data.realm" + ], + "extensions.'x-iam-ext'.browser_agent": [ + "data.devicetype" + ], + "extensions.'x-iam-ext'.provider_id": [ + "data.providerid" + ], + "extensions.'x-iam-ext'.application_id": [ + "data.applicationid" + ], + "extensions.'x-iam-ext'.application_type": [ + "data.applicationtype" + ], + "extensions.'x-iam-ext'.application_name": [ + "data.applicationname" + ], + "extensions.'x-iam-ext'.cause": [ + "data.cause" + ], + "extensions.'x-iam-ext'.target": [ + "data.target" + ], + "extensions.'x-iam-ext'.deleted": [ + "data.deleted" + ], + "extensions.'x-iam-ext'.performedby_clientname": [ + "data.performedby_clientname" + ], + "extensions.'x-iam-ext'.performedby_realm": [ + "data.performedby_realm" + ], + "extensions.'x-iam-ext'.performedby_username": [ + "data.performedby_username" + ], + "extensions.'x-iam-ext'.targetid": [ + "data.targetid" + ], + "extensions.'x-iam-ext'.targetid_realm": [ + "data.targetid_realm" + ], + "extensions.'x-iam-ext'.targetid_username": [ + "data.targetid_username" + ], + "extensions.'x-iam-ext'.continent_name": [ + "geoip.continent_name" + ], + "extensions.'x-iam-ext'.country_iso_code": [ + "geoip.country_iso_code" + ], + "extensions.'x-iam-ext'.country_name": [ + "geoip.country_name" + ], + "extensions.x-iam-ext.location_lon": [ + "geoip.lon" + ], + "extensions.'x-iam-ext'.location_lat": [ + "geoip.lat" + ], + "extensions.'x-iam-ext'.city_name": [ + "geoip.city_name" + ], + "extensions.'x-iam-ext'.policy_action": [ + "data.policy_action" + ], + "extensions.'x-iam-ext'.policy_name": [ + "data.policy_name" + ], + "extensions.'x-iam-ext'.rule_name": [ + "data.rule_name" + ], + "extensions.'x-iam-ext'.decision_reason": [ + "data.decision_reason" + ], + "extensions.'x-iam-ext'.risk_level": [ + "data.risk_level" + ], + "extensions.'x-iam-ext'.risk_score": [ + "data.risk_score" + ], + "extensions.'x-iam-ext'.deviceid": [ + "data.deviceid" + ], + "extensions.'x-iam-ext'.is_device_compliant": [ + "data.mdmiscompliant" + ], + "extensions.'x-iam-ext'.is_device_managed": [ + "data.mdmismanaged" + ], + "extensions.'x-iam-ext'.mdm_customerid": [ + "data.billingid" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..1d5d8365c --- /dev/null +++ b/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,219 @@ +{ + "username": [ + { + "key": "user-account.user_id", + "object": "useraccount" + }, + { + "key": "user-account.account_login", + "object": "useraccount" + }, + { + "key": "user-account.account_type", + "object": "useraccount" + }, + { + "key": "x-oca-event.user_ref", + "object": "ocaevent", + "references": "useraccount" + } + ], + "servicename": { + "key": "x-oca-event.module", + "object": "ocaevent" + }, + "sourcetype": { + "key": "x-oca-event.agent", + "object": "ocaevent" + }, + "ip": [ + { + "key": "ipv4-addr.value", + "object": "ipdata" + }, + { + "key": "x-oca-event.ip_refs", + "object": "ocaevent", + "references": [ + "ipdata" + ], + "group": true + } + ], + "tenantname": [ + { + "key": "domain-name.value", + "object": "domaindata", + "transformer": "ToDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "ocaevent", + "references": "domaindata" + } + ], + "result": [ + { + "object": "ocaevent", + "key": "x-oca-event.outcome" + } + ], + "subtype": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.subcategory" + }, + "cause": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.cause" + }, + "action": [ + { + "key": "x-oca-event.action", + "object": "ocaevent" + } + ], + "realm": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.realm" + }, + "devicetype": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.browser_agent" + }, + "applicationid": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.application_id" + }, + "applicationtype": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.application_type" + }, + "applicationname": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.applicationname" + }, + "target": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.target" + }, + "event_type": [ + { + "key": "x-oca-event.category", + "object": "ocaevent" + }, + { + "key": "x-oca-event.provider", + "object": "ocaevent", + "transformer": "VerifyStaticTransformer" + } + ], + "time": [ + { + "key": "x-oca-event.created", + "transformer": "EpochToTimestamp", + "object": "ocaevent" + } + ], + "performedby_username": { + "key": "x-oca-event.extensions.x-iam-ext.performedby_username", + "object": "ocaevent" + }, + "deleted": { + "key": "x-oca-event.extensions.x-iam-ext.deleted", + "object": "ocaevent" + }, + "performedby_clientname": { + "key": "x-oca-event.extensions.x-iam-ext.performedby_clientname", + "object": "ocaevent" + }, + "performedby_realm": { + "key": "x-oca-event.extensions.x-iam-ext.performedby_realm", + "object": "ocaevent" + }, + "targetid": { + "key": "x-oca-event.extensions.x-iam-ext.targetid", + "object": "ocaevent" + }, + "targetid_realm": { + "key": "x-oca-event.extensions.x-iam-ext.targetid_realm", + "object": "ocaevent" + }, + "targetid_username": { + "key": "x-oca-event.extensions.x-iam-ext.taregetid_username", + "object": "ocaevent" + }, + "userid": [ + { + "key": "user-account.user_id", + "object": "useraccount" + } + ], + "continent_name": { + "key": "x-oca-event.extensions.x-iam-ext.continent_name", + "object": "ocaevent" + }, + "city_name": { + "key": "x-oca-event.extensions.x-iam-ext.city_name", + "object": "ocaevent" + }, + "country_iso_code": { + "key": "x-oca-event.extensions.x-iam-ext.country_iso_code", + "object": "ocaevent" + }, + "country_name": { + "key": "x-oca-event.extensions.x-iam-ext.country_name", + "object": "ocaevent" + }, + "providerid": { + "key": "x-oca-event.extensions.x-iam-ext.provider_id", + "object": "ocaevent" + }, + "rule_name": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.rule_name" + }, + "policy_name": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.policy_name" + }, + "decision_reason": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.decision_reason" + }, + "policy_action": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.policy_action" + }, + "risk_level": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.risk_level" + }, + "risk_score": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.risk_score" + }, + "deviceid": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.deviceid" + }, + "mdmiscompliant": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.is_device_compliant" + }, + "mdmismanaged": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.is_device_managed" + }, + "billingid": { + "object": "ocaevent", + "key": "x-oca-event.extensions.x-iam-ext.mdm_customerid" + }, + "lat": { + "key": "x-oca-event.extensions.x-iam-ext.location_lat", + "object": "ocaevent" + }, + "lon": { + "key": "x-oca-event.extensions.x-iam-ext.location_lon", + "object": "ocaevent" + } +} \ No newline at end of file