From 524d1b75cf18496b4d0e5c15ea91f0c31bc246c3 Mon Sep 17 00:00:00 2001 From: Sharmila-MS Date: Fri, 31 May 2024 16:08:53 +0000 Subject: [PATCH 1/4] Trellix Endpoint Security HX Connector Added connector files for Trellix Endpoint Security HX data source --- ...trellix_endpoint_security_hx_05302024.json | 337 ++++++++ .../trellix_endpoint_security_hx/README.md | 697 +++++++++++++++++ .../trellix_endpoint_security_hx/__init__.py | 0 .../configuration/config.json | 48 ++ .../configuration/lang_en.json | 42 + .../entry_point.py | 11 + .../stix_translation/__init__.py | 0 .../stix_translation/json/config_map.json | 13 + .../stix_translation/json/from_stix_map.json | 77 ++ .../stix_translation/json/operators.json | 13 + .../json/stix_2_1/from_stix_map.json | 73 ++ .../json/stix_2_1/to_stix_map.json | 329 ++++++++ .../stix_translation/json/to_stix_map.json | 349 +++++++++ .../stix_translation/query_constructor.py | 467 +++++++++++ .../stix_translation/query_translator.py | 26 + .../stix_transmission/__init__.py | 0 .../stix_transmission/api_client.py | 117 +++ .../stix_transmission/delete_connector.py | 71 ++ .../stix_transmission/error_mapper.py | 40 + .../stix_transmission/ping_connector.py | 72 ++ .../stix_transmission/query_connector.py | 128 +++ .../stix_transmission/results_connector.py | 374 +++++++++ .../stix_transmission/status_connector.py | 135 ++++ ...ellix_endpoint_security_hx_json_to_stix.py | 292 +++++++ ...llix_endpoint_security_hx_stix_to_query.py | 446 +++++++++++ .../test_trellix_endpoint_security_hx.py | 731 ++++++++++++++++++ ...lix_endpoint_security_hx_supported_stix.md | 143 ++++ 27 files changed, 5031 insertions(+) create mode 100644 data/cybox/trellix_endpoint_security_hx/trellix_endpoint_security_hx_05302024.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/README.md create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/__init__.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/configuration/config.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/configuration/lang_en.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/entry_point.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/__init__.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/config_map.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/from_stix_map.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/operators.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/to_stix_map.json create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/query_constructor.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/query_translator.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/__init__.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/api_client.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/delete_connector.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/error_mapper.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/ping_connector.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/query_connector.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/results_connector.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/status_connector.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/test/stix_translation/test_trellix_endpoint_security_hx_json_to_stix.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/test/stix_translation/test_trellix_endpoint_security_hx_stix_to_query.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/test/stix_transmission/test_trellix_endpoint_security_hx.py create mode 100644 stix_shifter_modules/trellix_endpoint_security_hx/trellix_endpoint_security_hx_supported_stix.md diff --git a/data/cybox/trellix_endpoint_security_hx/trellix_endpoint_security_hx_05302024.json b/data/cybox/trellix_endpoint_security_hx/trellix_endpoint_security_hx_05302024.json new file mode 100644 index 000000000..e466d590e --- /dev/null +++ b/data/cybox/trellix_endpoint_security_hx/trellix_endpoint_security_hx_05302024.json @@ -0,0 +1,337 @@ +{ + "type": "bundle", + "id": "bundle--536c62be-2b7c-4140-9a57-80e2dcb9a1cd", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "trellix_endpoint_security_hx", + "identity_class": "system", + "created": "2024-05-30T00:22:50.336Z", + "modified": "2024-05-30T06:22:50.336Z" + }, + { + "id": "observed-data--2a1c6cbd-9c9a-41fb-93ed-3fa008d30d8c", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-05-30T16:27:58.300Z", + "modified": "2024-05-30T16:27:58.300Z", + "objects": { + "0": { + "type": "process", + "name": "pycharm64.exe", + "pid": 4336, + "creator_user_ref": "2", + "binary_ref": "7" + }, + "1": { + "type": "x-oca-event", + "process_ref": "0", + "user_ref": "2", + "created": "2024-05-28T16:31:22.206Z", + "modified": "2024-05-28T16:31:22.206Z", + "host_ref": "3", + "action": "File Write Event", + "file_ref": "4" + }, + "2": { + "type": "user-account", + "user_id": "user1" + }, + "3": { + "type": "x-oca-asset", + "device_id": "device1", + "hostname": "EC21", + "x_host_set": "my_comp_host_set" + }, + "4": { + "type": "file", + "name": "IdIndex.storage.values", + "x_path": "C:\\Users\\IdIndex.storage.values", + "parent_directory_ref": "5", + "content_ref": "6", + "x_bytes_written": 198376 + }, + "5": { + "type": "directory", + "path": "C:\\Users" + }, + "6": { + "type": "artifact", + "payload_bin": "[file content base 64 encoded]" + }, + "7": { + "type": "file", + "size": 0, + "name": "pycharm64.exe" + } + }, + "first_observed": "2024-05-28T16:31:22.206Z", + "last_observed": "2024-05-28T16:31:22.206Z", + "number_observed": 1 + }, + { + "id": "observed-data--ad8f4f30-b237-4a36-83ab-741ba88312a3", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-05-30T16:27:58.305Z", + "modified": "2024-05-30T16:27:58.305Z", + "objects": { + "0": { + "type": "file", + "name": "WmiPrvSE.exe", + "x_path": "C:\\Windows\\WmiPrvSE.exe", + "parent_directory_ref": "3", + "hashes": { + "MD5": "01010101001011011001010101010101" + } + }, + "1": { + "type": "process", + "binary_ref": "0", + "name": "WmiPrvSE.exe", + "parent_ref": "4", + "x_event_type": "start", + "pid": 9184, + "creator_user_ref": "5", + "command_line": "C:\\Windows\\wmiprvse.exe -secured -Embedding" + }, + "2": { + "type": "x-oca-event", + "process_ref": "1", + "parent_process_ref": "4", + "user_ref": "5", + "created": "2024-05-17T15:06:53.984Z", + "modified": "2024-05-17T15:06:53.984Z", + "x_last_run": "2024-05-17T15:06:53.984Z", + "x_accessed_time": "2024-05-17T15:06:53.984Z", + "start": "2024-05-17T15:06:53.984Z", + "host_ref": "6", + "action": "Process Event" + }, + "3": { + "type": "directory", + "path": "C:\\Windows" + }, + "4": { + "type": "process", + "name": "svchost.exe", + "cwd": "C:\\Windows" + }, + "5": { + "type": "user-account", + "user_id": "NT AUTHORITY\\NETWORK SERVICE" + }, + "6": { + "type": "x-oca-asset", + "device_id": "device1", + "hostname": "EC21", + "x_host_set": "my_comp_host_set" + } + }, + "first_observed": "2024-05-17T15:06:53.984Z", + "last_observed": "2024-05-17T15:06:53.984Z", + "number_observed": 1 + }, + { + "id": "observed-data--7dadc551-8952-47cd-a66d-58bd03cba0e6", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-05-30T16:35:57.200Z", + "modified": "2024-05-30T16:35:57.200Z", + "objects": { + "0": { + "type": "process", + "name": "chrome.exe", + "pid": 7600, + "creator_user_ref": "2", + "binary_ref": "7" + }, + "1": { + "type": "x-oca-event", + "process_ref": "0", + "user_ref": "2", + "ip_refs": [ + "3", + "6" + ], + "network_ref": "4", + "created": "2024-05-28T09:04:04.751Z", + "modified": "2024-05-28T09:04:04.751Z", + "x_accessed_time": "2024-05-28T09:04:04.751Z", + "host_ref": "5", + "action": "IPv4 Network Event" + }, + "2": { + "type": "user-account", + "user_id": "user2" + }, + "3": { + "type": "ipv4-addr", + "value": "1.2.3.4" + }, + "4": { + "type": "network-traffic", + "src_ref": "3", + "dst_ref": "6", + "src_port": 57896, + "dst_port": 443, + "protocols": [ + "ipv4" + ] + }, + "5": { + "type": "x-oca-asset", + "ip_refs": [ + "3" + ], + "device_id": "dev1", + "hostname": "EC23", + "x_host_set": "my_comp_host_set" + }, + "6": { + "type": "ipv4-addr", + "value": "9.8.0.0" + }, + "7": { + "type": "file", + "name": "chrome.exe" + } + }, + "first_observed": "2024-05-28T09:04:04.751Z", + "last_observed": "2024-05-28T09:04:04.751Z", + "number_observed": 1 + }, + { + "id": "observed-data--4640de62-4b95-4166-adef-8102e860f404", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-05-30T16:41:03.931Z", + "modified": "2024-05-30T16:41:03.931Z", + "objects": { + "0": { + "type": "process", + "name": "cortex-xdr-payload.exe", + "pid": 5536, + "creator_user_ref": "2", + "binary_ref": "7" + }, + "1": { + "type": "x-oca-event", + "process_ref": "0", + "user_ref": "2", + "ip_refs": [ + "3" + ], + "network_ref": "4", + "domain_ref": "5", + "created": "2024-05-02T04:48:00.463Z", + "modified": "2024-05-02T04:48:00.463Z", + "x_accessed_time": "2024-05-02T04:48:00.463Z", + "host_ref": "6", + "action": "URL Event" + }, + "2": { + "type": "user-account", + "user_id": "NT AUTHORITY\\SYSTEM" + }, + "3": { + "type": "ipv4-addr", + "value": "2.2.3.3" + }, + "4": { + "type": "network-traffic", + "dst_ref": "3", + "src_port": 49736, + "dst_port": 80, + "extensions": { + "http-request-ext": { + "request_value": "/latest/meta-data//ami-id", + "request_header": { + "Host": "2.2.3.3", + "User-Agent": "python-requests/2.26.0", + "Accept-Encoding": "gzip, deflate" + }, + "request_method": "GET" + } + }, + "protocols": [ + "http" + ] + }, + "5": { + "type": "domain-name", + "value": "2.2.3.3" + }, + "6": { + "type": "x-oca-asset", + "device_id": "dev56", + "hostname": "EC212", + "x_host_set": "my_comp_host_set" + }, + "7": { + "type": "file", + "name": "cortex-xdr-payload.exe" + } + }, + "first_observed": "2024-05-02T04:48:00.463Z", + "last_observed": "2024-05-02T04:48:00.463Z", + "number_observed": 1 + }, + { + "id": "observed-data--150c8d52-e20a-4930-8ec0-e5703ef704e6", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-05-30T16:53:02.956Z", + "modified": "2024-05-30T16:53:02.956Z", + "objects": { + "0": { + "type": "process", + "name": "lsass.exe", + "pid": 828, + "creator_user_ref": "2", + "binary_ref": "5" + }, + "1": { + "type": "x-oca-event", + "process_ref": "0", + "user_ref": "2", + "registry_ref": "3", + "created": "2024-05-24T13:34:08.114Z", + "modified": "2024-05-24T13:34:08.114Z", + "host_ref": "4", + "action": "Registry Event" + }, + "2": { + "type": "user-account", + "user_id": "NT AUTHORITY\\SYSTEM" + }, + "3": { + "type": "windows-registry-key", + "key": "HKEY_LOCAL_MACHINE\\SYSTEM\\SecureTimeHigh", + "values": [ + { + "name": "SecureTimeHigh", + "data_type": "REG_QWORD", + "data": "....o,d(" + } + ] + }, + "4": { + "type": "x-oca-asset", + "device_id": "device-1", + "hostname": "EC2-15", + "x_host_set": "test_host_set1" + }, + "5": { + "type": "file", + "name": "lsass.exe" + } + }, + "first_observed": "2024-05-29T04:15:12.428Z", + "last_observed": "2024-05-29T04:15:12.428Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} \ No newline at end of file diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/README.md b/stix_shifter_modules/trellix_endpoint_security_hx/README.md new file mode 100644 index 000000000..1aaa9d99f --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/README.md @@ -0,0 +1,697 @@ +# Trellix Endpoint Security HX + +## Supported STIX Mappings + +See the [table of mappings](trellix_endpoint_security_hx_supported_stix.md) for the STIX objects and operators supported by this connector. + +**Table of Contents** +- [Trellix Endpoint Security HX API Endpoints](#trellix-endpoint-security-hx-api-endpoints) +- [Curl Command to test the API Endpoints](#curl-command-to-test-api-endpoints) +- [Format of calling Stix shifter from Command Line](#format-for-calling-stix-shifter-from-the-command-line) +- [Pattern expression with STIX attributes and CUSTOM attributes - Single Observation](#single-observation) +- [Pattern expression with STIX and CUSTOM attributes - Multiple Observation](#multiple-observation) +- [STIX Execute Query](#stix-execute-query) +- [Usage of Host Sets in Command Line](#usage-of-host-sets-in-command-line) +- [Recommendations](#recommendations) +- [Limitations](#limitations) +- [References](#references) + +### Trellix Endpoint Security HX API Endpoints + + | Connector Method | Trellix Endpoint Security HX API Endpoint | Method | + |------------------|----------------------------------------------------------|--------| + | Ping Endpoint | Agents System Information API - hx/api/v3/agents/sysinfo | GET | + | Query Endpoint | Enterprise Search API - hx/api/v3/searches | POST | + | Status Endpoint | Enterprise Search API details - hx/api/v3/searches/{id} | GET | + | Results Endpoint | Search Results API - hx/api/v3/searches/{id}/results | GET | + | Delete Endpoint | Delete Search API - hx/api/v3/searches/{id} | DELETE | + + +### CURL command to test API Endpoints +#### Ping +``` +curl --location --request GET 'https://{host}:{port}/hx/api/v3/agents/sysinfo' \ +--cacert {PEM file} \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Basic {base 64 encoded token of user name and password}' +``` +#### Query +``` +curl --location --request POST 'https://{host}:{port}/hx/api/v3/searches' \ +--cacert {PEM file} \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Basic {base 64 encoded token of user name and password}' \ +--data '{ + "host_set": { + "_id": 1002 + }, + "query": [ + { + "field": "Local IP Address", + "operator": "not equals", + "value": "1.1.1.1" + } + ] +}' +``` + +#### Results +``` +curl --location --request GET 'https://{host}:{port}/hx/api/v3/searches/{search id}/results' \ +--cacert {PEM file} \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Basic {base 64 encoded token of user name and password}' +``` + +### Format for calling stix-shifter from the command line +``` +python main.py `` `` `` `` + +``` + +### Pattern expression with STIX and CUSTOM attributes + +#### Single Observation + +#### STIX Translate query using OR operator +```shell +translate trellix_endpoint_security_hx query "{}" "[domain-name:value LIKE 'download' OR ipv4-addr:value ='1.1.1.1'] START t'2024-05-01T00:00:00.000Z' STOP t'2024-05-29T00:00:00.000Z'" +"{\"host_sets\": \"host_set1,host_set2\"}" +``` +#### STIX Translate query - Output +```json +{ + "queries": [ + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "Local IP Address", + "value": "1.1.1.1", + "operator": "equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + } + ] + }, + { + "host_set": { + "_id": "host_set2" + }, + "query": [ + { + "field": "Local IP Address", + "value": "1.1.1.1", + "operator": "equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + } + ] + }, + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "DNS Hostname", + "value": "download", + "operator": "contains" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + } + ] + }, + { + "host_set": { + "_id": "host_set2" + }, + "query": [ + { + "field": "DNS Hostname", + "value": "download", + "operator": "contains" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + } + ] + }, + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "Remote IP Address", + "value": "1.1.1.1", + "operator": "equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + } + ] + }, + { + "host_set": { + "_id": "host_set2" + }, + "query": [ + { + "field": "Remote IP Address", + "value": "1.1.1.1", + "operator": "equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + } + ] + } + ] +} +``` +#### STIX Transmit Query +```shell +transmit +trellix_endpoint_security_hx +"{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" +"{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" +query +"{ \"host_set\": { \"_id\": \"host_set1\" }, \"query\": [ { \"field\": \"DNS Hostname\", \"value\": \"download\", +\"operator\": \"contains\" }, { \"field\": \"Timestamp - Event\", \"operator\": \"between\", \"value\": +[ \"2024-05-01T00:00:00.000Z\", \"2024-05-29T00:00:00.000Z\" ] } ] } +``` +#### STIX Transmit Query - Output + +```json +{ + "success": true, + "search_id": "2493:host_set1" +} +``` +#### STIX Transmit Status +```shell +transmit +trellix_endpoint_security_hx +"{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" +"{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" +status "2493:host_set1" +``` + +#### STIX Transmit Status - Output +```json +{ + "success": true, + "status": "COMPLETED", + "progress": 100 +} +``` +#### STIX Transmit Results +```shell +transmit +trellix_endpoint_security_hx +"{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" +"{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" +results "2493:host_set1" +``` + +#### STIX Transmit Results - Output +```json +{ + "success": true, + "data": [ + { + "Process Name": "svchost.exe", + "Process ID": "7996", + "Username": "NT AUTHORITY\\SYSTEM", + "Remote IP Address": "2.2.2.2", + "IP Address": "2.2.2.2", + "Port": "80", + "Local Port": "49985", + "Remote Port": "80", + "DNS Hostname": "download.windowsupdate.com", + "URL": "/c/msdownload.cab", + "HTTP Header": { + "User-Agent": "Windows-Update-Agent/10.0.10011.16384 Client-Protocol/2.0", + "Host": "download.windowsupdate.com" + }, + "HTTP Method": "GET", + "Timestamp - Event": "2024-05-02T04:57:11.644Z", + "Timestamp - Modified": "2024-05-02T04:57:11.644Z", + "Timestamp - Accessed": "2024-05-02T04:57:11.644Z", + "Host ID": "host1", + "Hostname": "EC2", + "Event Type": "URL Event", + "Host Set": "host_set1", + "Port Protocol": "http", + "File Name": "svchost.exe" + } + ], + "metadata": { + "host_offset": 0, + "host_record_index": 2 + } +} +``` + +#### STIX Translate results +```json +{ + "type": "bundle", + "id": "bundle--294ab994-fbd4-46cb-bcdc-77d50747f617", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "trellix_endpoint_security_hx", + "identity_class": "events", + "created": "2023-05-30T00:00:50.336Z", + "modified": "2024-05-30T00:01:50.336Z" + }, + { + "id": "observed-data--53bc6881-ba03-4cad-8235-2f9695e4695f", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-05-30T01:07:30.991Z", + "modified": "2024-05-30T01:07:30.991Z", + "objects": { + "0": { + "type": "process", + "name": "svchost.exe", + "pid": 7996, + "creator_user_ref": "2", + "binary_ref": "7" + }, + "1": { + "type": "x-oca-event", + "process_ref": "0", + "user_ref": "2", + "ip_refs": [ + "3" + ], + "network_ref": "4", + "domain_ref": "5", + "created": "2024-05-02T04:57:11.644Z", + "modified": "2024-05-02T04:57:11.644Z", + "x_accessed_time": "2024-05-02T04:57:11.644Z", + "host_ref": "6", + "action": "URL Event" + }, + "2": { + "type": "user-account", + "user_id": "NT AUTHORITY\\SYSTEM" + }, + "3": { + "type": "ipv4-addr", + "value": "2.2.2.2" + }, + "4": { + "type": "network-traffic", + "dst_ref": "3", + "src_port": 49985, + "dst_port": 80, + "extensions": { + "http-request-ext": { + "request_value": "/c/msdownload.cab", + "request_header": { + "User-Agent": "Windows-Update-Agent/10.0.10011.16384 Client-Protocol/2.0", + "Host": "download.windowsupdate.com" + }, + "request_method": "GET" + } + }, + "protocols": [ + "http" + ] + }, + "5": { + "type": "domain-name", + "value": "download.windowsupdate.com" + }, + "6": { + "type": "x-oca-asset", + "device_id": "host1", + "hostname": "EC2", + "x_host_set": "host_set1" + }, + "7": { + "type": "file", + "name": "svchost.exe" + } + }, + "first_observed": "2024-05-02T04:57:11.644Z", + "last_observed": "2024-05-02T04:57:11.644Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` +#### Multiple Observation +```shell +translate trellix_endpoint_security_hx query "{}" "[x-oca-event:file_ref.name IN ('conhost.exe','Svc.log') AND user-account:user_id = 'NT AUTHORITY\\SYSTEM'] OR [network-traffic: src_port < 100 AND ipv4-addr:value != '1.1.1.1']START t'2024-05-01T00:00:00.000Z' STOP t'2024-05-29T00:00:00.000Z'" "{\"host_sets\": \"host_set1\"}" +``` +#### STIX Multiple observation - Output +```json +{ + "queries": [ + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "Username", + "value": "NT AUTHORITY\\SYSTEM", + "operator": "equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-31T13:12:49.751Z", + "2024-05-31T13:17:49.751Z" + ] + }, + { + "field": "File Name", + "value": "conhost.exe", + "operator": "equals" + }, + { + "field": "File Name", + "value": "Svc.log", + "operator": "equals" + } + ] + }, + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "Local IP Address", + "value": "1.1.1.1", + "operator": "not equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + }, + { + "field": "Local Port", + "value": 100, + "operator": "less than" + } + ] + }, + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "Remote IP Address", + "value": "1.1.1.1", + "operator": "not equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + }, + { + "field": "Local Port", + "value": 100, + "operator": "less than" + } + ] + } + ] +} +``` +#### STIX Translate query using AND operator +```shell +translate trellix_endpoint_security_hx query "{}" "[network-traffic:src_port > 30 AND domain-name:value LIKE 'dns'] +START t'2024-05-01T00:00:00.000Z' STOP t'2024-05-29T00:00:00.000Z'" +"{\"host_sets\": \"host_set1,host_set2\"}" +``` +#### STIX Translate query - Output +```json +{ + "queries": [ + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "DNS Hostname", + "value": "dns", + "operator": "contains" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + }, + { + "field": "Local Port", + "value": 30, + "operator": "greater than" + } + ] + }, + { + "host_set": { + "_id": "host_set2" + }, + "query": [ + { + "field": "DNS Hostname", + "value": "dns", + "operator": "contains" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-05-01T00:00:00.000Z", + "2024-05-29T00:00:00.000Z" + ] + }, + { + "field": "Local Port", + "value": 30, + "operator": "greater than" + } + ] + } + ] +} +``` + +### STIX Execute query +```shell +execute +trellix_endpoint_security_hx +trellix_endpoint_security_hx +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"trellix_endpoint_security_hx\",\"identity_class\":\"system\",\"created\":\"2024-04-29T00:22:50.336Z\",\"modified\":\"2024-04-29T06:22:50.336Z\"}" +"{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" +"{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" +"[process:name NOT MATCHES 'explorer' AND windows-registry-key:key LIKE 'HKEY_USERS\\Microsoft\\Internet Explorer\\Toolbar\\ShellBrowser\\ITBar7Layout' OR file:size > 10]START t'2024-05-01T00:00:00.000Z' STOP t'2024-05-30T00:00:00.000Z'" +``` + +#### STIX Execute query - Output +```json +{ + "type": "bundle", + "id": "bundle--ebddf348-de7e-4913-80e9-9fd1021abf8e", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "trellix_endpoint_security_hx", + "identity_class": "system", + "created": "2024-04-29T00:22:50.336Z", + "modified": "2024-04-29T06:22:50.336Z" + }, + { + "id": "observed-data--a782a1e9-b853-44f5-916b-c159e4e00bdf", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-05-30T02:36:03.353Z", + "modified": "2024-05-30T02:36:03.353Z", + "objects": { + "0": { + "type": "process", + "name": "explorer.exe", + "pid": 3816, + "creator_user_ref": "2", + "binary_ref": "5" + }, + "1": { + "type": "x-oca-event", + "process_ref": "0", + "user_ref": "2", + "registry_ref": "3", + "created": "2024-05-29T15:54:50.795Z", + "modified": "2024-05-29T15:54:50.795Z", + "x_modified_time": "2024-05-29T15:54:50.795Z", + "host_ref": "4", + "action": "Registry Event" + }, + "2": { + "type": "user-account", + "user_id": "USER1" + }, + "3": { + "type": "windows-registry-key", + "key": "HKEY_USERS\\Microsoft\\Internet Explorer\\Toolbar\\ShellBrowser\\ITBar7Layout", + "values": [ + { + "name": "ITBar7Layout", + "data_type": "REG_BINARY", + "data": "............ ...................^....." + } + ] + }, + "4": { + "type": "x-oca-asset", + "device_id": "device1", + "hostname": "host1", + "x_host_set": "host_set1" + }, + "5": { + "type": "file", + "name": "explorer.exe" + } + }, + "first_observed": "2024-05-29T15:54:50.795Z", + "last_observed": "2024-05-29T15:54:50.795Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +### Usage of host sets in Command Line + +- The host set name should be passed as comma\(,\) separated values in options parameter under connections + while running the queries in command line. + +#### STIX Translate query + +translate trellix_endpoint_security_hx query "{}" "[domain-name:value LIKE 'download'] START t'2024-05-01T00:00:00.000Z' STOP t'2024-05-29T00:00:00.000Z'" +"{\"host_sets\": \"host_set1,host_set2\"}" + +#### STIX Transmit Query + +transmit +trellix_endpoint_security_hx +"{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" +"{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" +query +{input query} + +#### STIX Transmit Status + +transmit +trellix_endpoint_security_hx +"{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" +"{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" +status {search id} + +#### STIX Transmit Results + +transmit +trellix_endpoint_security_hx +"{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" +"{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" +results {search id} + +#### STIX Translate Results + +translate +trellix_endpoint_security_hx +results +"{\"type\": \"identity\", \"id\": \"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\", +\"name\": \"trellix_endpoint_security_hx\", \"identity_class\": \"events\", \"created\": \"2023-05-30T00:00:50.336Z\", +\"modified\": \"2024-05-30T00:01:50.336Z\"}" +"{transmit result response}" "{\"host_sets\": \"host_set1,host_set2\"}" + + +### Recommendations + +- Use multiple host sets in the input to fetch records from more than 1000 hosts. +- Progress threshold input parameter depends on the number of hosts in the host set that responds to the server. + The default value of this parameter in the connector is set to 50. This value can be updated based on the number + of hosts that responds. + +### Limitations + +- A maximum of 25 conditions using AND operator can be specified. Exception will be raised if more than 25 conditions + are specified +- Based on the concurrent search limit of the data source, the concurrent search limit of the connector should be + less than or equal to the data source. +- A maximum of 15 searches only can be created in data source. In order to create more searches, the existing + search ids created needs to be deleted. +- As per datasource limitation, If a host set containing more than 1000 hosts, the 1000 hosts that responds first + will be returned in response. Configure multiple host sets in the datasource to fetch records from more than 1000 hosts. +- Supported operators for IP address fields and File hash fields are =, !=, IN, NOT IN. +- Supported operators for Directory, file:parent_directory_ref.path, process:parent_ref.cwd are LIKE, MATCHES, NOT LIKE, NOT MATCHES. +- Supported operators for network traffic : extensions.'http-request-ext'.request_header fields are LIKE, MATCHES, NOT LIKE, NOT MATCHES. +- LIKE/MATCHES operator supports only substring search. Wild card character search is not supported. + +### References +- [Authentication | FireEye Developer Hub](https://fireeye.dev/docs/endpoint/authentication/) +- [Search limits](https://docs.trellix.com/bundle/hx_5.3.0_ug/page/UUID-bb2cc194-22e0-4501-95d8-6a73458db012.html) +- [API Documentation](https://fireeye.dev/apis/lighthouse/) +- [Enterprise Search](https://docs.trellix.com/bundle/hx_5.3.0_ug/page/UUID-e81232c3-a871-c015-f191-9fbd431bdb59.html) \ No newline at end of file diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/__init__.py b/stix_shifter_modules/trellix_endpoint_security_hx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/configuration/config.json b/stix_shifter_modules/trellix_endpoint_security_hx/configuration/config.json new file mode 100644 index 000000000..908688201 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/configuration/config.json @@ -0,0 +1,48 @@ +{ + "connection": { + "type": { + "displayName": "Trellix Endpoint Security HX", + "group": "Trellix" + }, + "host": { + "type": "text", + "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_:/\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_:/\\-]*[A-Za-z0-9])$" + }, + "port": { + "type": "number", + "min": 1, + "max": 65535 + }, + "help": { + "type": "link", + "default": "data-sources.html" + }, + "selfSignedCert": { + "type": "password", + "optional": true + }, + "options": { + "host_sets": { + "type": "text" + }, + "progress_threshold": { + "default": 50, + "min": 10, + "max": 100, + "type": "number", + "optional": true + } + } + }, + "configuration": { + "auth": { + "type" : "fields", + "username": { + "type": "password" + }, + "password": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/configuration/lang_en.json b/stix_shifter_modules/trellix_endpoint_security_hx/configuration/lang_en.json new file mode 100644 index 000000000..7ade97923 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/configuration/lang_en.json @@ -0,0 +1,42 @@ +{ + "connection": { + "host": { + "label": "Management IP address or hostname", + "description": "Specify the IP address or hostname of the data source" + }, + "port": { + "label": "Host port", + "description": "Set the port number that is associated with the hostname or IP address" + }, + "help": { + "label": "Need additional help?", + "description": "More details on the data source setting can be found in the specified link" + }, + "selfSignedCert": { + "label": "PEM Formatted SSL certificate(s)", + "description": "Provide a self-signed or CA-signed certificate to securely communicate with the data source." + }, + "options": { + "host_sets": { + "label": "Host Sets", + "description": "Provide one or more host set name(group of hosts) separated by comma. Example: Host_set1,Host_set2" + }, + "progress_threshold": { + "label": "Progress Threshold Percentage", + "description": "The Progress Percentage that needs to be verified for search status" + } + } + }, + "configuration": { + "auth": { + "username": { + "label": "Username", + "description": "Username with access to the search API" + }, + "password": { + "label": "Password", + "description": "Password of the user with access to the search API" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/entry_point.py b/stix_shifter_modules/trellix_endpoint_security_hx/entry_point.py new file mode 100644 index 000000000..b9ab8f9c0 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/entry_point.py @@ -0,0 +1,11 @@ +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint + + +class EntryPoint(BaseEntryPoint): + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + self.set_async(True) + if connection: + self.setup_transmission_simple(connection, configuration) + self.setup_translation_simple(dialect_default='default') diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/__init__.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/config_map.json b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/config_map.json new file mode 100644 index 000000000..6c08dd575 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/config_map.json @@ -0,0 +1,13 @@ +{ + "int_supported_fields": [ + "Size in bytes", + "Local Port", + "Remote Port" + ], + "hash_supported_fields": [ + "File MD5 Hash" + ], + "ip_supported_fields": [ + "Local IP Address", "Remote IP Address" + ] +} \ No newline at end of file diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/from_stix_map.json b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/from_stix_map.json new file mode 100644 index 000000000..1a1642668 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/from_stix_map.json @@ -0,0 +1,77 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["Local IP Address", "Remote IP Address"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["Local IP Address", "Remote IP Address"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["Local Port"], + "dst_port": ["Remote Port"], + "src_ref.value": ["Local IP Address"], + "dst_ref.value": ["Remote IP Address"], + "extensions.'http-request-ext'.request_header.'Accept-Encoding'": ["HTTP Header"], + "extensions.'http-request-ext'.request_header.'User-Agent'": ["HTTP Header"], + "extensions.'http-request-ext'.request_header.Host": ["HTTP Header"], + "extensions.'http-request-ext'.request_value": ["URL"] + } + }, + "user-account": { + "fields": { + "user_id": ["Username"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["Registry Key Full Path"], + "values[*].name": ["Registry Key Value Name"], + "values[*].data": ["Registry Key Value Text"] + } + }, + "domain-name": { + "fields": { + "value": ["DNS Hostname"] + } + }, + "file": { + "fields": { + "name": ["File Name"], + "size": ["Size in bytes"], + "hashes.MD5": ["File MD5 Hash"], + "parent_directory_ref.path": ["File Full Path"], + "x_path": ["File Full Path"] + } + }, + "directory": { + "fields": { + "path": ["File Full Path"] + } + }, + "process": { + "fields": { + "name": ["Process Name","Parent Process Name"], + "command_line": ["Process Arguments"], + "creator_user_ref.user_id": ["Username"], + "binary_ref.name": ["File Name"], + "parent_ref.name": ["Parent Process Name"], + "parent_ref.cwd": ["Parent Process Path"] + } + }, + "x-oca-event": { + "fields": { + "file_ref.name": ["File Name"], + "process_ref.name": ["Process Name","Parent Process Name"], + "parent_process_ref.name": ["Parent Process Name"], + "domain_ref.value": ["DNS Hostname"], + "registry_ref.key": ["Registry Key Full Path"], + "network_ref.src_port": ["Local Port"], + "ip_refs[*].value": ["Local IP Address", "Remote IP Address"], + "user_ref.user_id": ["Username"] + } + } +} diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/operators.json b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/operators.json new file mode 100644 index 000000000..82d7d0b7d --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/operators.json @@ -0,0 +1,13 @@ +{ + "ComparisonExpressionOperators.And": "and", + "ComparisonExpressionOperators.Or": "or", + "ComparisonComparators.GreaterThan": "greater than", + "ComparisonComparators.LessThan": "less than", + "ComparisonComparators.Equal": "equals", + "ComparisonComparators.NotEqual": "not equals", + "ComparisonComparators.Like": "contains", + "ComparisonComparators.In": "equals", + "ComparisonComparators.Matches": "contains", + "ObservationOperators.Or": "or", + "ObservationOperators.And": "or" +} \ No newline at end of file diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..f58b72e2f --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,73 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["Local IP Address", "Remote IP Address"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["Local IP Address", "Remote IP Address"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["Local Port"], + "dst_port": ["Remote Port"], + "src_ref.value": ["Local IP Address"], + "dst_ref.value": ["Remote IP Address"], + "extensions.'http-request-ext'.request_header.'Accept-Encoding'": ["HTTP Header"], + "extensions.'http-request-ext'.request_header.'User-Agent'": ["HTTP Header"], + "extensions.'http-request-ext'.request_header.Host": ["HTTP Header"], + "extensions.'http-request-ext'.request_value": ["URL"] + } + }, + "user-account": { + "fields": { + "user_id": ["Username"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["Registry Key Full Path"], + "values[*].name": ["Registry Key Value Name"], + "values[*].data": ["Registry Key Value Text"] + } + }, + "domain-name": { + "fields": { + "value": ["DNS Hostname"] + } + }, + "file": { + "fields": { + "name": ["File Name"], + "size": ["Size in bytes"], + "hashes.MD5": ["File MD5 Hash"], + "parent_directory_ref.path": ["File Full Path"], + "x_path": ["File Full Path"] + } + }, + "directory": { + "fields": { + "path": ["File Full Path"] + } + }, + "process": { + "fields": { + "command_line": ["Process Arguments"], + "creator_user_ref.user_id": ["Username"], + "image_ref.name": ["File Name"], + "parent_ref.cwd": ["Parent Process Path"] + } + }, + "x-oca-event": { + "fields": { + "file_ref.name": ["File Name"], + "domain_ref.value": ["DNS Hostname"], + "registry_ref.key": ["Registry Key Full Path"], + "network_ref.src_port": ["Local Port"], + "ip_refs[*].value": ["Local IP Address", "Remote IP Address"], + "user_ref.user_id": ["Username"] + } + } +} diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..8d16d5247 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,329 @@ +{ + "Event Type": { + "key": "x-oca-event.x_action", + "object": "event" + }, + "Timestamp - Last Run": { + "key": "x-oca-event.x_last_run", + "object": "event" + }, + "Timestamp - Started": { + "key": "x-oca-event.start", + "object": "event" + }, + "Timestamp - Accessed": { + "key": "x-oca-event.x_accessed_time", + "object": "event" + }, + "Timestamp - Modified": { + "key": "x-oca-event.modified", + "object": "event" + }, + "Timestamp - Event": [ + { + "key": "first_observed" + }, + { + "key": "last_observed" + }, + { + "key": "x-oca-event.created", + "object": "event" + } + ], + "Hostname": [ + { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "asset" + } + ], + "Host ID": { + "key": "x-oca-asset.device_id", + "object": "asset" + }, + "Username": [ + { + "key": "user-account.user_id", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "user" + } + ], + "Local IP Address": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ] + }, + { + "key": "x-oca-event.ip_refs", + "object": "event", + "references": [ + "src_ip" + ], + "group": true + } + ], + "Remote IP Address": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "x-oca-event.ip_refs", + "object": "event", + "references": [ + "dst_ip" + ], + "group": true + } + ], + "Remote Port": [ + { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "Local Port": [ + { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "Port Protocol": [ + { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "HTTP Header": { + "key": "network-traffic.extensions.http-request-ext.request_header", + "object": "nt" + }, + "URL": + { + "key": "network-traffic.extensions.http-request-ext.request_value", + "object": "nt" + }, + "HTTP Method": + { + "key": "network-traffic.extensions.http-request-ext.request_method", + "object": "nt" + }, + "File Name": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file" + } + ], + "Host Set": { + "key": "x-oca-asset.x_host_set", + "object": "asset" + }, + "File MD5 Hash": { + "key": "file.hashes.MD5", + "object": "file" + }, + "Size in bytes": { + "key": "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "File Bytes Written": { + "key": "file.x_bytes_written", + "object": "file", + "transformer": "ToInteger" + }, + "File Full Path": [ + { + "key": "file.x_path", + "object": "file" + }, + { + "key": "directory.path", + "object": "dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "dir" + } + ], + "Parent Process Path": [ + { + "key": "process.cwd", + "object": "parent_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "Registry Key Full Path": [ + { + "key": "windows-registry-key.key", + "object": "registry" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "registry" + } + ], + "Registry Key Values": { + "key": "windows-registry-key.values", + "object": "registry" + }, + "DNS Hostname": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain" + } + ], + "Process ID": [ + { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "Process Event Type": { + "key": "process.x_event_type", + "object": "process" + }, + "Process Arguments": { + "key": "process.command_line", + "object": "process" + }, + "Write Event File Name": [ + { + "key": "file.name", + "object": "file_write" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file_write" + } + ], + "Write Event File Full Path": [ + { + "key": "file.x_path", + "object": "file_write" + }, + { + "key": "directory.path", + "object": "dir1", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_write", + "references": "dir1" + } + ], + "Write Event File Text Written": [ + { + "key": "artifact.payload_bin", + "object": "artifact", + "transformer": "ToBase64" + }, + { + "key": "file.content_ref", + "object": "file_write", + "references": "artifact" + } + ], + "Write Event File Bytes Written": { + "key": "file.x_bytes_written", + "object": "file_write", + "transformer": "ToInteger" + }, + "Write Event File MD5 Hash": { + "key": "file.hashes.MD5", + "object": "file_write" + }, + "Write Event Size in bytes": { + "key": "file.size", + "object": "file_write", + "transformer": "ToInteger" + } +} \ No newline at end of file diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/to_stix_map.json b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/to_stix_map.json new file mode 100644 index 000000000..7ecd77d1b --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/to_stix_map.json @@ -0,0 +1,349 @@ +{ + "Event Type": { + "key": "x-oca-event.action", + "object": "event" + }, + "Timestamp - Last Run": { + "key": "x-oca-event.x_last_run", + "object": "event" + }, + "Host Set": { + "key": "x-oca-asset.x_host_set", + "object": "asset" + }, + "Timestamp - Started": { + "key": "x-oca-event.start", + "object": "event" + }, + "Timestamp - Accessed": { + "key": "x-oca-event.x_accessed_time", + "object": "event" + }, + "Timestamp - Modified": { + "key": "x-oca-event.modified", + "object": "event" + }, + "Timestamp - Event": [ + { + "key": "first_observed" + }, + { + "key": "last_observed" + }, + { + "key": "x-oca-event.created", + "object": "event" + } + ], + "Hostname": [ + { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "asset" + } + ], + "Host ID": { + "key": "x-oca-asset.device_id", + "object": "asset" + }, + "Username": [ + { + "key": "user-account.user_id", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "user" + } + ], + "Local IP Address": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ] + }, + { + "key": "x-oca-event.ip_refs", + "object": "event", + "references": [ + "src_ip" + ], + "group": true + } + ], + "Remote IP Address": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "x-oca-event.ip_refs", + "object": "event", + "references": [ + "dst_ip" + ], + "group": true + } + ], + "Remote Port": [ + { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "Local Port": [ + { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "Port Protocol": [ + { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "HTTP Header": { + "key": "network-traffic.extensions.http-request-ext.request_header", + "object": "nt" + }, + "URL": + { + "key": "network-traffic.extensions.http-request-ext.request_value", + "object": "nt" + }, + "HTTP Method": + { + "key": "network-traffic.extensions.http-request-ext.request_method", + "object": "nt" + }, + "File Name": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "file" + } + ], + "File MD5 Hash": { + "key": "file.hashes.MD5", + "object": "file" + }, + "Size in bytes": { + "key": "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "File Bytes Written": { + "key": "file.x_bytes_written", + "object": "file", + "transformer": "ToInteger" + }, + "File Full Path": [ + { + "key": "file.x_path", + "object": "file" + }, + { + "key": "directory.path", + "object": "dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "dir" + } + ], + "Parent Process Path": [ + { + "key": "process.cwd", + "object": "parent_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "Registry Key Full Path": [ + { + "key": "windows-registry-key.key", + "object": "registry" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "registry" + } + ], + "Registry Key Values": { + "key": "windows-registry-key.values", + "object": "registry" + }, + "DNS Hostname": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain" + } + ], + "Process Name": [ + { + "key": "process.name", + "object": "process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "Parent Process Name": [ + { + "key": "process.name", + "object": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "Process ID": { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + "Process Event Type": { + "key": "process.x_event_type", + "object": "process" + }, + "Process Arguments": { + "key": "process.command_line", + "object": "process" + }, + "Write Event File Name": [ + { + "key": "file.name", + "object": "file_write" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file_write" + } + ], + "Write Event File Full Path": [ + { + "key": "file.x_path", + "object": "file_write" + }, + { + "key": "directory.path", + "object": "dir1", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_write", + "references": "dir1" + } + ], + "Write Event File Text Written": [ + { + "key": "artifact.payload_bin", + "object": "artifact", + "transformer": "ToBase64" + }, + { + "key": "file.content_ref", + "object": "file_write", + "references": "artifact" + } + ], + "Write Event File Bytes Written": { + "key": "file.x_bytes_written", + "object": "file_write", + "transformer": "ToInteger" + }, + "Write Event File MD5 Hash": { + "key": "file.hashes.MD5", + "object": "file_write" + }, + "Write Event Size in bytes": { + "key": "file.size", + "object": "file_write", + "transformer": "ToInteger" + } +} \ No newline at end of file diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/query_constructor.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/query_constructor.py new file mode 100644 index 000000000..86abcc79b --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/query_constructor.py @@ -0,0 +1,467 @@ +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, \ + ComparisonExpression, ComparisonComparators, Pattern, \ + CombinedComparisonExpression, CombinedObservationExpression +import logging +import re +import json +from datetime import datetime, timedelta +from os import path +import copy + +logger = logging.getLogger(__name__) + +START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)" +STOP_TIME = datetime.utcnow() +CONFIG_MAP_PATH = "json/config_map.json" + + +class FileNotFoundException(Exception): + pass + + +class StartStopQualifierValueException(Exception): + pass + + +class SimilarExpressionForAndOperatorException(Exception): + pass + + +class QueryStringPatternTranslator: + + def __init__(self, pattern: Pattern, data_model_mapper, options): + logger.info("Trellix Endpoint Security HX Connector") + self.dmm = data_model_mapper + self.options = options + self.qualified_queries = [] + self.config_map = self.load_json(CONFIG_MAP_PATH) + self.comparator_lookup = self.dmm.map_comparator() + self.parse_expression(pattern) + + @staticmethod + def load_json(rel_path_of_file): + """ + Consumes a json file and returns a dictionary + :param rel_path_of_file: str + :return: dict + """ + _json_path = path.dirname(path.abspath(__file__)) + "/" + rel_path_of_file + try: + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundException + except FileNotFoundException as e: + raise FileNotFoundError(f'{rel_path_of_file} not found') from e + + @staticmethod + def _format_value_type(expression, value, mapped_field_type): + """ + Converts input value that matches with the mapped field value type + :param expression + :param value + :param mapped_field_type: str + :return formatted value + """ + stix_object, stix_field = expression.object_path.split(':') + converted_value = str(value) + if mapped_field_type == "int": + if not converted_value.isdigit(): + raise NotImplementedError(f'string type input - {converted_value} is not supported for ' + f'integer type field {stix_object}:{stix_field}') + converted_value = int(value) + return converted_value + + @staticmethod + def _format_in(expression, values, mapped_field_type): + """ + Formatting value in the event of IN operation + :param expression + :param values: str or int + :param mapped_field_type: str + :return: list of formatted values + """ + gen = values.element_iterator() + formatted_values = [] + for value in gen: + formatted_value = QueryStringPatternTranslator._escape_value( + QueryStringPatternTranslator._format_value_type(expression, value, mapped_field_type)) + formatted_values.append(formatted_value) + return formatted_values + + @staticmethod + def _format_equality(expression, value, mapped_field_type, comparator): + """ + Formatting value in the event of equality operation + :param expression + :param value: str or int + :param mapped_field_type: str + :return: list of formatted values + """ + value = QueryStringPatternTranslator._escape_value( + QueryStringPatternTranslator._format_value_type(expression, value, mapped_field_type)) + if comparator in [ComparisonComparators.GreaterThan, ComparisonComparators.LessThan]: + return value + return value + + @staticmethod + def _escape_value(value): + """ + Format the value with escape characters + :param value: str or int + :return: str or int + """ + if isinstance(value, str): + return '{}'.format(value.replace('"', '\"')) + return value + + @staticmethod + def _negate_comparator(comparator): + """ + returns negation of input operator + :param comparator:str + :return str + """ + negate_comparator = { + "equals": "not equals", + "not equals": "equals", + "less than": "greater than", + "contains": "not contains", + "greater than": "less than", + } + return negate_comparator[comparator] + + def _lookup_comparison_operator(self, expression_operator): + """ + lookup operators support in trellix_endpoint_security_hx + :return: str + """ + if str(expression_operator) not in self.comparator_lookup: + raise NotImplementedError( + f'Comparison operator {expression_operator.name} unsupported for trellix_endpoint_security_hx') + + return self.comparator_lookup[str(expression_operator)] + + @staticmethod + def _or_operator_query(previous_all_queries, current_all_queries): + """ + Create individual queries for different fields and merge the values in case of similar fields + :param previous_all_queries:list + :param current_all_queries:list + :return: list + """ + merged_query = [] + similar_query = [] + individual_query = [] + already_merged_query = [] + for previous_queries in previous_all_queries: + for current_queries in current_all_queries: + current_query = copy.deepcopy(current_queries) + previous_query = copy.deepcopy(previous_queries) + previous_key = [i['field'] for i in previous_query['query'] if i['field'] != 'Timestamp - Event'] + current_key = [j['field'] for j in current_query['query'] if j['field'] != 'Timestamp - Event'] + if current_key == previous_key or current_key[0] in previous_key: + # merge queries in case of same attribute combined by OR operator + merged_similar_query = copy.deepcopy(previous_query) + c_query = [j for j in current_query['query'] if j['field'] == current_key[0]] + merged_similar_query['query'].extend(c_query) + if previous_query in individual_query: + individual_query.remove(previous_query) + if current_query in individual_query: + individual_query.remove(current_query) + if previous_query not in already_merged_query: + already_merged_query.append(previous_query) + if current_query not in already_merged_query: + already_merged_query.append(current_query) + if merged_similar_query not in similar_query: + similar_query.append(merged_similar_query) + + else: + # create individual queries in case of different attributes + if previous_query not in individual_query and previous_query not in already_merged_query: + individual_query.append(previous_query) + if current_query not in individual_query and current_query not in already_merged_query: + individual_query.append(current_query) + + merged_query.extend(individual_query) + merged_query.extend(similar_query) + return merged_query + + @staticmethod + def _and_operator_query(previous_all_queries, current_all_queries, expression): + """ + Merge previous query with current query, and log the error in case of similar fields + :param expression + :param previous_all_queries:list + :param current_all_queries:list + :return: list + """ + merged_query = [] + for previous_queries in previous_all_queries: + for current_queries in current_all_queries: + current_query = copy.deepcopy(current_queries) + previous_query = copy.deepcopy(previous_queries) + previous_key = [i['field'] for i in previous_query['query'] if i['field'] != 'Timestamp - Event'] + current_key = [j['field'] for j in current_query['query'] if j['field'] != 'Timestamp - Event'] + if current_key == previous_key or current_key[0] in previous_key: + comparison = str(expression).split(" ") + raise SimilarExpressionForAndOperatorException(f'The expression [{comparison[0][21:]}] has same ' + f'data source field mapping with another expression ' + f'in the pattern which has only AND comparison ' + f'operator. Recommended to Use OR operator. ') + # merge multiple queries into a single query + c_query = [j for j in current_query['query'] if j['field'] != 'Timestamp - Event'] + for new in c_query: + previous_query['query'].append(new) + if previous_query not in merged_query: + merged_query.append(previous_query) + return merged_query + + def _create_single_comparison_query(self, formatted_value, mapped_fields_array, expression, + qualifier): + """ + Create a query for a comparison expression + :param formatted_value, str or int or boolean + :param mapped_fields_array, list + :param expression + :param qualifier, str + :return: list + """ + comparator = self._lookup_comparison_operator(expression.comparator) + if expression.negated: + comparator = QueryStringPatternTranslator._negate_comparator(comparator) + time_range_list = QueryStringPatternTranslator._parse_time_range(qualifier, self.options["time_range"]) + stix_object, stix_field = expression.object_path.split(':') + queries = [] + for field_name in mapped_fields_array: + if field_name == "HTTP Header" and comparator not in ["contains", "not contains"]: + raise NotImplementedError( + f'{str(expression.comparator).split(".")[1]} operator is not supported for ' + f'{stix_object}:{stix_field}.' + f'Possible supported operators are LIKE, NOT LIKE, MATCHES, NOT MATCHES') + if expression.comparator == ComparisonComparators.In: + format_query = [] + for val in formatted_value: + query = {"field": field_name, "value": val, "operator": comparator} + format_query.append(query) + time_format = { + "field": "Timestamp - Event", + "operator": "between", + "value": time_range_list + } + format_query.append(time_format) + queries.append({"query": format_query}) + else: + query = {"field": field_name, "value": formatted_value, "operator": comparator} + time_format = {"field": "Timestamp - Event", + "operator": "between", + "value": time_range_list + } + format_query = {"query": [query, time_format]} + + queries.append(format_query) + return queries + + def _eval_comparison_value(self, expression, mapped_field_type): + """ + Function for parsing comparison expression value + :param expression, expression object + :param mapped_field_type, str + :return: formatted value + """ + self._check_value_comparator_support(expression, expression.comparator, mapped_field_type) + if expression.comparator == ComparisonComparators.In: + value = QueryStringPatternTranslator._format_in(expression, expression.value, mapped_field_type) + + elif expression.comparator in [ComparisonComparators.GreaterThan, ComparisonComparators.Like, + ComparisonComparators.LessThan, ComparisonComparators.Matches, + ComparisonComparators.Equal, ComparisonComparators.NotEqual]: + value = QueryStringPatternTranslator._format_equality(expression, expression.value, mapped_field_type, + expression.comparator) + + else: + raise NotImplementedError('Unknown comparator expression operator') + return value + + @staticmethod + def _parse_time_range(qualifier, time_range): + """ + Converts qualifier to timestamp format + :param qualifier: str + :param time_range: int + return: list of formatted timestamps + """ + try: + compile_timestamp_regex = re.compile(START_STOP_PATTERN) + if qualifier and compile_timestamp_regex.search(qualifier): + time_range_iterator = compile_timestamp_regex.finditer(qualifier) + time_range_list = [each.group() for each in time_range_iterator] + else: + start_time = STOP_TIME - timedelta(minutes=time_range) + converted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + # limit 3 digit value for millisecond + converted_stop_time = STOP_TIME.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + time_range_list = [converted_start_time, converted_stop_time] + utc_timestamp = STOP_TIME.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + for timestamp in time_range_list: + if timestamp > utc_timestamp: + raise StartStopQualifierValueException('Start/Stop time should not be in the future UTC timestamp') + if time_range_list[0] >= time_range_list[1]: + raise StartStopQualifierValueException('Start time should be lesser than Stop time') + return time_range_list + except (KeyError, IndexError, TypeError) as e: + raise e + + def _check_value_comparator_support(self, expression, comparator, mapped_field_type): + """ + checks the comparator and value support + :param comparator + :param mapped_field_type: str + :return: None + """ + stix_object, stix_field = expression.object_path.split(':') + comparator_str = str(comparator).split(".")[1] + if (((stix_object == "file" and stix_field == "parent_directory_ref.path") or + (stix_object == "directory" and stix_field == "path") or + (stix_object == "process" and stix_field == "parent_ref.cwd")) and comparator not in + [ComparisonComparators.Like, ComparisonComparators.Matches]): + raise NotImplementedError( + f'{comparator_str} operator is not supported for {stix_object}:' + f'{stix_field}.Possible supported operators are LIKE, MATCHES, NOT LIKE, NOT MATCHES') + elif mapped_field_type == "string" and comparator in [ComparisonComparators.GreaterThan, + ComparisonComparators.LessThan]: + raise NotImplementedError(f'{comparator_str} operator is not supported for string type field {stix_object}:' + f'{stix_field}. Possible supported operators are =, !=, IN, NOT IN, LIKE, ' + f'NOT LIKE,MATCHES,NOT MATCHES') + elif mapped_field_type == "int" and comparator in [ComparisonComparators.Like, ComparisonComparators.Matches]: + raise NotImplementedError(f'{comparator_str} operator is not supported for integer ' + f'type field {stix_object}:' + f'{stix_field}.Possible supported operators are =, !=, IN, NOT IN, <, >') + elif mapped_field_type in ["hash", "ip"] and comparator in [ComparisonComparators.GreaterThan, + ComparisonComparators.LessThan, + ComparisonComparators.Like, + ComparisonComparators.Matches]: + m_type = "IP Address" if mapped_field_type == "ip" else "File Hash" + raise NotImplementedError( + f'{comparator_str} operator is not supported for {m_type} type field {stix_object}:' + f'{stix_field}.Possible supported operators are =, !=, IN, NOT IN') + + def _get_mapped_field_type(self, mapped_field_array): + """ + Returns the type of mapped field array + :param mapped_field_array: list + :return: str + """ + mapped_field = mapped_field_array[0] + mapped_field_type = "string" + for key, value in self.config_map.items(): + if mapped_field in value and key in ["int_supported_fields"]: + mapped_field_type = key.split('_')[0] + break + elif mapped_field in value and key in ["hash_supported_fields", "ip_supported_fields"]: + mapped_field_type = key.split('_')[0] + break + return mapped_field_type + + def _parse_mapped_fields(self, value, mapped_fields_array, expression, qualifier, or_operator): + """ + Creates queries based on combined comparison expression. + """ + current_query = self._create_single_comparison_query(value, mapped_fields_array, expression, qualifier) + if not self.qualified_queries[-1]: + self.qualified_queries[-1] = current_query + else: + previous_query = self.qualified_queries.pop() + if or_operator: + merged_query = QueryStringPatternTranslator._or_operator_query(previous_query, current_query) + else: + merged_query = QueryStringPatternTranslator._and_operator_query(previous_query, current_query, + expression) + self.qualified_queries.append(merged_query) + + @staticmethod + def verify_common_stix_attributes(comparison_expression): + """ + Raise Exception if similar six attributes are used in a pattern which has only AND operator + :param comparison_expression + """ + comparison_expression_str = str(comparison_expression) + comparison_pattern_1 = re.finditer(pattern=r'\(ComparisonExpression\(', string=comparison_expression_str) + comparison_pattern_2 = re.finditer(pattern=r' ComparisonExpression\(', string=comparison_expression_str) + indices = [index.start() for index in comparison_pattern_1] + [index.start() for index in comparison_pattern_2] + indices.sort() + for i in indices: + end_index = comparison_expression_str.find(')', i) + exp = comparison_expression_str[i:end_index + 1] + comparison = exp.split(" ") + if comparison[0] != "" and comparison_expression_str.find(comparison[0][1:], end_index) != -1: + raise SimilarExpressionForAndOperatorException( + f'Multiple [{comparison[0][22:]}] expression is used in the pattern which has only AND comparison ' + f'operator. Recommended to Use OR operator for similar STIX attributes.') + + def _parse_expression(self, expression, qualifier=None, or_operator=None): + """ + parse ANTLR pattern to TRELLIX ENDPOINT SECURITY HX query format + :param expression: expression object, ANTLR parsed expression object + :param qualifier: str, default is None + :param or_operator: boolean + """ + if isinstance(expression, ComparisonExpression): # Base Case + stix_object, stix_field = expression.object_path.split(':') + mapped_fields_array = self.dmm.map_field(stix_object, stix_field) + mapped_field_type = self._get_mapped_field_type(mapped_fields_array) + value = self._eval_comparison_value(expression, mapped_field_type) + + self._parse_mapped_fields(value, mapped_fields_array, expression, qualifier, + or_operator) + + elif isinstance(expression, CombinedComparisonExpression): + if self.or_operator_enabled: + self._parse_expression(expression.expr1, qualifier, True) + self._parse_expression(expression.expr2, qualifier, True) + else: + self._parse_expression(expression.expr1, qualifier) + self._parse_expression(expression.expr2, qualifier) + + elif isinstance(expression, ObservationExpression): + self.or_operator_enabled = False + self.qualified_queries.append([]) + if 'ComparisonExpressionOperators.Or' in str(expression.comparison_expression): + self.or_operator_enabled = True + else: + QueryStringPatternTranslator.verify_common_stix_attributes(expression.comparison_expression) + self._parse_expression(expression.comparison_expression, qualifier) + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + if isinstance(expression.observation_expression, CombinedObservationExpression): + self._parse_expression(expression.observation_expression.expr1, expression.qualifier) + self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + else: + self._parse_expression(expression.observation_expression, expression.qualifier) + + elif isinstance(expression, CombinedObservationExpression): + self._parse_expression(expression.expr1, qualifier) + self._parse_expression(expression.expr2, qualifier) + + elif isinstance(expression, Pattern): + self._parse_expression(expression.expression) + else: + raise RuntimeError(f"Unknown Recursion Case for expression={expression}, " + f"type(expression)={type(expression)}") + + def parse_expression(self, pattern: Pattern): + self._parse_expression(pattern) + + +def translate_pattern(pattern: Pattern, data_model_mapping, options): + """ + Conversion of ANTLR pattern to TRELLIX ENDPOINT SECURITY HX query + :param pattern: expression object, ANTLR parsed expression object + :param data_model_mapping: DataMapper object, mapping object obtained by parsing json + :param options: dict + :return: list + """ + query = QueryStringPatternTranslator(pattern, data_model_mapping, options).qualified_queries + final_queries = [item for sublist in query for item in sublist] + updated_queries = [{"host_set": {"_id": host}, "query": item['query']} for item in final_queries + for host in options['host_sets'].split(",") if host] + return updated_queries diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/query_translator.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/query_translator.py new file mode 100644 index 000000000..554327d64 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/query_translator.py @@ -0,0 +1,26 @@ +import logging + +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator +from . import query_constructor + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file + :param data: + :param antlr_parsing_object: Antlr parsing objects for the STIX pattern + :type antlr_parsing_object: object + query into another format. This should default to something if one isn't passed in + :return: transformed query string + :rtype: str + """ + + logger.info("Converting STIX2 Pattern to data source query") + + query_string = query_constructor.translate_pattern( + antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/__init__.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/api_client.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/api_client.py new file mode 100644 index 000000000..6e085cab9 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/api_client.py @@ -0,0 +1,117 @@ +from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync +import base64 +import json + +SERVER = 'hx/api/v3' +TOKEN_ENDPOINT = '/token' +PING_ENDPOINT = '/agents/sysinfo' +HOST_SET = '/host_sets' +QUERY_ENDPOINT = '/searches' +STATUS_ENDPOINT = '/searches/{search_id}' +RESULTS_ENDPOINT = '/searches/{search_id}/results' +DELETE_ENDPOINT = '/searches/{search_id}' +STOP_SEARCH_ENDPOINT = '/searches/{search_id}/actions' + + +class APIClient: + + def __init__(self, connection, configuration): + + auth = configuration.get('auth') + token_decoded = auth['username'] + ':' + auth['password'] + token = base64.b64encode(token_decoded.encode('ascii')) + self.encoded_token = f'Basic {token.decode("ascii")}' + self.headers = {'Content-Type': 'application/json'} + self.client = RestApiClientAsync(connection.get('host'), + connection.get('port'), + self.headers, + cert_verify=connection.get('selfSignedCert', None) + ) + self.timeout = connection['options'].get('timeout') + self.result_limit = connection['options'].get('result_limit') + self.progress_threshold = connection['options'].get('progress_threshold') + + async def ping_data_source(self): + """ + Pings the data source + :return: response object + """ + return await self.client.call_api(SERVER + PING_ENDPOINT, 'GET', headers=self.headers, + timeout=self.timeout) + + async def create_search(self, query): + """ + Create an Enterprise search for the input query + :param query: dict + :return: response object + """ + + return await self.client.call_api(SERVER + QUERY_ENDPOINT, 'POST', headers=self.headers, + timeout=self.timeout, + data=json.dumps(query)) + + async def get_search_status(self, search_id): + """ + Fetch the status of the search + :param search_id: str + :return: response object + """ + new_status_endpoint = STATUS_ENDPOINT.replace('{search_id}', str(search_id)) + return await self.client.call_api(SERVER + new_status_endpoint, 'GET', headers=self.headers, + timeout=self.timeout) + + async def get_search_results(self, search_id, offset, limit): + """ + Fetch the results of the search + :param search_id: str + :param offset: int + :param limit: int + :return: response object + """ + params = {"offset": offset, "limit": limit} + new_results_endpoint = RESULTS_ENDPOINT.replace('{search_id}', str(search_id)) + return await self.client.call_api(SERVER + new_results_endpoint, 'GET', headers=self.headers, + timeout=self.timeout, urldata=params) + + async def delete_search(self, search_id): + """ + Delete the corresponding search + :param search_id: str + :return: response object + """ + new_del_endpoint = DELETE_ENDPOINT.replace('{search_id}', str(search_id)) + + return await self.client.call_api(SERVER + new_del_endpoint, 'DELETE', headers=self.headers, + timeout=self.timeout) + + async def stop_search(self, search_id): + """ Stops collecting results from the hosts + :param search_id: str + :return: response object + """ + new_stop_search_endpoint = STOP_SEARCH_ENDPOINT.replace('{search_id}', str(search_id)) + return await self.client.call_api(SERVER + new_stop_search_endpoint + '/stop', 'POST', headers=self.headers, + timeout=self.timeout) + + async def generate_token(self): + """ Generate new token""" + self.headers['Authorization'] = self.encoded_token + return await self.client.call_api(SERVER + TOKEN_ENDPOINT, 'GET', headers=self.headers, timeout=self.timeout) + + async def delete_token(self): + """ Delete the generated API token to close the session""" + if self.headers.get('X-FeApi-Token'): + response = await self.client.call_api(SERVER + TOKEN_ENDPOINT, 'DELETE', headers=self.headers, + timeout=self.timeout) + if response.code == 204: + self.headers.pop('X-FeApi-Token') + else: + raise Exception + + async def get_host_set(self, host_set): + """ + Fetching host set details + """ + params = {"name": host_set} + return await self.client.call_api(SERVER + HOST_SET, 'GET', headers=self.headers, timeout=self.timeout, + urldata=params) diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/delete_connector.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/delete_connector.py new file mode 100644 index 000000000..248a9c2ac --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/delete_connector.py @@ -0,0 +1,71 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + + +class DeleteConnector(BaseDeleteConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def delete_query_connection(self, search_id): + """ + Perform delete operation for the specific search id + :param search_id: str + :return: return_obj, dict + """ + return_obj = {} + try: + if not self.api_client.headers.get('X-FeApi-Token'): + token_obj = await self.__get_token() + if token_obj: + return token_obj + response = await self.api_client.delete_search(search_id.split(":")[0]) + response_code = response.code + if response_code == 204: + return_obj['success'] = True + else: + response_content = response.read().decode('utf-8') + return_obj = self.handle_api_exception(response_code, response_content) + + except Exception as err: + self.logger.error(f'Error when deleting search id in Trellix Endpoint Security HX:{err}') + return_obj = self.handle_api_exception(None, str(err)) + return return_obj + + def handle_api_exception(self, code, response_data): + """ + create the exception response + :param code, int + :param response_data, dict + :return: return_obj, dict + """ + return_obj = {} + try: + response_data = json.loads(response_data) + if response_data.get('details', []): + message = response_data['details'][0]['message'] + else: + message = response_data.get('message') + except json.JSONDecodeError: + message = response_data + response_dict = {'code': code, 'message': message} if code else {'message': message} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + async def __get_token(self): + """ + Generate a new API token + :return: + """ + return_obj = {} + response = await self.api_client.generate_token() + if response.code == 204 and response.headers.get('X-FeApi-Token'): + self.api_client.headers['X-FeApi-Token'] = response.headers['X-FeApi-Token'] + if self.api_client.headers.get('Authorization'): + self.api_client.headers.pop('Authorization') + else: + return_obj = self.handle_api_exception(response.code, response.read().decode('utf-8')) + return return_obj diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/error_mapper.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/error_mapper.py new file mode 100644 index 000000000..57b1cee8b --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/error_mapper.py @@ -0,0 +1,40 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + 100: ErrorCode.TRANSMISSION_INVALID_PARAMETER, + 401: ErrorCode.TRANSMISSION_AUTH_CREDENTIALS, + 404: ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS, + 408: ErrorCode.TRANSMISSION_CONNECT, + 409: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + 422: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + 500: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + 406: ErrorCode.TRANSMISSION_CONNECT, + 503: ErrorCode.TRANSMISSION_CONNECT + +} + + +class ErrorMapper: + + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + logger = logger.set_logger(__name__) + + @staticmethod + def set_error_code(json_data, return_obj, connector=None): + code = None + try: + code = int(json_data['code']) + except Exception: + pass + + error_code = ErrorMapper.DEFAULT_ERROR + + if code in error_mapping: + error_code = error_mapping.get(code) + + if error_code == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: " + str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_code, connector=connector) diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/ping_connector.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/ping_connector.py new file mode 100644 index 000000000..8088edff1 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/ping_connector.py @@ -0,0 +1,72 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + + +class PingConnector(BasePingConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def ping_connection(self): + """ + Pings the data source in the event of Ping operation + :return: return_obj, dict + """ + return_obj = {} + try: + + token_obj = await self.__get_token() + if token_obj: + return token_obj + response_wrapper = await self.api_client.ping_data_source() + response_code = response_wrapper.code + response_content = response_wrapper.read().decode('utf-8') + if response_code == 200: + return_obj['success'] = True + else: + return_obj = self.handle_api_exception(response_code, response_content) + except Exception as err: + self.logger.error('Error while pinging in Trellix endpoint Security HX: %s', err) + return_obj = self.handle_api_exception(None, str(err)) + + await self.api_client.delete_token() + return return_obj + + def handle_api_exception(self, code, response_data): + """ + create the exception response + :param code, int + :param response_data, dict + :return: return_obj, dict + """ + return_obj = {} + try: + response_data = json.loads(response_data) + if response_data.get('details', []): + message = response_data['details'][0]['message'] + else: + message = response_data.get('message') + except json.JSONDecodeError: + message = response_data + response_dict = {'code': code, 'message': message} if code else {'message': message} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + + return return_obj + + async def __get_token(self): + """ + Generate a new API token + :return: + """ + return_obj = {} + response = await self.api_client.generate_token() + if response.code == 204 and response.headers.get('X-FeApi-Token'): + self.api_client.headers['X-FeApi-Token'] = response.headers['X-FeApi-Token'] + if self.api_client.headers.get('Authorization'): + self.api_client.headers.pop('Authorization') + else: + return_obj = self.handle_api_exception(response.code, response.read().decode('utf-8')) + return return_obj diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/query_connector.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/query_connector.py new file mode 100644 index 000000000..2dbe0cea3 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/query_connector.py @@ -0,0 +1,128 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_query_connector import BaseQueryConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + + +class InvalidHostSetNameException(Exception): + pass + + +class QueryConnector(BaseQueryConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def create_query_connection(self, query): + """ + Fetches the host set details and creates enterprise search on the corresponding host set. + :param query: + :return: return_obj, dict + """ + return_obj = {} + try: + if not self.api_client.headers.get('X-FeApi-Token'): + token_obj = await self.__get_token() + if token_obj: + return token_obj + + if isinstance(query, str): + query = json.loads(query) + host_set_name = query['host_set']['_id'] + host_object = await self.fetch_host_details(host_set_name) + if not host_object.get('host_set_id'): + return host_object + query['host_set']['_id'] = host_object['host_set_id'] + + response = await self.api_client.create_search(query) + response_content = response.read().decode('utf-8') + response_code = response.code + if response_code == 201: + response_content = json.loads(response_content) + return_obj['success'] = True + return_obj['search_id'] = f"{str(response_content['data']['_id'])}:{host_set_name}" + else: + return_obj = self.handle_api_exception(response_code, response_content) + + except Exception as err: + self.logger.error(f'Error when creating search in Trellix Endpoint Security HX:{err}') + return_obj = self.handle_api_exception(None, str(err)) + + await self.api_client.delete_token() + return return_obj + + async def __get_token(self): + """ + Generate a new API token + :return: + """ + return_obj = {} + response = await self.api_client.generate_token() + if response.code == 204 and response.headers.get('X-FeApi-Token'): + self.api_client.headers['X-FeApi-Token'] = response.headers['X-FeApi-Token'] + if self.api_client.headers.get('Authorization'): + self.api_client.headers.pop('Authorization') + else: + return_obj = self.handle_api_exception(response.code, response.read().decode('utf-8')) + return return_obj + + async def fetch_host_details(self, host_set_name): + """ + Fetches the host set id for the input host set name + :return: return_obj, dict + """ + return_obj = {} + try: + host_response = await self.api_client.get_host_set(host_set_name) + host_response_content = host_response.read().decode('utf-8') + host_response_code = host_response.code + if host_response_code == 200: + return_obj = QueryConnector.fetch_success_response(return_obj, host_response_content) + else: + return_obj = self.handle_api_exception(host_response_code, host_response_content) + + except InvalidHostSetNameException: + return_obj = self.handle_api_exception(100, "Invalid Host Set Name") + + except Exception as e: + self.logger.error('Error while fetching host set details in Trellix Endpoint Security : %s', e) + return_obj = self.handle_api_exception(None, str(e)) + return return_obj + + @staticmethod + def fetch_success_response(return_obj, host_response_content): + """ + Creates the return object with success status and host set id + :param return_obj: + :param host_response_content: + :return: dict + """ + host_response_content = json.loads(host_response_content) + return_obj['success'] = True + entries = host_response_content.get('data', {}).get('entries', []) + if entries: + return_obj['host_set_id'] = entries[0]['_id'] + else: + raise InvalidHostSetNameException() + return return_obj + + def handle_api_exception(self, code, response_data): + """ + create the exception response + :param code, int + :param response_data, dict + :return: return_obj, dict + """ + return_obj = {} + try: + response_data = json.loads(response_data) + if response_data.get('details', []): + message = response_data['details'][0]['message'] + else: + message = response_data.get('message') + except json.JSONDecodeError: + message = response_data + response_dict = {'code': code, 'message': message} if code else {'message': message} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/results_connector.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/results_connector.py new file mode 100644 index 000000000..a73657fca --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/results_connector.py @@ -0,0 +1,374 @@ +import copy +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + +TRELLIX_PER_HOST_LIMIT = 20 + + +class InvalidMetadataException(Exception): + pass + + +class ResultsConnector(BaseJsonResultsConnector): + last_host_record_count = 0 + + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def create_results_connection(self, search_id, offset, length, metadata=None): + """ + Fetching the results of the search id using offset and length + :param search_id: str + :param offset: str + :param length: str + :param metadata: dict + :return: return_obj, dict + """ + + return_obj = {} + try: + data = [] + offset = int(offset) + length = int(length) + host_offset = host_length = -1 + + token_obj = await self.__get_token() + if token_obj: + return token_obj + input_search_id = search_id.split(":")[0] + host_set = search_id.split(":")[1] + + current_host_offset, host_index, total_records = await self.fetch_host_details(offset, length, metadata) + + while len(data) < total_records: + host_offset, host_length = ResultsConnector.calculate_host_offset_length(current_host_offset, + total_records, data, + host_offset, host_length, + metadata) + response_code, response_content = await self.make_api_call(input_search_id, host_offset, host_length) + if response_code == 200: + records = ResultsConnector.fetch_records(response_content, host_index, host_set) + if not records: + host_offset = None + break + data += records + host_index = 0 + else: + return_obj = self.handle_api_exception(response_code, response_content) + if return_obj.get('status'): + token_obj = await self.__get_token() + if token_obj: + return token_obj + continue + break + if data: + return_obj = self.prepare_data_and_metadata(data, total_records, offset, host_offset, host_length, + metadata) + else: + if not return_obj.get('success') and not return_obj.get('error'): + return_obj['success'] = True + return_obj['data'] = [] + # stop the searches on the host set. + if return_obj.get('data') and ((len(return_obj['data']) < total_records and metadata) or + (not metadata and len(return_obj['data']) < (total_records - offset))): + await self.api_client.stop_search(input_search_id) + # delete the token to end the session + await self.api_client.delete_token() + + except InvalidMetadataException: + return_obj = self.handle_api_exception(100, "Invalid Metadata") + + except Exception as err: + response_dict = {'message': str(err)} + if "timeout_error" in str(err): + response_dict['code'] = 408 + self.logger.error('error while getting results in Trellix Endpoint Security HX: %s', err) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + def prepare_data_and_metadata(self, data, total_records, offset, host_offset, host_length, metadata): + """ + Apply Offset and length in data, and create metadata + :param data: list + :param total_records: int + :param offset: int + :param host_offset: int + :param host_length: int + :param metadata: dict + :return: dict + """ + return_obj = {'success': True} + if len(data) > total_records: + additional_records = len(data) - total_records + data = data[:-additional_records] # Remove the additional records from the data + # calculate the next index of the last host set to be used in next iteration + index = (ResultsConnector.last_host_record_count - additional_records) + 1 + else: + index = 0 + + if metadata: + return_obj['data'] = data + else: + return_obj['data'] = data[offset:total_records] + if offset + len(return_obj['data']) < self.api_client.result_limit and host_offset is not None: + new_offset = (host_offset + host_length) if index == 0 else (host_offset + host_length) - 1 + return_obj['metadata'] = {'host_offset': new_offset, + 'host_record_index': index + } + + return return_obj + + async def fetch_host_details(self, offset, length, metadata): + """ + Calculate host index, host offset, total records + :param offset: int + :param length: int + :param metadata: dict + :return: int + """ + if metadata: + if (isinstance(metadata, dict) and 'host_record_index' in metadata.keys() + and 'host_offset' in metadata.keys()): + current_host_offset, host_index = metadata['host_offset'], metadata['host_record_index'] + total_records = length + records_fetched = offset + if abs(self.api_client.result_limit - records_fetched) < total_records: + total_records = abs(self.api_client.result_limit - records_fetched) + else: + raise InvalidMetadataException(f'Invalid Metadata{metadata}') + else: + current_host_offset = -1 + host_index = 0 + total_records = offset + length + if self.api_client.result_limit < total_records: + total_records = self.api_client.result_limit + return current_host_offset, host_index, total_records + + @staticmethod + def calculate_host_offset_length(current_host_offset, total_records, data, host_offset, host_length, metadata): + """ + Calculate the host offset and length for the API + :param current_host_offset: int + :param total_records: int + :param data: list + :param host_offset: int + :param host_length: int + :param metadata: dict + :return: int + """ + if not data: + records_to_be_fetched = total_records + if metadata: + host_offset = current_host_offset + else: + host_offset = 0 + else: + records_to_be_fetched = total_records - len(data) + host_offset = host_offset + host_length + + if records_to_be_fetched % TRELLIX_PER_HOST_LIMIT == 0: + host_length = records_to_be_fetched // TRELLIX_PER_HOST_LIMIT + elif records_to_be_fetched % TRELLIX_PER_HOST_LIMIT > 0: + host_length = (records_to_be_fetched // TRELLIX_PER_HOST_LIMIT) + 1 + + return host_offset, host_length + + async def __get_token(self): + """ + Generate a new Fireeye API token + :return: dict + """ + return_obj = {} + response = await self.api_client.generate_token() + if response.code == 204 and response.headers.get('X-FeApi-Token'): + self.api_client.headers['X-FeApi-Token'] = response.headers['X-FeApi-Token'] + if self.api_client.headers.get('Authorization'): + self.api_client.headers.pop('Authorization') + else: + return_obj = self.handle_api_exception(response.code, response.read().decode('utf-8')) + return return_obj + + async def make_api_call(self, search_id, host_offset, host_length): + """ + Perform the API call and read the response + :param search_id: str + :param host_offset: int + :param host_length: int + :return: int, str + """ + response = await self.api_client.get_search_results(search_id, host_offset, host_length) + response_code = response.code + response_content = response.read().decode('utf-8') + return response_code, response_content + + @staticmethod + def fetch_records(response_content, host_index, host_set): + """ + Format the results and slice the records based on the host offset + :param response_content: str + :param host_index: int + :param host_set: str + :return: list + """ + data = json.loads(response_content) + if host_index: + formatted_data = ResultsConnector.format_results(data, host_set)[(host_index - 1):] + else: + formatted_data = ResultsConnector.format_results(data, host_set) + + return formatted_data + + @staticmethod + def format_results(response_data, host_set): + """ + Formats the response + :param response_data: dict + :param host_set : str + :return: list + """ + data = [] + if response_data.get('data') and response_data['data'].get('total', 0) > 0: + if response_data['data'].get('entries'): + for record in response_data['data']['entries']: + if record.get('host') and record.get('results'): + host_id = record['host']['_id'] + host_name = record['host']['hostname'] + for event in record['results']: + temp_event = copy.deepcopy(event['data']) + temp_event['Host ID'] = host_id + temp_event['Hostname'] = host_name + temp_event['Event Type'] = event['type'] + temp_event['Host Set'] = host_set + temp_event = ResultsConnector.format_events(temp_event) + if not temp_event.get('Timestamp - Modified'): + temp_event['Timestamp - Modified'] = temp_event['Timestamp - Event'] + data.append(temp_event) + if data: + # Fetch the record count of the last host + total_host = len(response_data['data']['entries']) + ResultsConnector.last_host_record_count = len(response_data['data']['entries'][total_host - 1] + ['results']) + return data + + @staticmethod + def format_events(temp_event): + """ + Format the events + :param temp_event: + :return: dict + """ + if temp_event.get("Registry Key Value Name"): + temp_event = ResultsConnector.format_registry(temp_event) + if 'network' in temp_event['Event Type'].lower(): + temp_event['Port Protocol'] = 'ipv4' if 'IPv4' in temp_event['Event Type'] else 'ipv6' + if temp_event.get('HTTP Header'): + temp_event = ResultsConnector.format_http_header(temp_event) + temp_event['Port Protocol'] = 'http' + if temp_event.get('File Text Written'): + temp_event = ResultsConnector.format_file(temp_event) + if temp_event.get('Process Name') and not temp_event.get('File Name'): + temp_event = ResultsConnector.format_process(temp_event) + + return temp_event + + @staticmethod + def format_file(temp_event): + """ + Add file name for binary references if file not found + :param temp_event: + :return: dict + """ + if temp_event.get('File Name'): + temp_event['Write Event File Name'] = temp_event['File Name'] + temp_event.pop('File Name') + if temp_event.get('File Full Path'): + temp_event['Write Event File Full Path'] = temp_event['File Full Path'] + temp_event.pop('File Full Path') + if temp_event.get('File Text Written'): + temp_event['Write Event File Text Written'] = temp_event['File Text Written'] + temp_event.pop('File Text Written') + if temp_event.get('File Bytes Written'): + temp_event['Write Event File Bytes Written'] = temp_event['File Bytes Written'] + temp_event.pop('File Bytes Written') + if temp_event.get('File MD5 Hash'): + temp_event['Write Event File MD5 Hash'] = temp_event['File MD5 Hash'] + temp_event.pop('File MD5 Hash') + if temp_event.get('Size in bytes'): + temp_event['Write Event Size in bytes'] = temp_event['Size in bytes'] + temp_event.pop('Size in bytes') + return temp_event + + @staticmethod + def format_process(temp_event): + """ + Add file name for binary references if file not found + :param temp_event: + :return: dict + """ + temp_event['File Name'] = copy.deepcopy(temp_event['Process Name']) + if temp_event.get('Parent Process Name'): + temp_event['Parent File Name'] = copy.deepcopy(temp_event['Parent Process Name']) + return temp_event + + @staticmethod + def format_http_header(temp_event): + """ + Modify the value of HTTP header string into a dictionary format + :param temp_event: + :return: + """ + header = temp_event.get('HTTP Header').split("\n") + temp_event['HTTP Header'] = {} + for value in header: + if "Host:" in value: + temp_event['HTTP Header']['Host'] = value.split("Host:")[1].strip() + elif 'User-Agent:' in value: + temp_event['HTTP Header']['User-Agent'] = value.split("User-Agent:")[1].strip() + elif 'Accept-Encoding:' in value: + temp_event['HTTP Header']['Accept-Encoding'] = value.split("Accept-Encoding:")[1].strip() + return temp_event + + @staticmethod + def format_registry(temp_event): + """ + Format the values for registry object + :param temp_event: + :return: + """ + registry_value = {"name": temp_event["Registry Key Value Name"]} + temp_event.pop("Registry Key Value Name") + if temp_event.get('Registry Key Value Type'): + registry_value['data_type'] = temp_event['Registry Key Value Type'] + temp_event.pop("Registry Key Value Type") + if temp_event.get('Registry Key Value Text'): + registry_value['data'] = temp_event['Registry Key Value Text'] + temp_event.pop("Registry Key Value Text") + temp_event['Registry Key Values'] = [registry_value] + return temp_event + + def handle_api_exception(self, code, response_data): + """ + create the exception response + :param code, int + :param response_data, dict + :return: dict + """ + return_obj = {} + try: + response_data = json.loads(response_data) + if response_data.get('details', []): + message = response_data['details'][0]['message'] + else: + message = response_data.get('message') + except json.JSONDecodeError: + message = response_data + if code == 401 and message == "Unauthorized": + return_obj['status'] = 'authorized' + else: + response_dict = {'code': code, 'message': message} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/status_connector.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/status_connector.py new file mode 100644 index 000000000..f3a2fe794 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/status_connector.py @@ -0,0 +1,135 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import BaseStatusConnector +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from enum import Enum +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + + +class UnsupportedHostSetException(Exception): + pass + + +class TrellixStatus(Enum): + RUNNING = 'RUNNING' + COMPLETED = 'COMPLETED' + + +class StatusConnector(BaseStatusConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + @staticmethod + def __get_status(status): + """ + Return the status of the search id + :param status: str, + :return: str + """ + switcher = { + TrellixStatus.COMPLETED.value: Status.COMPLETED, + TrellixStatus.RUNNING.value: Status.RUNNING, + } + return switcher.get(status).value + + async def create_status_connection(self, search_id): + """ + Fetching the progress and the status of the search id + :param search_id: str + :return: return_obj, dict + """ + return_obj = {} + try: + if not self.api_client.headers.get('X-FeApi-Token'): + token_obj = await self.__get_token() + if token_obj: + return token_obj + response_code, response_content = await self.make_api_call(search_id.split(":")[0]) + if response_code == 200: + return_obj = self.fetch_status_and_progress(return_obj, response_content) + else: + return_obj = await self.handle_api_exception(response_code, response_content) + + except UnsupportedHostSetException as ex: + return_obj = await self.handle_api_exception(100, f'The input query is not ' + f'supported for the host set {str(ex)}') + + except Exception as err: + self.logger.error(f'Error when getting search status in Trellix Endpoint Security: {err}') + return_obj = await self.handle_api_exception(None, str(err)) + + await self.api_client.delete_token() + return return_obj + + async def make_api_call(self, search_id): + """ + Make API call to fetch the status of search id + :return: + """ + response = await self.api_client.get_search_status(search_id) + response_code = response.code + response_content = response.read().decode('utf-8') + return response_code, response_content + + async def handle_api_exception(self, code, response_data): + """ + create the exception response + :param code, int + :param response_data, dict + :return: return_obj, dict + """ + return_obj = {} + try: + response_data = json.loads(response_data) + if response_data.get('details', []): + message = response_data['details'][0]['message'] + else: + message = response_data.get('message') + except json.JSONDecodeError: + message = response_data + response_dict = {'code': code, 'message': message} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + async def __get_token(self): + """ + Generate a new API token + :return: + """ + return_obj = {} + response = await self.api_client.generate_token() + if response.code == 204 and response.headers.get('X-FeApi-Token'): + self.api_client.headers['X-FeApi-Token'] = response.headers['X-FeApi-Token'] + if self.api_client.headers.get('Authorization'): + self.api_client.headers.pop('Authorization') + else: + return_obj = await self.handle_api_exception(response.code, response.read().decode('utf-8')) + return return_obj + + def fetch_status_and_progress(self, return_obj, response_content): + """ + Find the status and calculate the progress + :param return_obj: + :param response_content: + :return: return_obj, dict + """ + return_obj['success'] = True + response_content = json.loads(response_content) + return_obj['status'] = StatusConnector.__get_status("RUNNING") + return_obj['progress'] = 0 + + data = response_content.get("data", {}) + if data: + host_count = data.get("stats", {}).get("hosts", 0) + search_completed_count = data.get("stats", {}).get("running_state", {}).get('COMPLETE') + if host_count > 0: + return_obj['progress'] = int(search_completed_count / host_count * 100) + else: + raise UnsupportedHostSetException(data.get('host_set').get('name')) + if return_obj['progress'] >= self.api_client.progress_threshold: + return_obj['progress'] = 100 + return_obj['status'] = StatusConnector.__get_status("COMPLETED") + + return return_obj diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_translation/test_trellix_endpoint_security_hx_json_to_stix.py b/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_translation/test_trellix_endpoint_security_hx_json_to_stix.py new file mode 100644 index 000000000..7380e2cfd --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_translation/test_trellix_endpoint_security_hx_json_to_stix.py @@ -0,0 +1,292 @@ +""" test script to perform unit test case for trellix_endpoint_security_hx translate results """ +import unittest +from stix_shifter_modules.trellix_endpoint_security_hx.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers + +MODULE = "trellix_endpoint_security_hx" +entry_point = EntryPoint() +map_data = entry_point.get_results_translator().map_data +data_source = { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "trellix_endpoint_security_hx", + "identity_class": "events" +} +options = {} + +trellix_sample_response = { + "Process Name": "svchost.exe", + "Process ID": "3096", + "Username": "NT AUTHORITY\\SYSTEM", + "Remote IP Address": "111.111.111.111", + "IP Address": "111.111.111.111", + "Port": "80", + "Local Port": "12345", + "Remote Port": "80", + "DNS Hostname": "download.windowsupdate.com", + "URL": "/c/msdownload/update/others.cab", + "HTTP Header": { + "User-Agent": "Windows-Update-Agent/10.0 Client-Protocol/2.51", + "Host": "download.windowsupdate.com" + }, + "HTTP Method": "GET", + "Timestamp - Event": "2024-03-07T09:09:10.391Z", + "Timestamp - Accessed": "2024-03-07T09:09:10.391Z", + "Timestamp - Modified": "2024-03-07T09:09:10.391Z", + "Host ID": "TCJ", + "Hostname": "EC21", + "Event Type": "URL Event", + "Host Set": "test_host_set1", + "Port Protocol": "http", + "File Name": "svchost.exe" +} + + +class TestTrellixEndpointSecurityHxResultsToStix(unittest.TestCase): + """ + class to perform unit test case for trellix endpoint security HX translate results + """ + + @staticmethod + def get_first(itr, constraint): + """ + return the obj in the itr if constraint is true + """ + return next( + (obj for obj in itr if constraint(obj)), + None + ) + + @staticmethod + def get_first_of_type(itr, typ): + """ + to check whether the object belongs to respective stix object + """ + return TestTrellixEndpointSecurityHxResultsToStix.get_first(itr, lambda o: isinstance(o, dict) and o.get( + 'type') == typ) + + @staticmethod + def get_observed_data_objects(data): + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + return observed_data['objects'] + + def test_ipv4_addr_json_to_stix(self): + """ + to test ipv4-addr stix object properties + """ + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(trellix_sample_response) + ipv4_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), 'ipv4-addr') + assert (ipv4_obj.keys() == {'type', 'value'}) + assert ipv4_obj is not None + assert ipv4_obj['type'] == 'ipv4-addr' + assert ipv4_obj['value'] == '111.111.111.111' + + def test_network_traffic_json_to_stix(self): + """ + to test network_traffic stix object properties + """ + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(trellix_sample_response) + network_traffic_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), + 'network-traffic') + assert (network_traffic_obj.keys() == {'type', 'dst_ref', 'src_port', 'dst_port', 'extensions', 'protocols'}) + assert network_traffic_obj is not None + assert network_traffic_obj['type'] == 'network-traffic' + assert network_traffic_obj['src_port'] == 12345 + assert network_traffic_obj['dst_port'] == 80 + dst_ref = network_traffic_obj['dst_ref'] + assert (dst_ref in objects), f"dst_ref with key {network_traffic_obj['dst_ref']} " \ + f"not found" + assert (network_traffic_obj['extensions']['http-request-ext']['request_value'] == + '/c/msdownload/update/others.cab') + assert (network_traffic_obj['extensions']['http-request-ext']['request_header']['User-Agent'] == + 'Windows-Update-Agent/10.0 Client-Protocol/2.51') + assert network_traffic_obj['extensions']['http-request-ext']['request_method'] == "GET" + assert network_traffic_obj['protocols'] == ["http"] + + def test_process_json_to_stix(self): + """ + to test process stix object properties + """ + process_response = { + "File Name": "chrome.exe", + "File Full Path": "C:\\Google\\Application\\chrome.exe", + "Process Name": "chrome.exe", + "Parent Process Name": "explorer.exe", + "Parent Process Path": "\\Device\\explorer.exe", + "Process Event Type": "running", + "Process ID": "2460", + "Username": "user1", + "Timestamp - Event": "2024-01-25T14:40:20.105Z", + "Timestamp - Modified": "2024-01-25T14:40:20.105Z", + "Timestamp - Accessed": "2024-01-25T14:40:20.105Z", + "Host ID": "hostid11", + "Hostname": "ec21", + "Event Type": "Process Event", + "Host Set": "test_host_set1" + } + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(process_response) + process_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), + 'process') + assert (process_obj.keys() == {'type', 'name', 'pid', 'creator_user_ref', 'parent_ref', 'binary_ref', + 'x_event_type'}) + assert process_obj is not None + assert process_obj['type'] == 'process' + assert process_obj['name'] == 'chrome.exe' + assert process_obj['pid'] == 2460 + creator_user_ref = process_obj['creator_user_ref'] + assert (creator_user_ref in objects), f"creator_user_ref with key {process_obj['creator_user_ref']} " \ + f"not found" + parent_ref = process_obj['parent_ref'] + assert (parent_ref in objects), f"parent_ref with key {process_obj['parent_ref']} not found" + parent_obj = objects[parent_ref] + assert parent_obj['type'] == 'process' + assert parent_obj['name'] == 'explorer.exe' + assert parent_obj['cwd'] == "\\Device" + + def test_file_json_to_stix(self): + """ + to test file stix object properties + """ + data = { + "Process Name": "msedge.exe", + "Process ID": "5692", + "Username": "user-admin", + "Timestamp - Event": "2024-05-30T08:44:06.312Z", + "Timestamp - Modified": "2024-05-30T08:44:06.312Z", + "Host ID": "xfR", + "Hostname": "EC2-HCF", + "Event Type": "File Write Event", + "Host Set": "test_host_set1", + "Write Event File Name": "LOG", + "Write Event File Full Path": "C:\\Users\\indexeddb.leveldb\\LOG", + "Write Event File Text Written": "2024/05/30-08:44:06.310 1728 Reusing MANIFEST C:\\Users\\Administr", + "Write Event File Bytes Written": "609", + "Write Event File MD5 Hash": "010101010010101010010101010101010", + "Write Event Size in bytes": "392", + "File Name": "msedge.exe" + } + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(data) + file_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), + 'file') + assert (file_obj.keys() == {'type', 'name', 'x_path', 'parent_directory_ref', 'content_ref', + 'x_bytes_written', 'hashes', 'size'}) + assert file_obj is not None + assert file_obj['type'] == 'file' + assert file_obj['name'] == 'LOG' + assert file_obj['x_path'] == 'C:\\Users\\indexeddb.leveldb\\LOG' + assert file_obj['size'] == 392 + directory_ref = file_obj['parent_directory_ref'] + assert (directory_ref in objects), f"parent_ref with key {file_obj['parent_directory_ref']} not found" + dir_obj = objects[directory_ref] + assert dir_obj['type'] == 'directory' + assert dir_obj['path'] == 'C:\\Users\\indexeddb.leveldb' + content_ref = file_obj['content_ref'] + assert (content_ref in objects), f"parent_ref with key {file_obj['content_ref']} not found" + content_obj = objects[content_ref] + assert content_obj['type'] == 'artifact' + + event_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert('file_ref' in event_obj.keys()) + + def test_windows_registry_key_json_to_stix(self): + """ + to test windows-registry-key stix object properties + """ + data = { + "Process Name": "spoolsv.exe", + "Process ID": "2600", + "Username": "NT AUTHORITY\\SYSTEM", + "Registry Key Full Path": "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows", + "Registry Key Values": [{'name': 'ChangeID', 'data': '....', 'data_type': 'REG_DWORD'}], + "Timestamp - Event": "2024-02-27T15:30:52.926Z", + "Timestamp - Modified": "2024-02-27T15:30:52.926Z" + } + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(data) + windows_registry_key_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), + 'windows' + '-registry-key') + assert (windows_registry_key_obj.keys() == {'type', 'key', 'values'}) + assert windows_registry_key_obj is not None + assert windows_registry_key_obj['type'] == 'windows-registry-key' + assert windows_registry_key_obj['key'] == 'HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows' + assert windows_registry_key_obj['values'] == [{'name': 'ChangeID', 'data': '....', 'data_type': 'REG_DWORD'}] + + def test_domain_json_to_stix(self): + """ + to test domain name stix object properties + """ + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(trellix_sample_response) + domain_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), 'domain-name') + assert (domain_obj.keys() == {'type', 'value'}) + assert domain_obj is not None + assert domain_obj['type'] == 'domain-name' + assert domain_obj['value'] == 'download.windowsupdate.com' + + def test_user_account_json_to_stix(self): + """ + to test user account stix object properties + """ + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(trellix_sample_response) + user_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), 'user-account') + assert (user_obj.keys() == {'type', 'user_id'}) + assert user_obj is not None + assert user_obj['type'] == 'user-account' + assert user_obj['user_id'] == 'NT AUTHORITY\\SYSTEM' + + def test_asset_json_to_stix(self): + """ + to test x-oca-asset stix object properties + """ + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(trellix_sample_response) + asset_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), 'x-oca-asset') + assert (asset_obj.keys() == {'type', 'device_id', 'hostname', 'x_host_set'}) + assert asset_obj is not None + assert asset_obj['type'] == 'x-oca-asset' + assert asset_obj['device_id'] == 'TCJ' + assert asset_obj['x_host_set'] == "test_host_set1" + + def test_oca_event_json_to_stix(self): + """ + to test x-oca-event stix object properties + """ + objects = TestTrellixEndpointSecurityHxResultsToStix.get_observed_data_objects(trellix_sample_response) + event_obj = TestTrellixEndpointSecurityHxResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert (event_obj.keys() == {'type', 'process_ref', 'user_ref', 'ip_refs', 'network_ref', 'domain_ref', + 'created', 'x_accessed_time', 'host_ref', 'action', 'modified'}) + assert event_obj is not None + assert event_obj['type'] == 'x-oca-event' + assert event_obj['action'] == 'URL Event' + + process_ref = event_obj['process_ref'] + assert (process_ref in objects), f"process_ref with key {event_obj['process_ref']} not found" + process_obj = objects[process_ref] + assert process_obj['type'] == 'process' + + user_ref = event_obj['user_ref'] + assert (user_ref in objects), f"user_ref with key {event_obj['user_ref']} not found" + user_obj = objects[user_ref] + assert user_obj['type'] == 'user-account' + + network_ref = event_obj['network_ref'] + assert (network_ref in objects), f"network_ref with key {event_obj['network_ref']} not found" + network_obj = objects[network_ref] + assert network_obj['type'] == 'network-traffic' + + domain_ref = event_obj['domain_ref'] + assert (domain_ref in objects), f"domain_ref with key {event_obj['domain_ref']} not found" + domain_obj = objects[domain_ref] + assert domain_obj['type'] == 'domain-name' + + host_ref = event_obj['host_ref'] + assert (host_ref in objects), f"host_ref with key {event_obj['host_ref']} not found" + host_obj = objects[host_ref] + assert host_obj['type'] == 'x-oca-asset' diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_translation/test_trellix_endpoint_security_hx_stix_to_query.py b/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_translation/test_trellix_endpoint_security_hx_stix_to_query.py new file mode 100644 index 000000000..fa87d123b --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_translation/test_trellix_endpoint_security_hx_stix_to_query.py @@ -0,0 +1,446 @@ +from stix_shifter.stix_translation import stix_translation +import unittest + +translation = stix_translation.StixTranslation() + + +def _remove_timestamp_from_query(queries): + """ + remove the timestamp in the query + : param: queries: list + : return: queries: list + """ + for query in queries['queries']: + for item in query['query']: + if 'Timestamp - Event' in item['field']: + query['query'].remove(item) + break + return queries + + +class TestQueryTranslator(unittest.TestCase): + """ + class to perform unit test case for TRELLIX ENDPOINT SECURITY HX translate query + """ + if __name__ == "__main__": + unittest.main() + + def _test_query_assertions(self, query, queries): + """ + to assert each query in the list against expected result + """ + self.assertIsInstance(queries, dict) + self.assertIsInstance(query, dict) + self.assertIsInstance(query['queries'], list) + self.assertEqual(query, queries) + + def test_ipv4_query(self): + stix_pattern = "[ipv4-addr:value = '111.11.111.1']" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, + "query": [{"field": "Local IP Address", "value": "111.11.111.1", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-14T13:28:18.977Z", "2024-03-14T13:33:18.977Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [ + {"field": "Remote IP Address", "value": "111.11.111.1", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-14T13:28:18.977Z", "2024-03-14T13:33:18.977Z"]}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_ipv6_not_equal_operator(self): + stix_pattern = "[ipv6-addr:value != '1001:0db8:85a3:0000:0000:8a2e:0370:7334']" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, "query": [ + {"field": "Local IP Address", "value": "1001:0db8:85a3:0000:0000:8a2e:0370:7334", "operator": "not equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-14T13:36:20.952Z", "2024-03-14T13:41:20.952Z"]}]}, + {"host_set": {"_id": "host_set1"}, "query": [ + {"field": "Remote IP Address", "value": "1001:0db8:85a3:0000:0000:8a2e:0370:7334", + "operator": "not equals"}, {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-14T13:36:20.952Z", + "2024-03-14T13:41:20.952Z"]}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_traffic_gt_operator(self): + stix_pattern = "[network-traffic:src_port > 1234]" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, "query": [{"field": "Local Port", "value": 1234, + "operator": "greater than"}, + {"field": "Timestamp - Event", + "operator": "between", + "value": ["2024-03-14T13:40:04.718Z", + "2024-03-14T13:45:04.718Z"]}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_traffic_LIKE_operator(self): + stix_pattern = "[network-traffic:extensions.'http-request-ext'.request_value LIKE '/mail/u/0']" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [ + {"host_set": {"_id": "host_set1"}, "query": [{"field": "URL", "value": "/mail/u/0", "operator": "contains"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-18T09:03:51.908Z", + "2024-03-18T09:08:51.908Z"]}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_directory_MATCHES_operator(self): + stix_pattern = "[directory:path MATCHES 'C:\\\\Users\\\\Administrator\\\\AppData']" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, "query": [ + {"field": "File Full Path", "value": "C:\\Users\\Administrator\\AppData", "operator": "contains"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-18T09:15:35.409Z", "2024-03-18T09:20:35.409Z"]}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_domain_name_NOT_LIKE_operator(self): + stix_pattern = "[domain-name:value NOT LIKE 'dns']" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, + "query": [{"field": "DNS Hostname", "value": "dns", "operator": "not contains"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-18T09:21:27.261Z", "2024-03-18T09:26:27.261Z"]}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_process_IN_operator(self): + stix_pattern = "[process:name IN ('cmd.exe','conhost.exe')]" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, + "query": [{"field": "Process Name", "operator": "equals", "value": "cmd.exe"}, + {"field": "Process Name", "operator": "equals", "value": "conhost.exe"}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Parent Process Name", "operator": "equals", "value": "cmd.exe"}, + {"field": "Parent Process Name", "operator": "equals", + "value": "conhost.exe"}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_for_stix_attributes_joined_by_OR(self): + stix_pattern = "[process:name='svchost.exe' OR file:name IN( 'svchost1','svchost2') OR file:name NOT IN(" \ + "'file1'," \ + "'file2')] START t'2024-02-10T16:43:26.000Z' STOP t'2024-03-10T16:43:26.003Z'" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, + "query": [{"field": "File Name", "value": "file1", "operator": "not equals"}, + {"field": "File Name", "value": "file2", "operator": "not equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-02-10T16:43:26.000Z", "2024-03-10T16:43:26.003Z"]}, + {"field": "File Name", "value": "svchost1", "operator": "equals"}, + {"field": "File Name", "value": "svchost2", "operator": "equals"}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Process Name", "value": "svchost.exe", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-02-10T16:43:26.000Z", "2024-03-10T16:43:26.003Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [ + {"field": "Parent Process Name", "value": "svchost.exe", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-02-10T16:43:26.000Z", "2024-03-10T16:43:26.003Z"]}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_for_morethan_two_comparison_expressions_joined_by_AND(self): + stix_pattern = ("[ipv4-addr:value = '1.1.1.1' AND network-traffic:src_port > 1234 AND " + "process:name= 'cmd.exe' AND file:name='wpndatabase.db-wal' AND " + "user-account:user_id = 'NT AUTHORITY\\\\SYSTEM' ]") + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [ + {"host_set": {"_id": "host_set1"}, "query": [ + {"field": "Username", "value": "NT AUTHORITY\\SYSTEM", + "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-21T08:45:53.437Z", + "2024-03-21T08:50:53.437Z"]}, + {"field": "File Name", "value": "wpndatabase.db-wal", + "operator": "equals"}, + {"field": "Process Name", "value": "cmd.exe", + "operator": "equals"}, + {"field": "Local Port", "value": 1234, + "operator": "greater than"}, + {"field": "Local IP Address", "value": "1.1.1.1", + "operator": "equals"}]}, + {"host_set": {"_id": "host_set1"}, "query": [ + {"field": "Username", "value": "NT AUTHORITY\\SYSTEM", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-21T08:45:53.437Z", "2024-03-21T08:50:53.437Z"]}, + {"field": "File Name", "value": "wpndatabase.db-wal", "operator": "equals"}, + {"field": "Process Name", "value": "cmd.exe", "operator": "equals"}, + {"field": "Local Port", "value": 1234, "operator": "greater than"}, + {"field": "Remote IP Address", "value": "1.1.1.1", "operator": "equals"}]}, + {"host_set": {"_id": "host_set1"}, "query": [ + {"field": "Username", "value": "NT AUTHORITY\\SYSTEM", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-21T08:45:53.437Z", "2024-03-21T08:50:53.437Z"]}, + {"field": "File Name", "value": "wpndatabase.db-wal", "operator": "equals"}, + {"field": "Parent Process Name", "value": "cmd.exe", "operator": "equals"}, + {"field": "Local Port", "value": 1234, "operator": "greater than"}, + {"field": "Local IP Address", "value": "1.1.1.1", "operator": "equals"}]}, + {"host_set": {"_id": "host_set1"}, "query": [ + {"field": "Username", "value": "NT AUTHORITY\\SYSTEM", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-21T08:45:53.437Z", "2024-03-21T08:50:53.437Z"]}, + {"field": "File Name", "value": "wpndatabase.db-wal", "operator": "equals"}, + {"field": "Parent Process Name", "value": "cmd.exe", "operator": "equals"}, + {"field": "Local Port", "value": 1234, "operator": "greater than"}, + {"field": "Remote IP Address", "value": "1.1.1.1", "operator": "equals"}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_with_multiple_comparison_expressions_with_AND_OR_combinations(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1' OR network-traffic:src_port > 1234 OR process:name= " \ + "'amazon-ssm-agent.exe' AND file:name='wpndatabase.db-wal']" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, + "query": [{"field": "File Name", "value": "wpndatabase.db-wal", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-15T09:56:31.088Z", "2024-03-15T10:01:31.088Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Local IP Address", "value": "1.1.1.1", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-15T09:56:31.088Z", "2024-03-15T10:01:31.088Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Remote IP Address", "value": "1.1.1.1", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-15T09:56:31.088Z", "2024-03-15T10:01:31.088Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Local Port", "value": 1234, "operator": "greater than"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-15T09:56:31.088Z", "2024-03-15T10:01:31.088Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Process Name", "value": "amazon-ssm-agent.exe", + "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-15T09:56:31.088Z", "2024-03-15T10:01:31.088Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Parent Process Name", "value": "amazon-ssm-agent.exe", + "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-15T09:56:31.088Z", "2024-03-15T10:01:31.088Z"]}]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_for_multiple_observation_with_and_without_qualifier(self): + stix_pattern = "[network-traffic:src_port > 50 OR domain-name:value='dns'] AND [ " \ + "windows-registry-key:key='HKEY_LOCAL_MACHINE\\\\SOFTWARE\\\\Microsoft\\\\Windows' AND " \ + "user-account:user_id = 'NT AUTHORITY\\\\SYSTEM'] OR " \ + "[network-traffic:extensions.'http-request-ext'.request_header.'Accept-Encoding' LIKE 'gzip'] " \ + "START t'2024-02-10T11:00:00.000Z'STOP t'2024-03-01T11:00:00.003Z'" + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = {"queries": [{"host_set": {"_id": "host_set1"}, + "query": [{"field": "DNS Hostname", "value": "dns", "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-21T09:09:37.814Z", "2024-03-21T09:14:37.814Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Local Port", "value": 50, "operator": "greater than"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-21T09:09:37.814Z", "2024-03-21T09:14:37.814Z"]}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "Username", "value": "NT AUTHORITY\\SYSTEM", + "operator": "equals"}, + {"field": "Timestamp - Event", "operator": "between", + "value": ["2024-03-21T09:09:37.814Z", "2024-03-21T09:14:37.814Z"]}, + {"field": "Registry Key Full Path", + "value": "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows", + "operator": "equals"}]}, + {"host_set": {"_id": "host_set1"}, + "query": [{"field": "HTTP Header", "value": "gzip", + "operator": "contains"}, + {"field": "Timestamp - Event", + "operator": "between", + "value": ["2024-02-10T11:00:00.000Z", + "2024-03-01T11:00:00.003Z"]} + ]}]} + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_single_qualifier_with_precedence_bracket(self): + stix_pattern = ("([network-traffic:extensions.'http-request-ext'.request_header.'User-Agent' MATCHES 'header' " + "AND windows-registry-key:values[*].name = 'ChangeId'] OR [file:hashes.MD5 = 'abc123' " + "OR network-traffic:extensions.'http-request-ext'.request_value " + "!='/latest/meta-data/iam/security-credentials/']) " + "START t'2024-02-15T11:20:35.000Z'STOP t'2024-03-10T11:00:00.003Z'") + query = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + query = _remove_timestamp_from_query(query) + queries = { + "queries": [ + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "Registry Key Value Name", + "value": "ChangeId", + "operator": "equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-02-15T11:20:35.000Z", + "2024-03-10T11:00:00.003Z" + ] + }, + { + "field": "HTTP Header", + "value": "header", + "operator": "contains" + } + ] + }, + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "URL", + "value": "/latest/meta-data/iam/security-credentials/", + "operator": "not equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-02-15T11:20:35.000Z", + "2024-03-10T11:00:00.003Z" + ] + } + ] + }, + { + "host_set": { + "_id": "host_set1" + }, + "query": [ + { + "field": "File MD5 Hash", + "value": "abc123", + "operator": "equals" + }, + { + "field": "Timestamp - Event", + "operator": "between", + "value": [ + "2024-02-15T11:20:35.000Z", + "2024-03-10T11:00:00.003Z" + ] + } + ] + } + ] + } + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_future_timestamp_qualifier(self): + stix_pattern = "[network-traffic:src_port < 53]START t'2024-09-19T11:00:00.000Z' " \ + "STOP t'2024-02-07T11:00:00.003Z'" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "translation_error" == result['code'] + assert 'Start/Stop time should not be in the future UTC timestamp' in result['error'] + + def test_stop_time_lesser_than_start_time(self): + stix_pattern = "[network-traffic:src_port > 32794]START t'2024-01-19T11:00:00.000Z' " \ + "STOP t'2023-02-07T11:00:00.003Z'" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "translation_error" == result['code'] + assert 'Start time should be lesser than Stop time' in result['error'] + + def test_invalid_operator_for_trellix(self): + stix_pattern = "[file:size >= 50]" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "mapping_error" == result['code'] + assert 'data mapping error : Unable to map the following STIX Operators: [GreaterThanOrEqual] to data source ' \ + 'fields' in \ + result['error'] + + def test_invalid_operator_for_string_fields(self): + stix_pattern = "[process:name < 'process']" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "not_implemented" == result['code'] + assert 'LessThan operator is not supported for string type field ' in result['error'] + + def test_un_supported_operator(self): + stix_pattern = "[file:size NOT MATCHES '50']" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "not_implemented" == result['code'] + assert 'Matches operator is not supported for integer type field ' in result['error'] + + def test_similar_stix_attributes_for_and_operator(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1' AND network-traffic:src_ref.value = '2.2.2.2' AND process:name= " \ + "'cmd.exe' AND file:name='file.exe']" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "translation_error" == result['code'] + assert 'The expression [ipv4-addr:value] has same data source field mapping with another expression in the ' \ + 'pattern which has only AND comparison operator. Recommended to Use OR operator.' in \ + result['error'] + + def test_similar_mapping_fields_in_different_attributes_for_and_operator(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1' AND ipv4-addr:value != '2.2.2.2']" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "translation_error" == result['code'] + assert ('Multiple [ipv4-addr:value] expression is used in the pattern which has only AND comparison operator. ' + 'Recommended to Use OR operator for similar STIX attributes.') in result['error'] + + def test_ipv4_addr_not_supported_operator(self): + stix_pattern = "[ipv4-addr:value LIKE '1.1.1.1']" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "not_implemented" == result['code'] + assert ('Like operator is not supported for IP Address type field ipv4-addr:value.Possible supported ' + 'operators are =, !=, IN, NOT IN') in \ + result['error'] + + def test_stix_field_not_supported_operator(self): + stix_pattern = "[file:hashes.MD5 MATCHES 'abc123']" + result = translation.translate('trellix_endpoint_security_hx', 'query', '{}', stix_pattern, + {"host_sets": "host_set1"}) + assert result['success'] is False + assert "not_implemented" == result['code'] + assert ('Matches operator is not supported for File Hash type field file:hashes.MD5.Possible ' + 'supported operators are =, !=, IN, NOT IN') in \ + result['error'] diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_transmission/test_trellix_endpoint_security_hx.py b/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_transmission/test_trellix_endpoint_security_hx.py new file mode 100644 index 000000000..843c61546 --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/test/stix_transmission/test_trellix_endpoint_security_hx.py @@ -0,0 +1,731 @@ +from stix_shifter_modules.trellix_endpoint_security_hx.entry_point import EntryPoint +from stix_shifter.stix_transmission.stix_transmission import run_in_thread +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from tests.utils.async_utils import get_mock_response +from unittest.mock import patch +import unittest +import json +import copy + + +class TestTrellixEndpointSecurityHxConnection(unittest.TestCase, object): + + def connection(self): + return { + "host": "hostbla", + "port": 1, + "selfSignedCert": "cert", + "options": {"host_sets": "hostset1"} + } + + def connection_with_result_limit(self): + return { + "host": "hostbla", + "port": 1, + "selfSignedCert": "cert", + "options": {"host_sets": "hostset1", "result_limit": 2} + } + + def configuration(self): + return { + "auth": { + "username": "user", + "password": "123" + } + } + + mock_host_set_response = { + "data": { + "total": 1, + "query": { + "name": "host_set1" + }, + "sort": {}, + "offset": 0, + "limit": 50, + "entries": [ + { + "_id": 1001, + "name": "host_set1", + "type": "static", + "_revision": "20240214070201917815249051", + "url": "/hx/api/v3/host_sets/1001" + } + ] + }, + "message": "OK", + "details": [], + "route": "/hx/api/v3/host_sets" + } + + mocked_ping_response = { + "data": { + "total": 2, + "query": {}, + "sort": {}, + "offset": 0, + "limit": 50, + "entries": [ + { + "data": {"hostname": "EC2AMAZ1"} + }, + { + "data": {"hostname": "EC2AMAZ2"} + } + ]}} + + mocked_status_response = { + "data": { + "_id": 2285, + "state": "RUNNING", + "stats": { + "running_state": { + "NEW": 0, + "QUEUED": 0, + "FAILED": 0, + "COMPLETE": 3, + "ABORTED": 0, + "DELETED": 0, + "REFRESH": 0, + "CANCELLED": 0 + }, + "hosts": 5 + }, + "settings": { + "query_terms": { + "terms": [ + {"field": "Local Port", "value": 50, "operator": "greater than"}, + {"field": "Timestamp - Event", "operator": "between", "value": ["2024-05-27T11:10:17.682Z", + "2024-05-27T11:15:17.682Z"]} + ] + }, + "mode": "HOST" + } + } + } + + mocked_result_response = { + "data": { + "total": 5, + "query": {}, + "sort": {}, + "offset": 0, + "limit": 1, + "entries": [ + { + "host": { + "_id": "abcd", + "url": "/hx/api/v3/hosts/abcd", + "hostname": "Ec21" + }, + "results": [ + { + "id": 1, + "type": "IPv4 Network Event", + "data": { + "Process Name": "chrome.exe", + "Process ID": "6536", + "Username": "user1", + "Local IP Address": "1.1.1.1", + "Remote IP Address": "5.6.7.8", + "IP Address": "5.6.7.8", + "Port": "443", + "Local Port": "53842", + "Remote Port": "443", + "Timestamp - Event": "2024-05-24T07:10:03.554Z", + "Timestamp - Accessed": "2024-05-24T07:10:03.554Z" + } + } + + ] + }] + }, + "message": "OK", + "details": [], + "route": "/hx/api/v3/searches/id/results" + } + + mocked_result_response_2 = { + "data": { + "total": 5, + "query": {}, + "sort": {}, + "offset": 1, + "limit": 1, + "entries": [ + { + "host": { + "_id": "efgh", + "url": "/hx/api/v3/hosts/efgh", + "hostname": "Ec22" + }, + "results": [ + { + "id": 2, + "type": "IPv4 Network Event", + "data": { + "Process Name": "svchost.exe", + "Process ID": "1956", + "Username": "NT AUTHORITY\\NETWORK SERVICE", + "Local IP Address": "2.2.2.2", + "Remote IP Address": "3.2.2.3", + "IP Address": "3.2.2.3", + "Port": "53", + "Local Port": "64013", + "Remote Port": "53", + "Timestamp - Event": "2024-05-24T07:10:05.041Z", + "Timestamp - Accessed": "2024-05-24T07:10:05.041Z" + } + }, + { + "id": 3, + "type": "IPv4 Network Event", + "data": { + "Process Name": "ssm-agent-worker.exe", + "Process ID": "4932", + "Username": "NT AUTHORITY\\SYSTEM", + "Local IP Address": "2.2.2.2", + "Remote IP Address": "1.0.0.1", + "IP Address": "1.0.0.1", + "Port": "443", + "Local Port": "53843", + "Remote Port": "443", + "Timestamp - Event": "2024-05-24T07:10:05.042Z", + "Timestamp - Accessed": "2024-05-24T07:10:05.042Z" + } + }, + { + "id": 4, + "type": "IPv4 Network Event", + "data": { + "Process Name": "chrome.exe", + "Process ID": "6536", + "Username": "user1", + "Local IP Address": "2.2.2.2", + "Remote IP Address": "5.6.7.8", + "IP Address": "5.6.7.8", + "Port": "443", + "Local Port": "53844", + "Remote Port": "443", + "Timestamp - Event": "2024-05-24T07:10:19.559Z", + "Timestamp - Accessed": "2024-05-24T07:10:19.559Z" + } + } + ] + } + ] + }, + "message": "OK", + "details": [], + "route": "/hx/api/v3/searches/id/results" + } + + def test_is_async(self): + """ test async""" + entry_point = EntryPoint() + check_async = entry_point.is_async() + assert check_async + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_ping_success(self, mock_ssl, mock_ping_response): + """ test success response for ping""" + mock_ping_response.side_effect = [ + get_mock_response(204, "", 'byte', + headers={'X-FeApi-Token': "****"}), + get_mock_response(200, json.dumps(TestTrellixEndpointSecurityHxConnection.mocked_ping_response), 'byte'), + get_mock_response(204, "", 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_result = run_in_thread(entry_point.ping_connection) + assert ping_result["success"] is True + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_query_success(self, mock_ssl, mock_query_response): + """ test success response for query""" + query = {"host_set": {"_id": "host_set1"}, + "query": [ + {"field": "Local Port", "value": 50, "operator": "greater than"}, + {"field": "Timestamp - Event", "operator": "between", "value": ["2024-05-27T11:10:17.682Z", + "2024-05-27T11:15:17.682Z"]}] + } + query_response = '{"data": {"_id": 2285, "state": "RUNNING"}}' + mock_query_response.side_effect = \ + [get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(200, json.dumps(TestTrellixEndpointSecurityHxConnection.mock_host_set_response), 'byte'), + get_mock_response(201, query_response, 'byte'), + get_mock_response(204, "", 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + query_response = run_in_thread(entry_point.create_query_connection, json.dumps(query)) + assert query_response['success'] is True + assert query_response['search_id'] == "2285:host_set1" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_status_completed(self, mock_ssl, mock_status_response): + """" test success response for status with completed status""" + search_id = "2285:host_set1" + mock_status_response.side_effect = \ + [get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(200, json.dumps(TestTrellixEndpointSecurityHxConnection.mocked_status_response), + 'byte'), + get_mock_response(204, "", 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, search_id) + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.COMPLETED.value + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_status_running(self, mock_ssl, mock_status_response): + """ test success response for status with running status""" + search_id = "2285:host_set1" + mock_status_response_1 = copy.deepcopy(TestTrellixEndpointSecurityHxConnection.mocked_status_response) + mock_status_response_1['data']["stats"]["running_state"]["COMPLETE"] = 2 + mock_status_response.side_effect = \ + [get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(200, json.dumps(mock_status_response_1), + 'byte'), + get_mock_response(204, "", 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, search_id) + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.RUNNING.value + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_results_success_response(self, mock_ssl, mock_results_response): + """ test results with success response""" + search_id = "2285:host_set1" + first_response = get_mock_response(200, + json.dumps(TestTrellixEndpointSecurityHxConnection.mocked_result_response), + 'byte') + second_response = get_mock_response(200, json.dumps( + TestTrellixEndpointSecurityHxConnection.mocked_result_response_2), 'byte') + mock_results_response.side_effect = [ + get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + first_response, second_response, + get_mock_response(204, "", 'byte'), + get_mock_response(204, "", 'byte') + ] + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 2) + success = results_response["success"] + assert success + data = results_response["data"] + assert data + assert len(data) == 2 + assert results_response['metadata'] == {'host_offset': 1, 'host_record_index': 2} + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_results_success_response_with_metadata(self, mock_ssl, mock_results_response): + """ test results with success response with metadata""" + search_id = "2285:host_set1" + metadata = {'host_offset': 1, 'host_record_index': 2} + first_response = get_mock_response(200, + json.dumps(TestTrellixEndpointSecurityHxConnection.mocked_result_response_2), + 'byte') + mock_results_response.side_effect = [ + get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + first_response, + get_mock_response(204, "", 'byte'), + get_mock_response(204, "", 'byte') + ] + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 2, 1, metadata) + success = results_response["success"] + assert success + data = results_response["data"] + assert data + assert results_response['metadata'] == {'host_offset': 1, 'host_record_index': 3} + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_results_success_response_with_result_limit_less_than_length(self, mock_ssl, mock_results_response): + """ test results with success response with modified result limit""" + search_id = "2285:host_set1" + metadata = {'host_offset': 1, 'host_record_index': 1} + first_response = get_mock_response(200, + json.dumps(TestTrellixEndpointSecurityHxConnection.mocked_result_response_2), + 'byte') + mock_results_response.side_effect = [ + get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + first_response, + get_mock_response(204, "", 'byte'), + get_mock_response(204, "", 'byte') + ] + entry_point = EntryPoint(self.connection_with_result_limit(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 3, metadata) + success = results_response["success"] + assert success + data = results_response["data"] + assert data + assert len(data) == 2 + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_results_and_format_url_registry_event(self, mock_ssl, mock_results_response): + """ test results to verify the formatted url and registry events""" + search_id = "2286:host_set2" + mocked_response = { + "data": { + "total": 5, + "query": {}, + "sort": {}, + "offset": 0, + "limit": 1, + "entries": [ + { + "host": { + "_id": "abcd", + "url": "/hx/api/v3/hosts/abcd", + "hostname": "Ec21" + }, + "results": [ + { + "id": 1, + "type": "URL Event", + "data": { + "Process Name": "EC2Launch.exe", + "Process ID": "3664", + "Username": "NT AUTHORITY\\SYSTEM", + "Remote IP Address": "10.20.10.20", + "IP Address": "10.20.10.20", + "Port": "80", + "Local Port": "49726", + "Remote Port": "80", + "DNS Hostname": "10.20.10.20", + "URL": "/latest/meta-data//hibernation/configured", + "HTTP Header": "GET /latest/meta-data//hibernation/configured HTTP/1.1" + "\nHost: 10.20.10.20\nUser-Agent: Go-http-client/1.1\nX-Aws-" + "Ec2-Metadata-Token: AQAAACbkzU35XT2PUWZWf6WfwHTdKFFF4n_dottlg8D_0-" + "PIkAsoOA==\nAccept-Encoding: gzip\n\n", + "HTTP Method": "GET", + "Timestamp - Event": "2024-03-07T09:09:10.836Z", + "Timestamp - Accessed": "2024-03-07T09:09:10.836Z" + } + }, + { + "id": 2, + "type": "Registry Event", + "data": { + "Process Name": "svchost.exe", + "Process ID": "1484", + "Username": "NT AUTHORITY\\LOCAL SERVICE", + "Registry Key Full Path": "HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services" + "\\Tcpip\\Parameters\\Interfaces\\{d7f2a7dc-ea23-44ef-" + "8ebe-84bad862a9d8}\\LeaseObtainedTime", + "Registry Key Value Name": "LeaseObtainedTime", + "Registry Key Value Type": "REG_DWORD", + "Registry Key Value Text": "fJm.", + "Timestamp - Event": "2024-05-19T21:24:02.365Z", + "Timestamp - Modified": "2024-05-19T21:24:02.365Z" + } + }, + { + "id": 3, + "type": "File Write Event", + "data": { + "File Name": "SRU.chk", + "File Full Path": "C:\\Windows\\SRU.chk", + "File Text Written": "C:\\Windows\\syste", + "File Bytes Written": "8192", + "Size in bytes": "8192", + "File MD5 Hash": "5b9eb9cdf514a50f676d0cb63118ff41", + "Process Name": "svchost.exe", + "Process ID": "4344", + "Username": "NT AUTHORITY\\LOCAL SERVICE", + "Timestamp - Event": "2024-05-21T05:41:06.141Z", + "Timestamp - Modified": "2024-05-21T05:41:06.141Z" + } + } + + ] + }] + }, + "message": "OK", + "details": [], + "route": "/hx/api/v3/searches/id/results" + } + mock_results_response.side_effect = [ + get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(200, + json.dumps(mocked_response), + 'byte'), + get_mock_response(204, "", 'byte'), + get_mock_response(204, "", 'byte') + ] + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 3) + success = results_response["success"] + assert success + data = results_response["data"] + assert data[0]['HTTP Header']['Host'] == '10.20.10.20' + assert data[0]['HTTP Header']['User-Agent'] == 'Go-http-client/1.1' + assert data[0]['HTTP Header']['Accept-Encoding'] == 'gzip' + assert (data[1]['Registry Key Values'] == + [{'name': 'LeaseObtainedTime', 'data': 'fJm.', 'data_type': 'REG_DWORD'}]) + assert data[2]['Write Event File Bytes Written'] == "8192" + assert data[2]["Write Event File Text Written"] == "C:\\Windows\\syste" + assert data[2]["Write Event File Name"] == "SRU.chk" + assert data[2]["File Name"] == "svchost.exe" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_empty_records_in_results(self, mock_ssl, mock_result_response): + """ test empty records in results""" + search_id = "2287:host_set3" + mocked_response = { + "data": { + "total": 5, + "query": {}, + "sort": {}, + "offset": 0, + "limit": 1, + "entries": [] + }, + "message": "OK", + "details": [], + "route": "/hx/api/v3/searches/id/results" + } + mock_result_response.side_effect = [ + get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(200, + json.dumps(mocked_response), + 'byte'), + get_mock_response(204, "", 'byte'), + get_mock_response(204, "", 'byte') + ] + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 2) + assert results_response is not None + success = results_response["success"] + assert success + data = results_response["data"] + assert data == [] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_deletion_success(self, mock_ssl, mock_delete_response): + """ test delete query success response""" + search_id = "2287:host_set3" + mock_delete_response.side_effect = [ + get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(204, "", 'byte'), + get_mock_response(204, "", 'byte') + ] + entry_point = EntryPoint(self.connection(), self.configuration()) + del_response = run_in_thread(entry_point.delete_query_connection, search_id) + assert del_response is not None + success = del_response["success"] + assert success + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_deletion_failure(self, mock_ssl, mock_delete_response): + """ test invalid search id in delete results""" + search_id = "2289:host_set3" + mock_del_response = { + "details": [{"type": "error", "code": 1005, "message": "Search not found.", "path": "id"}], + "route": "/hx/api/v3/searches/id", "message": "Not Found"} + mock_delete_response.side_effect = [ + get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(404, json.dumps(mock_del_response), 'byte'), + get_mock_response(204, "", 'byte') + ] + entry_point = EntryPoint(self.connection(), self.configuration()) + del_response = run_in_thread(entry_point.delete_query_connection, search_id) + assert del_response is not None + assert del_response['success'] is False + assert 'error' in del_response + assert 'Search not found' in del_response['error'] + assert del_response['code'] == 'no_results' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_timeout_for_results(self, mock_ssl, mock_result_response): + """ test timeout for results""" + search_id = "2286:host_set2" + mock_result_response.side_effect = [Exception("'server timeout_error (2 sec)'")] + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 2) + assert results_response is not None + assert results_response['success'] is False + assert 'error' in results_response + assert 'timeout_error' in results_response['error'] + assert results_response['code'] == 'service_unavailable' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_invalid_authentication_for_results(self, mock_ssl, mock_result_response): + """ test invalid authentication for results""" + search_id = "2286:host_set2" + mock_result_response.side_effect = \ + [get_mock_response(401, json.dumps( + {'details': [{'code': 1105, 'message': 'Incorrect user id or password.', 'type': 'error'}], + 'message': 'Unauthorized'}), 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 2) + assert results_response is not None + assert results_response['success'] is False + assert 'error' in results_response + assert 'Incorrect user id or password.' in results_response['error'] + assert results_response['code'] == 'authentication_fail' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_timeout_for_status(self, mock_ssl, mock_status_response): + """ test timeout for status""" + + mock_status_response.side_effect = [Exception("'server timeout_error (2 sec)'")] + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, "1255") + assert status_response is not None + assert status_response['success'] is False + assert 'error' in status_response + assert 'timeout_error' in status_response['error'] + assert status_response['code'] == 'service_unavailable' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_invalid_search_id_for_status(self, mock_ssl, mock_status_response): + """ test invalid search id for status""" + mock_search_response = { + "details": [{"type": "error", "code": 1005, "message": "Search not found.", "path": "id"}], + "route": "/hx/api/v3/searches/id", "message": "Not Found"} + mock_status_response.side_effect = \ + [get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(404, json.dumps(mock_search_response), 'byte'), + get_mock_response(204, "", 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, "1255") + assert status_response is not None + assert status_response['success'] is False + assert 'error' in status_response + assert 'Search not found' in status_response['error'] + assert status_response['code'] == 'no_results' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_invalid_host_set_for_query(self, mock_ssl, mock_query_response): + """ test invalid host set name for query""" + mock_host_set_response = { + "data": { + "total": 1, + "query": { + "name": "invalid host set" + }, + "entries": [] + }, + "route": "/hx/api/v3/host_sets" + } + mock_query_response.side_effect = \ + [get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(200, json.dumps(mock_host_set_response), 'byte'), + get_mock_response(204, "", 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + query = {"host_set": {"_id": "invalid host set"}, + "query": [ + {"field": "Local Port", "value": 50, "operator": "greater than"}] + } + query_response = run_in_thread(entry_point.create_query_connection, json.dumps(query)) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert 'Invalid Host Set Name' in query_response['error'] + assert query_response['code'] == 'invalid_parameter' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_invalid_authentication_for_query(self, mock_ssl, mock_query_response): + """ test invalid authentication for query""" + + mock_query_response.side_effect = \ + [get_mock_response(401, json.dumps( + {'details': [{'code': 1105, 'message': 'Incorrect user id or password.', 'type': 'error'}], + 'message': 'Unauthorized'}), 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + query = {"host_set": {"_id": "invalid host set"}, + "query": [ + {"field": "Local Port", "value": 50, "operator": "greater than"}] + } + query_response = run_in_thread(entry_point.create_query_connection, json.dumps(query)) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert 'Incorrect user id or password.' in query_response['error'] + assert query_response['code'] == 'authentication_fail' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_time_out_exception_for_query(self, mock_ssl, mock_query_response): + """ test timeout exception for query""" + query = {"host_set": {"_id": "host set 1"}, + "query": [ + {"field": "Local Port", "value": 50, "operator": "greater than"}] + } + mock_query_response.side_effect = Exception("'server timeout_error (2 sec)'") + entry_point = EntryPoint(self.connection(), self.configuration()) + query_response = run_in_thread(entry_point.create_query_connection, json.dumps(query)) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert 'timeout_error' in query_response['error'] + assert query_response['code'] == 'service_unavailable' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_invalid_query(self, mock_ssl, mock_query_response): + """ test invalid field name in query""" + mock_response = {"details": [{"type": "error", "code": 1006, "message": "Invalid search term field.", + "path": "query"}], "route": "/hx/api/v3/searches", + "message": "Unprocessable Entity"} + + mock_query_response.side_effect = \ + [get_mock_response(204, "", 'byte', headers={'X-FeApi-Token': "****"}), + get_mock_response(422, json.dumps(mock_response), 'byte'), + get_mock_response(204, "", 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + query = {"host_set": {"_id": "host_set1"}, + "query": [ + {"field": "Local Port1", "value": 50, "operator": "greater than"}] + } + query_response = run_in_thread(entry_point.create_query_connection, json.dumps(query)) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert 'Invalid search term field.' in query_response['error'] + assert query_response['code'] == 'invalid_query' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_invalid_authentication_for_ping(self, mock_ssl, mock_ping_response): + """ test invalid authentication for ping""" + mock_ping_response.side_effect = [get_mock_response(401, json.dumps( + {'details': [{'code': 1105, 'message': 'Incorrect user id or password.', 'type': 'error'}], + 'message': 'Unauthorized'}), 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_result = run_in_thread(entry_point.ping_connection) + assert ping_result is not None + assert ping_result['success'] is False + assert 'error' in ping_result + assert 'Incorrect user id or password' in ping_result['error'] + assert ping_result['code'] == 'authentication_fail' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + @patch('ssl.SSLContext.load_verify_locations') + def test_time_out_exception_for_ping(self, mock_ssl, mock_ping_response): + """ test timeout exception for ping""" + mock_ping_response.side_effect = Exception("'server timeout_error (2 sec)'") + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_result = run_in_thread(entry_point.ping_connection) + assert ping_result is not None + assert ping_result['success'] is False + assert 'error' in ping_result + assert 'timeout_error' in ping_result['error'] + assert ping_result['code'] == 'service_unavailable' diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/trellix_endpoint_security_hx_supported_stix.md b/stix_shifter_modules/trellix_endpoint_security_hx/trellix_endpoint_security_hx_supported_stix.md new file mode 100644 index 000000000..d7838347e --- /dev/null +++ b/stix_shifter_modules/trellix_endpoint_security_hx/trellix_endpoint_security_hx_supported_stix.md @@ -0,0 +1,143 @@ +##### Updated on 05/30/24 +## Trellix Endpoint Security HX +### Results STIX Domain Objects +* Identity +* Observed Data + +### Supported STIX Operators +*Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).* + +| STIX Operator | Data Source Operator | +|-------------------|----------------------| +| AND (Comparison) | and | +| OR (Comparison) | or | +| \> | greater than | +| < | less than | +| = | equals | +| LIKE | contains | +| IN | equals | +| MATCHES | contains | +| != | not equals | +| OR (Observation) | or | +| AND (Observation) | or | +|
| | +### Searchable STIX objects and properties +| STIX Object and Property | Mapped Data Source Fields | +|------------------------------------------------------------------------------------|-------------------------------------| +| **ipv4-addr**:value | Local IP Address, Remote IP Address | +| **ipv6-addr**:value | Local IP Address, Remote IP Address | +| **network-traffic**:src_port | Local Port | +| **network-traffic**:dst_port | Remote Port | +| **network-traffic**:src_ref.value | Local IP Address | +| **network-traffic**:dst_ref.value | Remote IP Address | +| **network-traffic**:extensions.'http-request-ext'.request_header.'Accept-Encoding' | HTTP Header | +| **network-traffic**:extensions.'http-request-ext'.request_header.'User-Agent' | HTTP Header | +| **network-traffic**:extensions.'http-request-ext'.request_header.Host | HTTP Header | +| **network-traffic**:extensions.'http-request-ext'.request_value | URL | +| **process**:name | Process Name,Parent Process Name | +| **process**:command_line | Process Arguments | +| **process**:parent_ref.name | Parent Process Name | +| **process**:parent_ref.cwd | Parent Process Path | +| **process**:binary_ref.name | File Name | +| **process**:creator_user_ref.user_id | Username | +| **file**:name | File Name | +| **file**:hashes.MD5 | File MD5 Hash | +| **file**:size | Size in bytes | +| **file**:x_path | File Full Path | +| **file**:parent_directory_ref.path | File Full Path | +| **directory**:path | File Full Path | +| **user-account**:user_id | Username | +| **windows-registry-key**:key | Registry Key Full Path | +| **windows-registry-key**:values[*].name | Registry Key Value Name | +| **windows-registry-key**:values[*].data | Registry Key Value Text | +| **domain-name**:value | DNS Hostname | +| **x-oca-event**:file_ref.name | File Name | +| **x-oca-event**:process_ref.name | Process Name, Parent Process Name | +| **x-oca-event**:parent_process_ref.name | Parent Process Name | +| **x-oca-event**:domain_ref.value | DNS Hostname | +| **x-oca-event**:registry_ref.key | Registry Key Full Path | +| **x-oca-event**:network_ref.src_port | Local Port | +| **x-oca-event**:ip_refs[*].value | Local IP Address, Remote IP Address | +| **x-oca-event**:user_ref.user_id | Username | + +### Supported STIX Objects and Properties for Query Results +| STIX Object | STIX Property | Data Source Field | +|----------------------|--------------------------------------------|--------------------------------| +| domain-name | value | DNS Hostname | +|
| | | +| ipv4-addr | value | Local IP Address | +| ipv4-addr | value | Remote IP Address | +|
| | | +| ipv6-addr | value | Local IP Address | +| ipv6-addr | value | Remote IP Address | +|
| | | +| network-traffic | src_ref | Local IP Address | +| network-traffic | dst_ref | Remote IP Address | +| network-traffic | src_port | Local Port | +| network-traffic | dst_port | Remote Port | +| network-traffic | protocols | Port Protocol | +| network-traffic | extensions.http-request-ext.request_header | HTTP Header | +| network-traffic | extensions.http-request-ext.request_value | URL | +| network-traffic | extensions.http-request-ext.request_method | HTTP Method | +|
| | | +| process | name | Process Name | +| process | name | Parent Process Name | +| process | pid | Process ID | +| process | cwd | Parent Process Path | +| process | command_line | Process Arguments | +| process | parent_ref | Parent Process Path | +| process | parent_ref | Parent Process Name | +| process | binary_ref | File Name | +| process | creator_user_ref | Username | +| process | x_event_type | Process Event Type | +|
| | | +| artifact | payload_bin | Write Event File Text Written | +|
| | | +| file | name | File Name | +| file | name | Write Event File Name | +| file | hashes.MD5 | File MD5 Hash | +| file | hashes.MD5 | Write Event File MD5 Hash | +| file | size | Size in bytes | +| file | size | Write Event Size in bytes | +| file | content_ref | Write Event File Text Written | +| file | parent_directory_ref | File Full Path | +| file | parent_directory_ref | Write Event File Full Path | +| file | x_path | File Full Path | +| file | x_path | Write Event File Full Path | +| file | x_bytes_written | File Bytes Written | +| file | x_bytes_written | Write Event File Bytes Written | +|
| | | +| directory | path | File Full Path | +| directory | path | Write Event File Full Path | +|
| | | +| user-account | user_id | Username | +|
| | | +| windows-registry-key | key | Registry Key Full Path | +| windows-registry-key | values | Registry Key Values | +|
| | | +| x-oca-asset | hostname | Hostname | +| x-oca-asset | device_id | Host ID | +| x-oca-asset | ip_refs | Local IP Address | +| x-oca-asset | x_host_set | Host Set | +|
| | | +| x-oca-event | action | Event Type | +| x-oca-event | start | Timestamp - Started | +| x-oca-event | modified | Timestamp - Modified | +| x-oca-event | created | Timestamp - Event | +| x-oca-event | host_ref | Hostname | +| x-oca-event | user_ref | Username | +| x-oca-event | ip_refs | Local IP Address | +| x-oca-event | ip_refs | Remote IP Address | +| x-oca-event | network_ref | Remote Port | +| x-oca-event | network_ref | Local Port | +| x-oca-event | network_ref | Port Protocol | +| x-oca-event | file_ref | File Name | +| x-oca-event | parent_process_ref | Parent Process Path | +| x-oca-event | parent_process_ref | Parent Process Name | +| x-oca-event | registry_ref | Registry Key Full Path | +| x-oca-event | domain_ref | DNS Hostname | +| x-oca-event | process_ref | Process Name | +| x-oca-event | x_file_write_ref | Write Event File Name | +| x-oca-event | x_accessed_time | Timestamp - Accessed | +| x-oca-event | x_last_run | Timestamp - Last Run | +|
| | | From 14a5fa2962594888a4b884db8ef3ac87f7ae3113 Mon Sep 17 00:00:00 2001 From: Sharmila-MS Date: Tue, 4 Jun 2024 13:53:51 +0000 Subject: [PATCH 2/4] updated readme --- stix_shifter_modules/trellix_endpoint_security_hx/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/README.md b/stix_shifter_modules/trellix_endpoint_security_hx/README.md index 1aaa9d99f..87bee2557 100644 --- a/stix_shifter_modules/trellix_endpoint_security_hx/README.md +++ b/stix_shifter_modules/trellix_endpoint_security_hx/README.md @@ -247,7 +247,7 @@ transmit trellix_endpoint_security_hx "{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" "{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" -results "2493:host_set1" +results "2493:host_set1" 0 1 ``` #### STIX Transmit Results - Output From 969cfe7cf47c04024e825a8e3aa2de38023589c9 Mon Sep 17 00:00:00 2001 From: Sharmila-MS Date: Wed, 26 Jun 2024 09:41:56 +0000 Subject: [PATCH 3/4] updated review comments updated suggestions in read me file --- .../trellix_endpoint_security_hx/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/README.md b/stix_shifter_modules/trellix_endpoint_security_hx/README.md index 87bee2557..b4c505624 100644 --- a/stix_shifter_modules/trellix_endpoint_security_hx/README.md +++ b/stix_shifter_modules/trellix_endpoint_security_hx/README.md @@ -655,7 +655,7 @@ transmit trellix_endpoint_security_hx "{\"host\":\"1.2.3.4\",\"port\":123,\"selfSignedCert\":\"cert\",\"options\":{\"host_sets\":\"host_set1,host_set2\"}}" "{\"auth\":{\"username\":\"xxx\",\"password\": \"yyyy\"}}" -results {search id} +results {search id} offset length #### STIX Translate Results @@ -683,15 +683,16 @@ results less than or equal to the data source. - A maximum of 15 searches only can be created in data source. In order to create more searches, the existing search ids created needs to be deleted. -- As per datasource limitation, If a host set containing more than 1000 hosts, the 1000 hosts that responds first - will be returned in response. Configure multiple host sets in the datasource to fetch records from more than 1000 hosts. +- As per the Trellix data source limitation, if a host set contains more than 1000 hosts, the 1000 hosts that respond + first will be returned in the response. To avoid this scenario, configure multiple host sets in the data source + to fetch the records from more than 1000 hosts. - Supported operators for IP address fields and File hash fields are =, !=, IN, NOT IN. - Supported operators for Directory, file:parent_directory_ref.path, process:parent_ref.cwd are LIKE, MATCHES, NOT LIKE, NOT MATCHES. - Supported operators for network traffic : extensions.'http-request-ext'.request_header fields are LIKE, MATCHES, NOT LIKE, NOT MATCHES. - LIKE/MATCHES operator supports only substring search. Wild card character search is not supported. ### References -- [Authentication | FireEye Developer Hub](https://fireeye.dev/docs/endpoint/authentication/) +- [Authentication | Developer Hub](https://fireeye.dev/docs/endpoint/authentication/) - [Search limits](https://docs.trellix.com/bundle/hx_5.3.0_ug/page/UUID-bb2cc194-22e0-4501-95d8-6a73458db012.html) - [API Documentation](https://fireeye.dev/apis/lighthouse/) - [Enterprise Search](https://docs.trellix.com/bundle/hx_5.3.0_ug/page/UUID-e81232c3-a871-c015-f191-9fbd431bdb59.html) \ No newline at end of file From a3e30f0a1bae7b8289789e9486122c7094163998 Mon Sep 17 00:00:00 2001 From: Sharmila-MS Date: Thu, 27 Jun 2024 08:48:38 +0000 Subject: [PATCH 4/4] Updated the code related to Exception Updated the code related to Exceeded limit exception. added the mapping of parent file name in to-stix files. --- .../stix_translation/json/stix_2_1/to_stix_map.json | 11 +++++++++++ .../stix_translation/json/to_stix_map.json | 11 +++++++++++ .../stix_transmission/query_connector.py | 8 ++++++++ .../trellix_endpoint_security_hx_supported_stix.md | 2 ++ 4 files changed, 32 insertions(+) diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/to_stix_map.json index 8d16d5247..3caadb4f8 100644 --- a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/stix_2_1/to_stix_map.json @@ -175,6 +175,17 @@ "object": "process", "references": "file" } + ], + "Parent File Name": [ + { + "key": "file.name", + "object": "p_file" + }, + { + "key": "process.binary_ref", + "object": "parent_process", + "references": "p_file" + } ], "Host Set": { "key": "x-oca-asset.x_host_set", diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/to_stix_map.json b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/to_stix_map.json index 7ecd77d1b..b30a3c007 100644 --- a/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_translation/json/to_stix_map.json @@ -180,6 +180,17 @@ "references": "file" } ], + "Parent File Name": [ + { + "key": "file.name", + "object": "p_file" + }, + { + "key": "process.binary_ref", + "object": "parent_process", + "references": "p_file" + } + ], "File MD5 Hash": { "key": "file.hashes.MD5", "object": "file" diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/query_connector.py b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/query_connector.py index 2dbe0cea3..51376f13f 100644 --- a/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/query_connector.py +++ b/stix_shifter_modules/trellix_endpoint_security_hx/stix_transmission/query_connector.py @@ -123,6 +123,14 @@ def handle_api_exception(self, code, response_data): message = response_data.get('message') except json.JSONDecodeError: message = response_data + + if 'exceeded limit on existing searches' in message.lower(): + if response_data.get('details', []) and response_data['details'][0]['details']['existing_search_limit']: + message = (f"The total search limit - {response_data['details'][0]['details']['existing_search_limit']}" + f" is reached. Delete an existing search to create a new one") + else: + message = f"The total search limit is reached. Delete an existing search to create a new one" + response_dict = {'code': code, 'message': message} if code else {'message': message} ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) return return_obj diff --git a/stix_shifter_modules/trellix_endpoint_security_hx/trellix_endpoint_security_hx_supported_stix.md b/stix_shifter_modules/trellix_endpoint_security_hx/trellix_endpoint_security_hx_supported_stix.md index d7838347e..dc2054d88 100644 --- a/stix_shifter_modules/trellix_endpoint_security_hx/trellix_endpoint_security_hx_supported_stix.md +++ b/stix_shifter_modules/trellix_endpoint_security_hx/trellix_endpoint_security_hx_supported_stix.md @@ -88,6 +88,7 @@ | process | parent_ref | Parent Process Path | | process | parent_ref | Parent Process Name | | process | binary_ref | File Name | +| process | binary_ref | Parent File Name | | process | creator_user_ref | Username | | process | x_event_type | Process Event Type | |
| | | @@ -95,6 +96,7 @@ |
| | | | file | name | File Name | | file | name | Write Event File Name | +| file | name | Parent File Name | | file | hashes.MD5 | File MD5 Hash | | file | hashes.MD5 | Write Event File MD5 Hash | | file | size | Size in bytes |