From 3ddcbe15dc3085c72235f9139793d442330a5d11 Mon Sep 17 00:00:00 2001 From: lakshmi51974368 <92726459+lakshmi51974368@users.noreply.github.com> Date: Fri, 18 Mar 2022 14:48:05 +0000 Subject: [PATCH 1/7] Added Palo Alto Cortext XDR UDI Connector Added Palo Alto Cortext XDR UDI Connector with updated fork. --- stix_shifter_modules/paloalto/README.md | 987 ++++++++++++++++++ stix_shifter_modules/paloalto/__init__.py | 0 .../paloalto/configuration/config.json | 41 + .../paloalto/configuration/lang_en.json | 36 + stix_shifter_modules/paloalto/entry_point.py | 14 + .../paloalto/stix_translation/__init__.py | 0 .../stix_translation/json/config_map.json | 65 ++ .../stix_translation/json/fields_map.json | 45 + .../stix_translation/json/operators.json | 15 + .../json/stix_2_1/to_stix_map.json | 820 +++++++++++++++ .../json/stix_2_1/xdr_data_from_stix_map.json | 287 +++++ .../stix_translation/json/to_stix_map.json | 838 +++++++++++++++ .../json/xdr_data_from_stix_map.json | 293 ++++++ .../stix_translation/query_constructor.py | 477 +++++++++ .../stix_translation/query_translator.py | 27 + .../stix_translation/results_translator.py | 5 + .../paloalto/stix_transmission/__init__.py | 0 .../paloalto/stix_transmission/api_client.py | 176 ++++ .../stix_transmission/delete_connector.py | 30 + .../stix_transmission/error_mapper.py | 44 + .../stix_transmission/ping_connector.py | 49 + .../stix_transmission/query_connector.py | 53 + .../stix_transmission/response_mapper.py | 104 ++ .../stix_transmission/results_connector.py | 248 +++++ .../stix_transmission/status_connector.py | 121 +++ .../test_paloalto_json_to_stix.py | 443 ++++++++ .../test_paloalto_stix_to_query.py | 560 ++++++++++ .../tests/stix_transmission/test_paloalto.py | 652 ++++++++++++ 28 files changed, 6430 insertions(+) create mode 100644 stix_shifter_modules/paloalto/README.md create mode 100644 stix_shifter_modules/paloalto/__init__.py create mode 100644 stix_shifter_modules/paloalto/configuration/config.json create mode 100644 stix_shifter_modules/paloalto/configuration/lang_en.json create mode 100644 stix_shifter_modules/paloalto/entry_point.py create mode 100644 stix_shifter_modules/paloalto/stix_translation/__init__.py create mode 100644 stix_shifter_modules/paloalto/stix_translation/json/config_map.json create mode 100644 stix_shifter_modules/paloalto/stix_translation/json/fields_map.json create mode 100644 stix_shifter_modules/paloalto/stix_translation/json/operators.json create mode 100644 stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json create mode 100644 stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json create mode 100644 stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json create mode 100644 stix_shifter_modules/paloalto/stix_translation/query_constructor.py create mode 100644 stix_shifter_modules/paloalto/stix_translation/query_translator.py create mode 100644 stix_shifter_modules/paloalto/stix_translation/results_translator.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/__init__.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/api_client.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/delete_connector.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/error_mapper.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/ping_connector.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/query_connector.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/response_mapper.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/results_connector.py create mode 100644 stix_shifter_modules/paloalto/stix_transmission/status_connector.py create mode 100644 stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py create mode 100644 stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py create mode 100644 stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py diff --git a/stix_shifter_modules/paloalto/README.md b/stix_shifter_modules/paloalto/README.md new file mode 100644 index 000000000..3389ab9fc --- /dev/null +++ b/stix_shifter_modules/paloalto/README.md @@ -0,0 +1,987 @@ +# PaloAlto Cortex XDR Connector + +**Table of Contents** + +- [PaloAlto Cortex XDR API Endpoints](#paloalto-cortex-xdr-api-endpoints) +- [Pattern expression with STIX attributes - Single Observation](#single-observation) +- [Pattern expression with STIX attributes - Multiple Observation](#multiple-observation) +- [Pattern expression with STIX attributes - Execute Query](#stix-execute-query) +- [Limitations](#limitations) +- [References](#references) + +### PaloAlto Cortex XDR API Endpoints + + |Connector Method|PaloAlto Cortex XDR API Endpoint| Method + | ---- | ------ | -----| + |Query Endpoint |https:///public_api/v1/xql/start_xql_query/|POST + |Result Endpoint|https:///public_api/v1/xql/get_query_results/|POST + |Quota Endpoint|https:///public_api/v1/xql/get_quota/|POST + +### Format for calling stix-shifter from the command line +``` +$ stix-shifter translate query "" "" "" +``` +### Pattern expression with STIX attributes + +### Single Observation + +#### STIX Translate query +```shell +translate paloalto query '{}' "[ipv4-addr:value = '1.1.0.0'] START t'2022-03-01T11:00:00.000Z' STOP t'2023-03-08T11:00:00.003Z'" +``` +#### STIX Translate query - output +```json +{ + "queries": [ + { + "xdr_data": { + "query": "dataset = xdr_data | filter ((action_local_ip = \"1.1.0.0\" or action_remote_ip = \"1.1.0.0\") and (to_epoch(_time,\"millis\") >= 1646132400000 and to_epoch(_time,\"millis\") <= 1678273200003)) | alter dataset_name = \"xdr_data\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 1000 ", + "timeframe": { + "from": 1646132400000, + "to": 1678273200003 + } + } + } + ] +} +``` +#### STIX Transmit ping + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +ping +``` + +#### STIX Transmit ping - output +```json +{ + "success": true +} +``` +#### STIX Transmit query + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +query +"{ \"xdr_data\": { \"query\": \"dataset = xdr_data | filter ((action_local_ip = \\"1.1.0.0\\" or action_remote_ip = \\"1.1.0.0\\") and (to_epoch(_time,\\"millis\\") >= 1646132400000 and to_epoch(_time,\\"millis\\") <= 1678273200003)) | alter dataset_name = \\"xdr_data\\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 1000 \", \"timeframe\": { \"from\": 1646132400000, \"to\": 1678273200003 } } }" +``` + +#### STIX Transmit query - output +```json +{ + "success": true, + "search_id": "d84862b9b7254a_22989_inv" +} +``` +#### STIX Transmit status + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +status +"d84862b9b7254a_22989_inv" +``` + +#### STIX Transmit status - output +```json +{ + "success": true, + "status": "COMPLETED" +} +``` +#### STIX Transmit results + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +results +"d84862b9b7254a_22989_inv" 0 2 +``` + +#### STIX Transmit results - output +```json +{ + "success": true, + "data": [ + { + "xdr_data": { + "action_network_protocol": "TCP", + "action_local_ip": "193.93.62.47", + "action_remote_ip": "1.1.0.0", + "action_local_port": 31029, + "action_remote_port": 3389, + "agent_id": "193.93.62.47", + "event_id": "MTM4NTk0MDcwNDA4NDQ0MzIwNw==", + "event_timestamp": 1646799589433, + "action_network_creation_time": 1646799589433, + "action_network_connection_id": "AdgzbOqYRXkAAAAAAB+exg==", + "action_proxy": "FALSE", + "event_type": "STORY", + "event_sub_type": "event_sub_type_4" + } + }, + { + "xdr_data": { + "action_network_protocol": "TCP", + "action_local_ip": "193.93.62.6", + "action_remote_ip": "1.1.0.0", + "action_local_port": 47275, + "action_remote_port": 3389, + "agent_id": "193.93.62.6", + "event_id": "Nzc2MzU5MjE1NjE4NTYzNDI4OA==", + "event_timestamp": 1646799556894, + "action_network_creation_time": 1646799556894, + "action_network_connection_id": "AdgzbNczNawAAAAAAB+euA==", + "action_proxy": "FALSE", + "event_type": "STORY", + "event_sub_type": "event_sub_type_4" + } + } + ] +} +``` + + +#### STIX Translate results + +```shell +translate +paloalto +results +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"events\"}" +" [ { \"xdr_data\": { \"action_network_protocol\": \"TCP\", \"action_local_ip\": \"193.93.62.47\", \"action_remote_ip\": \"1.1.0.0\", \"action_local_port\": 31029, \"action_remote_port\": 3389, \"agent_id\": \"193.93.62.47\", \"event_id\": \"MTM4NTk0MDcwNDA4NDQ0MzIwNw==\", \"event_timestamp\": 1646799589433, \"action_network_creation_time\": 1646799589433, \"action_network_connection_id\": \"AdgzbOqYRXkAAAAAAB+exg==\", \"action_proxy\": \"FALSE\", \"event_type\": \"STORY\", \"event_sub_type\": \"event_sub_type_4\" } }, { \"xdr_data\": { \"action_network_protocol\": \"TCP\", \"action_local_ip\": \"193.93.62.6\", \"action_remote_ip\": \"1.1.0.0\", \"action_local_port\": 47275, \"action_remote_port\": 3389, \"agent_id\": \"193.93.62.6\", \"event_id\": \"Nzc2MzU5MjE1NjE4NTYzNDI4OA==\", \"event_timestamp\": 1646799556894, \"action_network_creation_time\": 1646799556894, \"action_network_connection_id\": \"AdgzbNczNawAAAAAAB+euA==\", \"action_proxy\": \"FALSE\", \"event_type\": \"STORY\", \"event_sub_type\": \"event_sub_type_4\" } } ] " +"{\"stix_validator\": true}" +``` + +#### STIX Translate results - output +```json +{ + "type": "bundle", + "id": "bundle--c7d5cb7b-bfc2-4964-be1a-596a4eb37f72", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "paloalto", + "identity_class": "events" + }, + { + "id": "observed-data--875927c9-575f-4410-bdf6-9bc6060b204c", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-03-09T08:08:41.394Z", + "modified": "2022-03-09T08:08:41.394Z", + "objects": { + "0": { + "type": "network-traffic", + "protocols": [ + "tcp" + ], + "src_ref": "1", + "dst_ref": "2", + "src_port": 31029, + "dst_port": 3389, + "extensions": { + "x-paloalto-network": { + "creation_time": "2022-03-09T04:19:49.433Z", + "connection_id": "AdgzbOqYRXkAAAAAAB+exg==", + "is_proxy": false + } + } + }, + "1": { + "type": "ipv4-addr", + "value": "193.93.62.47" + }, + "2": { + "type": "ipv4-addr", + "value": "1.1.0.0" + }, + "3": { + "type": "x-oca-asset", + "asset_id": "193.93.62.47" + }, + "4": { + "type": "x-oca-event", + "event_id": "MTM4NTk0MDcwNDA4NDQ0MzIwNw==", + "time": "2022-03-09T04:19:49.433Z", + "event_type": "STORY", + "sub_type": "event_sub_type_4" + } + }, + "first_observed": "2022-03-09T08:08:41.394Z", + "last_observed": "2022-03-09T08:08:41.394Z", + "number_observed": 1 + }, + { + "id": "observed-data--3b1d6e6b-95ea-406f-aafb-aa5e4056640b", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-03-09T08:08:51.467Z", + "modified": "2022-03-09T08:08:51.467Z", + "objects": { + "0": { + "type": "network-traffic", + "protocols": [ + "tcp" + ], + "src_ref": "1", + "dst_ref": "2", + "src_port": 47275, + "dst_port": 3389, + "extensions": { + "x-paloalto-network": { + "creation_time": "2022-03-09T04:19:16.894Z", + "connection_id": "AdgzbNczNawAAAAAAB+euA==", + "is_proxy": false + } + } + }, + "1": { + "type": "ipv4-addr", + "value": "193.93.62.6" + }, + "2": { + "type": "ipv4-addr", + "value": "1.1.0.0" + }, + "3": { + "type": "x-oca-asset", + "asset_id": "193.93.62.6" + }, + "4": { + "type": "x-oca-event", + "event_id": "Nzc2MzU5MjE1NjE4NTYzNDI4OA==", + "time": "2022-03-09T04:19:16.894Z", + "event_type": "STORY", + "sub_type": "event_sub_type_4" + } + }, + "first_observed": "2022-03-09T08:08:51.467Z", + "last_observed": "2022-03-09T08:08:51.467Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +### Multiple Observation + +#### STIX Translate query +```shell +translate paloalto query {} "([x-oca-event:event_type = 'file'] AND [file:size > 3000] OR [x-oca-asset:agent_version = '7.6.1.46600'] )START t'2022-02-10T00:00:00.000000Z' STOP t'2022-02-15T00:00:00.000000Z'" +``` + +#### STIX Translate query - output + +```json +{ + "queries": [ + { + "xdr_data": { + "query": "dataset = xdr_data | filter (event_type = ENUM.FILE and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or (action_file_size > 3000 and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or (agent_version = \"7.6.1.46600\" and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) | alter dataset_name = \"xdr_data\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 1000 ", + "timeframe": { + "from": 1644451200000, + "to": 1644883200000 + } + } + } + ] +} +``` + + +#### STIX Transmit query + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +query +" { \"xdr_data\": { \"query\": \"dataset = xdr_data | filter (event_type = ENUM.FILE and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) or (action_file_size > 3000 and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) or (agent_version = \\"7.6.1.46600\\" and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) | alter dataset_name = \\"xdr_data\\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 1000 \", \"timeframe\": { \"from\": 1644451200000, \"to\": 1644883200000 } } }" +``` + +#### STIX Transmit query - output + +```json +{ + "success": true, + "search_id": "8c1c048e556c4b_23040_inv" +} +``` +#### STIX Transmit status + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +status +"8c1c048e556c4b_23040_inv" +``` + +#### STIX Transmit status - output +```json +{ + "success": true, + "status": "COMPLETED" +} +``` +#### STIX Transmit results + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +results +"8c1c048e556c4b_23040_inv" 0 2 +``` + +#### STIX Transmit results - output +```json +{ + "success": true, + "data": [ + { + "xdr_data": { + "action_file_name": "MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "actor_process_image_name": "System", + "causality_actor_process_image_name": "System", + "os_actor_process_image_name": "System", + "action_file_path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Support\\MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "actor_process_image_path": "System", + "causality_actor_process_image_path": "System", + "os_actor_process_image_path": "System", + "actor_process_os_pid": 4, + "causality_actor_process_os_pid": 4, + "os_actor_process_os_pid": 4, + "actor_primary_user_sid": "S-1-5-18", + "actor_primary_username": "NT AUTHORITY\\SYSTEM", + "actor_process_logon_id": "1003", + "action_file_extension": "bin", + "action_file_type": 18, + "manifest_file_version": 5, + "actor_process_instance_id": "AdgdeB26mNQAAAAEAAAAAA==", + "actor_process_causality_id": "AdgdeB26mNQAAAAEAAAAAA==", + "actor_process_auth_id": "999", + "actor_process_signature_vendor": "Microsoft Corporation", + "actor_process_signature_product": "Microsoft Windows", + "actor_process_execution_time": 1644385473948, + "agent_version": "7.6.1.46600", + "agent_hostname": "EC2AMAZ-IQFSLIL", + "agent_content_version": "380-82571", + "agent_session_start_time": 1644385496543, + "agent_id": "37a92aad549d41d184ec9fbdafbff55c", + "agent_os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "event_id": "AAABfvB4P2Qqn9UKABvBDw==", + "event_timestamp": 1644711919520, + "event_version": 25, + "actor_process_is_native": "TRUE", + "agent_is_vdi": "FALSE", + "agent_os_type": "AGENT_OS_WINDOWS", + "event_type": "FILE", + "event_sub_type": "FILE_WRITE", + "actor_process_signature_status": "SIGNED" + } + }, + { + "xdr_data": { + "action_file_name": "MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "actor_process_image_name": "System", + "causality_actor_process_image_name": "System", + "os_actor_process_image_name": "System", + "action_file_path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Support\\MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "actor_process_image_path": "System", + "causality_actor_process_image_path": "System", + "os_actor_process_image_path": "System", + "actor_process_os_pid": 4, + "causality_actor_process_os_pid": 4, + "os_actor_process_os_pid": 4, + "actor_primary_user_sid": "S-1-5-18", + "actor_primary_username": "NT AUTHORITY\\SYSTEM", + "actor_process_logon_id": "1003", + "action_file_extension": "bin", + "action_file_type": 18, + "manifest_file_version": 5, + "actor_process_instance_id": "AdgdeB26mNQAAAAEAAAAAA==", + "actor_process_causality_id": "AdgdeB26mNQAAAAEAAAAAA==", + "actor_process_auth_id": "999", + "actor_process_signature_vendor": "Microsoft Corporation", + "actor_process_signature_product": "Microsoft Windows", + "actor_process_execution_time": 1644385473948, + "agent_version": "7.6.1.46600", + "agent_hostname": "EC2AMAZ-IQFSLIL", + "agent_content_version": "380-82571", + "agent_session_start_time": 1644385496543, + "agent_id": "37a92aad549d41d184ec9fbdafbff55c", + "agent_os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "event_id": "AAABfvB7dS4qn9UKABvHTA==", + "event_timestamp": 1644712129898, + "event_version": 25, + "actor_process_is_native": "TRUE", + "agent_is_vdi": "FALSE", + "agent_os_type": "AGENT_OS_WINDOWS", + "event_type": "FILE", + "event_sub_type": "FILE_WRITE", + "actor_process_signature_status": "SIGNED" + } + } + ] +} +``` + + +#### STIX Translate results + +```shell +translate +paloalto +results +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"events\"}" +" [ { \"xdr_data\": { \"action_file_name\": \"MpWppTracing-20220210-055530-00000003-ffffffff.bin\", \"actor_process_image_name\": \"System\", \"causality_actor_process_image_name\": \"System\", \"os_actor_process_image_name\": \"System\", \"action_file_path\": \"C:\\ProgramData\\Microsoft\\Windows Defender\\Support\\MpWppTracing-20220210-055530-00000003-ffffffff.bin\", \"actor_process_image_path\": \"System\", \"causality_actor_process_image_path\": \"System\", \"os_actor_process_image_path\": \"System\", \"actor_process_os_pid\": 4, \"causality_actor_process_os_pid\": 4, \"os_actor_process_os_pid\": 4, \"actor_primary_user_sid\": \"S-1-5-18\", \"actor_primary_username\": \"NT AUTHORITY\\SYSTEM\", \"actor_process_logon_id\": \"1003\", \"action_file_extension\": \"bin\", \"action_file_type\": 18, \"manifest_file_version\": 5, \"actor_process_instance_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_causality_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_auth_id\": \"999\", \"actor_process_signature_vendor\": \"Microsoft Corporation\", \"actor_process_signature_product\": \"Microsoft Windows\", \"actor_process_execution_time\": 1644385473948, \"agent_version\": \"7.6.1.46600\", \"agent_hostname\": \"EC2AMAZ-IQFSLIL\", \"agent_content_version\": \"380-82571\", \"agent_session_start_time\": 1644385496543, \"agent_id\": \"37a92aad549d41d184ec9fbdafbff55c\", \"agent_os_sub_type\": \"Windows Server 2016 [10.0 (Build 17763)]\", \"event_id\": \"AAABfvB4P2Qqn9UKABvBDw==\", \"event_timestamp\": 1644711919520, \"event_version\": 25, \"actor_process_is_native\": \"TRUE\", \"agent_is_vdi\": \"FALSE\", \"agent_os_type\": \"AGENT_OS_WINDOWS\", \"event_type\": \"FILE\", \"event_sub_type\": \"FILE_WRITE\", \"actor_process_signature_status\": \"SIGNED\" } }, { \"xdr_data\": { \"action_file_name\": \"MpWppTracing-20220210-055530-00000003-ffffffff.bin\", \"actor_process_image_name\": \"System\", \"causality_actor_process_image_name\": \"System\", \"os_actor_process_image_name\": \"System\", \"action_file_path\": \"C:\\ProgramData\\Microsoft\\Windows Defender\\Support\\MpWppTracing-20220210-055530-00000003-ffffffff.bin\", \"actor_process_image_path\": \"System\", \"causality_actor_process_image_path\": \"System\", \"os_actor_process_image_path\": \"System\", \"actor_process_os_pid\": 4, \"causality_actor_process_os_pid\": 4, \"os_actor_process_os_pid\": 4, \"actor_primary_user_sid\": \"S-1-5-18\", \"actor_primary_username\": \"NT AUTHORITY\\SYSTEM\", \"actor_process_logon_id\": \"1003\", \"action_file_extension\": \"bin\", \"action_file_type\": 18, \"manifest_file_version\": 5, \"actor_process_instance_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_causality_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_auth_id\": \"999\", \"actor_process_signature_vendor\": \"Microsoft Corporation\", \"actor_process_signature_product\": \"Microsoft Windows\", \"actor_process_execution_time\": 1644385473948, \"agent_version\": \"7.6.1.46600\", \"agent_hostname\": \"EC2AMAZ-IQFSLIL\", \"agent_content_version\": \"380-82571\", \"agent_session_start_time\": 1644385496543, \"agent_id\": \"37a92aad549d41d184ec9fbdafbff55c\", \"agent_os_sub_type\": \"Windows Server 2016 [10.0 (Build 17763)]\", \"event_id\": \"AAABfvB7dS4qn9UKABvHTA==\", \"event_timestamp\": 1644712129898, \"event_version\": 25, \"actor_process_is_native\": \"TRUE\", \"agent_is_vdi\": \"FALSE\", \"agent_os_type\": \"AGENT_OS_WINDOWS\", \"event_type\": \"FILE\", \"event_sub_type\": \"FILE_WRITE\", \"actor_process_signature_status\": \"SIGNED\" } } ] " +"{\"stix_validator\": true}" +``` + +#### STIX Translate results - output +```json +{ + "type": "bundle", + "id": "bundle--a1d2533a-11e1-4ee8-9c96-4c298f87a823", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "paloalto", + "identity_class": "events" + }, + { + "id": "observed-data--734f2fdb-7777-4a8f-8fb1-c08dceec94fa", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-03-09T08:56:29.407Z", + "modified": "2022-03-09T08:56:29.407Z", + "objects": { + "0": { + "type": "file", + "name": "MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "parent_directory_ref": "9", + "extensions": { + "x-paloalto-file": { + "extension": "bin", + "type": 18, + "manifest_version": 5 + } + } + }, + "1": { + "type": "file", + "name": "System" + }, + "2": { + "type": "process", + "name": "System", + "binary_ref": "1", + "pid": 4, + "extensions": { + "x-paloalto-process": { + "instance_id": "AdgdeB26mNQAAAAEAAAAAA==", + "causality_id": "AdgdeB26mNQAAAAEAAAAAA==", + "auth_id": "999", + "signature_vendor": "Microsoft Corporation", + "signature_product": "Microsoft Windows", + "execution_time": "2022-02-09T05:44:33.948Z", + "is_native": true, + "signature_status": "SIGNED" + } + } + }, + "3": { + "type": "file", + "name": "System" + }, + "4": { + "type": "process", + "name": "System" + }, + "5": { + "type": "process", + "parent_ref": "4", + "binary_ref": "3", + "pid": 4 + }, + "6": { + "type": "file", + "name": "System" + }, + "7": { + "type": "process", + "name": "System" + }, + "8": { + "type": "process", + "parent_ref": "7", + "binary_ref": "6", + "pid": 4 + }, + "9": { + "type": "directory", + "path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Support" + }, + "10": { + "type": "user-account", + "user_id": "S-1-5-18", + "display_name": "NT AUTHORITY\\SYSTEM", + "account_login": "1003" + }, + "11": { + "type": "x-oca-asset", + "agent_version": "7.6.1.46600", + "hostname": "EC2AMAZ-IQFSLIL", + "content_version": "380-82571", + "start_time": "2022-02-09T05:44:56.543Z", + "asset_id": "37a92aad549d41d184ec9fbdafbff55c", + "os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "is_vdi": false, + "os_type": "AGENT_OS_WINDOWS" + }, + "12": { + "type": "x-oca-event", + "event_id": "AAABfvB4P2Qqn9UKABvBDw==", + "time": "2022-02-13T00:25:19.520Z", + "version": 25, + "event_type": "FILE", + "sub_type": "FILE_WRITE" + } + }, + "first_observed": "2022-03-09T08:56:29.407Z", + "last_observed": "2022-03-09T08:56:29.407Z", + "number_observed": 1 + }, + { + "id": "observed-data--65f9e698-ecd2-41f9-b370-cf9dae876eeb", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-03-09T08:56:29.746Z", + "modified": "2022-03-09T08:56:29.746Z", + "objects": { + "0": { + "type": "file", + "name": "MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "parent_directory_ref": "9", + "extensions": { + "x-paloalto-file": { + "extension": "bin", + "type": 18, + "manifest_version": 5 + } + } + }, + "1": { + "type": "file", + "name": "System" + }, + "2": { + "type": "process", + "name": "System", + "binary_ref": "1", + "pid": 4, + "extensions": { + "x-paloalto-process": { + "instance_id": "AdgdeB26mNQAAAAEAAAAAA==", + "causality_id": "AdgdeB26mNQAAAAEAAAAAA==", + "auth_id": "999", + "signature_vendor": "Microsoft Corporation", + "signature_product": "Microsoft Windows", + "execution_time": "2022-02-09T05:44:33.948Z", + "is_native": true, + "signature_status": "SIGNED" + } + } + }, + "3": { + "type": "file", + "name": "System" + }, + "4": { + "type": "process", + "name": "System" + }, + "5": { + "type": "process", + "parent_ref": "4", + "binary_ref": "3", + "pid": 4 + }, + "6": { + "type": "file", + "name": "System" + }, + "7": { + "type": "process", + "name": "System" + }, + "8": { + "type": "process", + "parent_ref": "7", + "binary_ref": "6", + "pid": 4 + }, + "9": { + "type": "directory", + "path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Support" + }, + "10": { + "type": "user-account", + "user_id": "S-1-5-18", + "display_name": "NT AUTHORITY\\SYSTEM", + "account_login": "1003" + }, + "11": { + "type": "x-oca-asset", + "agent_version": "7.6.1.46600", + "hostname": "EC2AMAZ-IQFSLIL", + "content_version": "380-82571", + "start_time": "2022-02-09T05:44:56.543Z", + "asset_id": "37a92aad549d41d184ec9fbdafbff55c", + "os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "is_vdi": false, + "os_type": "AGENT_OS_WINDOWS" + }, + "12": { + "type": "x-oca-event", + "event_id": "AAABfvB7dS4qn9UKABvHTA==", + "time": "2022-02-13T00:28:49.898Z", + "version": 25, + "event_type": "FILE", + "sub_type": "FILE_WRITE" + } + }, + "first_observed": "2022-03-09T08:56:29.746Z", + "last_observed": "2022-03-09T08:56:29.746Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` +#### STIX Execute query +```shell +execute +paloalto +paloalto +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class \":\"events\"}" +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +"[file:name = 'chrome.exe' ] START t'2022-01-16T11:00:00.000Z' STOP t'2022-01-20T11:00:00.003Z'" +``` +#### STIX Execute query - output +```json +{ + "type": "bundle", + "id": "bundle--899d573b-3fcb-4561-bedd-6038fd4da192", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "paloalto", + "identity_class ": "events" + }, + { + "id": "observed-data--59220f58-f8ee-4d2b-a2df-367958e132b7", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-03-09T09:04:32.272Z", + "modified": "2022-03-09T09:04:32.272Z", + "objects": { + "0": { + "type": "file", + "name": "LICENSE", + "parent_directory_ref": "9", + "extensions": { + "x-paloalto-file": { + "type": 18, + "manifest_version": 5 + } + } + }, + "1": { + "type": "file", + "name": "chrome.exe", + "accessed": "2021-12-15T22:05:27.664Z", + "modified": "2021-12-12T08:19:30.068Z", + "parent_directory_ref": "10" + }, + "2": { + "type": "process", + "name": "chrome.exe", + "binary_ref": "1", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4572, + "extensions": { + "x-paloalto-process": { + "instance_id": "AdgAs0qzIE0AABHcAAAAAA==", + "causality_id": "AdgAs0qzIE0AABHcAAAAAA==", + "auth_id": "999", + "signature_vendor": "Google LLC", + "signature_product": "Google LLC", + "extension": "exe", + "execution_time": "2022-01-03T15:05:05.983Z", + "is_native": true, + "signature_status": "SIGNED" + } + } + }, + "3": { + "type": "file", + "name": "chrome.exe", + "parent_directory_ref": "11" + }, + "4": { + "type": "process", + "name": "chrome.exe" + }, + "5": { + "type": "process", + "parent_ref": "4", + "binary_ref": "3", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4572 + }, + "6": { + "type": "file", + "name": "chrome.exe", + "accessed": "2021-12-15T22:05:27.664Z", + "modified": "2021-12-12T08:19:30.068Z", + "parent_directory_ref": "12" + }, + "7": { + "type": "process", + "name": "chrome.exe" + }, + "8": { + "type": "process", + "parent_ref": "7", + "binary_ref": "6", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4572 + }, + "9": { + "type": "directory", + "path": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\2\\4572_137263248" + }, + "10": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "11": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "12": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "13": { + "type": "user-account", + "user_id": "S-1-5-21-3039464837-300237904-2407637926-500", + "display_name": "EC2AMAZ-65BN1IK\\Administrator", + "account_login": "1003" + }, + "14": { + "type": "x-oca-asset", + "agent_version": "7.6.1.46600", + "hostname": "EC2AMAZ-65BN1IK", + "content_version": "350-80787", + "start_time": "2022-01-19T10:02:28.366Z", + "asset_id": "f344796340f84d0ca7e0fdaedbcbd594", + "os_sub_type": "Windows Server 2016 [10.0 (Build 14393)]", + "is_vdi": false, + "os_type": "AGENT_OS_WINDOWS" + }, + "15": { + "type": "x-oca-event", + "event_id": "AAABfnQpgUfunpI7AANI6Q==", + "time": "2022-01-19T21:06:24.409Z", + "version": 25, + "event_type": "FILE", + "sub_type": "FILE_WRITE" + } + }, + "x_actor_process_last_observed": "2021-12-15T22:05:27.664Z", + "x_os_actor_process_last_observed": "2021-12-15T22:05:27.664Z", + "x_process_actor_first_observed": "2021-11-10T06:59:32.948Z", + "x_process_causality_actor_first_observed": "2021-11-10T06:59:32.948Z", + "x_process_os_actor_first_observed": "2021-11-10T06:59:32.948Z", + "first_observed": "2022-03-09T09:04:32.272Z", + "last_observed": "2022-03-09T09:04:32.272Z", + "number_observed": 1 + }, + { + "id": "observed-data--1bd94f55-3b0d-4057-b281-f8c17cdc6cfa", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-03-09T09:04:32.272Z", + "modified": "2022-03-09T09:04:32.272Z", + "objects": { + "0": { + "type": "file", + "name": "22c34728-a90d-4fe4-8774-21563d2be0f3.tmp", + "size": 117576, + "parent_directory_ref": "9", + "extensions": { + "x-paloalto-file": { + "extension": "tmp", + "writer": "AdgAs0qzIE0AABHcAAAAAA==", + "type": 0, + "manifest_version": 5 + } + } + }, + "1": { + "type": "file", + "name": "chrome.exe", + "accessed": "2021-12-15T22:05:27.664Z", + "modified": "2021-12-12T08:19:30.068Z", + "parent_directory_ref": "10" + }, + "2": { + "type": "process", + "name": "chrome.exe", + "binary_ref": "1", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4572, + "extensions": { + "x-paloalto-process": { + "instance_id": "AdgAs0qzIE0AABHcAAAAAA==", + "causality_id": "AdgAs0qzIE0AABHcAAAAAA==", + "auth_id": "999", + "signature_vendor": "Google LLC", + "signature_product": "Google LLC", + "extension": "exe", + "execution_time": "2022-01-03T15:05:05.983Z", + "is_native": true, + "signature_status": "SIGNED" + } + } + }, + "3": { + "type": "file", + "name": "chrome.exe", + "parent_directory_ref": "11" + }, + "4": { + "type": "process", + "name": "chrome.exe" + }, + "5": { + "type": "process", + "parent_ref": "4", + "binary_ref": "3", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4572 + }, + "6": { + "type": "file", + "name": "chrome.exe", + "accessed": "2021-12-15T22:05:27.664Z", + "modified": "2021-12-12T08:19:30.068Z", + "parent_directory_ref": "12" + }, + "7": { + "type": "process", + "name": "chrome.exe" + }, + "8": { + "type": "process", + "parent_ref": "7", + "binary_ref": "6", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4572 + }, + "9": { + "type": "directory", + "path": "C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data" + }, + "10": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "11": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "12": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "13": { + "type": "user-account", + "user_id": "S-1-5-21-3039464837-300237904-2407637926-500", + "display_name": "EC2AMAZ-65BN1IK\\Administrator", + "account_login": "1003" + }, + "14": { + "type": "x-oca-asset", + "agent_version": "7.6.1.46600", + "hostname": "EC2AMAZ-65BN1IK", + "content_version": "350-80787", + "start_time": "2022-01-19T10:02:28.366Z", + "asset_id": "f344796340f84d0ca7e0fdaedbcbd594", + "os_sub_type": "Windows Server 2016 [10.0 (Build 14393)]", + "is_vdi": false, + "os_type": "AGENT_OS_WINDOWS" + }, + "15": { + "type": "x-oca-event", + "event_id": "AAABfnRfbAPunpI7AANyKA==", + "time": "2022-01-19T22:05:17.974Z", + "version": 25, + "event_type": "FILE", + "sub_type": "FILE_WRITE" + } + }, + "x_actor_process_last_observed": "2021-12-15T22:05:27.664Z", + "x_os_actor_process_last_observed": "2021-12-15T22:05:27.664Z", + "x_process_actor_first_observed": "2021-11-10T06:59:32.948Z", + "x_process_causality_actor_first_observed": "2021-11-10T06:59:32.948Z", + "x_process_os_actor_first_observed": "2021-11-10T06:59:32.948Z", + "first_observed": "2022-03-09T09:04:32.272Z", + "last_observed": "2022-03-09T09:04:32.272Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +### Limitations +- Each XQL API query entails a cost of query units calculated according to the complexity and number of search results. +- Queries called without enough quota will fail. + +### Observations +- The Maximum result limit of the connector is 1000, as the connector doesnt support the stream data feature which returns more than 1000 results. + +### References +- [Cortex XDR](https://docs.paloaltonetworks.com/cortex/cortex-xdr.html) +- [Get Started with XQL](https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-xql-language-reference/get-started-with-xql.html) +- [Cortex XDR™ XQL Schema Reference](https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-xql-schema-reference.html) +- [Cortex XDR API Overview](https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-api/cortex-xdr-api-overview.html) diff --git a/stix_shifter_modules/paloalto/__init__.py b/stix_shifter_modules/paloalto/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/paloalto/configuration/config.json b/stix_shifter_modules/paloalto/configuration/config.json new file mode 100644 index 000000000..692db5e17 --- /dev/null +++ b/stix_shifter_modules/paloalto/configuration/config.json @@ -0,0 +1,41 @@ +{ + "connection": { + "type": { + "displayName": "Palo Alto Cortex XDR" + }, + "host": { + "type": "text" + }, + "help": { + "type": "link", + "default": "data-sources.html" + }, + "options": { + "type": "fields", + "result_limit": { + "default": 1000, + "min": 1, + "max": 1000, + "type": "number" + } + }, + "quota_threshold": { + "default": 3.00, + "type": "float" + } + }, + "configuration": { + "auth": { + "type" : "fields", + "tenant": { + "type": "password" + }, + "api_key": { + "type": "password" + }, + "api_key_id": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/configuration/lang_en.json b/stix_shifter_modules/paloalto/configuration/lang_en.json new file mode 100644 index 000000000..a5855e75e --- /dev/null +++ b/stix_shifter_modules/paloalto/configuration/lang_en.json @@ -0,0 +1,36 @@ +{ + "connection": { + "host": { + "label": "Management IP address or Hostname", + "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" + }, + "help": { + "label": "Need additional help?", + "description": "More details on the data source setting can be found in the specified link" + }, + "result_limit": { + "label": "The Result limit for the connector", + "description": "The total number of results returned from the connector" + }, + "quota_threshold": { + "label": "The quota limit for the API", + "description": "Prevents the connector from exceeding the quota threshold limit" +} + }, + "configuration": { + "auth": { + "tenant": { + "label": "Tenant", + "description": "Tenant Id of Palo Alto Cortex XDR Application" + }, + "api_key": { + "label": "API Key", + "description": "The API Key is a unique identifier required for authenticating API calls." + }, + "api_key_id": { + "label": "API Key Id", + "description": "The API Key ID is a unique token used to authenticate the API Key" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/entry_point.py b/stix_shifter_modules/paloalto/entry_point.py new file mode 100644 index 000000000..8c8fedd99 --- /dev/null +++ b/stix_shifter_modules/paloalto/entry_point.py @@ -0,0 +1,14 @@ +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint + + +class EntryPoint(BaseEntryPoint): + + # python main.py translate paloalto query '{}' "[ipv4-addr:value = '127.0.0.1']" + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + self.set_async(True) + if connection: + self.setup_transmission_simple(connection, configuration) + + self.setup_translation_simple(dialect_default='xdr_data') diff --git a/stix_shifter_modules/paloalto/stix_translation/__init__.py b/stix_shifter_modules/paloalto/stix_translation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/paloalto/stix_translation/json/config_map.json b/stix_shifter_modules/paloalto/stix_translation/json/config_map.json new file mode 100644 index 000000000..0e87bc6f8 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/config_map.json @@ -0,0 +1,65 @@ +{ + "int_supported_fields": [ + "action_local_port", + "action_remote_port", + "action_pkts_sent", + "action_pkts_received", + "action_file_size", + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid", + "action_process_requested_parent_pid", + "action_thread_parent_pid", + "action_thread_child_pid" + ], + "timestamp_supported_fields": [ + "action_file_access_time", + "actor_process_file_access_time", + "os_actor_process_file_access_time", + "action_file_mod_time", + "actor_process_file_mod_time", + "os_actor_process_file_mod_time", + "action_file_create_time", + "action_process_file_create_time", + "actor_process_file_create_time", + "causality_actor_process_file_create_time", + "os_actor_process_file_create_time", + "action_process_instance_execution_time", + "actor_process_execution_time", + "action_network_creation_time" + ], + "mac_supported_fields": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" + ], + "enum_supported_fields": [ + "action_network_protocol", + "agent_os_type", + "event_type" + ], + "enum_supported_values": { + "action_network_protocol": ["TCP", "UDP"], + "agent_os_type": ["AGENT_OS_LINUX", "AGENT_OS_MAC", "AGENT_OS_WINDOWS"], + "event_type": [ + "AGENT_STATUS","CLOUD_AUDIT_LOGS","DEVICE","EVENT_LOG","FILE","GAP","HOST_FIREWALL", + "HOST_METADATA","HOST_STATUS","INJECTION","LOAD_IMAGE","LOGIN_EVENT","MOUNT","NAMESPACE", + "NETWORK","PROCESS","PROCESS_HANDLE","REGISTRY","RPC_CALL","STORY","SYSTEM_CALL", + "THREAD","USER_STATUS_CHANGE","VPN_EVENT"] + }, + "timestamp_supported_dataset": { + "xdr_data": "_time" + }, + "mandatory_properties_to_stix": { + "user": ["actor_primary_user_sid","action_process_user_sid"], + "file": ["action_file_name"], + "file_action_process" : ["action_process_image_name"], + "file_actor_process": ["actor_process_image_name"], + "file_causality_process": ["causality_actor_process_image_name"], + "file_os_actor": ["os_actor_process_image_name"], + "nt": [["action_local_ip","action_remote_ip"],"action_network_protocol"] + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json b/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json new file mode 100644 index 000000000..6e43b80e1 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json @@ -0,0 +1,45 @@ +{ + "all_fields": [ + "action_local_ip","action_remote_ip","agent_ip_addresses_v6","dst_agent_ip_addresses_v6", + "action_local_port","action_remote_port","action_network_protocol","action_pkts_sent", + "action_pkts_received","action_file_name","action_process_image_name","actor_process_image_name", + "causality_actor_process_image_name","os_actor_process_image_name","action_file_size","action_file_md5", + "action_module_md5","action_process_image_md5","action_file_authenticode_sha1", + "action_file_authenticode_sha2","action_file_sha256","action_module_sha256", + "action_process_image_sha256","action_file_access_time","actor_process_file_access_time", + "os_actor_process_file_access_time","action_file_mod_time","actor_process_file_mod_time", + "os_actor_process_file_mod_time","action_file_create_time","action_file_path", + "action_process_image_path","action_registry_file_path","actor_process_image_path", + "causality_actor_process_image_path","os_actor_process_image_path","action_process_image_command_line", + "actor_process_command_line","causality_actor_process_command_line","os_actor_process_command_line", + "action_process_file_create_time","actor_process_file_create_time", + "causality_actor_process_file_create_time","os_actor_process_file_create_time", + "action_module_process_os_pid","action_process_os_pid","actor_process_os_pid", + "causality_actor_process_os_pid","os_actor_process_os_pid","action_process_requested_parent_pid", + "action_thread_parent_pid","action_thread_child_pid","action_process_username","auth_domain", + "dst_host_metadata_domain","host_metadata_domain","dst_action_url_category","action_registry_key_name", + "action_registry_value_name","mac","associated_mac","dst_associated_mac","dst_mac", + "actor_primary_user_sid","action_process_user_sid","actor_primary_username","actor_process_logon_id", + "action_file_info_company","action_file_extension","action_file_attributes", + "action_file_internal_zipped_files","action_file_last_writer_actor","action_file_signature_status", + "action_file_signature_vendor","action_file_signature_product","action_file_info_description", + "action_file_group","action_file_group_name","action_file_type","action_file_info_file_version", + "manifest_file_version","action_file_info_product_version","action_file_owner", + "action_file_owner_name","action_file_info_product_name","action_file_id", + "action_file_wildfire_verdict","action_file_hash_control_verdict","actor_process_instance_id", + "actor_process_causality_id","actor_process_auth_id","actor_process_container_id", + "actor_process_signature_vendor","actor_process_signature_status","actor_process_signature_product", + "actor_process_image_extension","action_process_termination_code","action_process_termination_date", + "action_remote_process_thread_id","action_process_instance_execution_time", + "actor_process_execution_time","action_process_handle_is_kernel","action_process_is_container_root", + "actor_process_is_native","agent_version","agent_hostname","agent_content_version", + "agent_session_start_time","agent_id","agent_os_type","agent_os_sub_type", + "agent_is_vdi","action_user_agent","http_req_user_agent_header","action_evtlog_data_fields", + "action_evtlog_description","action_evtlog_source","action_evtlog_event_id","action_evtlog_level", + "action_evtlog_tid","action_evtlog_uid","action_evtlog_pid","action_evtlog_message", + "action_evtlog_version","event_id","vpn_event_description","event_timestamp","event_version", + "event_rpc_interface_uuid","event_address_mapped_image_path","event_type","event_sub_type", + "action_network_creation_time","action_network_connection_id","action_network_packet_data", + "action_proxy","host_metadata_hostname","action_external_hostname" + ] +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/operators.json b/stix_shifter_modules/paloalto/stix_translation/json/operators.json new file mode 100644 index 000000000..c4048192e --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/operators.json @@ -0,0 +1,15 @@ +{ + "ComparisonExpressionOperators.And": "and", + "ComparisonExpressionOperators.Or": "or", + "ComparisonComparators.Equal": "=", + "ComparisonComparators.NotEqual": "!=", + "ComparisonComparators.Like": "contains", + "ComparisonComparators.Matches": "~=", + "ComparisonComparators.GreaterThan": ">", + "ComparisonComparators.GreaterThanOrEqual": ">=", + "ComparisonComparators.LessThan": "<", + "ComparisonComparators.LessThanOrEqual": "<=", + "ComparisonComparators.In": "in", + "ObservationOperators.Or": "or", + "ObservationOperators.And": "or" + } \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..16d6a3d69 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,820 @@ +{ + "xdr_data": { + "action_local_ip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], + "action_remote_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "agent_ip_addresses_v6": { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + "dst_agent_ip_addresses_v6": { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + "action_local_port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "action_remote_port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "action_network_protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + "action_pkts_sent": { + "key": "network-traffic.src_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "action_pkts_received": { + "key": "network-traffic.dst_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "action_file_name": { + "key": "file.name", + "object": "file" + }, + "action_process_image_name": [ + { + "key": "file.name", + "object": "file_action_process" + }, + { + "key": "process.image_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "actor_process_image_name": [ + { + "key": "file.name", + "object": "file_actor_process" + }, + { + "key": "process.image_ref", + "object": "process_actor", + "references": "file_actor_process" + } + ], + "causality_actor_process_image_name": [ + { + "key": "file.name", + "object": "file_causality_process" + }, + { + "key": "process.parent_ref", + "object": "process_causality_actor" + } + ], + "os_actor_process_image_name": [ + { + "key": "file.name", + "object": "file_os_actor" + }, + { + "key": "process.parent_ref", + "object": "process_os_actor" + } + ], + "action_file_size": { + "key": "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_authenticode_sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "action_file_authenticode_sha2": { + "key": "file.hashes.SHA-2", + "object": "file" + }, + "action_file_md5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "action_module_md5": { + "key": "file.hashes.MD5", + "object": "file_action_module" + }, + "action_process_image_md5": [ + { + "key": "file.hashes.MD5", + "object": "file_action_process" + }, + { + "key": "process.image_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "action_file_sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "action_module_sha256": { + "key": "file.hashes.SHA-256", + "object": "file_action_module" + }, + "action_process_image_sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "file_action_process" + }, + { + "key": "process.image_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "action_file_access_time": [ + { + "key": "file.atime", + "object": "file", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_action_file_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_access_time": [ + { + "key": "file.atime", + "object": "file_actor_process", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_actor_process_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_access_time": [ + { + "key": "file.atime", + "object": "file_os_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_os_actor_process_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_file_mod_time": [ + { + "key": "file.mtime", + "object": "file", + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_mod_time": [ + { + "key": "file.mtime", + "object": "file_actor_process", + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_mod_time": [ + { + "key": "file.mtime", + "object": "file_os_actor", + "transformer": "EpochToTimestamp" + } + ], + "action_file_create_time": [ + { + "key": "file.ctime", + "object": "file", + "transformer": "EpochToTimestamp" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_file_path": [ + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "action_registry_file_path": [ + { + "key": "directory.path", + "object": "directory_action_registry", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_action_registry", + "references": "directory_action_registry" + } + ], + "action_process_image_path": [ + { + "key": "directory.path", + "object": "directory_action_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_action_process", + "references": "directory_action_process" + }, + { + "key": "process.image_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_actor_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_actor_process", + "references": "directory_actor_process" + }, + { + "key": "process.image_ref", + "object": "process_actor", + "references": "file_actor_process" + } + ], + "causality_actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_causality_actor", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_causality_process", + "references": "directory_causality_actor" + }, + { + "key": "process.image_ref", + "object": "process_causality_actor", + "references": "file_causality_process" + } + ], + "os_actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_os_actor", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_os_actor", + "references": "directory_os_actor" + }, + { + "key": "process.image_ref", + "object": "process_os_actor", + "references": "file_os_actor" + } + ], + "action_process_image_command_line": { + "key": "process.command_line", + "object": "process_action" + }, + "actor_process_command_line": { + "key": "process.command_line", + "object": "process_actor" + }, + "causality_actor_process_command_line": { + "key": "process.command_line", + "object": "process_causality_actor" + }, + "os_actor_process_command_line": { + "key": "process.command_line", + "object": "process_os_actor" + }, + "action_process_file_create_time": [ + { + "key": "process.created_time", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_create_time": [ + { + "key": "process.created_time", + "object": "process_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "causality_actor_process_file_create_time": [ + { + "key": "process.created_time", + "object": "process_causality_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_causality_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_create_time": [ + { + "key": "process.created_time", + "object": "process_os_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_os_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_module_process_os_pid": { + "key": "process.pid", + "object": "process_action_module", + "transformer": "ToInteger" + }, + "action_process_os_pid": { + "key": "process.pid", + "object": "process_action", + "transformer": "ToInteger" + }, + "actor_process_os_pid": { + "key": "process.pid", + "object": "process_actor", + "transformer": "ToInteger" + }, + "causality_actor_process_os_pid": { + "key": "process.pid", + "object": "process_causality_actor", + "transformer": "ToInteger" + }, + "os_actor_process_os_pid": { + "key": "process.pid", + "object": "process_os_actor", + "transformer": "ToInteger" + }, + "action_process_requested_parent_pid": [ + { + "key": "process.pid", + "object": "action_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process_action", + "references": "action_parent_process" + } + ], + "action_thread_parent_pid": [ + { + "key": "process.pid", + "object": "action_parent_thread", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process_action_thread", + "references": "action_parent_thread" + } + ], + "action_thread_child_pid": [ + { + "key": "process.pid", + "object": "action_child_thread", + "transformer": "ToInteger" + }, + { + "key": "process.child_refs", + "object": "process_action", + "references": "action_child_thread" + } + ], + "auth_domain": { + "key": "domain-name.value", + "object": "domain" + }, + "dst_host_metadata_domain": { + "key": "domain-name.value", + "object": "domain_dst" + }, + "host_metadata_domain": { + "key": "domain-name.value", + "object": "domain_host" + }, + "dst_action_url_category": { + "key": "url.value", + "object": "url" + }, + "action_registry_key_name": { + "key": "windows-registry-key.key", + "object": "windowsregistry" + }, + "action_registry_value_name": { + "key": "windows-registry-key.values", + "object": "windowsregistry" + }, + "mac": { + "key": "mac-addr.value", + "object": "mac" + }, + "associated_mac": { + "key": "mac-addr.value", + "object": "ass_mac" + }, + "dst_associated_mac": { + "key": "mac-addr.value", + "object": "dst_ass_mac" + }, + "dst_mac": { + "key": "mac-addr.value", + "object": "dst_mac" + }, + "actor_primary_user_sid": { + "key": "user-account.user_id", + "object": "user" + }, + "action_process_user_sid": { + "key": "user-account.extensions.x-paloalto-user.process_user_id", + "object": "user" + }, + "actor_primary_username": { + "key": "user-account.display_name", + "object": "user" + }, + "action_process_username": [ + { + "key": "user-account.extensions.x-paloalto-user.process_user_name", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process_action", + "references": "user" + } + ], + "actor_process_logon_id": { + "key": "user-account.account_login", + "object": "user" + }, + "action_file_info_company": { + "key": "file.extensions.x-paloalto-file.company", + "object": "file" + }, + "action_file_extension": { + "key": "file.extensions.x-paloalto-file.extension", + "object": "file" + }, + "action_file_attributes": { + "key": "file.extensions.x-paloalto-file.attributes", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_internal_zipped_files": { + "key": "file.extensions.x-paloalto-file.zipped_files", + "object": "file" + }, + "action_file_last_writer_actor": { + "key": "file.extensions.x-paloalto-file.writer", + "object": "file" + }, + "action_file_mode": { + "key": "file.extensions.x-paloalto-file.mode", + "object": "file" + }, + "action_file_signature_status": { + "key": "file.extensions.x-paloalto-file.signature_status", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_signature_vendor": { + "key": "file.extensions.x-paloalto-file.signature_vendor", + "object": "file" + }, + "action_file_signature_product": { + "key": "file.extensions.x-paloalto-file.signature_product", + "object": "file" + }, + "action_file_info_description": { + "key": "file.extensions.x-paloalto-file.file_description", + "object": "file" + }, + "action_file_group": { + "key": "file.extensions.x-paloalto-file.group", + "object": "file" + }, + "action_file_group_name": { + "key": "file.extensions.x-paloalto-file.group_name", + "object": "file" + }, + "action_file_type": { + "key": "file.extensions.x-paloalto-file.type", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_info_file_version": { + "key": "file.extensions.x-paloalto-file.version", + "object": "file" + }, + "manifest_file_version": { + "key": "file.extensions.x-paloalto-file.manifest_version", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_info_product_version": { + "key": "file.extensions.x-paloalto-file.product_version", + "object": "file" + }, + "action_file_owner": { + "key": "file.extensions.x-paloalto-file.owner", + "object": "file" + }, + "action_file_owner_name": { + "key": "file.extensions.x-paloalto-file.owner_name", + "object": "file" + }, + "action_file_info_product_name": { + "key": "file.extensions.x-paloalto-file.product_name", + "object": "file" + }, + "action_file_id": { + "key": "file.extensions.x-paloalto-file.id", + "object": "file" + }, + "action_file_wildfire_verdict": { + "key": "file.extensions.x-paloalto-file.wildfire_verdict", + "object": "file" + }, + "action_file_hash_control_verdict": { + "key": "file.extensions.x-paloalto-process.control_verdict", + "object": "file" + }, + "actor_process_instance_id": { + "key": "process.extensions.x-paloalto-process.instance_id", + "object": "process_actor" + }, + "actor_process_causality_id": { + "key": "process.extensions.x-paloalto-process.causality_id", + "object": "process_actor" + }, + "actor_process_auth_id": { + "key": "process.extensions.x-paloalto-process.auth_id", + "object": "process_actor" + }, + "actor_process_container_id": { + "key": "process.extensions.x-paloalto-process.container_id", + "object": "process_actor" + }, + "actor_process_signature_vendor": { + "key": "process.extensions.x-paloalto-process.signature_vendor", + "object": "process_actor" + }, + "actor_process_signature_status": { + "key": "process.extensions.x-paloalto-process.signature_status", + "object": "process_actor" + }, + "actor_process_signature_product": { + "key": "process.extensions.x-paloalto-process.signature_product", + "object": "process_actor" + }, + "actor_process_image_extension": { + "key": "process.extensions.x-paloalto-process.extension", + "object": "process_actor" + }, + "action_process_termination_code": { + "key": "process.extensions.x-paloalto-process.termination_code", + "object": "process_action", + "transformer": "ToInteger" + }, + "action_process_termination_date": { + "key": "process.extensions.x-paloalto-process.termination_date", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + "action_remote_process_thread_id": { + "key": "process.extensions.x-paloalto-process.tid", + "object": "process_action", + "transformer": "ToInteger" + }, + "action_process_instance_execution_time": { + "key": "process.extensions.x-paloalto-process.instance_exec_time", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + "actor_process_execution_time": { + "key": "process.extensions.x-paloalto-process.execution_time", + "object": "process_actor", + "transformer": "EpochToTimestamp" + }, + "action_process_handle_is_kernel": { + "key": "process.extensions.x-paloalto-process.is_kernel", + "object": "process_action", + "transformer": "StringToBool" + }, + "action_process_is_container_root": { + "key": "process.extensions.x-paloalto-process.is_root", + "object": "process_action", + "transformer": "StringToBool" + }, + "actor_process_is_native": { + "key": "process.extensions.x-paloalto-process.is_native", + "object": "process_actor", + "transformer": "StringToBool" + }, + "agent_version": { + "key": "x-oca-asset.agent_version", + "object": "asset" + }, + "agent_hostname": { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + "agent_content_version": { + "key": "x-oca-asset.content_version", + "object": "asset" + }, + "agent_session_start_time": { + "key": "x-oca-asset.start_time", + "object": "asset", + "transformer": "EpochToTimestamp" + }, + "agent_id": { + "key": "x-oca-asset.asset_id", + "object": "asset" + }, + "agent_os_type": { + "key": "x-oca-asset.os_type", + "object": "asset" + }, + "agent_os_sub_type": { + "key": "x-oca-asset.os_sub_type", + "object": "asset" + }, + "agent_is_vdi": { + "key": "x-oca-asset.is_vdi", + "object": "asset", + "transformer": "StringToBool" + }, + "action_user_agent": { + "key": "x-oca-asset.user_agent", + "object": "asset" + }, + "http_req_user_agent_header": { + "key": "x-oca-asset.agent_header", + "object": "asset" + }, + "action_evtlog_data_fields": { + "key": "x-paloalto-evtlog.data_fields", + "object": "evtlog" + }, + "action_evtlog_description": { + "key": "x-paloalto-evtlog.description", + "object": "evtlog" + }, + "action_evtlog_source": { + "key": "x-paloalto-evtlog.source", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_event_id": { + "key": "x-paloalto-evtlog.evtlog_id", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_level": { + "key": "x-paloalto-evtlog.level", + "object": "evtlog" + }, + "action_evtlog_tid": { + "key": "x-paloalto-evtlog.tid", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_uid": { + "key": "x-paloalto-evtlog.uid", + "object": "evtlog" + }, + "action_evtlog_pid": { + "key": "x-paloalto-evtlog.pid", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_message": { + "key": "x-paloalto-evtlog.message", + "object": "evtlog" + }, + "action_evtlog_version": { + "key": "x-paloalto-evtlog.version", + "object": "evtlog", + "transformer": "ToInteger" + }, + "event_id": { + "key": "x-oca-event.event_id", + "object": "event" + }, + "vpn_event_description": { + "key": "x-oca-event.event_description", + "object": "event" + }, + "event_timestamp": { + "key": "x-oca-event.time", + "object": "event", + "transformer": "EpochToTimestamp" + }, + "event_version": { + "key": "x-oca-event.version", + "object": "event", + "transformer": "ToInteger" + }, + "event_rpc_interface_uuid": { + "key": "x-oca-event.uuid", + "object": "event" + }, + "event_address_mapped_image_path": { + "key": "x-oca-event.path", + "object": "event" + }, + "event_type": { + "key": "x-oca-event.event_type", + "object": "event" + }, + "event_sub_type": { + "key": "x-oca-event.sub_type", + "object": "event" + }, + "action_network_creation_time": { + "key": "network-traffic.extensions.x-paloalto-network.creation_time", + "object": "nt", + "transformer": "EpochToTimestamp" + }, + "action_network_connection_id": { + "key": "network-traffic.extensions.x-paloalto-network.connection_id", + "object": "nt" + }, + "action_network_packet_data": { + "key": "network-traffic.extensions.x-paloalto-network.packet_data", + "object": "nt" + }, + "action_proxy": { + "key": "network-traffic.extensions.x-paloalto-network.is_proxy", + "object": "nt", + "transformer": "StringToBool" + }, + "host_metadata_hostname": { + "key": "network-traffic.extensions.x-paloalto-network.metadata_hostname", + "object": "nt" + }, + "action_external_hostname": { + "key": "network-traffic.extensions.x-paloalto-network.external_hostname", + "object": "nt" + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json new file mode 100644 index 000000000..a59ebf0eb --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json @@ -0,0 +1,287 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "action_local_ip", + "action_remote_ip" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "agent_ip_addresses_v6", + "dst_agent_ip_addresses_v6" + ] + } + }, + "network-traffic": { + "fields": { + "src_port": [ + "action_local_port" + ], + "dst_port": [ + "action_remote_port" + ], + "protocols[*]": [ + "action_network_protocol" + ], + "src_ref.value": [ + "action_local_ip" + ], + "dst_ref.value": [ + "action_remote_ip" + ], + "src_packets": [ + "action_pkts_sent" + ], + "dst_packets": [ + "action_pkts_received" + ] + } + }, + "file": { + "fields": { + "name": [ + "action_file_name", + "action_process_image_name", + "actor_process_image_name", + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "size": [ + "action_file_size" + ], + "hashes.MD5": [ + "action_file_md5", + "action_module_md5", + "action_process_image_md5" + ], + "hashes.'SHA-1'": [ + "action_file_authenticode_sha1" + ], + "hashes.'SHA-2'": [ + "action_file_authenticode_sha2" + ], + "hashes.'SHA-256'": [ + "action_file_sha256", + "action_module_sha256", + "action_process_image_sha256" + ], + "atime": [ + "action_file_access_time", + "actor_process_file_access_time", + "os_actor_process_file_access_time" + ], + "mtime": [ + "action_file_mod_time", + "actor_process_file_mod_time", + "os_actor_process_file_mod_time" + ], + "ctime": [ + "action_file_create_time" + ], + "parent_directory_ref.path": [ + "action_file_path", + "action_process_image_path", + "action_registry_file_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "directory": { + "fields": { + "path": [ + "action_file_path", + "action_process_image_path", + "action_registry_file_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "process": { + "fields": { + "command_line": [ + "action_process_image_command_line", + "actor_process_command_line", + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "created_time": [ + "action_process_file_create_time", + "actor_process_file_create_time", + "causality_actor_process_file_create_time", + "os_actor_process_file_create_time" + ], + "pid": [ + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid", + "action_process_requested_parent_pid", + "action_thread_parent_pid", + "action_thread_child_pid" + ], + "parent_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "child_refs.pid": [ + "action_thread_child_pid" + ], + "creator_user_ref.user_id": [ + "action_process_username" + ], + "parent_ref.name": [ + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "image_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "image_ref.hashes.MD5": [ + "action_process_image_md5" + ], + "image_ref.hashes.'SHA-256'": [ + "action_process_image_sha256" + ], + "image_ref.parent_directory_ref.path": [ + "action_process_image_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "auth_domain", + "dst_host_metadata_domain", + "host_metadata_domain" + ] + } + }, + "url": { + "fields": { + "value": [ + "dst_action_url_category" + ] + } + }, + "windows-registry-key": { + "fields": { + "key": [ + "action_registry_key_name" + ], + "values[*]": [ + "action_registry_value_name" + ] + } + }, + "mac-addr": { + "fields": { + "value": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" + ] + } + }, + "user-account": { + "fields": { + "user_id": [ + "actor_primary_user_sid", + "action_process_user_sid" + ], + "display_name": [ + "actor_primary_username", + "action_process_username" + ], + "account_login": [ + "actor_process_logon_id" + ] + } + }, + "x-paloalto-file": { + "fields": { + "extension": [ + "action_file_extension" + ], + "file_description": [ + "action_file_info_description" + ] + } + }, + "x-paloalto-process": { + "fields": { + "extension": [ + "actor_process_image_extension" + ], + "execution_time": [ + "action_process_instance_execution_time", + "actor_process_execution_time" + ] + } + }, + "x-oca-asset": { + "fields": { + "hostname": [ + "agent_hostname" + ], + "asset_id": [ + "agent_id" + ], + "os_type": [ + "agent_os_type" + ], + "user_agent": [ + "action_user_agent" + ] + } + }, + "x-paloalto-evtlog": { + "fields": { + "description": [ + "action_evtlog_description" + ], + "message": [ + "action_evtlog_message" + ] + } + }, + "x-oca-event": { + "fields": { + "event_id": [ + "event_id" + ], + "event_description": [ + "vpn_event_description" + ], + "path": [ + "event_address_mapped_image_path" + ], + "event_type": [ + "event_type" + ] + } + }, + "x-paloalto-network": { + "fields": { + "creation_time": [ + "action_network_creation_time" + ], + "hostname": [ + "host_metadata_hostname", + "action_external_hostname" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json new file mode 100644 index 000000000..39d17fb7f --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json @@ -0,0 +1,838 @@ +{ + "xdr_data": { + "action_local_ip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], + "action_remote_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "agent_ip_addresses_v6": { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + "dst_agent_ip_addresses_v6": { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + "action_local_port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "action_remote_port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "action_network_protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + "action_pkts_sent": { + "key": "network-traffic.src_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "action_pkts_received": { + "key": "network-traffic.dst_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "action_file_name": { + "key": "file.name", + "object": "file" + }, + "action_process_image_name": [ + { + "key": "file.name", + "object": "file_action_process" + }, + { + "key": "process.name", + "object": "process_action" + }, + { + "key": "process.binary_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "actor_process_image_name": [ + { + "key": "file.name", + "object": "file_actor_process" + }, + { + "key": "process.name", + "object": "process_actor" + }, + { + "key": "process.binary_ref", + "object": "process_actor", + "references": "file_actor_process" + } + ], + "causality_actor_process_image_name": [ + { + "key": "file.name", + "object": "file_causality_process" + }, + { + "key": "process.name", + "object": "causality_parent_process" + }, + { + "key": "process.parent_ref", + "object": "process_causality_actor", + "references": "causality_parent_process" + } + ], + "os_actor_process_image_name": [ + { + "key": "file.name", + "object": "file_os_actor" + }, + { + "key": "process.name", + "object": "os_parent_process" + }, + { + "key": "process.parent_ref", + "object": "process_os_actor", + "references": "os_parent_process" + } + ], + "action_file_size": { + "key": "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_authenticode_sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "action_file_authenticode_sha2": { + "key": "file.hashes.SHA-2", + "object": "file" + }, + "action_file_md5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "action_module_md5": { + "key": "file.hashes.MD5", + "object": "file_action_module" + }, + "action_process_image_md5": [ + { + "key": "file.hashes.MD5", + "object": "file_action_process" + }, + { + "key": "process.binary_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "action_file_sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "action_module_sha256": { + "key": "file.hashes.SHA-256", + "object": "file_action_module" + }, + "action_process_image_sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "file_action_process" + }, + { + "key": "process.binary_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "action_file_access_time": [ + { + "key": "file.accessed", + "object": "file", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_action_file_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_access_time": [ + { + "key": "file.accessed", + "object": "file_actor_process", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_actor_process_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_access_time": [ + { + "key": "file.accessed", + "object": "file_os_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_os_actor_process_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_file_mod_time": [ + { + "key": "file.modified", + "object": "file", + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_mod_time": [ + { + "key": "file.modified", + "object": "file_actor_process", + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_mod_time": [ + { + "key": "file.modified", + "object": "file_os_actor", + "transformer": "EpochToTimestamp" + } + ], + "action_file_create_time": [ + { + "key": "file.created", + "object": "file", + "transformer": "EpochToTimestamp" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_file_path": [ + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "action_registry_file_path": [ + { + "key": "directory.path", + "object": "directory_action_registry", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_action_registry", + "references": "directory_action_registry" + } + ], + "action_process_image_path": [ + { + "key": "directory.path", + "object": "directory_action_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_action_process", + "references": "directory_action_process" + }, + { + "key": "process.binary_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_actor_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_actor_process", + "references": "directory_actor_process" + }, + { + "key": "process.binary_ref", + "object": "process_actor", + "references": "file_actor_process" + } + ], + "causality_actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_causality_actor", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_causality_process", + "references": "directory_causality_actor" + }, + { + "key": "process.binary_ref", + "object": "process_causality_actor", + "references": "file_causality_process" + } + ], + "os_actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_os_actor", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_os_actor", + "references": "directory_os_actor" + }, + { + "key": "process.binary_ref", + "object": "process_os_actor", + "references": "file_os_actor" + } + ], + "action_process_image_command_line": { + "key": "process.command_line", + "object": "process_action" + }, + "actor_process_command_line": { + "key": "process.command_line", + "object": "process_actor" + }, + "causality_actor_process_command_line": { + "key": "process.command_line", + "object": "process_causality_actor" + }, + "os_actor_process_command_line": { + "key": "process.command_line", + "object": "process_os_actor" + }, + "action_process_file_create_time": [ + { + "key": "process.created", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_create_time": [ + { + "key": "process.created", + "object": "process_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "causality_actor_process_file_create_time": [ + { + "key": "process.created", + "object": "process_causality_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_causality_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_create_time": [ + { + "key": "process.created", + "object": "process_os_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_os_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_module_process_os_pid": { + "key": "process.pid", + "object": "process_action_module", + "transformer": "ToInteger" + }, + "action_process_os_pid": { + "key": "process.pid", + "object": "process_action", + "transformer": "ToInteger" + }, + "actor_process_os_pid": { + "key": "process.pid", + "object": "process_actor", + "transformer": "ToInteger" + }, + "causality_actor_process_os_pid": { + "key": "process.pid", + "object": "process_causality_actor", + "transformer": "ToInteger" + }, + "os_actor_process_os_pid": { + "key": "process.pid", + "object": "process_os_actor", + "transformer": "ToInteger" + }, + "action_process_requested_parent_pid": [ + { + "key": "process.pid", + "object": "action_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process_action", + "references": "action_parent_process" + } + ], + "action_thread_parent_pid": [ + { + "key": "process.pid", + "object": "action_parent_thread", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process_action_thread", + "references": "action_parent_thread" + } + ], + "action_thread_child_pid": [ + { + "key": "process.pid", + "object": "action_child_thread", + "transformer": "ToInteger" + }, + { + "key": "process.child_refs", + "object": "process_action", + "references": "action_child_thread" + } + ], + "auth_domain": { + "key": "domain-name.value", + "object": "domain" + }, + "dst_host_metadata_domain": { + "key": "domain-name.value", + "object": "domain_dst" + }, + "host_metadata_domain": { + "key": "domain-name.value", + "object": "domain_host" + }, + "dst_action_url_category": { + "key": "url.value", + "object": "url" + }, + "action_registry_key_name": { + "key": "windows-registry-key.key", + "object": "windowsregistry" + }, + "action_registry_value_name": { + "key": "windows-registry-key.values", + "object": "windowsregistry" + }, + "mac": { + "key": "mac-addr.value", + "object": "mac" + }, + "associated_mac": { + "key": "mac-addr.value", + "object": "ass_mac" + }, + "dst_associated_mac": { + "key": "mac-addr.value", + "object": "dst_ass_mac" + }, + "dst_mac": { + "key": "mac-addr.value", + "object": "dst_mac" + }, + "actor_primary_user_sid": { + "key": "user-account.user_id", + "object": "user" + }, + "action_process_user_sid": { + "key": "user-account.extensions.x-paloalto-user.process_user_id", + "object": "user" + }, + "actor_primary_username": { + "key": "user-account.display_name", + "object": "user" + }, + "action_process_username": [ + { + "key": "user-account.extensions.x-paloalto-user.process_user_name", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process_action", + "references": "user" + } + ], + "actor_process_logon_id": { + "key": "user-account.account_login", + "object": "user" + }, + "action_file_info_company": { + "key": "file.extensions.x-paloalto-file.company", + "object": "file" + }, + "action_file_extension": { + "key": "file.extensions.x-paloalto-file.extension", + "object": "file" + }, + "action_file_attributes": { + "key": "file.extensions.x-paloalto-file.attributes", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_internal_zipped_files": { + "key": "file.extensions.x-paloalto-file.zipped_files", + "object": "file" + }, + "action_file_last_writer_actor": { + "key": "file.extensions.x-paloalto-file.writer", + "object": "file" + }, + "action_file_mode": { + "key": "file.extensions.x-paloalto-file.mode", + "object": "file" + }, + "action_file_signature_status": { + "key": "file.extensions.x-paloalto-file.signature_status", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_signature_vendor": { + "key": "file.extensions.x-paloalto-file.signature_vendor", + "object": "file" + }, + "action_file_signature_product": { + "key": "file.extensions.x-paloalto-file.signature_product", + "object": "file" + }, + "action_file_info_description": { + "key": "file.extensions.x-paloalto-file.file_description", + "object": "file" + }, + "action_file_group": { + "key": "file.extensions.x-paloalto-file.group", + "object": "file" + }, + "action_file_group_name": { + "key": "file.extensions.x-paloalto-file.group_name", + "object": "file" + }, + "action_file_type": { + "key": "file.extensions.x-paloalto-file.type", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_info_file_version": { + "key": "file.extensions.x-paloalto-file.version", + "object": "file" + }, + "manifest_file_version": { + "key": "file.extensions.x-paloalto-file.manifest_version", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_info_product_version": { + "key": "file.extensions.x-paloalto-file.product_version", + "object": "file" + }, + "action_file_owner": { + "key": "file.extensions.x-paloalto-file.owner", + "object": "file" + }, + "action_file_owner_name": { + "key": "file.extensions.x-paloalto-file.owner_name", + "object": "file" + }, + "action_file_info_product_name": { + "key": "file.extensions.x-paloalto-file.product_name", + "object": "file" + }, + "action_file_id": { + "key": "file.extensions.x-paloalto-file.id", + "object": "file" + }, + "action_file_wildfire_verdict": { + "key": "file.extensions.x-paloalto-file.wildfire_verdict", + "object": "file" + }, + "action_file_hash_control_verdict": { + "key": "file.extensions.x-paloalto-process.control_verdict", + "object": "file" + }, + "actor_process_instance_id": { + "key": "process.extensions.x-paloalto-process.instance_id", + "object": "process_actor" + }, + "actor_process_causality_id": { + "key": "process.extensions.x-paloalto-process.causality_id", + "object": "process_actor" + }, + "actor_process_auth_id": { + "key": "process.extensions.x-paloalto-process.auth_id", + "object": "process_actor" + }, + "actor_process_container_id": { + "key": "process.extensions.x-paloalto-process.container_id", + "object": "process_actor" + }, + "actor_process_signature_vendor": { + "key": "process.extensions.x-paloalto-process.signature_vendor", + "object": "process_actor" + }, + "actor_process_signature_status": { + "key": "process.extensions.x-paloalto-process.signature_status", + "object": "process_actor" + }, + "actor_process_signature_product": { + "key": "process.extensions.x-paloalto-process.signature_product", + "object": "process_actor" + }, + "actor_process_image_extension": { + "key": "process.extensions.x-paloalto-process.extension", + "object": "process_actor" + }, + "action_process_termination_code": { + "key": "process.extensions.x-paloalto-process.termination_code", + "object": "process_action", + "transformer": "ToInteger" + }, + "action_process_termination_date": { + "key": "process.extensions.x-paloalto-process.termination_date", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + "action_remote_process_thread_id": { + "key": "process.extensions.x-paloalto-process.tid", + "object": "process_action", + "transformer": "ToInteger" + }, + "action_process_instance_execution_time": { + "key": "process.extensions.x-paloalto-process.instance_exec_time", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + "actor_process_execution_time": { + "key": "process.extensions.x-paloalto-process.execution_time", + "object": "process_actor", + "transformer": "EpochToTimestamp" + }, + "action_process_handle_is_kernel": { + "key": "process.extensions.x-paloalto-process.is_kernel", + "object": "process_action", + "transformer": "StringToBool" + }, + "action_process_is_container_root": { + "key": "process.extensions.x-paloalto-process.is_root", + "object": "process_action", + "transformer": "StringToBool" + }, + "actor_process_is_native": { + "key": "process.extensions.x-paloalto-process.is_native", + "object": "process_actor", + "transformer": "StringToBool" + }, + "agent_version": { + "key": "x-oca-asset.agent_version", + "object": "asset" + }, + "agent_hostname": { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + "agent_content_version": { + "key": "x-oca-asset.content_version", + "object": "asset" + }, + "agent_session_start_time": { + "key": "x-oca-asset.start_time", + "object": "asset", + "transformer": "EpochToTimestamp" + }, + "agent_id": { + "key": "x-oca-asset.asset_id", + "object": "asset" + }, + "agent_os_type": { + "key": "x-oca-asset.os_type", + "object": "asset" + }, + "agent_os_sub_type": { + "key": "x-oca-asset.os_sub_type", + "object": "asset" + }, + "agent_is_vdi": { + "key": "x-oca-asset.is_vdi", + "object": "asset", + "transformer": "StringToBool" + }, + "action_user_agent": { + "key": "x-oca-asset.user_agent", + "object": "asset" + }, + "http_req_user_agent_header": { + "key": "x-oca-asset.agent_header", + "object": "asset" + }, + "action_evtlog_data_fields": { + "key": "x-paloalto-evtlog.data_fields", + "object": "evtlog" + }, + "action_evtlog_description": { + "key": "x-paloalto-evtlog.description", + "object": "evtlog" + }, + "action_evtlog_source": { + "key": "x-paloalto-evtlog.source", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_event_id": { + "key": "x-paloalto-evtlog.evtlog_id", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_level": { + "key": "x-paloalto-evtlog.level", + "object": "evtlog" + }, + "action_evtlog_tid": { + "key": "x-paloalto-evtlog.tid", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_uid": { + "key": "x-paloalto-evtlog.uid", + "object": "evtlog" + }, + "action_evtlog_pid": { + "key": "x-paloalto-evtlog.pid", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_message": { + "key": "x-paloalto-evtlog.message", + "object": "evtlog" + }, + "action_evtlog_version": { + "key": "x-paloalto-evtlog.version", + "object": "evtlog", + "transformer": "ToInteger" + }, + "event_id" :{ + "key": "x-oca-event.event_id", + "object": "event" + }, + "vpn_event_description": { + "key": "x-oca-event.event_description", + "object": "event" + }, + "event_timestamp": { + "key": "x-oca-event.time", + "object": "event", + "transformer": "EpochToTimestamp" + }, + "event_version": { + "key": "x-oca-event.version", + "object": "event", + "transformer": "ToInteger" + }, + "event_rpc_interface_uuid": { + "key": "x-oca-event.uuid", + "object": "event" + }, + "event_address_mapped_image_path": { + "key": "x-oca-event.path", + "object": "event" + }, + "event_type": { + "key": "x-oca-event.event_type", + "object": "event" + }, + "event_sub_type": { + "key": "x-oca-event.sub_type", + "object": "event" + }, + "action_network_creation_time": { + "key": "network-traffic.extensions.x-paloalto-network.creation_time", + "object": "nt", + "transformer": "EpochToTimestamp" + }, + "action_network_connection_id": { + "key": "network-traffic.extensions.x-paloalto-network.connection_id", + "object": "nt" + }, + "action_network_packet_data": { + "key": "network-traffic.extensions.x-paloalto-network.packet_data", + "object": "nt" + }, + "action_proxy": { + "key": "network-traffic.extensions.x-paloalto-network.is_proxy", + "object": "nt", + "transformer": "StringToBool" + }, + "host_metadata_hostname": { + "key": "network-traffic.extensions.x-paloalto-network.metadata_hostname", + "object": "nt" + }, + "action_external_hostname": { + "key": "network-traffic.extensions.x-paloalto-network.external_hostname", + "object": "nt" + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json new file mode 100644 index 000000000..089e7c897 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json @@ -0,0 +1,293 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "action_local_ip", + "action_remote_ip" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "agent_ip_addresses_v6", + "dst_agent_ip_addresses_v6" + ] + } + }, + "network-traffic": { + "fields": { + "src_port": [ + "action_local_port" + ], + "dst_port": [ + "action_remote_port" + ], + "protocols[*]": [ + "action_network_protocol" + ], + "src_ref.value": [ + "action_local_ip" + ], + "dst_ref.value": [ + "action_remote_ip" + ], + "src_packets": [ + "action_pkts_sent" + ], + "dst_packets": [ + "action_pkts_received" + ] + } + }, + "file": { + "fields": { + "name": [ + "action_file_name", + "action_process_image_name", + "actor_process_image_name", + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "size": [ + "action_file_size" + ], + "hashes.MD5": [ + "action_file_md5", + "action_module_md5", + "action_process_image_md5" + ], + "hashes.'SHA-1'": [ + "action_file_authenticode_sha1" + ], + "hashes.'SHA-2'": [ + "action_file_authenticode_sha2" + ], + "hashes.'SHA-256'": [ + "action_file_sha256", + "action_module_sha256", + "action_process_image_sha256" + ], + "accessed": [ + "action_file_access_time", + "actor_process_file_access_time", + "os_actor_process_file_access_time" + ], + "modified": [ + "action_file_mod_time", + "actor_process_file_mod_time", + "os_actor_process_file_mod_time" + ], + "created": [ + "action_file_create_time" + ], + "parent_directory_ref.path": [ + "action_file_path", + "action_process_image_path", + "action_registry_file_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "directory": { + "fields": { + "path": [ + "action_file_path", + "action_process_image_path", + "action_registry_file_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "process": { + "fields": { + "command_line": [ + "action_process_image_command_line", + "actor_process_command_line", + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "created": [ + "action_process_file_create_time", + "actor_process_file_create_time", + "causality_actor_process_file_create_time", + "os_actor_process_file_create_time" + ], + "name": [ + "action_process_image_name", + "actor_process_image_name", + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "pid": [ + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid", + "action_process_requested_parent_pid", + "action_thread_parent_pid", + "action_thread_child_pid" + ], + "parent_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "child_refs.pid": [ + "action_thread_child_pid" + ], + "creator_user_ref.user_id": [ + "action_process_username" + ], + "parent_ref.name": [ + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "binary_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "binary_ref.hashes.MD5": [ + "action_process_image_md5" + ], + "binary_ref.hashes.'SHA-256'": [ + "action_process_image_sha256" + ], + "binary_ref.parent_directory_ref.path": [ + "action_process_image_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "auth_domain", + "dst_host_metadata_domain", + "host_metadata_domain" + ] + } + }, + "url": { + "fields": { + "value": [ + "dst_action_url_category" + ] + } + }, + "windows-registry-key": { + "fields": { + "key": [ + "action_registry_key_name" + ], + "values[*]": [ + "action_registry_value_name" + ] + } + }, + "mac-addr": { + "fields": { + "value": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" + ] + } + }, + "user-account": { + "fields": { + "user_id": [ + "actor_primary_user_sid", + "action_process_user_sid" + ], + "display_name": [ + "actor_primary_username", + "action_process_username" + ], + "account_login": [ + "actor_process_logon_id" + ] + } + }, + "x-paloalto-file": { + "fields": { + "extension": [ + "action_file_extension" + ], + "file_description": [ + "action_file_info_description" + ] + } + }, + "x-paloalto-process": { + "fields": { + "extension": [ + "actor_process_image_extension" + ], + "execution_time": [ + "action_process_instance_execution_time", + "actor_process_execution_time" + ] + } + }, + "x-oca-asset": { + "fields": { + "hostname": [ + "agent_hostname" + ], + "asset_id": [ + "agent_id" + ], + "os_type": [ + "agent_os_type" + ], + "user_agent": [ + "action_user_agent" + ] + } + }, + "x-paloalto-evtlog": { + "fields": { + "description": [ + "action_evtlog_description" + ], + "message": [ + "action_evtlog_message" + ] + } + }, + "x-oca-event": { + "fields": { + "event_id": [ + "event_id" + ], + "event_description": [ + "vpn_event_description" + ], + "path": [ + "event_address_mapped_image_path" + ], + "event_type": [ + "event_type" + ] + } + }, + "x-paloalto-network": { + "fields": { + "creation_time": [ + "action_network_creation_time" + ], + "hostname": [ + "host_metadata_hostname", + "action_external_hostname" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/query_constructor.py b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py new file mode 100644 index 000000000..6b40d16ab --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py @@ -0,0 +1,477 @@ +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, \ + ComparisonExpression, ComparisonComparators, Pattern, CombinedComparisonExpression, CombinedObservationExpression +import logging +import re +from datetime import datetime, timedelta +from os import path +import json + +logger = logging.getLogger(__name__) + +START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)" +MAC = '^(([0-9a-fA-F]{2}[:-]){5}([0-9a-fA-F]{2}))$' +CONFIG_MAP_PATH = "json/config_map.json" +FIELDS_MAP_PATH = "json/fields_map.json" + +STOP_TIME = datetime.utcnow() + + +class StartStopQualifierValueException(Exception): + pass + + +class QueryStringPatternTranslator: + """ + translate stix pattern to native data source query language + """ + def __init__(self, pattern: Pattern, data_model_mapper, options): + logger.info("Palo Alto Cortex XDR Connector") + self.dmm = data_model_mapper + self.options = options + self.timeframe = [] + self.comparator_lookup = self.dmm.map_comparator() + self.config_map = self.load_json(CONFIG_MAP_PATH) + self.all_fields_map = self.load_json(FIELDS_MAP_PATH) + self.translated_query = self.parse_expression(pattern) + self.qualified_query = self._create_formatted_query(self.dmm, self.translated_query, + self.timeframe, self.all_fields_map, options) + + @staticmethod + def load_json(rel_path_of_file): + """ + Consumes a json file and returns a dictionary + :param rel_path_of_file: str + :return: dict + """ + _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundError + + @staticmethod + def _create_formatted_query(dmm, translated_query, timeframe, all_fields_map, options): + """ + Formation of Palo Alto - native query language structure + :param dmm + :param translated_query:str + :param timeframe:list + :param all_fields_map:dict + :param options:dict + :return: formatted_query :list + """ + limit = options["result_limit"] # The default result_limit is changed to 1000, + # since stream data feature is not supported + dataset_name = dmm.dialect + all_fields = all_fields_map["all_fields"] # all_fields included in to_stix_mapping + fields = ','.join(all_fields) + query = f'dataset = {dataset_name} | filter {translated_query} | alter dataset_name = \"{dataset_name}\" ' \ + f'| fields dataset_name,{fields} | limit {limit} ' # adding custom field 'dataset_name' to query, + # since this 'dataset_name' field will be used in translate results + formatted_query = { + dataset_name: {"query": query, "timeframe": {"from": min(timeframe), "to": max(timeframe)}}} + return [formatted_query] + + def _format_set(self, values, mapped_field_type, mapped_fields_array) -> str: + """ + Formats value in the event of set operation + :param values + :param mapped_field_type: str + :param mapped_fields_array: list + :return formatted value + """ + gen = values.element_iterator() + formatted_value = ','.join(QueryStringPatternTranslator._escape_value( + self._format_value_type(value, mapped_field_type, mapped_fields_array), mapped_field_type) + for value in gen) + return f'({formatted_value})' + + @staticmethod + def _format_match(value, mapped_field_type) -> str: + """ + Formats value in the event of match operation + :param value + :param mapped_field_type:str + :return formatted string type value + """ + if mapped_field_type != "string": + raise NotImplementedError('MATCHES operators is supported only for string type input') + if '^' in value and value.index('^') != 0: + raise NotImplementedError('^ symbol should be at the starting position of the expression') + if '$' in value and value.index('$') != len(value) - 1: + raise NotImplementedError('$ symbol should be at the ending position of the expression') + value = '{}'.format(value.replace('\\', '\\\\').replace('\"', '\\"').replace('(', '\\(').replace(')', '\\)')) + return f'\"{value}\"' + + @staticmethod + def _format_equality(value, mapped_field_type) -> str: + """ + Formats value in the event of equality operation + :param value + :param mapped_field_type: str + :return formatted value + """ + return QueryStringPatternTranslator._escape_value(value, mapped_field_type) + + @staticmethod + def _format_like(value, mapped_field_type) -> str: + """ + Formats value in the event of LIKE operation + :param value + :param mapped_field_type: str + :return formatted value + """ + if mapped_field_type != "string": + raise NotImplementedError("LIKE operator is supported only for string type input") + return QueryStringPatternTranslator._escape_value(value, mapped_field_type) + + @staticmethod + def _escape_value(value, mapped_field_type) -> str: + """ + adds escape characters to string type value + :param value + :param mapped_field_type: str + :return formatted value + """ + if isinstance(value, str) and mapped_field_type != "enum": + value = f'\"{value}\"' + return str(value) + + def _check_enum_supported_values(self, converted_value, mapped_fields_array): + """ + checks for enum supported values + :param mapped_fields_array: list + :param converted_value:str + :return enum formatted value :str + """ + try: + formatted_values = converted_value.upper() + if formatted_values not in self.config_map["enum_supported_values"][mapped_fields_array[0]]: + raise NotImplementedError(f'Unsupported ENUM values provided. Possible supported enum values are' + f'{self.config_map["enum_supported_values"][mapped_fields_array[0]]}') + return f'ENUM.{formatted_values}' + except (KeyError, IndexError, TypeError) as e: + raise KeyError(f'{mapped_fields_array[0]} is not found in enum_supported_values') from e + + def _format_value_type(self, value, mapped_field_type, mapped_fields_array): + """ + check input value format that matches with the mapped field value type + :param value + :param mapped_field_type: str + :param mapped_fields_array: list + :return formatted value + """ + converted_value = str(value) + if mapped_field_type == "enum": + converted_value = self._check_enum_supported_values(converted_value, mapped_fields_array) + elif mapped_field_type == "timestamp": + converted_value = QueryStringPatternTranslator._format_datetime(converted_value) + elif mapped_field_type == "mac": + compile_mac_regex = re.compile(MAC) + if not compile_mac_regex.search(converted_value): + raise NotImplementedError(f'Invalid mac address - {converted_value} provided') + elif mapped_field_type == "int": + if not converted_value.isdigit(): + raise NotImplementedError(f'string type input - {converted_value} is not supported for ' + f'integer type fields') + converted_value = int(value) + elif mapped_field_type == "boolean": + converted_value = QueryStringPatternTranslator._check_boolean_value(converted_value) + return converted_value + + def _check_value_comparator_support(self, value, comparator, mapped_field_type): + """ + checks the comparator and value support + :param value + :param comparator + :param mapped_field_type: str + """ + operator = self.comparator_lookup[str(comparator)] + if mapped_field_type == "enum" and (comparator not in [ComparisonComparators.Equal, ComparisonComparators.In, + ComparisonComparators.NotEqual]): + raise NotImplementedError(f'{operator} operator is not supported for Enum type input. Possible supported ' + f'operator are [=,!=,in,not in]') + if isinstance(value, str) and comparator not in [ComparisonComparators.Equal, ComparisonComparators.NotEqual, + ComparisonComparators.Like, ComparisonComparators.Matches]: + raise NotImplementedError(f'{operator} operator is not supported for string type input') + if isinstance(value, bool) and comparator not in [ComparisonComparators.Equal, ComparisonComparators.NotEqual]: + raise NotImplementedError(f'{operator} operator is not supported for Boolean type input') + + @staticmethod + def _check_boolean_value(converted_value): + """ + returns boolean value of input + :param converted_value:str + :return bool + """ + if converted_value.lower() == "true" or (converted_value.isdigit() and converted_value == "1"): + boolean_value = bool(True) + elif converted_value.lower() == "false" or (converted_value.isdigit() and converted_value == "0"): + boolean_value = bool(False) + else: + raise NotImplementedError('Invalid boolean type input') + return boolean_value + + @staticmethod + def _format_negate(comparator): + """ + returns negation of input operator + :param comparator:str + :return str + """ + negate_comparator = { + ">": "<=", + ">=": "<", + "<": ">=", + "<=": ">", + "=": "!=", + "!=": "=", + "contains": "not contains", + "in": "not in", + "~=": "!~=" + } + return negate_comparator[comparator] + + @staticmethod + def _format_datetime(value): + """ + Converts timestamp to milliseconds + :param value + :return: int, converted epoch value + """ + try: + time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ' + epoch = datetime(1970, 1, 1) + converted_time = int(((datetime.strptime(value, + time_pattern) - epoch).total_seconds()) * 1000) + return converted_time + except ValueError: + pass + raise NotImplementedError(f'cannot convert the timestamp {value} to milliseconds') + + @staticmethod + def _check_time_range_values(converted_timestamp): + """ + checks for valid start and stop time + :param converted_timestamp: list + """ + if converted_timestamp[0] > converted_timestamp[1]: + raise StartStopQualifierValueException('Start time should be lesser than Stop time') + + @staticmethod + def _parse_time_range(qualifier, time_range): + """ + Converts qualifier timestamp to epoch + :param qualifier: str + :param time_range: int + return: list of converted epoch values + """ + try: + compile_timestamp_regex = re.compile(START_STOP_PATTERN) + converted_timestamp = [] + if qualifier and compile_timestamp_regex.search(qualifier): + time_range_iterator = compile_timestamp_regex.finditer(qualifier) + time_range_list = [each.group() for each in time_range_iterator] + else: + start_time = STOP_TIME - timedelta(minutes=time_range) + converted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + # limit 3 digit value for millisecond + converted_stop_time = STOP_TIME.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + time_range_list = [converted_start_time, converted_stop_time] + for timestamp in time_range_list: + converted_time = QueryStringPatternTranslator._format_datetime(timestamp) + converted_timestamp.append(converted_time) + return converted_timestamp + except (KeyError, IndexError, TypeError) as e: + raise e + + def _add_timestamp_to_query(self, query, qualifier): + """ + adds timestamp filter to Palo Alto Cortex XDR query + :param query: str + :param qualifier + :return str + """ + converted_timestamp = QueryStringPatternTranslator._parse_time_range(qualifier, self.options["time_range"]) + QueryStringPatternTranslator._check_time_range_values(converted_timestamp) # check timestamp value range + self.timeframe += converted_timestamp + if self.dmm.dialect in self.config_map["timestamp_supported_dataset"].keys(): + timestamp_field_name = self.config_map["timestamp_supported_dataset"][self.dmm.dialect] + timestamp_field = f' and (to_epoch({timestamp_field_name},\"millis\") >= {converted_timestamp[0]} and ' \ + f'to_epoch({timestamp_field_name},\"millis\") <= {converted_timestamp[1]})' + if timestamp_field not in query: + query = f'({query} {timestamp_field})' + else: + query = f'({query})' + return query + + def _check_mapped_field_type(self, mapped_field_array): + """ + Returns the type of mapped field array + :param mapped_field_array: list + :return: str + """ + mapped_field = mapped_field_array[0] + mapped_field_type = "string" + for key, value in self.config_map.items(): + if mapped_field in value and key in ["int_supported_fields", "enum_supported_fields", + "boolean_supported_fields", "timestamp_supported_fields", + "mac_supported_fields"]: + mapped_field_type = key.split('_')[0] + break + return mapped_field_type + + @staticmethod + def _parse_mapped_fields(value, comparator, mapped_fields_array): + """ + parse mapped fields into boolean expression + :param value: str + :param comparator: str + :param mapped_fields_array: list + :return: str + """ + comparison_string = "" + mapped_fields_count = len(mapped_fields_array) + for field_name in mapped_fields_array: + comparison_string += f'{field_name} {comparator} {value}' + if mapped_fields_count > 1: + comparison_string += " or " + mapped_fields_count -= 1 + if len(mapped_fields_array) > 1: + # More than one data source field maps to the STIX attribute, so group comparisons together. + grouped_comparison_string = "(" + comparison_string + ")" + comparison_string = grouped_comparison_string + return comparison_string + + def _lookup_comparison_operator(self, expression_operator): + """ + lookup operators support in Palo Alto Cortex XDR + :param expression_operator:enum object + :return str + """ + if str(expression_operator) not in self.comparator_lookup: + raise NotImplementedError( + f'Comparison operator {expression_operator.name} unsupported for Palo Alto Cortex XDR connector') + + return self.comparator_lookup[str(expression_operator)] + + def _eval_comparison_value(self, expression, mapped_field_type, mapped_fields_array): + """ + Function for parsing comparison expression value + :param expression: expression object + :param mapped_field_type:str + :param mapped_fields_array:list + :return: formatted expression value + """ + if expression.comparator == ComparisonComparators.Matches: + value = QueryStringPatternTranslator._format_match(expression.value, mapped_field_type) + elif expression.comparator == ComparisonComparators.In: + value = self._format_set(expression.value, mapped_field_type, mapped_fields_array) + elif expression.comparator in [ComparisonComparators.GreaterThan, ComparisonComparators.GreaterThanOrEqual, + ComparisonComparators.LessThan, ComparisonComparators.LessThanOrEqual, + ComparisonComparators.Equal, ComparisonComparators.NotEqual]: + value = self._format_value_type(expression.value, mapped_field_type, mapped_fields_array) + self._check_value_comparator_support(value, expression.comparator, mapped_field_type) + value = self._format_equality(value, mapped_field_type) + elif expression.comparator == ComparisonComparators.Like: + value = self._format_value_type(expression.value, mapped_field_type, mapped_fields_array) + value = self._format_like(value, mapped_field_type) + else: + raise NotImplementedError('Unknown comparator expression operator') + return value + + def _eval_combined_comparison_exp(self, expression): + """ + Function for parsing combined comparison expression + :param expression: expression object + """ + operator = self._lookup_comparison_operator(expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + if not expression_01 or not expression_02: + return '' + if isinstance(expression.expr1, CombinedComparisonExpression): + expression_01 = f'{expression_01}' + if isinstance(expression.expr2, CombinedComparisonExpression): + expression_02 = f'{expression_02}' + + query_string = f'{expression_01} {operator} {expression_02}' + return f'{query_string}' + + def _eval_combined_observation_exp(self, expression, qualifier=None): + """ + Function for parsing combined observation expression + :param expression: expression object + :param qualifier: qualifier + """ + operator = self._lookup_comparison_operator(expression.operator) + expression_01 = self._parse_expression(expression.expr1, qualifier) + expression_02 = self._parse_expression(expression.expr2, qualifier) + query = '' + if expression_01 and expression_02: + query = f'{expression_01} {operator} {expression_02}' + elif expression_01: + query = f'{expression_01}' + elif expression_02: + query = f'{expression_02}' + return query + + def _parse_expression(self, expression, qualifier=None) -> str: + """ + parse ANTLR pattern to Palo Alto Cortex XDR query + :param expression: expression object, ANTLR parsed expression object + :param qualifier: str, default in None + :return str + """ + if isinstance(expression, ComparisonExpression): # Base Case + stix_objects = expression.object_path.split(':') + mapped_fields_array = self.dmm.map_field(stix_objects[0], stix_objects[1]) + comparator = self._lookup_comparison_operator(expression.comparator) + if expression.negated: + comparator = QueryStringPatternTranslator._format_negate(comparator) + mapped_field_type = self._check_mapped_field_type(mapped_fields_array) + value = self._eval_comparison_value(expression, mapped_field_type, mapped_fields_array) + + comparison_string = self._parse_mapped_fields(value, comparator, mapped_fields_array) + return comparison_string + + elif isinstance(expression, CombinedComparisonExpression): + return self._eval_combined_comparison_exp(expression) + elif isinstance(expression, ObservationExpression): + query_string = self._parse_expression(expression.comparison_expression) + return self._add_timestamp_to_query(query_string, qualifier) + + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + if isinstance(expression.observation_expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(expression.observation_expression.operator) + expression_01 = self._parse_expression(expression.observation_expression.expr1, expression.qualifier) + expression_02 = self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + query_string = f'{expression_01} {operator} {expression_02}' + else: + query_string = self._parse_expression(expression.observation_expression, expression.qualifier) + if qualifier is not None: + query_string = self._add_timestamp_to_query(query_string, qualifier) + return query_string + elif isinstance(expression, CombinedObservationExpression): + return self._eval_combined_observation_exp(expression, qualifier) + elif isinstance(expression, Pattern): + return f'{self._parse_expression(expression.expression)}' + else: + raise RuntimeError(f'Unknown Recursion Case for expression={expression},' + f' type(expression)={type(expression)}') + + def parse_expression(self, pattern: Pattern): + return self._parse_expression(pattern) + + +def translate_pattern(pattern: Pattern, data_model_mapping, options): + """ + Conversion of ANTLR pattern to Palo ALTO CORTEX XDR query + :param pattern: expression object, ANTLR parsed expression object + :param data_model_mapping: DataMapper object, mapping object obtained by parsing json + :param options: dict, time_range defaults to 5 + :return: list, PALO ALTO queries + """ + + query = QueryStringPatternTranslator(pattern, data_model_mapping, options).qualified_query + return query diff --git a/stix_shifter_modules/paloalto/stix_translation/query_translator.py b/stix_shifter_modules/paloalto/stix_translation/query_translator.py new file mode 100644 index 000000000..845ad4f11 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/query_translator.py @@ -0,0 +1,27 @@ +import logging + +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator +from . import query_constructor + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file + :param antlr_parsing_object: Antlr parsing objects for the STIX pattern + :type antlr_parsing_object: object + :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into + another format. This should default to something if one isn't passed in + :type mapping: str (filepath) + :return: transformed query string + :rtype: str + """ + + logger.info("Converting STIX2 Pattern to data source query") + + query_string = query_constructor.translate_pattern( + antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/paloalto/stix_translation/results_translator.py b/stix_shifter_modules/paloalto/stix_translation/results_translator.py new file mode 100644 index 000000000..746b8300c --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/results_translator.py @@ -0,0 +1,5 @@ +from stix_shifter_utils.stix_translation.src.json_to_stix.json_to_stix import JSONToStix + + +class ResultsTranslator(JSONToStix): + pass diff --git a/stix_shifter_modules/paloalto/stix_transmission/__init__.py b/stix_shifter_modules/paloalto/stix_transmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/paloalto/stix_transmission/api_client.py b/stix_shifter_modules/paloalto/stix_transmission/api_client.py new file mode 100644 index 000000000..4961455c6 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/api_client.py @@ -0,0 +1,176 @@ +from stix_shifter_utils.stix_transmission.utils.RestApiClient import RestApiClient +from stix_shifter_utils.utils import logger +from stix_shifter_utils.utils.error_response import ErrorResponder +from .response_mapper import ResponseMapper +from datetime import datetime, timezone +import secrets +import string +import hashlib +import json + + +class MaxDailyQuotaException(Exception): + pass + + +class APIClient: + QUERY_ENDPOINT = 'public_api/v1/xql/start_xql_query/' + RESULT_ENDPOINT = 'public_api/v1/xql/get_query_results/' + STREAM_ENDPOINT = 'public_api/v1/xql/get_query_results_stream/' + QUOTA_ENDPOINT = '/public_api/v1/xql/get_quota/' + + def __init__(self, connection, configuration): + self.auth = configuration.get('auth') + self.logger = logger.set_logger(__name__) + nonce = "".join([secrets.choice(string.ascii_letters + string.digits) for _ in range(64)]) + timestamp = int(datetime.now(timezone.utc).timestamp()) * 1000 + self.auth = configuration.get('auth') + auth_key = f"{self.auth['api_key']}{nonce}{timestamp}" + auth_key = auth_key.encode("utf-8") + api_key_hash = hashlib.sha256(auth_key).hexdigest() + headers = { + "x-xdr-timestamp": str(timestamp), + "x-xdr-nonce": nonce, + "x-xdr-auth-id": str(self.auth['api_key_id']), + "Authorization": api_key_hash + } + + url_modifier_function = None + self.client = RestApiClient(connection.get('host'), + connection.get('port', None), + headers, + url_modifier_function=url_modifier_function, + ) + self.result_limit = connection['options'].get('result_limit') + self.timeout = connection['options']['timeout'] + self.quota_threshold = connection['quota_threshold'] + self.connector = __name__.split('.')[1] + + def ping_data_source(self): + """ + Ping the Data Source + :return: Response object + """ + data = { + "request_data": {} + } + + return self.client.call_api(self.QUOTA_ENDPOINT, 'POST', headers=self.client.headers, + data=json.dumps(data), timeout=self.timeout) + + def get_remaining_quota(self): + """ + Pings the quota endpoint to fetch the remaining quota + :return: Response object + """ + return_obj = {} + response_dict = {} + data = { + "request_data": {} + } + quota_wrapper = None + + try: + quota_wrapper = self.client.call_api(self.QUOTA_ENDPOINT, 'POST', headers=self.client.headers, + data=json.dumps(data), timeout=self.timeout) + quota_response_code = quota_wrapper.response.status_code + quota_response_text = json.loads(quota_wrapper.read().decode('utf-8')) + if quota_response_code == 200: + if 'reply' in quota_response_text.keys() and quota_response_text['reply']['used_quota'] >= \ + self.quota_threshold: + raise MaxDailyQuotaException + return_obj['success'] = True + else: + return_obj = ResponseMapper().status_code_mapping(quota_response_code, quota_response_text) + + except ValueError as ex: + if quota_wrapper is not None: + self.logger.debug(quota_wrapper.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + except MaxDailyQuotaException: + response_dict['type'] = "MaxDailyQuotaException" + response_dict['message'] = "query usage exceeded max daily quota" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + except Exception as ex: + response_dict['type'] = ex.__class__.__name__ + response_dict['message'] = ex + self.logger.error('error when getting search results: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + + return return_obj + + def create_search(self, query): + """ + Queries the data source + :return: Response object + """ + return_obj = self.get_remaining_quota() + if return_obj['success']: + if not isinstance(query, dict): + query = json.loads(query) + + for dataset in query.keys(): + query[dataset]["tenants"] = self.auth['tenant'].split(",") + data = { + "request_data": + query[dataset] + } + return self.client.call_api(self.QUERY_ENDPOINT, 'POST', headers=self.client.headers, + data=json.dumps(data), timeout=self.timeout) + return return_obj + + def get_search_status(self, search_id): + """ + Queries the data source to fetch the status of api call + :return: Response object + """ + data = { + "request_data": { + "query_id": search_id, + "pending_flag": True, + "limit": self.result_limit, + "format": "json" + } + } + return self.client.call_api(self.RESULT_ENDPOINT, 'POST', headers=self.client.headers, data=json.dumps(data), + timeout=self.timeout) + + def get_search_results(self, search_id): + """ + Return the search results + :param search_id:str + :return: Response object + """ + data = { + "request_data": { + "query_id": search_id, + "pending_flag": False, + "limit": self.result_limit, + "format": "json" + } + } + return self.client.call_api(self.RESULT_ENDPOINT, 'POST', headers=self.client.headers, data=json.dumps(data), + timeout=self.timeout) + + @staticmethod + def delete_search(): + """ + Delete operation of a search id is not supported in Palo Alto Cortex XDR + :return dict + """ + return {"code": 200, "success": True} + + # *** The stream data feature is not used currently. It may be implemented in future *** + # def get_stream_results(self, stream_id): + # """ + # Return the stream results + # :return: Raw Json data + # """ + # data = { + # "request_data": + # {"stream_id": stream_id, + # "is_gzip_compressed": False + # } + # } + # return self.client.call_api(self.STREAM_ENDPOINT, 'POST', headers=self.client.headers, data=json.dumps(data), + # timeout=self.timeout) diff --git a/stix_shifter_modules/paloalto/stix_transmission/delete_connector.py b/stix_shifter_modules/paloalto/stix_transmission/delete_connector.py new file mode 100644 index 000000000..3baf0d4a5 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/delete_connector.py @@ -0,0 +1,30 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + + +class DeleteConnector(BaseDeleteConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def delete_query_connection(self, search_id): + """ + Delete operation of a search id is not supported in Palo Alto Cortex XDR + :param search_id:str + :return dict + """ + try: + response_dict = self.api_client.delete_search() + response_code = response_dict['code'] + # Construct a response object + return_obj = {} + if response_code == 200: + return_obj['success'] = response_dict['success'] + return_obj['message'] = 'Delete operation of a search id is not supported in Palo Alto Cortex XDR' + else: + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj + except Exception as err: + self.logger.error('error when deleting search %s:', err) + raise NotImplementedError('Error in delete search id operation') from err diff --git a/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py b/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py new file mode 100644 index 000000000..3a6b02c23 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py @@ -0,0 +1,44 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + "ConnectionError": ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + "AttributeError": ErrorCode.TRANSMISSION_INVALID_PARAMETER, + "AuthenticationError": ErrorCode.TRANSMISSION_AUTH_CREDENTIALS, + "SyntaxError": ErrorCode.TRANSMISSION_QUERY_LOGICAL_ERROR, + "EmptyResultException": ErrorCode.TRANSMISSION_RESPONSE_EMPTY_RESULT, + "APIPermissionException": ErrorCode.TRANSMISSION_FORBIDDEN, + "InvalidLicense": ErrorCode.TRANSMISSION_CONNECT, + "InvalidJsonException": ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + "MaxDailyQuotaException": ErrorCode.TRANSMISSION_CONNECT, + "TimeoutError": ErrorCode.TRANSMISSION_CONNECT, + "GeneralQuotaException": ErrorCode.TRANSMISSION_INVALID_PARAMETER, + "StreamNotImplemented": ErrorCode.TRANSMISSION_INVALID_PARAMETER +} + + +class ErrorMapper: + """ + Set Error Code + """ + logger = logger.set_logger(__name__) + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + + @staticmethod + def set_error_code(json_data, return_obj): + err_type = None + try: + err_type = json_data['type'] + except KeyError: + pass + + error_type = ErrorMapper.DEFAULT_ERROR + + if err_type in error_mapping: + error_type = error_mapping.get(err_type) + + if error_type == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: %s", str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_type) diff --git a/stix_shifter_modules/paloalto/stix_transmission/ping_connector.py b/stix_shifter_modules/paloalto/stix_transmission/ping_connector.py new file mode 100644 index 000000000..adac8f908 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/ping_connector.py @@ -0,0 +1,49 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +from requests.exceptions import ConnectionError +from .response_mapper import ResponseMapper + + +class PingConnector(BasePingConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def ping_connection(self): + """ + Ping the endpoint + :return: dict + """ + return_obj = {} + response_dict = {} + response_wrapper = None + try: + response_wrapper = self.api_client.ping_data_source() + response_code = response_wrapper.response.status_code + response_text = json.loads(response_wrapper.read().decode('utf-8')) + if response_code == 200 and 'reply' in response_text.keys(): + return_obj['success'] = True + else: + return_obj = ResponseMapper().status_code_mapping(response_code, response_text) + + except ValueError as ex: + if response_wrapper is not None: + self.logger.debug(response_wrapper.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + + except Exception as exe: + if 'timeout_error' in str(exe): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = exe.__class__.__name__ + response_dict['message'] = exe + self.logger.error('error when getting search results: %s', exe) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj diff --git a/stix_shifter_modules/paloalto/stix_transmission/query_connector.py b/stix_shifter_modules/paloalto/stix_transmission/query_connector.py new file mode 100644 index 000000000..4e895c0e1 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/query_connector.py @@ -0,0 +1,53 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_query_connector import BaseQueryConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +from requests.exceptions import ConnectionError +from .response_mapper import ResponseMapper + + +class QueryConnector(BaseQueryConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def create_query_connection(self, query): + """ + Function to create query connection + :param query: dict, Query + :return: dict + """ + return_obj = {} + response_dict = {} + response_wrapper = None + try: + response_wrapper = self.api_client.create_search(query) + if isinstance(response_wrapper, dict): + return response_wrapper + response_code = response_wrapper.response.status_code + response_text = json.loads(response_wrapper.read().decode('utf-8')) + if response_code == 200 and 'reply' in response_text.keys(): + return_obj['success'] = True + return_obj['search_id'] = response_text['reply'] + else: + return_obj = ResponseMapper().status_code_mapping(response_code, response_text) + + except ValueError as ex: + if response_wrapper is not None and not isinstance(response_wrapper, dict): + self.logger.debug(response_wrapper.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + + except Exception as exep: + if 'timeout_error' in str(exep): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = exep.__class__.__name__ + response_dict['message'] = exep + self.logger.error('error when getting search results: %s', exep) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj diff --git a/stix_shifter_modules/paloalto/stix_transmission/response_mapper.py b/stix_shifter_modules/paloalto/stix_transmission/response_mapper.py new file mode 100644 index 000000000..850b52649 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/response_mapper.py @@ -0,0 +1,104 @@ +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + + +class InvalidAuthenticationException(Exception): + pass + + +class InternalServerErrorException(Exception): + pass + + +class InvalidLicenseException(Exception): + pass + + +class APIPermissionException(Exception): + pass + + +class InvalidJsonException(Exception): + pass + + +class InvalidQueryException(Exception): + pass + + +class ResponseMapper: + logger = logger.set_logger(__name__) + connector = __name__.split('.')[1] + + @staticmethod + def status_code_mapping(response_code, response_text): + """ + raise exceptions for the error response code + :return dict + """ + return_obj = {} + response_dict = {} + + try: + if response_code == 500: + return_obj = ResponseMapper.handle_response_500(response_text, return_obj, response_dict) + else: + response_codes_match = { + 400: InvalidJsonException(response_text['reply']['err_msg']), + 401: InvalidAuthenticationException, + 402: InvalidLicenseException, + 403: APIPermissionException + } + raise response_codes_match[response_code] + except InvalidAuthenticationException: + response_dict['type'] = "AuthenticationError" + response_dict['message'] = "Invalid api_key" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except InvalidLicenseException: + response_dict['type'] = "InvalidLicense" + response_dict['message'] = "User does not have the required license type to run this API" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except APIPermissionException: + response_dict['type'] = "APIPermissionException" + response_dict['message'] = "The provided API Key does not have the required RBAC permissions to run " \ + "this API" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except InvalidJsonException as ex: + response_dict['type'] = "InvalidJsonException" + response_dict['message'] = ex + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except Exception as ex: + response_dict['type'] = ex.__class__.__name__ + response_dict['message'] = ex + ResponseMapper.logger.error('error when getting search results: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + return return_obj + + @staticmethod + def handle_response_500(response_text, return_obj, response_dict): + """ + Handle 500 error code response + :param response_text: dict + :param response_dict: dict + :param return_obj: dict + """ + try: + if isinstance(response_text['reply']['err_extra'], dict) and \ + 'parse_err' in response_text['reply']['err_extra'].keys(): + raise InvalidQueryException + raise InternalServerErrorException(response_text['reply']['err_msg']) + + except InvalidQueryException: + response_dict['type'] = "SyntaxError" + response_dict['message'] = 'Bad query Syntax' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except InternalServerErrorException as ex: + response_dict['type'] = "AttributeError" + response_dict['message'] = ex + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except Exception as exp: + response_dict['type'] = exp.__class__.__name__ + response_dict['message'] = exp + ResponseMapper.logger.error('error when getting search results: %s', exp) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + return return_obj diff --git a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py new file mode 100644 index 000000000..8a5fdaa02 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py @@ -0,0 +1,248 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +from requests.exceptions import ConnectionError +from .response_mapper import ResponseMapper +from os import path + +TO_STIX_PATH = "../stix_translation/json/to_stix_map.json" +CONFIG_MAP_PATH = "../stix_translation/json/config_map.json" + + +class InvalidQueryException(Exception): + pass + + +class StreamNotImplementedException(Exception): + pass + + +class ResultsConnector(BaseResultsConnector): + + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + @staticmethod + def load_json(rel_path_of_file): + """ + Consumes a json file and returns a dictionary + :param rel_path_of_file: str + :return: dict + """ + _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundError + + def create_results_connection(self, search_id, offset, length): + """ + Fetching the results using search id, offset and length + :param search_id: str, search id generated in transmit query + :param offset: str, offset value + :param length: str, length value + :return: dict + """ + + response_dict = {} + return_obj = {} + results = [] + response_wrapper = None + try: + min_range = int(offset) + max_range = int(offset) + int(length) + + # Grab the response, extract the response code, and convert it to readable json + response_wrapper = self.api_client.get_search_results(search_id) + response_code = response_wrapper.code + response_text = json.loads(response_wrapper.read().decode('utf-8')) + if response_code != 200: + return_obj = ResponseMapper().status_code_mapping(response_code, response_text) + else: + if 'status' in response_text['reply'].keys() and response_text['reply']['status'] \ + in ('SUCCESS', 'PARTIAL_SUCCESS'): + if 'data' in response_text['reply']['results'].keys() and \ + response_text['reply']['number_of_results'] > 0: + results = ResultsConnector.format_results_data(response_text['reply']['results']['data']) + elif 'stream_id' in response_text['reply']['results'].keys(): + raise StreamNotImplementedException + # *** The stream data feature is not used currently. It may be implemented in future *** + # stream_wrapper = self.api_client.get_stream_results(response_text['reply']['results'] + # ['stream_id']) + # results = ResultsConnector.format_stream_data(stream_wrapper.read().decode('utf-8')) + return_obj['success'] = True + return_obj['data'] = results[min_range:max_range] if results else [] + + elif 'status' in response_text['reply'].keys() and response_text['reply']['status'] == 'FAIL': + raise InvalidQueryException + + except ValueError as ex: + if response_wrapper is not None: + self.logger.debug(response_wrapper.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + except InvalidQueryException: + response_dict['type'] = "SyntaxError" + response_dict['message'] = 'Tenant Query Failed' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + except StreamNotImplementedException: + response_dict['type'] = "StreamNotImplemented" + response_dict['message'] = "Getting results from Stream Id is not implemented" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + except Exception as ex: + if 'timeout_error' in str(ex): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = ex.__class__.__name__ + response_dict['message'] = ex + self.logger.error('error when getting search results: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj + + @staticmethod + def format_results_data(result_data): + """ + Format the results in json format + :param result_data: list of dictionary items + :return: list + """ + results = [] + to_stix_mapping = ResultsConnector.load_json(TO_STIX_PATH) + mandatory_map = ResultsConnector.load_json(CONFIG_MAP_PATH)["mandatory_properties_to_stix"] + dataset_map = result_data[0]['dataset_name'] + try: + for log in result_data: + data = {} + results_dict = {} + dataset = "" + for field, value in log.items(): + if value is not None and value != "NULL" and value != '' and field != 'dataset_name' \ + and (field in to_stix_mapping[dataset_map].keys()): + stix_data_map = to_stix_mapping[dataset_map][field] + data = ResultsConnector.check_object(stix_data_map, mandatory_map, data, log, + field, value) + elif field == 'dataset_name': + dataset = value + + results_dict[dataset] = data + results.append(results_dict) + except (KeyError, IndexError, TypeError) as e: + raise e + return results + + @staticmethod + def check_object(stix_data_map, mandatory_map, data, log, field, value): + """ + The mandatory stix fields are removed, if no value is received from data source + :param stix_data_map: list of dictionary items + :param mandatory_map : list of dictionary items + :param data : dictionary + :param log: dictionary + :param field : string + :param value: string/integer + :return: list of dictionary items + """ + try: + + if isinstance(stix_data_map, list): + obj_list = [obj["object"] for obj in stix_data_map if "object" in obj.keys()] + for obj in obj_list: + if obj in mandatory_map.keys(): + data = ResultsConnector.check_mandatory_map(mandatory_map, obj, log, data, field, value) + else: + data[field] = value + elif isinstance(stix_data_map, dict): + if stix_data_map["object"] in mandatory_map.keys(): + data = ResultsConnector.check_mandatory_map(mandatory_map, stix_data_map["object"], + log, data, field, value) + else: + data[field] = value + except (KeyError, IndexError, TypeError) as e: + raise e + return data + + @staticmethod + def check_mandatory_map(mandatory_map, obj, log, data, field, value): + """ + Donot add the the fields which belongs to the object of mandatory field , + if data source doesnt return any value if the mandatory field + :param mandatory_map: list of dictionary items + :param obj : string + :param log : dictionary + :param data : dictionary + :param field : string + :param value : string/integer + """ + try: + if "file" in obj: + if log[mandatory_map[obj][0]] != "NULL" and log[mandatory_map[obj][0]] != '' and \ + log[mandatory_map[obj][0]] is not None: + data[field] = value + elif obj in ["user", "nt"]: + if obj == "nt": + data = ResultsConnector.process_nt_obj(mandatory_map, obj, data, log, value, field) + else: + if any((log[item] != "NULL" and log[item] != '' and log[item] is not None) + for item in mandatory_map[obj]): + data[field] = value + except (KeyError, IndexError, TypeError) as e: + raise e + return data + + @staticmethod + def process_nt_obj(mandatory_map, obj, data, log, value, field): + """ + check if any of the fields in the list is not null + :param mandatory_map: list of dictionary items + :param obj : string + :param log : dictionary + :param data : dictionary + :param field : string + :param value : string/integer + """ + nt_obj = False + try: + for i in mandatory_map[obj]: + if isinstance(i, list): + if any((log[item] != "NULL" and log[item] != '' and log[item] is not None) + for item in i): + nt_obj = True + else: + if nt_obj: + data[i] = log[i] if (log[i] != "NULL" and log[i] != '' and log[i] is not None) else "NULL" + data[field] = value + + except (KeyError, IndexError, TypeError) as e: + raise e + return data + + # *** The stream data feature is not used currently. It may be implemented in future *** + # @staticmethod + # def format_stream_data(stream_data): + # """ + # Format the stream data into json format + # :param stream_data: string + # :return: list + # """ + # results = [] + # + # temp_data = stream_data.split("\n") + # for log in temp_data: + # if log > " ": + # data = {} + # results_dict = {} + # dataset = "" + # for field, value in json.loads(log).items(): + # if value is not None and (isinstance(value, str) and value.lower() != "null") \ + # and field != 'dataset_name': + # data[field] = value + # elif field == 'dataset_name': + # dataset = value + # results_dict[dataset] = data + # results.append(results_dict) + # return results diff --git a/stix_shifter_modules/paloalto/stix_transmission/status_connector.py b/stix_shifter_modules/paloalto/stix_transmission/status_connector.py new file mode 100644 index 000000000..6c02861d5 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/status_connector.py @@ -0,0 +1,121 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import BaseStatusConnector +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from enum import Enum +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +from requests.exceptions import ConnectionError +from .response_mapper import ResponseMapper + + +class InvalidResponseException(Exception): + pass + + +class InvalidQueryException(Exception): + pass + + +class PaloAltoStatus(Enum): + PENDING = 'PENDING' + COMPLETED = 'COMPLETED' + FAIL = 'FAIL' + + +class StatusConnector(BaseStatusConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + # Map data source status to connector status + @staticmethod + def get_status(status): + """ + Return the status of the search id + :param status: str, + :return: str + """ + switcher = { + PaloAltoStatus.PENDING.value: Status.RUNNING, + PaloAltoStatus.COMPLETED.value: Status.COMPLETED, + PaloAltoStatus.FAIL.value: Status.ERROR + } + return switcher.get(status).value + + def create_status_connection(self, search_id): + """ + Fetching the progress and the status of the search id + :param search_id: str, search id + :return: dict + """ + return_obj = {} + response_dict = {} + response = None + try: + response = self.api_client.get_search_status(search_id) + # Based on the response + response_code = response.code + response_read = response.read() + response_text = json.loads(response_read) + + if response_code == 200: + if 'status' in response_text['reply'].keys(): + if response_text['reply']['status'] == "PENDING": + return_obj['success'] = True + return_obj['status'] = StatusConnector.get_status('PENDING') + return_obj['progress'] = 100 + elif response_text['reply']['status'] == "SUCCESS": + if 'number_of_results' in response_text['reply'].keys(): + return_obj['success'] = True + return_obj['status'] = StatusConnector.get_status('COMPLETED') + else: + raise InvalidResponseException + else: + return_obj = self.handle_fail_response(response_text, return_obj, response_dict) + else: + return_obj = ResponseMapper().status_code_mapping(response_code, response_text) + + except ValueError as ex: + if response is not None: + self.logger.debug(response.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + + except InvalidResponseException: + response_dict['type'] = 'EmptyResultException' + response_dict['message'] = 'Empty results received from Tenant' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + except Exception as err: + if 'timeout_error' in str(err): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = err.__class__.__name__ + response_dict['message'] = err + self.logger.error('error when getting search results: %s', err) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj + + def handle_fail_response(self, response_text, return_obj, response_dict): + """ + Handle partial success or failed status response + :param return_obj: dict + :param response_dict: dict + :param response_text: dict + """ + try: + if response_text['reply']['status'] == "PARTIAL_SUCCESS": + return_obj['success'] = True + return_obj['status'] = StatusConnector.get_status('COMPLETED') + return_obj['message'] = "Partial Success -At least one tenant failed to execute the query" + elif response_text['reply']['status'] == "FAIL": + raise InvalidQueryException + except InvalidQueryException: + + response_dict['type'] = "SyntaxError" + response_dict['message'] = 'Tenant Query Failed' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py new file mode 100644 index 000000000..5f68d5758 --- /dev/null +++ b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py @@ -0,0 +1,443 @@ +import unittest +from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator +from stix_shifter_modules.paloalto.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers + +MODULE = "paloalto" +entry_point = EntryPoint() +map_data = entry_point.get_results_translator().map_data +data_source = { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "paloalto", + "identity_class": "events" +} +options = {} + + +class TestPaloaltoResultsToStix(unittest.TestCase): + """ + class to perform unit test case for paloalto translate results + """ + + @staticmethod + def get_first(itr, constraint): + """ + return the obj in the itr if constraint is true + """ + return next( + (obj for obj in itr if constraint(obj)), + None + ) + + @staticmethod + def get_first_of_type(itr, typ): + """ + to check whether the object belongs to respective stix object + """ + return TestPaloaltoResultsToStix.get_first(itr, lambda o: isinstance(o, dict) and o.get('type') == typ) + + def test_file_json_to_stix(self): + """to test File stix object properties""" + + data = {'xdr_data': {'action_file_name': 'arp_cache_periodic.py', + 'action_file_size': '7943', + 'action_file_md5': 'f9d1ba13674eaf75c129534aba30487e', + 'action_module_md5': 'a2f22af507acc1961cf9504491f3e4b0', + 'action_process_image_md5': '93f1eb1ed4475f58f6870a385021c92d', + 'action_file_sha256': 'f6c82fe48662c1e6fa1b4732c020672bd6a929d0e0102712a4064d5c1351bfaf', + 'action_module_sha256': '2e365914142b4c0f8340594e253e8fa25f452338238b87e7d303af89d0747d78', + 'action_process_image_sha256': 'b55e59f545cdb5866b538c46fc280e51b4955df6b3a\ + 76686a73e80333757f003', + 'actor_process_file_access_time': '1634092022616', + 'os_actor_process_file_access_time': '1634096052944', + 'actor_process_file_mod_time': '1640745638339', + 'os_actor_process_file_mod_time': '1634096052944', + 'actor_process_file_create_time': '1631299512000', + 'os_actor_process_file_create_time': '1644385345812', + 'action_file_path': 'C:\\Program Files (x86)\\Google\\Policies', + 'action_process_image_path': 'C:\\Windows\\System32\\wevtutil.exe', + 'action_registry_file_path': 'C:\\Windows\\AppCompat\\Programs\\Amcache.hve', + 'actor_process_image_path': 'C:\\Program Files\\Amazon\\SSM\\amazon-ssm-agent.exe', + 'causality_actor_process_image_path': 'C:\\Windows\\System32\\lsass.exe', + 'os_actor_process_image_path': 'C:\\Program Files\\Amazon\\SSM\\amazon-ssm-agent.exe' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + file_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'file') + assert file_obj is not None + assert file_obj['type'] == 'file' + assert file_obj['name'] == 'arp_cache_periodic.py' + assert file_obj['size'] == 7943 + + def test_process_json_to_stix(self): + """ to test process stix object properties """ + data = {'xdr_data': {'action_process_image_command_line': 'C:\Windows\system32\lsass.exe', + 'actor_process_command_line': 'C:\Windows\system32\lsass.exe', + 'causality_actor_process_command_line': 'C:\Windows\system32\lsass.exe', + 'os_actor_process_command_line': 'C:\Windows\system32\lsass.exe', + 'actor_process_file_create_time': '1631299512000', + 'causality_actor_process_file_create_time': '1536995564723', + 'os_actor_process_file_create_time': '1637334466002', + 'action_process_image_name': 'wevtutil.exe', + 'actor_process_image_name': 'amazon-ssm-agent.exe', + 'causality_actor_process_image_name': 'pycharm64.exe', + 'os_actor_process_image_name': 'lsass.exe', + 'action_process_os_pid': '6228', + 'actor_process_os_pid': '4300', + 'causality_actor_process_os_pid': '5144', + 'os_actor_process_os_pid': '2744', + 'action_process_requested_parent_pid': '5144', + 'action_process_username': 'NT AUTHORITY\\NETWORK SERVICE', + 'actor_effective_username': 'EC2AMAZ-4KPRAA7\\Administrator' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + process_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'process') + assert process_obj is not None + assert process_obj['type'] == 'process' + assert process_obj['name'] == 'wevtutil.exe' + assert process_obj['pid'] == 6228 + assert process_obj['command_line'] == 'C:\Windows\system32\lsass.exe' + + def test_network_traffic_json_to_stix(self): + """to test network-traffic stix object properties""" + + data = {'xdr_data': {'action_local_port': '54083', + 'action_remote_port': '80', + 'action_network_protocol': 'TCP', + 'action_local_ip': '172.28.32.1', + 'action_remote_ip': '23.205.106.166' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + network_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'network-traffic') + + assert network_obj is not None + assert network_obj["type"] == 'network-traffic' + assert network_obj["src_port"] == 54083 + assert network_obj['src_ref'] == '1' + assert 'tcp' in network_obj["protocols"] + assert network_obj["dst_port"] == 80 + assert network_obj['dst_ref'] == '2' + + def test_mac_addr_json_to_stix(self): + """ + to test mac stix object properties + """ + data = {'xdr_data': {'associated_mac': '00:15:5d:6a:79:52'}} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + mac_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'mac-addr') + assert mac_obj is not None, 'mac-aadr object type not found' + assert mac_obj.keys() == {'type', 'value'} + assert mac_obj['type'] == 'mac-addr' + assert mac_obj['value'] == '00:15:5d:6a:79:52' + + def test_user_account_json_to_stix(self): + """to test user-account stix object properties""" + + data = {'xdr_data': {'actor_process_logon_id': '794419589', + 'action_process_username': 'EC2AMAZ-IQFSLIL\\Administrator', + 'actor_primary_username': 'NT AUTHORITY\\SYSTEM', + 'actor_primary_user_sid': 'S-1-5-18' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + user_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'user-account') + + assert user_obj is not None + assert user_obj['type'] == 'user-account' + assert user_obj['user_id'] == "S-1-5-18" + assert user_obj['extensions']['x-paloalto-user']['process_user_name'] == 'EC2AMAZ-IQFSLIL\\Administrator' + + def test_domain_name_json_to_stix(self): + """to test domain-name stix object properties""" + + data = {'xdr_data': {'auth_domain': 'dl.delivery.mp.microsoft.com', + 'dst_host_metadata_domain': '8.tlu.dl.delivery.mp.microsoft.com', + 'host_metadata_domain': '7.tlu.dl.delivery.mp.microsoft.com' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + domain_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'domain-name') + assert domain_obj is not None + assert domain_obj['type'] == 'domain-name' + assert domain_obj['value'] == 'dl.delivery.mp.microsoft.com' + + def test_windows_registry_key_json_to_stix(self): + """to test windows registry stix object properties""" + + data = {'xdr_data': {'action_registry_key_name': 'HKEY_LOCAL_MACHINE\\SYSTEM\\ControlSet001\\Services\\terminpt' + '\\Enum', + 'action_registry_value_name': 'Start' + + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + windows_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'windows-registry-key') + assert windows_obj is not None + assert windows_obj['type'] == 'windows-registry-key' + assert windows_obj['key'] == 'HKEY_LOCAL_MACHINE\\SYSTEM\\ControlSet001\\Services\\terminpt\\Enum' + assert windows_obj['values'] == 'Start' + + def test_url_json_to_stix(self): + """to test url stix object properties""" + + data = {'xdr_data': {'dst_action_url_category': 'https://paloalto/index.com' + + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + url_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'url') + assert url_obj is not None + assert url_obj['type'] == 'url' + assert url_obj['value'] == 'https://paloalto/index.com' + + def test_custom_file_json_to_stix(self): + """to test custom file stix object properties""" + data = {'xdr_data': { + 'action_file_extension': 'json', + 'action_file_attributes': 128, + 'action_file_last_writer_actor': 'AdgAsdUgVlUAAAbYAAAAAA==', + 'action_file_signature_status': 3, + 'action_file_type': 18, + 'manifest_file_version': 5 + }} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + custom_file_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'file') + assert custom_file_obj is not None + assert custom_file_obj['type'] == 'file' + assert custom_file_obj['extensions']['x-paloalto-file']['extension'] == "json" + assert custom_file_obj['extensions']['x-paloalto-file']['attributes'] == 128 + assert custom_file_obj['extensions']['x-paloalto-file']['writer'] == "AdgAsdUgVlUAAAbYAAAAAA==" + assert custom_file_obj['extensions']['x-paloalto-file']['manifest_version'] == 5 + + def test_custom_process_json_to_stix(self): + """to test custom process stix object properties""" + data = {'xdr_data': { + 'actor_process_instance_id': 'AdgAsdUgVlUAAAbYAAAAAA==', + 'actor_process_causality_id': 'AdgdeB9glMcAAAOYAAAAAA==', + 'actor_process_signature_vendor': 'Microsoft Corporation', + 'actor_process_signature_status': 'SIGNED', + 'actor_process_signature_product': 'Microsoft Windows', + 'action_process_termination_date': 1646315246849, + 'actor_process_execution_time': 1641280255000, + 'actor_process_is_native': 'FALSE' + }} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + custom_process_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'process') + assert custom_process_obj is not None + assert custom_process_obj['type'] == 'process' + assert custom_process_obj['extensions']['x-paloalto-process']['instance_id'] == "AdgAsdUgVlUAAAbYAAAAAA==" + assert custom_process_obj['extensions']['x-paloalto-process']['signature_vendor'] == "Microsoft Corporation" + assert custom_process_obj['extensions']['x-paloalto-process']['signature_status'] == "SIGNED" + assert custom_process_obj['extensions']['x-paloalto-process']['execution_time'] == "2022-01-04T07:10:55.000Z" + assert custom_process_obj['extensions']['x-paloalto-process']['is_native'] == False + + def test_asset_json_to_stix(self): + """to test custom oca-asset stix object properties""" + data = {'xdr_data': { + 'agent_version': '7.6.1.46600', + 'agent_hostname': 'EC2AMAZ-IQFSLIL', + 'agent_content_version': '350-80787', + 'agent_session_start_time': 1642662241933, + 'agent_id': '45.9.20.38', + 'agent_os_sub_type': 'Windows Server 2016', + 'agent_is_vdi': 'FALSE' + }} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + asset_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'x-oca-asset') + assert asset_obj is not None + assert asset_obj['type'] == 'x-oca-asset' + assert asset_obj['agent_version'] == "7.6.1.46600" + assert asset_obj['hostname'] == "EC2AMAZ-IQFSLIL" + assert asset_obj['content_version'] == "350-80787" + assert asset_obj['start_time'] == "2022-01-20T07:04:01.933Z" + assert asset_obj['os_sub_type'] == "Windows Server 2016" + assert asset_obj['is_vdi'] == False + + def test_evtlog_json_to_stix(self): + """to test custom evtlog stix object properties""" + data = {'xdr_data': { + 'action_evtlog_description': 'An account was logged off', + 'action_evtlog_source': 3, + 'action_evtlog_event_id': 4625, + 'action_evtlog_uid': 'S-1-5-19', + 'action_evtlog_level': 'INFO', + 'action_evtlog_version': 2 + }} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + evtlog_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'x-paloalto-evtlog') + assert evtlog_obj is not None + assert evtlog_obj['type'] == 'x-paloalto-evtlog' + assert evtlog_obj['description'] == "An account was logged off" + assert evtlog_obj['source'] == 3 + assert evtlog_obj['evtlog_id'] == 4625 + assert evtlog_obj['uid'] == "S-1-5-19" + assert evtlog_obj['level'] == "INFO" + + def test_event_json_to_stix(self): + """to test custom event stix object properties""" + data = {'xdr_data': { + 'event_id': 'OTE0MTk5MTg2MDI1NzUyODc0NQ==', + 'event_timestamp': 164632333729, + 'event_version': 25, + 'event_rpc_interface_uuid': '{00000136-0000-0000-C000-000000000046}', + 'event_type': 'EVENT_LOG' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + event_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert event_obj is not None + assert event_obj['event_id'] == "OTE0MTk5MTg2MDI1NzUyODc0NQ==" + assert event_obj['time'] == '1975-03-21T11:12:13.729Z' + assert event_obj['version'] == 25 + assert event_obj['uuid'] == '{00000136-0000-0000-C000-000000000046}' + assert event_obj['event_type'] == "EVENT_LOG" + + def test_custom_network_json_to_stix(self): + """to test custom network stix object properties""" + data = {'xdr_data': { + 'action_network_creation_time': 164632333729, + 'action_network_connection_id': 'AdgAsdUgVlUAAAbYAAAAAA==', + 'action_proxy': 'FALSE', + 'action_external_hostname': 'Windows 8' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + network_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'network-traffic') + assert network_obj is not None + assert network_obj['extensions']['x-paloalto-network']['creation_time'] == '1975-03-21T11:12:13.729Z' + assert network_obj['extensions']['x-paloalto-network']['connection_id'] == "AdgAsdUgVlUAAAbYAAAAAA==" + assert network_obj['extensions']['x-paloalto-network']['is_proxy'] == False + assert network_obj['extensions']['x-paloalto-network']['external_hostname'] == 'Windows 8' diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py new file mode 100644 index 000000000..9d54062c8 --- /dev/null +++ b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py @@ -0,0 +1,560 @@ +from stix_shifter.stix_translation import stix_translation +from stix_shifter_utils.utils.error_response import ErrorCode +import unittest +import re + +translation = stix_translation.StixTranslation() + + +def _remove_timestamp_from_query(queries): + pattern1 = r"to_epoch\(_time,\"millis\"\)\s*.=\s*\d{0,13}\s*and\s*to_epoch\(_time,\"millis\"\)\s*.=\s*\d{0,13}" + pattern2 = r"\{'from':\s*\d{0,13},\s*'to':\s*\d{0,13}\}" + if isinstance(queries, list): + modified_queries = [] + for query in queries: + replace_pat1 = re.sub(pattern1, '', str(query)) + replace_pat2 = re.sub(pattern2, '{}', replace_pat1) + modified_queries.append(replace_pat2) + return modified_queries + elif isinstance(queries, str): + replace_pat1 = re.sub(pattern1, '', queries) + return re.sub(pattern2, '{}', replace_pat1) + + +all_fields = "dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6," \ + "action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received," \ + "action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name," \ + "os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5," \ + "action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2," \ + "action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time," \ + "actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time," \ + "actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path," \ + "action_process_image_path,action_registry_file_path,actor_process_image_path," \ + "causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line," \ + "actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line," \ + "action_process_file_create_time,actor_process_file_create_time," \ + "causality_actor_process_file_create_time,os_actor_process_file_create_time," \ + "action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid," \ + "os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid," \ + "action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain," \ + "host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac," \ + "associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid," \ + "actor_primary_username," \ + "actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes," \ + "action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status," \ + "action_file_signature_vendor,action_file_signature_product,action_file_info_description," \ + "action_file_group,action_file_group_name,action_file_type,action_file_info_file_version," \ + "manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name," \ + "action_file_info_product_name,action_file_id,action_file_wildfire_verdict," \ + "action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id," \ + "actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor," \ + "actor_process_signature_status,actor_process_signature_product,actor_process_image_extension," \ + "action_process_termination_code,action_process_termination_date,action_remote_process_thread_id," \ + "action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel," \ + "action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname," \ + "agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi," \ + "action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description," \ + "action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid," \ + "action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description," \ + "event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type," \ + "event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data," \ + "action_proxy,host_metadata_hostname,action_external_hostname" + + +class TestQueryTranslator(unittest.TestCase): + if __name__ == "__main__": + unittest.main() + + def _test_query_assertions(self, query, queries): + """ + to assert the each query in the list against expected result + """ + self.assertIsInstance(queries, list) + self.assertIsInstance(query, dict) + self.assertIsInstance(query['queries'], list) + for index, each_query in enumerate(query.get('queries'), start=0): + self.assertEqual(each_query, queries[index]) + + def test_ipv4_query(self): + stix_pattern = "[ipv4-addr:value = '172.31.90.48']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_local_ip = \"172.31.90.48\" or " + "action_remote_ip = \"172.31.90.48\") and " + "(to_epoch(_time,\"millis\") >= 1645615464114 and to_epoch(_time,\"millis\") <= 1645615764114)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', 'timeframe': {" + "'from': 1645615464114, 'to': " + "1645615764114}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_traffic_query(self): + stix_pattern = "[network-traffic:dst_port=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port = 53996 and " + "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" + " | alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', 'timeframe': {" + "'from': 1645615637334, 'to': " + "1645615937334}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_greater_than(self): + stix_pattern = "[network-traffic:dst_port>53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port > 53996 and " + "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" + " | alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', 'timeframe': {" + "'from': 1645615637334, 'to': " + "1645615937334}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_not_equals(self): + stix_pattern = "[network-traffic:dst_port!=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port != 53996 and " + "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" + " | alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', 'timeframe': {" + "'from': 1645615637334, 'to': " + "1645615937334}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_enum_type(self): + stix_pattern = "[network-traffic:protocols[*] IN ('TCP','udp')]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_network_protocol in (ENUM.TCP," + "ENUM.UDP) and " + "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645635857746, " + "'to': 1645636157746}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_not_greater_than(self): + stix_pattern = "[network-traffic:dst_port NOT > 53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port <= 53996 and " + "(to_epoch(_time,\"millis\") >= 1645636616556 and to_epoch(_time,\"millis\") <= 1645636916556)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1645636616556, 'to': " + "1645636916556}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_less_than(self): + stix_pattern = "[network-traffic:dst_port<53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port < 53996 and " + "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1645636370847, 'to': " + "1645636670847}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_lessthan_or_equals(self): + stix_pattern = "[network-traffic:dst_port<=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port <= 53996 and " + "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1645636370847, 'to': " + "1645636670847}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_greaterthan_or_equals(self): + stix_pattern = "[network-traffic:dst_port>=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port >= 53996 and " + "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1645636370847, 'to': " + "1645636670847}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_file_like_operator(self): + stix_pattern = "[file:name LIKE 'edr-2022-02-09_08-56-58-474-checksum.txt']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or action_process_image_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or actor_process_image_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or causality_actor_process_image_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or os_actor_process_image_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\") and " + "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645635327346, " + "'to': 1645635627346}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_file_IN_operator(self): + stix_pattern = "[file:name IN ('SentinelOne_1.binlog','edr-2022-02-09_08-56-58-47-checksum.txt')]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name in (" + "\"SentinelOne_1.binlog\",\"edr-2022-02-09_08-56-58-47-checksum.txt\") or " + "action_process_image_name in (\"SentinelOne_1.binlog\"," + "\"edr-2022-02-09_08-56-58-47-checksum.txt\") or actor_process_image_name in (" + "\"SentinelOne_1.binlog\",\"edr-2022-02-09_08-56-58-47-checksum.txt\") or " + "causality_actor_process_image_name in (\"SentinelOne_1.binlog\"," + "\"edr-2022-02-09_08-56-58-47-checksum.txt\") or os_actor_process_image_name in (" + "\"SentinelOne_1.binlog\",\"edr-2022-02-09_08-56-58-47-checksum.txt\")) and " + "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1645635327346, 'to': " + "1645635627346}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_file_matches_operator(self): + stix_pattern = "[file:name MATCHES '^g.{2}.exe']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name ~= \"^g.{2}.exe\" or " + "action_process_image_name ~= \"^g.{2}.exe\" or actor_process_image_name ~= \"^g.{2}.exe\" or " + "causality_actor_process_image_name ~= \"^g.{2}.exe\" or os_actor_process_image_name ~= \"^g.{" + "2}.exe\") and " + "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645635327346, " + "'to': 1645635627346}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_mac_address_query(self): + stix_pattern = "[mac-addr:value ='12:83:0e:be:f3:1d']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((mac = \"12:83:0e:be:f3:1d\" or " + "associated_mac = \"12:83:0e:be:f3:1d\" or dst_associated_mac = \"12:83:0e:be:f3:1d\" or dst_mac " + "= \"12:83:0e:be:f3:1d\") and " + "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645635857746, " + "'to': 1645636157746}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_process_not_matches_operator(self): + stix_pattern = "[file:name NOT MATCHES '^g.{2}.exe']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name !~= \"^g.{2}.exe\" or " + "action_process_image_name !~= \"^g.{2}.exe\" or actor_process_image_name !~= \"^g.{2}.exe\" or " + "causality_actor_process_image_name !~= \"^g.{2}.exe\" or os_actor_process_image_name !~= \"^g.{" + "2}.exe\") and " + "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645636692740, " + "'to': 1645636992740}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_process_not_in_operator(self): + stix_pattern = "[process:name NOT IN('conhost.exe','AtBroker.exe')]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\") or actor_process_image_name not in (\"conhost.exe\"," + "\"AtBroker.exe\") or causality_actor_process_image_name not in (\"conhost.exe\",\"AtBroker.exe\") " + "or os_actor_process_image_name not in (\"conhost.exe\",\"AtBroker.exe\")) and " + "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1645636692740, 'to': " + "1645636992740}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_windows_regkey_not_like_operator(self): + stix_pattern = "[windows-registry-key:values[*] NOT LIKE 'DeltaUpdateFailure']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_registry_value_name not contains " + "\"DeltaUpdateFailure\" and " + "(to_epoch(_time,\"millis\") >= 1645638324427 and to_epoch(_time,\"millis\") <= 1645638624427)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645638324427, " + "'to': 1645638624427}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_from_multiple_comparison_expressions_joined_by_AND(self): + stix_pattern = "[file:name LIKE 'arp_cache.py' AND windows-registry-key:values[*] NOT LIKE 'SlotPerRow']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_registry_value_name not contains " + "\"SlotPerRow\" and (action_file_name contains \"arp_cache.py\" or action_process_image_name " + "contains \"arp_cache.py\" or actor_process_image_name contains \"arp_cache.py\" or " + "causality_actor_process_image_name contains \"arp_cache.py\" or os_actor_process_image_name " + "contains \"arp_cache.py\") and " + "(to_epoch(_time,\"millis\") >= 1645638324427 and to_epoch(_time,\"millis\") <= 1645638624427)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645638324427, " + "'to': 1645638624427}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_from_multiple_comparison_expressions_joined_by_OR(self): + stix_pattern = "[network-traffic:dst_port!=53996 OR process:name NOT MATCHES '^G.{5}U.{5}.exe']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_process_image_name !~= \"^G.{5}U.{" + "5}.exe\" or actor_process_image_name !~= \"^G.{5}U.{5}.exe\" or " + "causality_actor_process_image_name !~= \"^G.{5}U.{5}.exe\" or os_actor_process_image_name !~= " + "\"^G.{5}U.{5}.exe\") or action_remote_port != 53996 and " + "(to_epoch(_time,\"millis\") >= 1645635761271 and to_epoch(_time,\"millis\") <= 1645636061271)) " + "| alter dataset_name = " + "\"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1645635761271," + " 'to': 1645636061271}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_for_morethan_two_comparison_expressions_joined_by_AND(self): + stix_pattern = "[file:name NOT LIKE 'arp_cache.py' AND windows-registry-key:values[*] IN" \ + "('SlotPerRow','DeltaUpdateFailure') AND network-traffic:dst_port!=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port != 53996 and " + "action_registry_value_name in (\"SlotPerRow\",\"DeltaUpdateFailure\") and (action_file_name not " + "contains \"arp_cache.py\" or action_process_image_name not contains \"arp_cache.py\" or " + "actor_process_image_name not contains \"arp_cache.py\" or causality_actor_process_image_name not " + "contains \"arp_cache.py\" or os_actor_process_image_name not contains \"arp_cache.py\") and " + "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) | " + "alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1645636692740, " + "'to': 1645636992740}}}"] + queries = _remove_timestamp_from_query(queries) + + self._test_query_assertions(query, queries) + + def test_query_for_morethan_two_comparison_expressions_joined_by_OR(self): + stix_pattern = "[network-traffic:dst_port!=53996 OR process:name NOT MATCHES '^G.{5}U.{5}.exe' OR " \ + "mac-addr:value ='12:83:0e:be:f3:1d']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((mac = \"12:83:0e:be:f3:1d\" or " + "associated_mac = \"12:83:0e:be:f3:1d\" or dst_associated_mac = \"12:83:0e:be:f3:1d\" or dst_mac " + "= \"12:83:0e:be:f3:1d\") or (action_process_image_name !~= \"^G.{5}U.{5}.exe\" or " + "actor_process_image_name !~= \"^G.{5}U.{5}.exe\" or causality_actor_process_image_name !~= \"^G.{" + "5}U.{5}.exe\" or os_actor_process_image_name !~= \"^G.{5}U.{5}.exe\") or action_remote_port != " + "53996 and " + "(to_epoch(_time,\"millis\") >= 1645636966650 and to_epoch(_time,\"millis\") <= 1645637266650)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645636966650, " + "'to': 1645637266650}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_and_without_qualifier_query(self): + stix_pattern = "[file:name LIKE 'metadata']START t'2022-01-01T00:00:00.030Z' STOP t'2022-02-07T00:00:00.030Z'" \ + " AND[windows-registry-key:values[*] NOT LIKE 'SlotPerRow'OR " \ + "process:name MATCHES '^G.{5}U.{5}.exe']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name contains \"metadata\" or " + "action_process_image_name contains \"metadata\" or actor_process_image_name contains \"metadata\" " + "or causality_actor_process_image_name contains \"metadata\" or os_actor_process_image_name " + "contains \"metadata\") and (to_epoch(_time,\"millis\") >= 1640995200030 and to_epoch(_time," + "\"millis\") <= 1644192000030)) or ((action_process_image_name ~= \"^G.{5}U.{5}.exe\" or " + "actor_process_image_name ~= \"^G.{5}U.{5}.exe\" or causality_actor_process_image_name ~= \"^G.{" + "5}U.{5}.exe\" or os_actor_process_image_name ~= \"^G.{5}U.{5}.exe\") or " + "action_registry_value_name not contains \"SlotPerRow\" and " + "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " + "| alter dataset_name = " + "\"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1640995200030, " + "'to': 1645636157746}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_observation_with_invalid_operator(self): + stix_pattern = "[network-traffic:dst_port LIKE '53996']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'LIKE operator is supported only for string type input' in result['error'] + + def test_invalid_mac_address(self): + stix_pattern = "[mac-addr:value = '00:00:00']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'Invalid mac address' in result['error'] + + def test_invalid_stix_pattern(self): + stix_pattern = "[not_a_valid_pattern]" + result = translation.translate('paloalto', 'query', '{}', stix_pattern, {'validate_pattern': 'true'}) + assert result['success'] is False + assert ErrorCode.TRANSLATION_STIX_VALIDATION.value == result['code'] + assert stix_pattern[1:-1] in result['error'] + + def test_combined_observation_AND(self): + stix_pattern = "([process:pid = '868' OR process:name = 'svchost.exe'] AND " \ + "[network-traffic:dst_port = '53996' AND ipv4-addr:value = '172.31.31.67']) " \ + "START t'2022-01-19T11:00:00.000Z' STOP t'2022-02-07T11:00:00.003Z'" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_process_image_name = \"svchost.exe\" " + "or actor_process_image_name = \"svchost.exe\" or causality_actor_process_image_name = " + "\"svchost.exe\" or os_actor_process_image_name = \"svchost.exe\") or (" + "action_module_process_os_pid = 868 or action_process_os_pid = 868 or actor_process_os_pid = 868 " + "or causality_actor_process_os_pid = 868 or os_actor_process_os_pid = 868 or " + "action_process_requested_parent_pid = 868 or action_thread_parent_pid = 868 or " + "action_thread_child_pid = 868) and " + "(to_epoch(_time,\"millis\") >= 1642590000000 and to_epoch(_time,\"millis\") <= 1644231600003)) " + "or ((action_local_ip = \"172.31.31.67\" or action_remote_ip " + "= \"172.31.31.67\") and action_remote_port = 53996 and " + "(to_epoch(_time,\"millis\") >= 1642590000000 and to_epoch(_time,\"millis\") <= 1644231600003)) " + "| alter dataset_name = " + "\"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': 1642590000000, " + "'to': 1644231600003}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combined_observation_OR(self): + stix_pattern = "([file:name NOT MATCHES '^g.{2}.exe' AND network-traffic:dst_port <= 137]OR " \ + "[mac-addr:value = '12:83:0e:be:f3:1d' OR process:name LIKE 'svchost.exe']) " \ + "START t'2022-01-10T11:00:00.000Z' STOP t'2022-02-05T11:00:00.003Z'" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port <= 137 and (" + "action_file_name !~= \"^g.{2}.exe\" or action_process_image_name !~= \"^g.{2}.exe\" or " + "actor_process_image_name !~= \"^g.{2}.exe\" or causality_actor_process_image_name !~= \"^g.{" + "2}.exe\" or os_actor_process_image_name !~= \"^g.{2}.exe\") and " + "(to_epoch(_time,\"millis\") >= 1641812400000 and to_epoch(_time,\"millis\") <= 1644058800003)) " + "or ((action_process_image_name " + "contains \"svchost.exe\" or actor_process_image_name contains \"svchost.exe\" or " + "causality_actor_process_image_name contains \"svchost.exe\" or os_actor_process_image_name " + "contains \"svchost.exe\") or (mac = \"12:83:0e:be:f3:1d\" or associated_mac = " + "\"12:83:0e:be:f3:1d\" or dst_associated_mac = \"12:83:0e:be:f3:1d\" or dst_mac = " + "\"12:83:0e:be:f3:1d\") and " + "(to_epoch(_time,\"millis\") >= 1641812400000 and to_epoch(_time,\"millis\") <= 1644058800003)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1641812400000, " + "'to': 1644058800003}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_invalid_data_for_matches(self): + stix_pattern = "[process:pid MATCHES '53996']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'MATCHES operators is supported only for string type input' in result['error'] + + def test_invalid_regex_for_matches(self): + stix_pattern = "[process:name MATCHES 'wild^fire$']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert '^ symbol should be at the starting position of the expression' in result['error'] + + def test_invalid_enum_values(self): + stix_pattern = "[network-traffic:protocols[*] = 'idp']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'Unsupported ENUM values provided' in result['error'] + + def test_matches_operator_for_enum(self): + stix_pattern = "[network-traffic:protocols[*] MATCHES 'tcp']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'MATCHES operators is supported only for string type input' in result['error'] + + def test_invalid_operator_for_enum(self): + stix_pattern = "[network-traffic:protocols[*] < 'tcp']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'operator is not supported for Enum type input' in result['error'] + + def test_invalid_operator_for_string_input(self): + stix_pattern = "[process:name < 'conhost.exe' ]" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'operator is not supported for string type input' in result['error'] + + def test_invalid_qualifier(self): + stix_pattern = "[process:pid < '53996'] START " \ + "t'2022-02-01T08:43:10.003Z' STOP t'2022-01-07T10:43:10.005Z' " + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert "invalid_parameter" == result['code'] + assert 'Start time should be lesser than Stop time' in result['error'] + + def test_invalid_dollar_regexp(self): + stix_pattern = "[process:name MATCHES '^wildfire$s']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert '$ symbol should be at the ending position of the expression' in result['error'] + + def test_invalid_value_for_timestamp_field(self): + stix_pattern = "[process:created = '2022-02-0108:43:10.003Z']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'cannot convert the timestamp' in result['error'] + + def test_format_timestamp_fields(self): + stix_pattern = "[process:created = '2022-02-01T08:43:10.003Z']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_process_file_create_time = " + "1643704990003 or actor_process_file_create_time = 1643704990003 or " + "causality_actor_process_file_create_time = 1643704990003 or os_actor_process_file_create_time = " + "1643704990003) and " + "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "'timeframe': {'from': " + "1645635857746, " + "'to': 1645636157746}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) diff --git a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py new file mode 100644 index 000000000..8ed6bbf50 --- /dev/null +++ b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py @@ -0,0 +1,652 @@ +import unittest +from unittest.mock import patch +from stix_shifter.stix_transmission import stix_transmission +from stix_shifter_modules.paloalto.entry_point import EntryPoint +import json + + +class PaloaltoMockResponse: + """ class for Palo Alto mock response""" + + def __init__(self, response_code, txt): + self.status_code = response_code + self.content = txt + + def read(self): + """ to read contents of results returned by api""" + return bytearray(self.content, 'utf-8') + + +class StatusResponse: + """ class for status response""" + + def __init__(self, code, txt): + self.code = code + self.content = txt + + def read(self): + return bytearray(self.content, 'utf-8') + + +class PingResponse: + """ class for ping outer response""" + + def __init__(self, response_object): + self.response = response_object + + def read(self): + return bytearray(self.response.content, 'utf-8') + + +class QuotaResponse: + """ class for ping outer response""" + + def __init__(self, response_object): + self.response = response_object + + +class TestPaloaltoConnection(unittest.TestCase, object): + + @staticmethod + def connection(): + return { + "host": "hostbla" + } + + @staticmethod + def configuration(): + return { + "auth": { + "api_key": "bla", + "api_key_id": "bla", + "tenant": "bla" + } + } + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping(self, mock_ping_response): + """test ping connection""" + mocked_return_value = '{"reply":{"used_quota": 0.08015277777777775}}' + mock_ping = PaloaltoMockResponse(200, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is True + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') + def test_query_response(self, mock_search_response): + """test create search query""" + mocked_return_value = '{"reply": {"search_id": "07f63c733f5946_15006_inv"}}' + mock_search = PaloaltoMockResponse(200, mocked_return_value) + search_response = PingResponse(mock_search) + mock_search_response.return_value = search_response + + query = json.dumps({"xdr_data": {"query": "dataset = xdr_data | filter ((action_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\") or actor_process_image_name not " + "in (\"conhost.exe\",\"AtBroker.exe\") or " + "causality_actor_process_image_name not in (\"conhost.exe\"," + "\"AtBroker.exe\") or os_actor_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\")) and (to_epoch(_time," + "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " + "1644883200000)) or ((action_process_file_create_time = " + "1643704990003 or actor_process_file_create_time = 1643704990003 or " + "causality_actor_process_file_create_time = 1643704990003 or " + "os_actor_process_file_create_time = 1643704990003) and (to_epoch(" + "_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") " + "<= 1644883200000)) or ((action_process_image_name ~= \"wildfire$\" " + "or actor_process_image_name ~= \"wildfire$\" or " + "causality_actor_process_image_name ~= \"wildfire$\" or " + "os_actor_process_image_name ~= \"wildfire$\") and (to_epoch(" + "_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") " + "<= 1644883200000)) | alter dataset_name = \"xdr_data\" | fields " + "dataset_name,action_local_ip,action_remote_ip," + "agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port," + "action_remote_port,action_network_protocol,action_file_name," + "action_file_size,action_file_md5,action_module_md5," + "action_process_image_md5,action_file_authenticode_sha1," + "action_file_authenticode_sha2,action_file_sha256," + "action_module_sha256,action_process_image_sha256," + "action_file_access_time,actor_process_file_access_time," + "os_actor_process_file_access_time,action_file_mod_time," + "actor_process_file_mod_time,os_actor_process_file_mod_time," + "action_file_create_time,action_file_path," + "action_process_image_path,action_registry_file_path," + "actor_process_image_path,causality_actor_process_image_path," + "os_actor_process_image_path,action_process_image_command_line," + "actor_process_command_line,causality_actor_process_command_line," + "os_actor_process_command_line,action_process_file_create_time," + "actor_process_file_create_time," + "causality_actor_process_file_create_time," + "os_actor_process_file_create_time,action_process_image_name," + "actor_process_image_name,causality_actor_process_image_name," + "os_actor_process_image_name,action_module_process_os_pid ," + "action_process_os_pid,actor_process_os_pid," + "causality_actor_process_os_pid,os_actor_process_os_pid," + "action_process_requested_parent_pid,action_thread_parent_pid," + "action_thread_child_pid,action_process_username,auth_domain," + "dst_host_metadata_domain,host_metadata_domain," + "dst_action_url_category ,action_registry_key_name," + "action_registry_value_name,mac,associated_mac,dst_associated_mac ," + "dst_mac,dst_user_id,user_id,action_username," + "actor_primary_username,actor_process_logon_id | limit 10000 ", + "timeframe": {"from": 1644451200000, "to": 1644883200000}}}) + + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query(query) + assert query_response is not None + assert 'success' in query_response + assert query_response['success'] is True + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_response(self, mock_status_response): + mocked_return_value = '{"reply" : {"status": "SUCCESS","number_of_results":100}}' + mock_status_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is True + assert status_response['status'] == "COMPLETED" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_result_response(self, mock_result_response): + """test for valid result response""" + mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", + "number_of_results": 1, + "results": {"data": [{"dataset_name": "xdr_data", + "causality_actor_process_image_name": + "taskhostw.exe"}]}}}) + mock_result_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'] == [{'xdr_data': {'causality_actor_process_image_name': 'taskhostw.exe'}}] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_result_with_empty_nt_response(self, mock_result_response): + """test for valid result response""" + mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", + "number_of_results": 1, + "results": {"data": [{"dataset_name": "xdr_data", + 'action_local_ip': '', 'action_remote_ip': '', + 'agent_ip_addresses_v6': None, + 'dst_agent_ip_addresses_v6': None, + 'action_local_port': 0, + 'action_remote_port': 0, + 'action_pkts_sent': None, + 'action_pkts_received': None, + 'action_network_protocol': "NULL", + 'actor_process_image_name': "lsass.exe"}]}}}) + mock_result_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "62428d95420f47_24655_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'] == [{'xdr_data': {'actor_process_image_name': 'lsass.exe'}}] + + def test_delete_response(self): + """test delete response""" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + search_id = "e1d1b56ca81845_15180_inv" + delete_response = transmission.delete(search_id) + assert delete_response is not None + assert delete_response['success'] is True + assert delete_response['message'] == "Delete operation of a search id is not supported in Palo Alto Cortex XDR" + + def test_is_async(self): + """check for synchronous or asynchronous""" + entry_point = EntryPoint(self.connection(), self.configuration()) + check_async = entry_point.is_async() + assert check_async is True + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_400_exception(self, mock_ping_response): + """Test ping response with 400 exception""" + mocked_return_value = '{"reply":{"err_msg": "InvalidJson"}}' + mock_ping = PaloaltoMockResponse(400, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert 'error' in ping_response + assert ping_response['code'] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_401_auth_exception(self, mock_ping_response): + """test 401 authentication error exception""" + mocked_return_value = '{"reply": { "err_msg" : "auth Error"}}' + mock_ping = PaloaltoMockResponse(401, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert ping_response['code'] == "authentication_fail" + assert "Invalid api_key" in ping_response['error'] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_500_exception(self, mock_ping_response): + """check 500 bad query exception""" + mocked_return_value = '{"reply": {"err_extra":{"parse_err" :"Internal server error" }}}' + mock_ping = PaloaltoMockResponse(500, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert "Bad query Syntax" in ping_response["error"] + assert ping_response["code"] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_internal_server_exception(self, mock_ping_response): + """check 500 bad query exception""" + mocked_return_value = '{"reply": {"err_msg":"Internal server error","err_extra":"server error"}}' + mock_ping = PaloaltoMockResponse(500, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert "Internal server error" in ping_response["error"] + assert ping_response["code"] == "invalid_parameter" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_402_license_exception(self, mock_ping_response): + """Check 402 invalid license exception""" + mocked_return_value = '{"reply": { "err_msg" : "Invalid license"}}' + mock_ping = PaloaltoMockResponse(402, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert 'error' in ping_response + assert ping_response['code'] == "service_unavailable" + assert "User does not have the required license type to run this API" in ping_response['error'] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_empty_result_exception(self, mock_status_response): + """Test empty results exception""" + mocked_return_value = '{"reply" : {"status": "SUCCESS"}}' + mock_status_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is False + assert 'error' in status_response + assert "Empty results received from Tenant" in status_response['error'] + assert status_response['code'] == "no_results" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_partial_success_exception(self, mock_status_response): + """Test partial success status response exception""" + mocked_return_value = '{"reply" : {"status": "PARTIAL_SUCCESS"}}' + mock_status_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is True + assert status_response['message'] == "Partial Success -At least one tenant failed to execute the query" + assert status_response['status'] == "COMPLETED" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_fail_exception(self, mock_search_response): + """Test status fail exception""" + mocked_return_value = '{"reply" : {"status": "FAIL"}}' + mock_search_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is False + assert "Tenant Query Failed" in status_response['error'] + assert status_response['code'] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_failed_query_response(self, mock_result_response): + """Test failed query result response""" + mocked_return_value = json.dumps({"reply": {"status": "FAIL"}}) + mock_result_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert "Tenant Query Failed" in result_response['error'] + assert result_response['code'] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') + def test_create_query_value_error_exception(self, mock_search_response): + """test create query value error with invalid json""" + mocked_return_value = "Invalid_json" + mock_search = PaloaltoMockResponse(200, mocked_return_value) + search_response = PingResponse(mock_search) + mock_search_response.return_value = search_response + query = "" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query(query) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert "Cannot parse response" in query_response["error"] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_value_error_exception(self, mock_search_response): + """test ping connector value error with invalid json""" + mocked_return_value = "Invalid_json" + mock_search = PaloaltoMockResponse(200, mocked_return_value) + search_response = PingResponse(mock_search) + mock_search_response.return_value = search_response + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.ping() + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert "Cannot parse response" in query_response["error"] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') + def test_create_query_exception(self, mock_search_response): + """Test search response with 400 exception""" + mocked_return_value = '{"reply": { "err_msg" : "400 error"}}' + mock_ping = PaloaltoMockResponse(400, mocked_return_value) + mock_search_response.return_value = PingResponse(mock_ping) + query = "{}" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.query(query) + assert ping_response is not None + assert ping_response['success'] is False + assert 'error' in ping_response + assert ping_response['code'] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_results_value_error_exception(self, get_search_results): + """test results connector value error with invalid json""" + mocked_return_value = "Invalid json" + get_search_results.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert "Cannot parse response" in result_response["error"] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_results_401_exception(self, get_search_results): + """test results with 401 exception""" + mocked_return_value = '{"reply": { "err_msg" : "auth Error"}}' + get_search_results.return_value = StatusResponse(401, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert result_response['code'] == "authentication_fail" + assert "Invalid api_key" in result_response['error'] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_pending_response(self, mock_search_response): + """Test status pending exception""" + mocked_return_value = '{"reply" : {"status": "PENDING"}}' + mock_search_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is True + assert status_response['status'] == "RUNNING" + assert status_response['progress'] == 100 + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_value_error_exception(self, mock_search_status): + """test status connector value error with invalid json""" + mocked_return_value = "Invalid json" + mock_search_status.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert status_response['success'] is False + assert 'error' in status_response + assert "Cannot parse response" in status_response["error"] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_403_exception(self, get_search_results): + """test results with 403 exception""" + mocked_return_value = '{"reply": { "err_msg" : "api permission exception"}}' + get_search_results.return_value = StatusResponse(403, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert status_response['success'] is False + assert status_response['code'] == "forbidden" + assert "The provided API Key does not have the required RBAC permissions to run this API" in \ + status_response['error'] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') + def test_ping_invalid_host(self, mock_quota_response): + """test ping connection with invalid host""" + mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') + mock_quota_response.return_value = PingResponse(mock_quota) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert any(i in ping_response['error'] for i in ["Invalid Host", "timeout error", + "Failed to establish a new connection"]) + assert any(i in ping_response['code'] for i in ["service_unavailable", "unknown"]) + + def test_ping_timeout_error(self): + """test ping connection timeout error""" + connection = self.connection() + connection["options"] = {"timeout": 1} + transmission = stix_transmission.StixTransmission('paloalto', connection, self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert "timeout_error (1 sec)" in ping_response['error'] + assert ping_response['code'] == "service_unavailable" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') + def test_query_search_timeout_error(self, mock_quota_response): + """test query search timeout error""" + mock_quota_response.return_value = {"success": True} + connection = self.connection() + connection["options"] = {"timeout": 1} + transmission = stix_transmission.StixTransmission('paloalto', connection, self.configuration()) + query = {"xdr_data": {"query": "dataset = xdr_data | filter ((action_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\") or actor_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\") or causality_actor_process_image_name not " + "in (\"conhost.exe\",\"AtBroker.exe\") or os_actor_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\")) and (to_epoch(_time,\"millis\") >= " + "1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or ((" + "action_process_file_create_time = 1643704990003 or " + "actor_process_file_create_time = 1643704990003 or " + "causality_actor_process_file_create_time = 1643704990003 or " + "os_actor_process_file_create_time = 1643704990003) and (to_epoch(_time," + "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " + "1644883200000)) or ((action_process_image_name ~= \"wildfire$\" or " + "actor_process_image_name ~= \"wildfire$\" or " + "causality_actor_process_image_name ~= \"wildfire$\" or " + "os_actor_process_image_name ~= \"wildfire$\") and (to_epoch(_time," + "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " + "1644883200000)) | alter dataset_name = \"xdr_data\" | fields dataset_name," + "action_local_ip,action_remote_ip,agent_ip_addresses_v6," + "dst_agent_ip_addresses_v6,action_local_port,action_remote_port," + "action_network_protocol,action_file_name,action_file_size,action_file_md5," + "action_module_md5,action_process_image_md5,action_file_authenticode_sha1," + "action_file_authenticode_sha2,action_file_sha256,action_module_sha256," + "action_process_image_sha256,action_file_access_time," + "actor_process_file_access_time,os_actor_process_file_access_time," + "action_file_mod_time,actor_process_file_mod_time," + "os_actor_process_file_mod_time,action_file_create_time,action_file_path," + "action_process_image_path,action_registry_file_path,actor_process_image_path," + "causality_actor_process_image_path,os_actor_process_image_path," + "action_process_image_command_line,actor_process_command_line," + "causality_actor_process_command_line,os_actor_process_command_line," + "action_process_file_create_time,actor_process_file_create_time," + "causality_actor_process_file_create_time,os_actor_process_file_create_time," + "action_process_image_name,actor_process_image_name," + "causality_actor_process_image_name,os_actor_process_image_name," + "action_module_process_os_pid ,action_process_os_pid,actor_process_os_pid," + "causality_actor_process_os_pid,os_actor_process_os_pid," + "action_process_requested_parent_pid,action_thread_parent_pid," + "action_thread_child_pid,action_process_username,auth_domain," + "dst_host_metadata_domain,host_metadata_domain,dst_action_url_category ," + "action_registry_key_name,action_registry_value_name,mac,associated_mac," + "dst_associated_mac ,dst_mac,dst_user_id,user_id,action_username," + "actor_primary_username,actor_process_logon_id | limit 10000 ", + "timeframe": {"from": 1644451200000, "to": 1644883200000}}} + + query = json.dumps(query) + ping_response = transmission.query(query) + assert ping_response is not None + assert ping_response['success'] is False + assert "timeout_error (1 sec)" in ping_response['error'] + assert ping_response['code'] == "service_unavailable" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_query_invalid_host(self, mock_ping_response): + """test query invalid host error""" + mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') + mock_ping_response.return_value = PingResponse(mock_quota) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query = {"xdr_data": {"query": "dataset = xdr_data | filter ((action_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\") or actor_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\") or causality_actor_process_image_name not " + "in (\"conhost.exe\",\"AtBroker.exe\") or os_actor_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\")) and (to_epoch(_time,\"millis\") >= " + "1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or ((" + "action_process_file_create_time = 1643704990003 or " + "actor_process_file_create_time = 1643704990003 or " + "causality_actor_process_file_create_time = 1643704990003 or " + "os_actor_process_file_create_time = 1643704990003) and (to_epoch(_time," + "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " + "1644883200000)) or ((action_process_image_name ~= \"wildfire$\" or " + "actor_process_image_name ~= \"wildfire$\" or " + "causality_actor_process_image_name ~= \"wildfire$\" or " + "os_actor_process_image_name ~= \"wildfire$\") and (to_epoch(_time," + "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " + "1644883200000)) | alter dataset_name = \"xdr_data\" | fields dataset_name," + "action_local_ip,action_remote_ip,agent_ip_addresses_v6," + "dst_agent_ip_addresses_v6,action_local_port,action_remote_port," + "action_network_protocol,action_file_name,action_file_size,action_file_md5," + "action_module_md5,action_process_image_md5,action_file_authenticode_sha1," + "action_file_authenticode_sha2,action_file_sha256,action_module_sha256," + "action_process_image_sha256,action_file_access_time," + "actor_process_file_access_time,os_actor_process_file_access_time," + "action_file_mod_time,actor_process_file_mod_time," + "os_actor_process_file_mod_time,action_file_create_time,action_file_path," + "action_process_image_path,action_registry_file_path,actor_process_image_path," + "causality_actor_process_image_path,os_actor_process_image_path," + "action_process_image_command_line,actor_process_command_line," + "causality_actor_process_command_line,os_actor_process_command_line," + "action_process_file_create_time,actor_process_file_create_time," + "causality_actor_process_file_create_time,os_actor_process_file_create_time," + "action_process_image_name,actor_process_image_name," + "causality_actor_process_image_name,os_actor_process_image_name," + "action_module_process_os_pid ,action_process_os_pid,actor_process_os_pid," + "causality_actor_process_os_pid,os_actor_process_os_pid," + "action_process_requested_parent_pid,action_thread_parent_pid," + "action_thread_child_pid,action_process_username,auth_domain," + "dst_host_metadata_domain,host_metadata_domain,dst_action_url_category ," + "action_registry_key_name,action_registry_value_name,mac,associated_mac," + "dst_associated_mac ,dst_mac,dst_user_id,user_id,action_username," + "actor_primary_username,actor_process_logon_id | limit 10000 ", + "timeframe": {"from": 1644451200000, "to": 1644883200000}}} + + query = json.dumps(query) + query_response = transmission.query(query) + assert query_response is not None + assert query_response['success'] is False + assert any(i in query_response['error'] for i in ["Invalid Host", "timeout_error", + "Failed to establish a new connection"]) + assert any(i in query_response['code'] for i in ["service_unavailable", "unknown"]) + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') + def test_status_timeout_error(self, mock_quota_response): + """test status timeout error""" + mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') + mock_quota_response.return_value = PingResponse(mock_quota) + connection = self.connection() + connection["options"] = {"timeout": 1} + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', connection, self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert status_response['success'] is False + assert "timeout_error (1 sec)" in status_response['error'] + assert status_response['code'] == "service_unavailable" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') + def test_status_invalid_host_error(self, mock_quota_response): + """test status invalid host error""" + mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') + mock_quota_response.return_value = PingResponse(mock_quota) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert status_response['success'] is False + assert any(i in status_response['error'] for i in ["Invalid Host", "timeout_error", + "Failed to establish a new connection"]) + assert any(i in status_response['code'] for i in ["service_unavailable", "unknown"]) + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') + def test_results_invalid_host_error(self, mock_quota_response): + """test results invalid error""" + mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') + mock_quota_response.return_value = PingResponse(mock_quota) + search_id = "e1d1b56ca81845_15180_inv" + offset = 0 + length = 3 + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + results_response = transmission.results(search_id, offset, length) + assert results_response is not None + assert results_response['success'] is False + assert any(i in results_response['error'] for i in ["Invalid Host", "timeout_error", + "Failed to establish a new connection"]) + assert any(i in results_response['code'] for i in ["service_unavailable", "unknown"]) + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') + def test_results_timeout_error(self, mock_quota_response): + """test results timeout error""" + mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') + mock_quota_response.return_value = PingResponse(mock_quota) + connection = self.connection() + connection["options"] = {"timeout": 1} + search_id = "e1d1b56ca81845_15180_inv" + offset = 0 + length = 3 + transmission = stix_transmission.StixTransmission('paloalto', connection, self.configuration()) + results_response = transmission.results(search_id, offset, length) + assert results_response is not None + assert results_response['success'] is False + assert "timeout_error (1 sec)" in results_response['error'] + assert results_response['code'] == "service_unavailable" From 76a3cfcbd8e9bcaa6676bc6194dced7bc60aaeef Mon Sep 17 00:00:00 2001 From: lakshmi51974368 <92726459+lakshmi51974368@users.noreply.github.com> Date: Tue, 22 Mar 2022 12:44:26 +0000 Subject: [PATCH 2/7] Updated PaloAlto Transmit Unit test Cases Updated PaloAlto Transmit Unit test Cases --- .../tests/stix_transmission/test_paloalto.py | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py index 8ed6bbf50..c75f94a96 100644 --- a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py +++ b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py @@ -45,6 +45,8 @@ def __init__(self, response_object): self.response = response_object +@patch('stix_shifter_modules.cybereason.stix_transmission.api_client' + '.APIClient.__init__') class TestPaloaltoConnection(unittest.TestCase, object): @staticmethod @@ -62,9 +64,9 @@ def configuration(): "tenant": "bla" } } - + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping(self, mock_ping_response): + def test_ping(self, mock_ping_response, mock_init): """test ping connection""" mocked_return_value = '{"reply":{"used_quota": 0.08015277777777775}}' mock_ping = PaloaltoMockResponse(200, mocked_return_value) @@ -75,7 +77,7 @@ def test_ping(self, mock_ping_response): assert ping_response['success'] is True @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') - def test_query_response(self, mock_search_response): + def test_query_response(self, mock_search_response, mock_init): """test create search query""" mocked_return_value = '{"reply": {"search_id": "07f63c733f5946_15006_inv"}}' mock_search = PaloaltoMockResponse(200, mocked_return_value) @@ -139,7 +141,7 @@ def test_query_response(self, mock_search_response): assert query_response['success'] is True @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_response(self, mock_status_response): + def test_status_response(self, mock_status_response, mock_init): mocked_return_value = '{"reply" : {"status": "SUCCESS","number_of_results":100}}' mock_status_response.return_value = StatusResponse(200, mocked_return_value) search_id = "e1d1b56ca81845_15180_inv" @@ -151,7 +153,7 @@ def test_status_response(self, mock_status_response): assert status_response['status'] == "COMPLETED" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_result_response(self, mock_result_response): + def test_result_response(self, mock_result_response, mock_init): """test for valid result response""" mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", "number_of_results": 1, @@ -170,7 +172,7 @@ def test_result_response(self, mock_result_response): assert result_response['data'] == [{'xdr_data': {'causality_actor_process_image_name': 'taskhostw.exe'}}] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_result_with_empty_nt_response(self, mock_result_response): + def test_result_with_empty_nt_response(self, mock_result_response, mock_init): """test for valid result response""" mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", "number_of_results": 1, @@ -195,7 +197,7 @@ def test_result_with_empty_nt_response(self, mock_result_response): assert 'data' in result_response assert result_response['data'] == [{'xdr_data': {'actor_process_image_name': 'lsass.exe'}}] - def test_delete_response(self): + def test_delete_response(self, mock_init): """test delete response""" transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) search_id = "e1d1b56ca81845_15180_inv" @@ -204,14 +206,14 @@ def test_delete_response(self): assert delete_response['success'] is True assert delete_response['message'] == "Delete operation of a search id is not supported in Palo Alto Cortex XDR" - def test_is_async(self): + def test_is_async(self, mock_init): """check for synchronous or asynchronous""" entry_point = EntryPoint(self.connection(), self.configuration()) check_async = entry_point.is_async() assert check_async is True @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_400_exception(self, mock_ping_response): + def test_ping_400_exception(self, mock_ping_response, mock_init): """Test ping response with 400 exception""" mocked_return_value = '{"reply":{"err_msg": "InvalidJson"}}' mock_ping = PaloaltoMockResponse(400, mocked_return_value) @@ -224,7 +226,7 @@ def test_ping_400_exception(self, mock_ping_response): assert ping_response['code'] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_401_auth_exception(self, mock_ping_response): + def test_ping_401_auth_exception(self, mock_ping_response, mock_init): """test 401 authentication error exception""" mocked_return_value = '{"reply": { "err_msg" : "auth Error"}}' mock_ping = PaloaltoMockResponse(401, mocked_return_value) @@ -237,7 +239,7 @@ def test_ping_401_auth_exception(self, mock_ping_response): assert "Invalid api_key" in ping_response['error'] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_500_exception(self, mock_ping_response): + def test_ping_500_exception(self, mock_ping_response, mock_init): """check 500 bad query exception""" mocked_return_value = '{"reply": {"err_extra":{"parse_err" :"Internal server error" }}}' mock_ping = PaloaltoMockResponse(500, mocked_return_value) @@ -250,7 +252,7 @@ def test_ping_500_exception(self, mock_ping_response): assert ping_response["code"] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_internal_server_exception(self, mock_ping_response): + def test_ping_internal_server_exception(self, mock_ping_response, mock_init): """check 500 bad query exception""" mocked_return_value = '{"reply": {"err_msg":"Internal server error","err_extra":"server error"}}' mock_ping = PaloaltoMockResponse(500, mocked_return_value) @@ -263,7 +265,7 @@ def test_ping_internal_server_exception(self, mock_ping_response): assert ping_response["code"] == "invalid_parameter" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_402_license_exception(self, mock_ping_response): + def test_ping_402_license_exception(self, mock_ping_response, mock_init): """Check 402 invalid license exception""" mocked_return_value = '{"reply": { "err_msg" : "Invalid license"}}' mock_ping = PaloaltoMockResponse(402, mocked_return_value) @@ -277,7 +279,7 @@ def test_ping_402_license_exception(self, mock_ping_response): assert "User does not have the required license type to run this API" in ping_response['error'] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_empty_result_exception(self, mock_status_response): + def test_status_empty_result_exception(self, mock_status_response, mock_init): """Test empty results exception""" mocked_return_value = '{"reply" : {"status": "SUCCESS"}}' mock_status_response.return_value = StatusResponse(200, mocked_return_value) @@ -292,7 +294,7 @@ def test_status_empty_result_exception(self, mock_status_response): assert status_response['code'] == "no_results" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_partial_success_exception(self, mock_status_response): + def test_status_partial_success_exception(self, mock_status_response, mock_init): """Test partial success status response exception""" mocked_return_value = '{"reply" : {"status": "PARTIAL_SUCCESS"}}' mock_status_response.return_value = StatusResponse(200, mocked_return_value) @@ -306,7 +308,7 @@ def test_status_partial_success_exception(self, mock_status_response): assert status_response['status'] == "COMPLETED" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_fail_exception(self, mock_search_response): + def test_status_fail_exception(self, mock_search_response, mock_init): """Test status fail exception""" mocked_return_value = '{"reply" : {"status": "FAIL"}}' mock_search_response.return_value = StatusResponse(200, mocked_return_value) @@ -320,7 +322,7 @@ def test_status_fail_exception(self, mock_search_response): assert status_response['code'] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_failed_query_response(self, mock_result_response): + def test_failed_query_response(self, mock_result_response, mock_init): """Test failed query result response""" mocked_return_value = json.dumps({"reply": {"status": "FAIL"}}) mock_result_response.return_value = StatusResponse(200, mocked_return_value) @@ -336,7 +338,7 @@ def test_failed_query_response(self, mock_result_response): assert result_response['code'] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') - def test_create_query_value_error_exception(self, mock_search_response): + def test_create_query_value_error_exception(self, mock_search_response, mock_init): """test create query value error with invalid json""" mocked_return_value = "Invalid_json" mock_search = PaloaltoMockResponse(200, mocked_return_value) @@ -351,7 +353,7 @@ def test_create_query_value_error_exception(self, mock_search_response): assert "Cannot parse response" in query_response["error"] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_value_error_exception(self, mock_search_response): + def test_ping_value_error_exception(self, mock_search_response, mock_init): """test ping connector value error with invalid json""" mocked_return_value = "Invalid_json" mock_search = PaloaltoMockResponse(200, mocked_return_value) @@ -365,7 +367,7 @@ def test_ping_value_error_exception(self, mock_search_response): assert "Cannot parse response" in query_response["error"] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') - def test_create_query_exception(self, mock_search_response): + def test_create_query_exception(self, mock_search_response, mock_init): """Test search response with 400 exception""" mocked_return_value = '{"reply": { "err_msg" : "400 error"}}' mock_ping = PaloaltoMockResponse(400, mocked_return_value) @@ -379,7 +381,7 @@ def test_create_query_exception(self, mock_search_response): assert ping_response['code'] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_results_value_error_exception(self, get_search_results): + def test_results_value_error_exception(self, get_search_results, mock_init): """test results connector value error with invalid json""" mocked_return_value = "Invalid json" get_search_results.return_value = StatusResponse(200, mocked_return_value) @@ -394,7 +396,7 @@ def test_results_value_error_exception(self, get_search_results): assert "Cannot parse response" in result_response["error"] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_results_401_exception(self, get_search_results): + def test_results_401_exception(self, get_search_results, mock_init): """test results with 401 exception""" mocked_return_value = '{"reply": { "err_msg" : "auth Error"}}' get_search_results.return_value = StatusResponse(401, mocked_return_value) @@ -409,7 +411,7 @@ def test_results_401_exception(self, get_search_results): assert "Invalid api_key" in result_response['error'] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_pending_response(self, mock_search_response): + def test_status_pending_response(self, mock_search_response, mock_init): """Test status pending exception""" mocked_return_value = '{"reply" : {"status": "PENDING"}}' mock_search_response.return_value = StatusResponse(200, mocked_return_value) @@ -423,7 +425,7 @@ def test_status_pending_response(self, mock_search_response): assert status_response['progress'] == 100 @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_value_error_exception(self, mock_search_status): + def test_status_value_error_exception(self, mock_search_status, mock_init): """test status connector value error with invalid json""" mocked_return_value = "Invalid json" mock_search_status.return_value = StatusResponse(200, mocked_return_value) @@ -436,7 +438,7 @@ def test_status_value_error_exception(self, mock_search_status): assert "Cannot parse response" in status_response["error"] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_403_exception(self, get_search_results): + def test_status_403_exception(self, get_search_results, mock_init): """test results with 403 exception""" mocked_return_value = '{"reply": { "err_msg" : "api permission exception"}}' get_search_results.return_value = StatusResponse(403, mocked_return_value) @@ -450,7 +452,7 @@ def test_status_403_exception(self, get_search_results): status_response['error'] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_ping_invalid_host(self, mock_quota_response): + def test_ping_invalid_host(self, mock_quota_response, mock_init): """test ping connection with invalid host""" mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') mock_quota_response.return_value = PingResponse(mock_quota) @@ -462,7 +464,7 @@ def test_ping_invalid_host(self, mock_quota_response): "Failed to establish a new connection"]) assert any(i in ping_response['code'] for i in ["service_unavailable", "unknown"]) - def test_ping_timeout_error(self): + def test_ping_timeout_error(self, mock_init): """test ping connection timeout error""" connection = self.connection() connection["options"] = {"timeout": 1} @@ -474,7 +476,7 @@ def test_ping_timeout_error(self): assert ping_response['code'] == "service_unavailable" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_query_search_timeout_error(self, mock_quota_response): + def test_query_search_timeout_error(self, mock_quota_response, mock_init): """test query search timeout error""" mock_quota_response.return_value = {"success": True} connection = self.connection() @@ -532,7 +534,7 @@ def test_query_search_timeout_error(self, mock_quota_response): assert ping_response['code'] == "service_unavailable" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_query_invalid_host(self, mock_ping_response): + def test_query_invalid_host(self, mock_ping_response, mock_init): """test query invalid host error""" mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') mock_ping_response.return_value = PingResponse(mock_quota) @@ -590,7 +592,7 @@ def test_query_invalid_host(self, mock_ping_response): assert any(i in query_response['code'] for i in ["service_unavailable", "unknown"]) @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_status_timeout_error(self, mock_quota_response): + def test_status_timeout_error(self, mock_quota_response, mock_init): """test status timeout error""" mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') mock_quota_response.return_value = PingResponse(mock_quota) @@ -605,7 +607,7 @@ def test_status_timeout_error(self, mock_quota_response): assert status_response['code'] == "service_unavailable" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_status_invalid_host_error(self, mock_quota_response): + def test_status_invalid_host_error(self, mock_quota_response, mock_init): """test status invalid host error""" mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') mock_quota_response.return_value = PingResponse(mock_quota) @@ -619,7 +621,7 @@ def test_status_invalid_host_error(self, mock_quota_response): assert any(i in status_response['code'] for i in ["service_unavailable", "unknown"]) @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_results_invalid_host_error(self, mock_quota_response): + def test_results_invalid_host_error(self, mock_quota_response, mock_init): """test results invalid error""" mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') mock_quota_response.return_value = PingResponse(mock_quota) @@ -635,7 +637,7 @@ def test_results_invalid_host_error(self, mock_quota_response): assert any(i in results_response['code'] for i in ["service_unavailable", "unknown"]) @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_results_timeout_error(self, mock_quota_response): + def test_results_timeout_error(self, mock_quota_response, mock_init): """test results timeout error""" mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') mock_quota_response.return_value = PingResponse(mock_quota) From 4ee3202d9ecc24e6b8b97df336abdae9ffd2594e Mon Sep 17 00:00:00 2001 From: lakshmi51974368 <92726459+lakshmi51974368@users.noreply.github.com> Date: Thu, 24 Mar 2022 10:14:59 +0000 Subject: [PATCH 3/7] Updated Transmit Unit Test file Updated Transmit unit test file --- .../tests/stix_transmission/test_paloalto.py | 295 ++++-------------- 1 file changed, 68 insertions(+), 227 deletions(-) diff --git a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py index c75f94a96..89ee58072 100644 --- a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py +++ b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py @@ -3,6 +3,21 @@ from stix_shifter.stix_transmission import stix_transmission from stix_shifter_modules.paloalto.entry_point import EntryPoint import json +from requests.exceptions import ConnectionError + + +class MockStatusObj: + def __init__(self, code): + self.status_code = code + + +class MockResponse: + def __init__(self, response_code, obj): + self.response = response_code + self.object = obj + + def read(self): + return bytearray(self.object, 'utf-8') class PaloaltoMockResponse: @@ -38,15 +53,6 @@ def read(self): return bytearray(self.response.content, 'utf-8') -class QuotaResponse: - """ class for ping outer response""" - - def __init__(self, response_object): - self.response = response_object - - -@patch('stix_shifter_modules.cybereason.stix_transmission.api_client' - '.APIClient.__init__') class TestPaloaltoConnection(unittest.TestCase, object): @staticmethod @@ -66,7 +72,7 @@ def configuration(): } @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping(self, mock_ping_response, mock_init): + def test_ping(self, mock_ping_response): """test ping connection""" mocked_return_value = '{"reply":{"used_quota": 0.08015277777777775}}' mock_ping = PaloaltoMockResponse(200, mocked_return_value) @@ -77,7 +83,7 @@ def test_ping(self, mock_ping_response, mock_init): assert ping_response['success'] is True @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') - def test_query_response(self, mock_search_response, mock_init): + def test_query_response(self, mock_search_response): """test create search query""" mocked_return_value = '{"reply": {"search_id": "07f63c733f5946_15006_inv"}}' mock_search = PaloaltoMockResponse(200, mocked_return_value) @@ -141,7 +147,8 @@ def test_query_response(self, mock_search_response, mock_init): assert query_response['success'] is True @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_response(self, mock_status_response, mock_init): + def test_status_response(self, mock_status_response): + """test status response""" mocked_return_value = '{"reply" : {"status": "SUCCESS","number_of_results":100}}' mock_status_response.return_value = StatusResponse(200, mocked_return_value) search_id = "e1d1b56ca81845_15180_inv" @@ -153,7 +160,7 @@ def test_status_response(self, mock_status_response, mock_init): assert status_response['status'] == "COMPLETED" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_result_response(self, mock_result_response, mock_init): + def test_result_response(self, mock_result_response): """test for valid result response""" mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", "number_of_results": 1, @@ -172,7 +179,7 @@ def test_result_response(self, mock_result_response, mock_init): assert result_response['data'] == [{'xdr_data': {'causality_actor_process_image_name': 'taskhostw.exe'}}] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_result_with_empty_nt_response(self, mock_result_response, mock_init): + def test_result_with_empty_nt_response(self, mock_result_response): """test for valid result response""" mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", "number_of_results": 1, @@ -197,7 +204,7 @@ def test_result_with_empty_nt_response(self, mock_result_response, mock_init): assert 'data' in result_response assert result_response['data'] == [{'xdr_data': {'actor_process_image_name': 'lsass.exe'}}] - def test_delete_response(self, mock_init): + def test_delete_response(self): """test delete response""" transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) search_id = "e1d1b56ca81845_15180_inv" @@ -206,14 +213,14 @@ def test_delete_response(self, mock_init): assert delete_response['success'] is True assert delete_response['message'] == "Delete operation of a search id is not supported in Palo Alto Cortex XDR" - def test_is_async(self, mock_init): + def test_is_async(self): """check for synchronous or asynchronous""" entry_point = EntryPoint(self.connection(), self.configuration()) check_async = entry_point.is_async() assert check_async is True @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_400_exception(self, mock_ping_response, mock_init): + def test_ping_400_exception(self, mock_ping_response): """Test ping response with 400 exception""" mocked_return_value = '{"reply":{"err_msg": "InvalidJson"}}' mock_ping = PaloaltoMockResponse(400, mocked_return_value) @@ -226,7 +233,7 @@ def test_ping_400_exception(self, mock_ping_response, mock_init): assert ping_response['code'] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_401_auth_exception(self, mock_ping_response, mock_init): + def test_ping_401_auth_exception(self, mock_ping_response): """test 401 authentication error exception""" mocked_return_value = '{"reply": { "err_msg" : "auth Error"}}' mock_ping = PaloaltoMockResponse(401, mocked_return_value) @@ -239,7 +246,7 @@ def test_ping_401_auth_exception(self, mock_ping_response, mock_init): assert "Invalid api_key" in ping_response['error'] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_500_exception(self, mock_ping_response, mock_init): + def test_ping_500_exception(self, mock_ping_response): """check 500 bad query exception""" mocked_return_value = '{"reply": {"err_extra":{"parse_err" :"Internal server error" }}}' mock_ping = PaloaltoMockResponse(500, mocked_return_value) @@ -252,7 +259,7 @@ def test_ping_500_exception(self, mock_ping_response, mock_init): assert ping_response["code"] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_internal_server_exception(self, mock_ping_response, mock_init): + def test_ping_internal_server_exception(self, mock_ping_response): """check 500 bad query exception""" mocked_return_value = '{"reply": {"err_msg":"Internal server error","err_extra":"server error"}}' mock_ping = PaloaltoMockResponse(500, mocked_return_value) @@ -265,7 +272,7 @@ def test_ping_internal_server_exception(self, mock_ping_response, mock_init): assert ping_response["code"] == "invalid_parameter" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_402_license_exception(self, mock_ping_response, mock_init): + def test_ping_402_license_exception(self, mock_ping_response): """Check 402 invalid license exception""" mocked_return_value = '{"reply": { "err_msg" : "Invalid license"}}' mock_ping = PaloaltoMockResponse(402, mocked_return_value) @@ -279,7 +286,7 @@ def test_ping_402_license_exception(self, mock_ping_response, mock_init): assert "User does not have the required license type to run this API" in ping_response['error'] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_empty_result_exception(self, mock_status_response, mock_init): + def test_status_empty_result_exception(self, mock_status_response): """Test empty results exception""" mocked_return_value = '{"reply" : {"status": "SUCCESS"}}' mock_status_response.return_value = StatusResponse(200, mocked_return_value) @@ -294,7 +301,7 @@ def test_status_empty_result_exception(self, mock_status_response, mock_init): assert status_response['code'] == "no_results" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_partial_success_exception(self, mock_status_response, mock_init): + def test_status_partial_success_exception(self, mock_status_response): """Test partial success status response exception""" mocked_return_value = '{"reply" : {"status": "PARTIAL_SUCCESS"}}' mock_status_response.return_value = StatusResponse(200, mocked_return_value) @@ -308,7 +315,7 @@ def test_status_partial_success_exception(self, mock_status_response, mock_init) assert status_response['status'] == "COMPLETED" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_fail_exception(self, mock_search_response, mock_init): + def test_status_fail_exception(self, mock_search_response): """Test status fail exception""" mocked_return_value = '{"reply" : {"status": "FAIL"}}' mock_search_response.return_value = StatusResponse(200, mocked_return_value) @@ -322,7 +329,7 @@ def test_status_fail_exception(self, mock_search_response, mock_init): assert status_response['code'] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_failed_query_response(self, mock_result_response, mock_init): + def test_failed_query_response(self, mock_result_response): """Test failed query result response""" mocked_return_value = json.dumps({"reply": {"status": "FAIL"}}) mock_result_response.return_value = StatusResponse(200, mocked_return_value) @@ -338,7 +345,7 @@ def test_failed_query_response(self, mock_result_response, mock_init): assert result_response['code'] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') - def test_create_query_value_error_exception(self, mock_search_response, mock_init): + def test_create_query_value_error_exception(self, mock_search_response): """test create query value error with invalid json""" mocked_return_value = "Invalid_json" mock_search = PaloaltoMockResponse(200, mocked_return_value) @@ -353,7 +360,7 @@ def test_create_query_value_error_exception(self, mock_search_response, mock_ini assert "Cannot parse response" in query_response["error"] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_ping_value_error_exception(self, mock_search_response, mock_init): + def test_ping_value_error_exception(self, mock_search_response): """test ping connector value error with invalid json""" mocked_return_value = "Invalid_json" mock_search = PaloaltoMockResponse(200, mocked_return_value) @@ -367,7 +374,7 @@ def test_ping_value_error_exception(self, mock_search_response, mock_init): assert "Cannot parse response" in query_response["error"] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') - def test_create_query_exception(self, mock_search_response, mock_init): + def test_create_query_exception(self, mock_search_response): """Test search response with 400 exception""" mocked_return_value = '{"reply": { "err_msg" : "400 error"}}' mock_ping = PaloaltoMockResponse(400, mocked_return_value) @@ -381,7 +388,7 @@ def test_create_query_exception(self, mock_search_response, mock_init): assert ping_response['code'] == "invalid_query" @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_results_value_error_exception(self, get_search_results, mock_init): + def test_results_value_error_exception(self, get_search_results): """test results connector value error with invalid json""" mocked_return_value = "Invalid json" get_search_results.return_value = StatusResponse(200, mocked_return_value) @@ -396,7 +403,7 @@ def test_results_value_error_exception(self, get_search_results, mock_init): assert "Cannot parse response" in result_response["error"] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') - def test_results_401_exception(self, get_search_results, mock_init): + def test_results_401_exception(self, get_search_results): """test results with 401 exception""" mocked_return_value = '{"reply": { "err_msg" : "auth Error"}}' get_search_results.return_value = StatusResponse(401, mocked_return_value) @@ -411,7 +418,7 @@ def test_results_401_exception(self, get_search_results, mock_init): assert "Invalid api_key" in result_response['error'] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_pending_response(self, mock_search_response, mock_init): + def test_status_pending_response(self, mock_search_response): """Test status pending exception""" mocked_return_value = '{"reply" : {"status": "PENDING"}}' mock_search_response.return_value = StatusResponse(200, mocked_return_value) @@ -422,10 +429,10 @@ def test_status_pending_response(self, mock_search_response, mock_init): assert 'success' in status_response assert status_response['success'] is True assert status_response['status'] == "RUNNING" - assert status_response['progress'] == 100 + assert status_response['progress'] == 50 @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_value_error_exception(self, mock_search_status, mock_init): + def test_status_value_error_exception(self, mock_search_status): """test status connector value error with invalid json""" mocked_return_value = "Invalid json" mock_search_status.return_value = StatusResponse(200, mocked_return_value) @@ -438,7 +445,7 @@ def test_status_value_error_exception(self, mock_search_status, mock_init): assert "Cannot parse response" in status_response["error"] @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') - def test_status_403_exception(self, get_search_results, mock_init): + def test_status_403_exception(self, get_search_results): """test results with 403 exception""" mocked_return_value = '{"reply": { "err_msg" : "api permission exception"}}' get_search_results.return_value = StatusResponse(403, mocked_return_value) @@ -451,204 +458,38 @@ def test_status_403_exception(self, get_search_results, mock_init): assert "The provided API Key does not have the required RBAC permissions to run this API" in \ status_response['error'] - @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_ping_invalid_host(self, mock_quota_response, mock_init): - """test ping connection with invalid host""" - mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') - mock_quota_response.return_value = PingResponse(mock_quota) + + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_quota_invalid_json_exception(self, mock_ping): + """test quota invalid json exception""" + response = "invalid json" + mock_ping.side_effect = [MockResponse(MockStatusObj(200), response)] + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query({}) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert "Cannot parse response" in query_response["error"] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_invalid_host(self, mock_ping): + """Test Invalid host""" + mock_ping.side_effect = ConnectionError("Invalid Host") transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) ping_response = transmission.ping() assert ping_response is not None assert ping_response['success'] is False - assert any(i in ping_response['error'] for i in ["Invalid Host", "timeout error", - "Failed to establish a new connection"]) - assert any(i in ping_response['code'] for i in ["service_unavailable", "unknown"]) - - def test_ping_timeout_error(self, mock_init): - """test ping connection timeout error""" - connection = self.connection() - connection["options"] = {"timeout": 1} - transmission = stix_transmission.StixTransmission('paloalto', connection, self.configuration()) - ping_response = transmission.ping() - assert ping_response is not None - assert ping_response['success'] is False - assert "timeout_error (1 sec)" in ping_response['error'] + assert "Invalid Host" in ping_response['error'] assert ping_response['code'] == "service_unavailable" - @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_query_search_timeout_error(self, mock_quota_response, mock_init): - """test query search timeout error""" - mock_quota_response.return_value = {"success": True} - connection = self.connection() - connection["options"] = {"timeout": 1} - transmission = stix_transmission.StixTransmission('paloalto', connection, self.configuration()) - query = {"xdr_data": {"query": "dataset = xdr_data | filter ((action_process_image_name not in (" - "\"conhost.exe\",\"AtBroker.exe\") or actor_process_image_name not in (" - "\"conhost.exe\",\"AtBroker.exe\") or causality_actor_process_image_name not " - "in (\"conhost.exe\",\"AtBroker.exe\") or os_actor_process_image_name not in (" - "\"conhost.exe\",\"AtBroker.exe\")) and (to_epoch(_time,\"millis\") >= " - "1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or ((" - "action_process_file_create_time = 1643704990003 or " - "actor_process_file_create_time = 1643704990003 or " - "causality_actor_process_file_create_time = 1643704990003 or " - "os_actor_process_file_create_time = 1643704990003) and (to_epoch(_time," - "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " - "1644883200000)) or ((action_process_image_name ~= \"wildfire$\" or " - "actor_process_image_name ~= \"wildfire$\" or " - "causality_actor_process_image_name ~= \"wildfire$\" or " - "os_actor_process_image_name ~= \"wildfire$\") and (to_epoch(_time," - "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " - "1644883200000)) | alter dataset_name = \"xdr_data\" | fields dataset_name," - "action_local_ip,action_remote_ip,agent_ip_addresses_v6," - "dst_agent_ip_addresses_v6,action_local_port,action_remote_port," - "action_network_protocol,action_file_name,action_file_size,action_file_md5," - "action_module_md5,action_process_image_md5,action_file_authenticode_sha1," - "action_file_authenticode_sha2,action_file_sha256,action_module_sha256," - "action_process_image_sha256,action_file_access_time," - "actor_process_file_access_time,os_actor_process_file_access_time," - "action_file_mod_time,actor_process_file_mod_time," - "os_actor_process_file_mod_time,action_file_create_time,action_file_path," - "action_process_image_path,action_registry_file_path,actor_process_image_path," - "causality_actor_process_image_path,os_actor_process_image_path," - "action_process_image_command_line,actor_process_command_line," - "causality_actor_process_command_line,os_actor_process_command_line," - "action_process_file_create_time,actor_process_file_create_time," - "causality_actor_process_file_create_time,os_actor_process_file_create_time," - "action_process_image_name,actor_process_image_name," - "causality_actor_process_image_name,os_actor_process_image_name," - "action_module_process_os_pid ,action_process_os_pid,actor_process_os_pid," - "causality_actor_process_os_pid,os_actor_process_os_pid," - "action_process_requested_parent_pid,action_thread_parent_pid," - "action_thread_child_pid,action_process_username,auth_domain," - "dst_host_metadata_domain,host_metadata_domain,dst_action_url_category ," - "action_registry_key_name,action_registry_value_name,mac,associated_mac," - "dst_associated_mac ,dst_mac,dst_user_id,user_id,action_username," - "actor_primary_username,actor_process_logon_id | limit 10000 ", - "timeframe": {"from": 1644451200000, "to": 1644883200000}}} - - query = json.dumps(query) - ping_response = transmission.query(query) + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_timeout_error(self, mock_ping): + """Test Timeout Error""" + mock_ping.side_effect = TimeoutError("timeout_error (30 sec)") + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() assert ping_response is not None assert ping_response['success'] is False - assert "timeout_error (1 sec)" in ping_response['error'] - assert ping_response['code'] == "service_unavailable" - - @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') - def test_query_invalid_host(self, mock_ping_response, mock_init): - """test query invalid host error""" - mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') - mock_ping_response.return_value = PingResponse(mock_quota) - transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) - query = {"xdr_data": {"query": "dataset = xdr_data | filter ((action_process_image_name not in (" - "\"conhost.exe\",\"AtBroker.exe\") or actor_process_image_name not in (" - "\"conhost.exe\",\"AtBroker.exe\") or causality_actor_process_image_name not " - "in (\"conhost.exe\",\"AtBroker.exe\") or os_actor_process_image_name not in (" - "\"conhost.exe\",\"AtBroker.exe\")) and (to_epoch(_time,\"millis\") >= " - "1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or ((" - "action_process_file_create_time = 1643704990003 or " - "actor_process_file_create_time = 1643704990003 or " - "causality_actor_process_file_create_time = 1643704990003 or " - "os_actor_process_file_create_time = 1643704990003) and (to_epoch(_time," - "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " - "1644883200000)) or ((action_process_image_name ~= \"wildfire$\" or " - "actor_process_image_name ~= \"wildfire$\" or " - "causality_actor_process_image_name ~= \"wildfire$\" or " - "os_actor_process_image_name ~= \"wildfire$\") and (to_epoch(_time," - "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " - "1644883200000)) | alter dataset_name = \"xdr_data\" | fields dataset_name," - "action_local_ip,action_remote_ip,agent_ip_addresses_v6," - "dst_agent_ip_addresses_v6,action_local_port,action_remote_port," - "action_network_protocol,action_file_name,action_file_size,action_file_md5," - "action_module_md5,action_process_image_md5,action_file_authenticode_sha1," - "action_file_authenticode_sha2,action_file_sha256,action_module_sha256," - "action_process_image_sha256,action_file_access_time," - "actor_process_file_access_time,os_actor_process_file_access_time," - "action_file_mod_time,actor_process_file_mod_time," - "os_actor_process_file_mod_time,action_file_create_time,action_file_path," - "action_process_image_path,action_registry_file_path,actor_process_image_path," - "causality_actor_process_image_path,os_actor_process_image_path," - "action_process_image_command_line,actor_process_command_line," - "causality_actor_process_command_line,os_actor_process_command_line," - "action_process_file_create_time,actor_process_file_create_time," - "causality_actor_process_file_create_time,os_actor_process_file_create_time," - "action_process_image_name,actor_process_image_name," - "causality_actor_process_image_name,os_actor_process_image_name," - "action_module_process_os_pid ,action_process_os_pid,actor_process_os_pid," - "causality_actor_process_os_pid,os_actor_process_os_pid," - "action_process_requested_parent_pid,action_thread_parent_pid," - "action_thread_child_pid,action_process_username,auth_domain," - "dst_host_metadata_domain,host_metadata_domain,dst_action_url_category ," - "action_registry_key_name,action_registry_value_name,mac,associated_mac," - "dst_associated_mac ,dst_mac,dst_user_id,user_id,action_username," - "actor_primary_username,actor_process_logon_id | limit 10000 ", - "timeframe": {"from": 1644451200000, "to": 1644883200000}}} - - query = json.dumps(query) - query_response = transmission.query(query) - assert query_response is not None - assert query_response['success'] is False - assert any(i in query_response['error'] for i in ["Invalid Host", "timeout_error", - "Failed to establish a new connection"]) - assert any(i in query_response['code'] for i in ["service_unavailable", "unknown"]) - - @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_status_timeout_error(self, mock_quota_response, mock_init): - """test status timeout error""" - mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') - mock_quota_response.return_value = PingResponse(mock_quota) - connection = self.connection() - connection["options"] = {"timeout": 1} - search_id = "e1d1b56ca81845_15180_inv" - transmission = stix_transmission.StixTransmission('paloalto', connection, self.configuration()) - status_response = transmission.status(search_id) - assert status_response is not None - assert status_response['success'] is False - assert "timeout_error (1 sec)" in status_response['error'] - assert status_response['code'] == "service_unavailable" - - @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_status_invalid_host_error(self, mock_quota_response, mock_init): - """test status invalid host error""" - mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') - mock_quota_response.return_value = PingResponse(mock_quota) - search_id = "e1d1b56ca81845_15180_inv" - transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) - status_response = transmission.status(search_id) - assert status_response is not None - assert status_response['success'] is False - assert any(i in status_response['error'] for i in ["Invalid Host", "timeout_error", - "Failed to establish a new connection"]) - assert any(i in status_response['code'] for i in ["service_unavailable", "unknown"]) - - @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_results_invalid_host_error(self, mock_quota_response, mock_init): - """test results invalid error""" - mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') - mock_quota_response.return_value = PingResponse(mock_quota) - search_id = "e1d1b56ca81845_15180_inv" - offset = 0 - length = 3 - transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) - results_response = transmission.results(search_id, offset, length) - assert results_response is not None - assert results_response['success'] is False - assert any(i in results_response['error'] for i in ["Invalid Host", "timeout_error", - "Failed to establish a new connection"]) - assert any(i in results_response['code'] for i in ["service_unavailable", "unknown"]) - - @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_remaining_quota') - def test_results_timeout_error(self, mock_quota_response, mock_init): - """test results timeout error""" - mock_quota = PaloaltoMockResponse(200, '{"reply":{"used_quota": 0.08015277777777775}}') - mock_quota_response.return_value = PingResponse(mock_quota) - connection = self.connection() - connection["options"] = {"timeout": 1} - search_id = "e1d1b56ca81845_15180_inv" - offset = 0 - length = 3 - transmission = stix_transmission.StixTransmission('paloalto', connection, self.configuration()) - results_response = transmission.results(search_id, offset, length) - assert results_response is not None - assert results_response['success'] is False - assert "timeout_error (1 sec)" in results_response['error'] - assert results_response['code'] == "service_unavailable" + assert "timeout_error (30 sec)" in ping_response['error'] + assert ping_response['code'] == "service_unavailable" \ No newline at end of file From e554ed8bc2a91a4a2e140d28d281b691935d2988 Mon Sep 17 00:00:00 2001 From: lakshmi51974368 <92726459+lakshmi51974368@users.noreply.github.com> Date: Thu, 24 Mar 2022 15:03:48 +0000 Subject: [PATCH 4/7] Updated PR comments 1. Added support for more than 1000 records using stream API as discussed. 2. Updated Stix 2.1 to_stix_map.json with the modified format extensions 3. Added Quota Limit for API calls based on License. --- stix_shifter_modules/paloalto/README.md | 8 +- .../paloalto/configuration/config.json | 25 ++-- .../paloalto/configuration/lang_en.json | 18 +-- .../json/stix_2_1/to_stix_map.json | 92 +++++++------- .../stix_translation/query_constructor.py | 20 ++- .../paloalto/stix_transmission/api_client.py | 57 +++++---- .../stix_transmission/error_mapper.py | 4 +- .../stix_transmission/query_connector.py | 1 - .../stix_transmission/results_connector.py | 116 ++++++++++-------- .../stix_transmission/status_connector.py | 5 +- .../test_paloalto_stix_to_query.py | 48 ++++---- .../tests/stix_transmission/test_paloalto.py | 96 ++++++++++++++- 12 files changed, 308 insertions(+), 182 deletions(-) diff --git a/stix_shifter_modules/paloalto/README.md b/stix_shifter_modules/paloalto/README.md index 3389ab9fc..f237f3cd3 100644 --- a/stix_shifter_modules/paloalto/README.md +++ b/stix_shifter_modules/paloalto/README.md @@ -14,8 +14,9 @@ |Connector Method|PaloAlto Cortex XDR API Endpoint| Method | ---- | ------ | -----| |Query Endpoint |https:///public_api/v1/xql/start_xql_query/|POST - |Result Endpoint|https:///public_api/v1/xql/get_query_results/|POST - |Quota Endpoint|https:///public_api/v1/xql/get_quota/|POST + |Result Endpoint |https:///public_api/v1/xql/get_query_results/|POST + |Stream Endpoint |https:///public_api/v1/xql/get_query_results_stream/|POST + |Quota Endpoint |https:///public_api/v1/xql/get_quota/|POST ### Format for calling stix-shifter from the command line ``` @@ -978,7 +979,8 @@ paloalto - Queries called without enough quota will fail. ### Observations -- The Maximum result limit of the connector is 1000, as the connector doesnt support the stream data feature which returns more than 1000 results. +- The quota limit range for the API call would be from 1 to 4 units for the Standard license, and it will be from 6 to 12 + if additional units are purchased. ### References - [Cortex XDR](https://docs.paloaltonetworks.com/cortex/cortex-xdr.html) diff --git a/stix_shifter_modules/paloalto/configuration/config.json b/stix_shifter_modules/paloalto/configuration/config.json index 692db5e17..b5b1a0681 100644 --- a/stix_shifter_modules/paloalto/configuration/config.json +++ b/stix_shifter_modules/paloalto/configuration/config.json @@ -6,22 +6,21 @@ "host": { "type": "text" }, + "quota_threshold": { + "default": 3, + "min": 1, + "max": 4, + "type": "number" + }, + "additional_unit_quota_threshold": { + "default": 12, + "min": 6, + "max": 12, + "type": "number" + }, "help": { "type": "link", "default": "data-sources.html" - }, - "options": { - "type": "fields", - "result_limit": { - "default": 1000, - "min": 1, - "max": 1000, - "type": "number" - } - }, - "quota_threshold": { - "default": 3.00, - "type": "float" } }, "configuration": { diff --git a/stix_shifter_modules/paloalto/configuration/lang_en.json b/stix_shifter_modules/paloalto/configuration/lang_en.json index a5855e75e..838e36bd4 100644 --- a/stix_shifter_modules/paloalto/configuration/lang_en.json +++ b/stix_shifter_modules/paloalto/configuration/lang_en.json @@ -4,18 +4,18 @@ "label": "Management IP address or Hostname", "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" }, + "quota_threshold": { + "label": "The quota limit for the API", + "description": "Prevents the connector from exceeding the quota threshold limit. The threshold range is from 1 to 4 units" + }, + "additional_unit_quota_threshold": { + "label": "The quota limit for the API for the additionally purchased units", + "description": "Prevents the connector from exceeding the quota threshold limit of the additional units along with the standard license quota. The Threshold range is from 6 to 12 units" + }, "help": { "label": "Need additional help?", "description": "More details on the data source setting can be found in the specified link" - }, - "result_limit": { - "label": "The Result limit for the connector", - "description": "The total number of results returned from the connector" - }, - "quota_threshold": { - "label": "The quota limit for the API", - "description": "Prevents the connector from exceeding the quota threshold limit" -} + } }, "configuration": { "auth": { diff --git a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json index 16d6a3d69..50c08b380 100644 --- a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json @@ -483,7 +483,7 @@ "object": "user" }, "action_process_user_sid": { - "key": "user-account.extensions.x-paloalto-user.process_user_id", + "key": "user-account.extensions.x-paloalto-user-ext.process_user_id", "object": "user" }, "actor_primary_username": { @@ -492,7 +492,7 @@ }, "action_process_username": [ { - "key": "user-account.extensions.x-paloalto-user.process_user_name", + "key": "user-account.extensions.x-paloalto-user-ext.process_user_name", "object": "user" }, { @@ -506,166 +506,166 @@ "object": "user" }, "action_file_info_company": { - "key": "file.extensions.x-paloalto-file.company", + "key": "file.extensions.x-paloalto-file-ext.company", "object": "file" }, "action_file_extension": { - "key": "file.extensions.x-paloalto-file.extension", + "key": "file.extensions.x-paloalto-file-ext.extension", "object": "file" }, "action_file_attributes": { - "key": "file.extensions.x-paloalto-file.attributes", + "key": "file.extensions.x-paloalto-file-ext.attributes", "object": "file", "transformer": "ToInteger" }, "action_file_internal_zipped_files": { - "key": "file.extensions.x-paloalto-file.zipped_files", + "key": "file.extensions.x-paloalto-file-ext.zipped_files", "object": "file" }, "action_file_last_writer_actor": { - "key": "file.extensions.x-paloalto-file.writer", + "key": "file.extensions.x-paloalto-file-ext.writer", "object": "file" }, "action_file_mode": { - "key": "file.extensions.x-paloalto-file.mode", + "key": "file.extensions.x-paloalto-file-ext.mode", "object": "file" }, "action_file_signature_status": { - "key": "file.extensions.x-paloalto-file.signature_status", + "key": "file.extensions.x-paloalto-file-ext.signature_status", "object": "file", "transformer": "ToInteger" }, "action_file_signature_vendor": { - "key": "file.extensions.x-paloalto-file.signature_vendor", + "key": "file.extensions.x-paloalto-file-ext.signature_vendor", "object": "file" }, "action_file_signature_product": { - "key": "file.extensions.x-paloalto-file.signature_product", + "key": "file.extensions.x-paloalto-file-ext.signature_product", "object": "file" }, "action_file_info_description": { - "key": "file.extensions.x-paloalto-file.file_description", + "key": "file.extensions.x-paloalto-file-ext.file_description", "object": "file" }, "action_file_group": { - "key": "file.extensions.x-paloalto-file.group", + "key": "file.extensions.x-paloalto-file-ext.group", "object": "file" }, "action_file_group_name": { - "key": "file.extensions.x-paloalto-file.group_name", + "key": "file.extensions.x-paloalto-file-ext.group_name", "object": "file" }, "action_file_type": { - "key": "file.extensions.x-paloalto-file.type", + "key": "file.extensions.x-paloalto-file-ext.type", "object": "file", "transformer": "ToInteger" }, "action_file_info_file_version": { - "key": "file.extensions.x-paloalto-file.version", + "key": "file.extensions.x-paloalto-file-ext.version", "object": "file" }, "manifest_file_version": { - "key": "file.extensions.x-paloalto-file.manifest_version", + "key": "file.extensions.x-paloalto-file-ext.manifest_version", "object": "file", "transformer": "ToInteger" }, "action_file_info_product_version": { - "key": "file.extensions.x-paloalto-file.product_version", + "key": "file.extensions.x-paloalto-file-ext.product_version", "object": "file" }, "action_file_owner": { - "key": "file.extensions.x-paloalto-file.owner", + "key": "file.extensions.x-paloalto-file-ext.owner", "object": "file" }, "action_file_owner_name": { - "key": "file.extensions.x-paloalto-file.owner_name", + "key": "file.extensions.x-paloalto-file-ext.owner_name", "object": "file" }, "action_file_info_product_name": { - "key": "file.extensions.x-paloalto-file.product_name", + "key": "file.extensions.x-paloalto-file-ext.product_name", "object": "file" }, "action_file_id": { - "key": "file.extensions.x-paloalto-file.id", + "key": "file.extensions.x-paloalto-file-ext.id", "object": "file" }, "action_file_wildfire_verdict": { - "key": "file.extensions.x-paloalto-file.wildfire_verdict", + "key": "file.extensions.x-paloalto-file-ext.wildfire_verdict", "object": "file" }, "action_file_hash_control_verdict": { - "key": "file.extensions.x-paloalto-process.control_verdict", + "key": "file.extensions.x-paloalto-process-ext.control_verdict", "object": "file" }, "actor_process_instance_id": { - "key": "process.extensions.x-paloalto-process.instance_id", + "key": "process.extensions.x-paloalto-process-ext.instance_id", "object": "process_actor" }, "actor_process_causality_id": { - "key": "process.extensions.x-paloalto-process.causality_id", + "key": "process.extensions.x-paloalto-process-ext.causality_id", "object": "process_actor" }, "actor_process_auth_id": { - "key": "process.extensions.x-paloalto-process.auth_id", + "key": "process.extensions.x-paloalto-process-ext.auth_id", "object": "process_actor" }, "actor_process_container_id": { - "key": "process.extensions.x-paloalto-process.container_id", + "key": "process.extensions.x-paloalto-process-ext.container_id", "object": "process_actor" }, "actor_process_signature_vendor": { - "key": "process.extensions.x-paloalto-process.signature_vendor", + "key": "process.extensions.x-paloalto-process-ext.signature_vendor", "object": "process_actor" }, "actor_process_signature_status": { - "key": "process.extensions.x-paloalto-process.signature_status", + "key": "process.extensions.x-paloalto-process-ext.signature_status", "object": "process_actor" }, "actor_process_signature_product": { - "key": "process.extensions.x-paloalto-process.signature_product", + "key": "process.extensions.x-paloalto-process-ext.signature_product", "object": "process_actor" }, "actor_process_image_extension": { - "key": "process.extensions.x-paloalto-process.extension", + "key": "process.extensions.x-paloalto-process-ext.extension", "object": "process_actor" }, "action_process_termination_code": { - "key": "process.extensions.x-paloalto-process.termination_code", + "key": "process.extensions.x-paloalto-process-ext.termination_code", "object": "process_action", "transformer": "ToInteger" }, "action_process_termination_date": { - "key": "process.extensions.x-paloalto-process.termination_date", + "key": "process.extensions.x-paloalto-process-ext.termination_date", "object": "process_action", "transformer": "EpochToTimestamp" }, "action_remote_process_thread_id": { - "key": "process.extensions.x-paloalto-process.tid", + "key": "process.extensions.x-paloalto-process-ext.tid", "object": "process_action", "transformer": "ToInteger" }, "action_process_instance_execution_time": { - "key": "process.extensions.x-paloalto-process.instance_exec_time", + "key": "process.extensions.x-paloalto-process-ext.instance_exec_time", "object": "process_action", "transformer": "EpochToTimestamp" }, "actor_process_execution_time": { - "key": "process.extensions.x-paloalto-process.execution_time", + "key": "process.extensions.x-paloalto-process-ext.execution_time", "object": "process_actor", "transformer": "EpochToTimestamp" }, "action_process_handle_is_kernel": { - "key": "process.extensions.x-paloalto-process.is_kernel", + "key": "process.extensions.x-paloalto-process-ext.is_kernel", "object": "process_action", "transformer": "StringToBool" }, "action_process_is_container_root": { - "key": "process.extensions.x-paloalto-process.is_root", + "key": "process.extensions.x-paloalto-process-ext.is_root", "object": "process_action", "transformer": "StringToBool" }, "actor_process_is_native": { - "key": "process.extensions.x-paloalto-process.is_native", + "key": "process.extensions.x-paloalto-process-ext.is_native", "object": "process_actor", "transformer": "StringToBool" }, @@ -791,29 +791,29 @@ "object": "event" }, "action_network_creation_time": { - "key": "network-traffic.extensions.x-paloalto-network.creation_time", + "key": "network-traffic.extensions.x-paloalto-network-ext.creation_time", "object": "nt", "transformer": "EpochToTimestamp" }, "action_network_connection_id": { - "key": "network-traffic.extensions.x-paloalto-network.connection_id", + "key": "network-traffic.extensions.x-paloalto-network-ext.connection_id", "object": "nt" }, "action_network_packet_data": { - "key": "network-traffic.extensions.x-paloalto-network.packet_data", + "key": "network-traffic.extensions.x-paloalto-network-ext.packet_data", "object": "nt" }, "action_proxy": { - "key": "network-traffic.extensions.x-paloalto-network.is_proxy", + "key": "network-traffic.extensions.x-paloalto-network-ext.is_proxy", "object": "nt", "transformer": "StringToBool" }, "host_metadata_hostname": { - "key": "network-traffic.extensions.x-paloalto-network.metadata_hostname", + "key": "network-traffic.extensions.x-paloalto-network-ext.metadata_hostname", "object": "nt" }, "action_external_hostname": { - "key": "network-traffic.extensions.x-paloalto-network.external_hostname", + "key": "network-traffic.extensions.x-paloalto-network-ext.external_hostname", "object": "nt" } } diff --git a/stix_shifter_modules/paloalto/stix_translation/query_constructor.py b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py index 6b40d16ab..3ff5fae8d 100644 --- a/stix_shifter_modules/paloalto/stix_translation/query_constructor.py +++ b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py @@ -20,10 +20,15 @@ class StartStopQualifierValueException(Exception): pass +class FileNotFoundException(Exception): + pass + + class QueryStringPatternTranslator: """ translate stix pattern to native data source query language """ + def __init__(self, pattern: Pattern, data_model_mapper, options): logger.info("Palo Alto Cortex XDR Connector") self.dmm = data_model_mapper @@ -43,11 +48,15 @@ def load_json(rel_path_of_file): :param rel_path_of_file: str :return: dict """ + _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file - if path.exists(_json_path): - with open(_json_path, encoding='utf-8') as f_obj: - return json.load(f_obj) - raise FileNotFoundError + try: + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundException + except FileNotFoundException as e: + raise FileNotFoundError(f'{rel_path_of_file} not found') from e @staticmethod def _create_formatted_query(dmm, translated_query, timeframe, all_fields_map, options): @@ -60,8 +69,7 @@ def _create_formatted_query(dmm, translated_query, timeframe, all_fields_map, op :param options:dict :return: formatted_query :list """ - limit = options["result_limit"] # The default result_limit is changed to 1000, - # since stream data feature is not supported + limit = options["result_limit"] dataset_name = dmm.dialect all_fields = all_fields_map["all_fields"] # all_fields included in to_stix_mapping fields = ','.join(all_fields) diff --git a/stix_shifter_modules/paloalto/stix_transmission/api_client.py b/stix_shifter_modules/paloalto/stix_transmission/api_client.py index 4961455c6..742479e0e 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/api_client.py +++ b/stix_shifter_modules/paloalto/stix_transmission/api_client.py @@ -7,6 +7,7 @@ import string import hashlib import json +from requests.exceptions import ConnectionError class MaxDailyQuotaException(Exception): @@ -34,16 +35,15 @@ def __init__(self, connection, configuration): "x-xdr-auth-id": str(self.auth['api_key_id']), "Authorization": api_key_hash } - - url_modifier_function = None self.client = RestApiClient(connection.get('host'), connection.get('port', None), headers, - url_modifier_function=url_modifier_function, + url_modifier_function=None, ) self.result_limit = connection['options'].get('result_limit') self.timeout = connection['options']['timeout'] self.quota_threshold = connection['quota_threshold'] + self.additional_quota_threshold = connection['additional_unit_quota_threshold'] self.connector = __name__.split('.')[1] def ping_data_source(self): @@ -76,10 +76,16 @@ def get_remaining_quota(self): quota_response_code = quota_wrapper.response.status_code quota_response_text = json.loads(quota_wrapper.read().decode('utf-8')) if quota_response_code == 200: - if 'reply' in quota_response_text.keys() and quota_response_text['reply']['used_quota'] >= \ - self.quota_threshold: - raise MaxDailyQuotaException - return_obj['success'] = True + if 'reply' in quota_response_text.keys(): + # The daily quota unit for standard license is 5. additional units up to 10 can be added. + if quota_response_text['reply']['license_quota'] == 5 and \ + quota_response_text['reply']['additional_purchased_quota'] == 0.0: + if quota_response_text['reply']['used_quota'] >= self.quota_threshold: + raise MaxDailyQuotaException + else: + if quota_response_text['reply']['used_quota'] >= self.additional_quota_threshold: + raise MaxDailyQuotaException + return_obj['success'] = True else: return_obj = ResponseMapper().status_code_mapping(quota_response_code, quota_response_text) @@ -91,8 +97,15 @@ def get_remaining_quota(self): response_dict['type'] = "MaxDailyQuotaException" response_dict['message'] = "query usage exceeded max daily quota" ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) except Exception as ex: - response_dict['type'] = ex.__class__.__name__ + if 'timeout_error' in str(ex): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = ex.__class__.__name__ response_dict['message'] = ex self.logger.error('error when getting search results: %s', ex) ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) @@ -160,17 +173,17 @@ def delete_search(): """ return {"code": 200, "success": True} - # *** The stream data feature is not used currently. It may be implemented in future *** - # def get_stream_results(self, stream_id): - # """ - # Return the stream results - # :return: Raw Json data - # """ - # data = { - # "request_data": - # {"stream_id": stream_id, - # "is_gzip_compressed": False - # } - # } - # return self.client.call_api(self.STREAM_ENDPOINT, 'POST', headers=self.client.headers, data=json.dumps(data), - # timeout=self.timeout) + def get_stream_results(self, stream_id): + """ + Return the stream results + :param stream_id: string + :return: Raw Json data + """ + data = { + "request_data": + {"stream_id": stream_id, + "is_gzip_compressed": False + } + } + return self.client.call_api(self.STREAM_ENDPOINT, 'POST', headers=self.client.headers, data=json.dumps(data), + timeout=self.timeout) diff --git a/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py b/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py index 3a6b02c23..b1fcff8cc 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py +++ b/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py @@ -12,9 +12,7 @@ "InvalidLicense": ErrorCode.TRANSMISSION_CONNECT, "InvalidJsonException": ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, "MaxDailyQuotaException": ErrorCode.TRANSMISSION_CONNECT, - "TimeoutError": ErrorCode.TRANSMISSION_CONNECT, - "GeneralQuotaException": ErrorCode.TRANSMISSION_INVALID_PARAMETER, - "StreamNotImplemented": ErrorCode.TRANSMISSION_INVALID_PARAMETER + "TimeoutError": ErrorCode.TRANSMISSION_CONNECT } diff --git a/stix_shifter_modules/paloalto/stix_transmission/query_connector.py b/stix_shifter_modules/paloalto/stix_transmission/query_connector.py index 4e895c0e1..f8a3a1e43 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/query_connector.py +++ b/stix_shifter_modules/paloalto/stix_transmission/query_connector.py @@ -31,7 +31,6 @@ def create_query_connection(self, query): return_obj['search_id'] = response_text['reply'] else: return_obj = ResponseMapper().status_code_mapping(response_code, response_text) - except ValueError as ex: if response_wrapper is not None and not isinstance(response_wrapper, dict): self.logger.debug(response_wrapper.read()) diff --git a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py index 8a5fdaa02..ae1d6557a 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py +++ b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py @@ -10,11 +10,11 @@ CONFIG_MAP_PATH = "../stix_translation/json/config_map.json" -class InvalidQueryException(Exception): +class FileNotFoundException(Exception): pass -class StreamNotImplementedException(Exception): +class InvalidQueryException(Exception): pass @@ -32,10 +32,13 @@ def load_json(rel_path_of_file): :return: dict """ _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file - if path.exists(_json_path): - with open(_json_path, encoding='utf-8') as f_obj: - return json.load(f_obj) - raise FileNotFoundError + try: + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundException + except FileNotFoundException as e: + raise FileNotFoundError(f'{rel_path_of_file} not found') from e def create_results_connection(self, search_id, offset, length): """ @@ -45,33 +48,36 @@ def create_results_connection(self, search_id, offset, length): :param length: str, length value :return: dict """ - response_dict = {} return_obj = {} results = [] response_wrapper = None + to_stix_mapping = ResultsConnector.load_json(TO_STIX_PATH) + mandatory_map = ResultsConnector.load_json(CONFIG_MAP_PATH)["mandatory_properties_to_stix"] try: min_range = int(offset) max_range = int(offset) + int(length) - # Grab the response, extract the response code, and convert it to readable json response_wrapper = self.api_client.get_search_results(search_id) - response_code = response_wrapper.code response_text = json.loads(response_wrapper.read().decode('utf-8')) - if response_code != 200: - return_obj = ResponseMapper().status_code_mapping(response_code, response_text) + if response_wrapper.code != 200: + return_obj = ResponseMapper().status_code_mapping(response_wrapper.code, response_text) else: if 'status' in response_text['reply'].keys() and response_text['reply']['status'] \ in ('SUCCESS', 'PARTIAL_SUCCESS'): if 'data' in response_text['reply']['results'].keys() and \ response_text['reply']['number_of_results'] > 0: - results = ResultsConnector.format_results_data(response_text['reply']['results']['data']) + results = ResultsConnector.format_results_data( + response_text['reply']['results']['data'], to_stix_mapping, mandatory_map) elif 'stream_id' in response_text['reply']['results'].keys(): - raise StreamNotImplementedException - # *** The stream data feature is not used currently. It may be implemented in future *** - # stream_wrapper = self.api_client.get_stream_results(response_text['reply']['results'] - # ['stream_id']) - # results = ResultsConnector.format_stream_data(stream_wrapper.read().decode('utf-8')) + stream_wrapper = self.api_client.get_stream_results( + response_text['reply']['results']['stream_id']) + if stream_wrapper.code != 200: + return_obj = ResponseMapper().status_code_mapping(stream_wrapper.code, + stream_wrapper.read().decode('utf-8')) + else: + results = ResultsConnector.format_stream_data(stream_wrapper.read().decode('utf-8'), + to_stix_mapping, mandatory_map) return_obj['success'] = True return_obj['data'] = results[min_range:max_range] if results else [] @@ -90,10 +96,6 @@ def create_results_connection(self, search_id, offset, length): response_dict['type'] = "ConnectionError" response_dict['message'] = "Invalid Host" ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) - except StreamNotImplementedException: - response_dict['type'] = "StreamNotImplemented" - response_dict['message'] = "Getting results from Stream Id is not implemented" - ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) except Exception as ex: if 'timeout_error' in str(ex): response_dict['type'] = 'TimeoutError' @@ -105,15 +107,15 @@ def create_results_connection(self, search_id, offset, length): return return_obj @staticmethod - def format_results_data(result_data): + def format_results_data(result_data, to_stix_mapping, mandatory_map): """ Format the results in json format :param result_data: list of dictionary items + :param to_stix_mapping: to stix mapping dictionary + :param mandatory_map: dictionary :return: list """ results = [] - to_stix_mapping = ResultsConnector.load_json(TO_STIX_PATH) - mandatory_map = ResultsConnector.load_json(CONFIG_MAP_PATH)["mandatory_properties_to_stix"] dataset_map = result_data[0]['dataset_name'] try: for log in result_data: @@ -135,6 +137,39 @@ def format_results_data(result_data): raise e return results + @staticmethod + def format_stream_data(stream_data, to_stix_mapping, mandatory_map): + """ + Format the stream data into json format + :param stream_data: string + :param to_stix_mapping: to stix mapping dictionary + :param mandatory_map: dictionary + :return: list + """ + results = [] + try: + temp_data = stream_data.split("\n") + dataset_map = json.loads(temp_data[0])['dataset_name'] + for log in temp_data: + if log > " ": + data = {} + results_dict = {} + dataset = "" + log_dict = json.loads(log) + for field, value in log_dict.items(): + if value is not None and value != "NULL" and value != '' and field != 'dataset_name' \ + and (field in to_stix_mapping[dataset_map].keys()): + stix_data_map = to_stix_mapping[dataset_map][field] + data = ResultsConnector.check_object(stix_data_map, mandatory_map, + data, log_dict, field, value) + elif field == 'dataset_name': + dataset = value + results_dict[dataset] = data + results.append(results_dict) + except (KeyError, IndexError, TypeError) as e: + raise e + return results + @staticmethod def check_object(stix_data_map, mandatory_map, data, log, field, value): """ @@ -180,14 +215,15 @@ def check_mandatory_map(mandatory_map, obj, log, data, field, value): """ try: if "file" in obj: - if log[mandatory_map[obj][0]] != "NULL" and log[mandatory_map[obj][0]] != '' and \ + if mandatory_map[obj][0] in log.keys() and log[mandatory_map[obj][0]] != "NULL" and \ + log[mandatory_map[obj][0]] != '' and \ log[mandatory_map[obj][0]] is not None: data[field] = value elif obj in ["user", "nt"]: if obj == "nt": data = ResultsConnector.process_nt_obj(mandatory_map, obj, data, log, value, field) else: - if any((log[item] != "NULL" and log[item] != '' and log[item] is not None) + if any((item in log.keys() and log[item] != "NULL" and log[item] != '' and log[item] is not None) for item in mandatory_map[obj]): data[field] = value except (KeyError, IndexError, TypeError) as e: @@ -209,7 +245,7 @@ def process_nt_obj(mandatory_map, obj, data, log, value, field): try: for i in mandatory_map[obj]: if isinstance(i, list): - if any((log[item] != "NULL" and log[item] != '' and log[item] is not None) + if any((item in log.keys() and log[item] != "NULL" and log[item] != '' and log[item] is not None) for item in i): nt_obj = True else: @@ -220,29 +256,3 @@ def process_nt_obj(mandatory_map, obj, data, log, value, field): except (KeyError, IndexError, TypeError) as e: raise e return data - - # *** The stream data feature is not used currently. It may be implemented in future *** - # @staticmethod - # def format_stream_data(stream_data): - # """ - # Format the stream data into json format - # :param stream_data: string - # :return: list - # """ - # results = [] - # - # temp_data = stream_data.split("\n") - # for log in temp_data: - # if log > " ": - # data = {} - # results_dict = {} - # dataset = "" - # for field, value in json.loads(log).items(): - # if value is not None and (isinstance(value, str) and value.lower() != "null") \ - # and field != 'dataset_name': - # data[field] = value - # elif field == 'dataset_name': - # dataset = value - # results_dict[dataset] = data - # results.append(results_dict) - # return results diff --git a/stix_shifter_modules/paloalto/stix_transmission/status_connector.py b/stix_shifter_modules/paloalto/stix_transmission/status_connector.py index 6c02861d5..b6b19ebc6 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/status_connector.py +++ b/stix_shifter_modules/paloalto/stix_transmission/status_connector.py @@ -60,14 +60,17 @@ def create_status_connection(self, search_id): if response_code == 200: if 'status' in response_text['reply'].keys(): + # Since PaloAlto API doesnt return the numerical value of progress, the value for Pending + # status is set to 50. If the status is completed, it is set to 100. if response_text['reply']['status'] == "PENDING": return_obj['success'] = True return_obj['status'] = StatusConnector.get_status('PENDING') - return_obj['progress'] = 100 + return_obj['progress'] = 50 elif response_text['reply']['status'] == "SUCCESS": if 'number_of_results' in response_text['reply'].keys(): return_obj['success'] = True return_obj['status'] = StatusConnector.get_status('COMPLETED') + return_obj['progress'] = 100 else: raise InvalidResponseException else: diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py index 9d54062c8..737266c74 100644 --- a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py +++ b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py @@ -82,7 +82,7 @@ def test_ipv4_query(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_local_ip = \"172.31.90.48\" or " "action_remote_ip = \"172.31.90.48\") and " "(to_epoch(_time,\"millis\") >= 1645615464114 and to_epoch(_time,\"millis\") <= 1645615764114)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', 'timeframe': {" + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" "'from': 1645615464114, 'to': " "1645615764114}}}"] @@ -96,7 +96,7 @@ def test_network_traffic_query(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port = 53996 and " "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" " | alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', 'timeframe': {" + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" "'from': 1645615637334, 'to': " "1645615937334}}}"] @@ -110,7 +110,7 @@ def test_network_query_greater_than(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port > 53996 and " "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" " | alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', 'timeframe': {" + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" "'from': 1645615637334, 'to': " "1645615937334}}}"] queries = _remove_timestamp_from_query(queries) @@ -123,7 +123,7 @@ def test_network_query_not_equals(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port != 53996 and " "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" " | alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', 'timeframe': {" + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" "'from': 1645615637334, 'to': " "1645615937334}}}"] queries = _remove_timestamp_from_query(queries) @@ -136,7 +136,7 @@ def test_network_query_enum_type(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_network_protocol in (ENUM.TCP," "ENUM.UDP) and " "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645635857746, " "'to': 1645636157746}}}"] @@ -150,7 +150,7 @@ def test_network_query_not_greater_than(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port <= 53996 and " "(to_epoch(_time,\"millis\") >= 1645636616556 and to_epoch(_time,\"millis\") <= 1645636916556)) " "| alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1645636616556, 'to': " "1645636916556}}}"] queries = _remove_timestamp_from_query(queries) @@ -163,7 +163,7 @@ def test_network_query_less_than(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port < 53996 and " "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " "| alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1645636370847, 'to': " "1645636670847}}}"] queries = _remove_timestamp_from_query(queries) @@ -176,7 +176,7 @@ def test_network_query_lessthan_or_equals(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port <= 53996 and " "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " "| alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1645636370847, 'to': " "1645636670847}}}"] queries = _remove_timestamp_from_query(queries) @@ -189,7 +189,7 @@ def test_network_query_greaterthan_or_equals(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port >= 53996 and " "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " "| alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1645636370847, 'to': " "1645636670847}}}"] queries = _remove_timestamp_from_query(queries) @@ -207,7 +207,7 @@ def test_file_like_operator(self): "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or os_actor_process_image_name contains " "\"edr-2022-02-09_08-56-58-474-checksum.txt\") and " "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645635327346, " "'to': 1645635627346}}}"] @@ -229,7 +229,7 @@ def test_file_IN_operator(self): "\"SentinelOne_1.binlog\",\"edr-2022-02-09_08-56-58-47-checksum.txt\")) and " "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " "| alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1645635327346, 'to': " "1645635627346}}}"] @@ -245,7 +245,7 @@ def test_file_matches_operator(self): "causality_actor_process_image_name ~= \"^g.{2}.exe\" or os_actor_process_image_name ~= \"^g.{" "2}.exe\") and " "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645635327346, " "'to': 1645635627346}}}"] @@ -261,7 +261,7 @@ def test_mac_address_query(self): "associated_mac = \"12:83:0e:be:f3:1d\" or dst_associated_mac = \"12:83:0e:be:f3:1d\" or dst_mac " "= \"12:83:0e:be:f3:1d\") and " "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645635857746, " "'to': 1645636157746}}}"] @@ -277,7 +277,7 @@ def test_process_not_matches_operator(self): "causality_actor_process_image_name !~= \"^g.{2}.exe\" or os_actor_process_image_name !~= \"^g.{" "2}.exe\") and " "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645636692740, " "'to': 1645636992740}}}"] @@ -294,7 +294,7 @@ def test_process_not_in_operator(self): "or os_actor_process_image_name not in (\"conhost.exe\",\"AtBroker.exe\")) and " "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) " "| alter " - "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1645636692740, 'to': " "1645636992740}}}"] queries = _remove_timestamp_from_query(queries) @@ -307,7 +307,7 @@ def test_windows_regkey_not_like_operator(self): queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_registry_value_name not contains " "\"DeltaUpdateFailure\" and " "(to_epoch(_time,\"millis\") >= 1645638324427 and to_epoch(_time,\"millis\") <= 1645638624427)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645638324427, " "'to': 1645638624427}}}"] @@ -324,7 +324,7 @@ def test_query_from_multiple_comparison_expressions_joined_by_AND(self): "causality_actor_process_image_name contains \"arp_cache.py\" or os_actor_process_image_name " "contains \"arp_cache.py\") and " "(to_epoch(_time,\"millis\") >= 1645638324427 and to_epoch(_time,\"millis\") <= 1645638624427)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645638324427, " "'to': 1645638624427}}}"] @@ -341,7 +341,7 @@ def test_query_from_multiple_comparison_expressions_joined_by_OR(self): "\"^G.{5}U.{5}.exe\") or action_remote_port != 53996 and " "(to_epoch(_time,\"millis\") >= 1645635761271 and to_epoch(_time,\"millis\") <= 1645636061271)) " "| alter dataset_name = " - "\"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "\"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1645635761271," " 'to': 1645636061271}}}"] @@ -360,7 +360,7 @@ def test_query_for_morethan_two_comparison_expressions_joined_by_AND(self): "actor_process_image_name not contains \"arp_cache.py\" or causality_actor_process_image_name not " "contains \"arp_cache.py\" or os_actor_process_image_name not contains \"arp_cache.py\") and " "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) | " - "alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1645636692740, " "'to': 1645636992740}}}"] queries = _remove_timestamp_from_query(queries) @@ -379,7 +379,7 @@ def test_query_for_morethan_two_comparison_expressions_joined_by_OR(self): "5}U.{5}.exe\" or os_actor_process_image_name !~= \"^G.{5}U.{5}.exe\") or action_remote_port != " "53996 and " "(to_epoch(_time,\"millis\") >= 1645636966650 and to_epoch(_time,\"millis\") <= 1645637266650)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645636966650, " "'to': 1645637266650}}}"] @@ -403,7 +403,7 @@ def test_multiple_observation_with_and_without_qualifier_query(self): "action_registry_value_name not contains \"SlotPerRow\" and " "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " "| alter dataset_name = " - "\"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "\"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1640995200030, " "'to': 1645636157746}}}"] queries = _remove_timestamp_from_query(queries) @@ -448,7 +448,7 @@ def test_combined_observation_AND(self): "= \"172.31.31.67\") and action_remote_port = 53996 and " "(to_epoch(_time,\"millis\") >= 1642590000000 and to_epoch(_time,\"millis\") <= 1644231600003)) " "| alter dataset_name = " - "\"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "\"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': 1642590000000, " "'to': 1644231600003}}}"] queries = _remove_timestamp_from_query(queries) @@ -472,7 +472,7 @@ def test_combined_observation_OR(self): "\"12:83:0e:be:f3:1d\" or dst_associated_mac = \"12:83:0e:be:f3:1d\" or dst_mac = " "\"12:83:0e:be:f3:1d\") and " "(to_epoch(_time,\"millis\") >= 1641812400000 and to_epoch(_time,\"millis\") <= 1644058800003)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1641812400000, " "'to': 1644058800003}}}"] @@ -552,7 +552,7 @@ def test_format_timestamp_fields(self): "causality_actor_process_file_create_time = 1643704990003 or os_actor_process_file_create_time = " "1643704990003) and " "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " - "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 1000 ', " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " "'timeframe': {'from': " "1645635857746, " "'to': 1645636157746}}}"] diff --git a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py index 89ee58072..73ad115cc 100644 --- a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py +++ b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py @@ -458,7 +458,31 @@ def test_status_403_exception(self, get_search_results): assert "The provided API Key does not have the required RBAC permissions to run this API" in \ status_response['error'] + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_max_quota_exception(self, mock_ping): + """test maximum quota threshold exception""" + response = { + 'reply': {'license_quota': 5, 'additional_purchased_quota': 0.0, 'used_quota': 4, 'eval_quota': 0.0}} + mock_ping.side_effect = [MockResponse(MockStatusObj(200), json.dumps(response))] + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query({}) + assert query_response is not None + assert query_response['success'] is False + assert query_response['code'] == "service_unavailable" + assert "query usage exceeded max daily quota" in query_response['error'] + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_max_additional_quota_exception(self, mock_ping): + """test maximum additional quota threshold exception""" + response = { + 'reply': {'license_quota': 5, 'additional_purchased_quota': 10.0, 'used_quota': 12, 'eval_quota': 0.0}} + mock_ping.side_effect = [MockResponse(MockStatusObj(200), json.dumps(response))] + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query({}) + assert query_response is not None + assert query_response['success'] is False + assert query_response['code'] == "service_unavailable" + assert "query usage exceeded max daily quota" in query_response['error'] @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') def test_quota_invalid_json_exception(self, mock_ping): @@ -483,6 +507,28 @@ def test_invalid_host(self, mock_ping): assert "Invalid Host" in ping_response['error'] assert ping_response['code'] == "service_unavailable" + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_invalid_host_for_status(self, mock_query): + """Test Invalid host for Status API""" + mock_query.side_effect = ConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status("123_inv") + assert status_response is not None + assert status_response['success'] is False + assert "Invalid Host" in status_response['error'] + assert status_response['code'] == "service_unavailable" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_invalid_host_for_results(self, mock_query): + """Test Invalid host for Results API""" + mock_query.side_effect = ConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + results_response = transmission.results("123_inv", 0, 2) + assert results_response is not None + assert results_response['success'] is False + assert "Invalid Host" in results_response['error'] + assert results_response['code'] == "service_unavailable" + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') def test_timeout_error(self, mock_ping): """Test Timeout Error""" @@ -492,4 +538,52 @@ def test_timeout_error(self, mock_ping): assert ping_response is not None assert ping_response['success'] is False assert "timeout_error (30 sec)" in ping_response['error'] - assert ping_response['code'] == "service_unavailable" \ No newline at end of file + assert ping_response['code'] == "service_unavailable" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_stream_results') + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_format_stream_data(self, mock_search_response, mock_stream_response): + stream_id = "123" + mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", + "number_of_results": 10001, + "results": {"stream_id": stream_id}}}) + mock_search_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + stream_return_value = '{"dataset_name":"xdr_data","action_local_ip":"65.0.202.35","action_remote_ip":' \ + '"172.31.90.48","action_local_port":"50893","action_remote_port":"3389",' \ + '"action_network_protocol":"TCP"}' + mock_stream_response.return_value = StatusResponse(200, stream_return_value) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'] == [{'xdr_data': {'action_local_ip': '65.0.202.35', + 'action_network_protocol': 'TCP', + 'action_remote_ip': '172.31.90.48', + 'action_local_port': '50893', + 'action_remote_port': '3389'}}] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_result_with_empty_user_response(self, mock_result_response): + """test for valid result response""" + mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", + "number_of_results": 1, + "results": {"data": [{"dataset_name": "xdr_data", + "actor_primary_user_sid": "S123", + "actor_primary_username": "username", + "actor_process_logon_id": "id12"}]}}}) + mock_result_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "62428d95420f47_24655_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'] == [{'xdr_data': {'actor_primary_user_sid': 'S123', + 'actor_primary_username': 'username', + 'actor_process_logon_id': 'id12'}}] From d2301c258ca9e34facf1251572fbc1c2895e7f5d Mon Sep 17 00:00:00 2001 From: lakshmi51974368 <92726459+lakshmi51974368@users.noreply.github.com> Date: Tue, 29 Mar 2022 07:21:11 +0000 Subject: [PATCH 5/7] Updated lang_en.json Updated the Standard daily License API quota details --- stix_shifter_modules/paloalto/configuration/lang_en.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix_shifter_modules/paloalto/configuration/lang_en.json b/stix_shifter_modules/paloalto/configuration/lang_en.json index 838e36bd4..5d7df0cf3 100644 --- a/stix_shifter_modules/paloalto/configuration/lang_en.json +++ b/stix_shifter_modules/paloalto/configuration/lang_en.json @@ -6,11 +6,11 @@ }, "quota_threshold": { "label": "The quota limit for the API", - "description": "Prevents the connector from exceeding the quota threshold limit. The threshold range is from 1 to 4 units" + "description": "Prevents the connector from exceeding the entire daily units(5) allocated for the Standard License. The threshold range is from 1 to 4 units" }, "additional_unit_quota_threshold": { "label": "The quota limit for the API for the additionally purchased units", - "description": "Prevents the connector from exceeding the quota threshold limit of the additional units along with the standard license quota. The Threshold range is from 6 to 12 units" + "description": "Prevents the connector from exceeding the quota threshold limit of the 15 units ( 5 unit for standard license and additional 10 units ). The Threshold range is from 6 to 12 units" }, "help": { "label": "Need additional help?", From f89f294e11f91cd84b5a127a87470377fa245366 Mon Sep 17 00:00:00 2001 From: lakshmi51974368 <92726459+lakshmi51974368@users.noreply.github.com> Date: Wed, 30 Mar 2022 16:01:52 +0000 Subject: [PATCH 6/7] Updated quota_threshold configuration value Updated 'quota_threshold' configuration value --- stix_shifter_modules/paloalto/README.md | 7 +++++-- .../paloalto/configuration/config.json | 13 ++++--------- .../paloalto/configuration/lang_en.json | 6 +----- .../paloalto/stix_transmission/api_client.py | 6 ++++-- .../tests/stix_transmission/test_paloalto.py | 2 +- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/stix_shifter_modules/paloalto/README.md b/stix_shifter_modules/paloalto/README.md index f237f3cd3..62b1fc5d3 100644 --- a/stix_shifter_modules/paloalto/README.md +++ b/stix_shifter_modules/paloalto/README.md @@ -979,8 +979,11 @@ paloalto - Queries called without enough quota will fail. ### Observations -- The quota limit range for the API call would be from 1 to 4 units for the Standard license, and it will be from 6 to 12 - if additional units are purchased. +- The quota limit range for the API call would be from 1 to 15 (Standard 5 + 10 Additional ) units. + The default value of quota thershold limit in CP4S UI is 5. +- If the user purchases standard license only(5 units), and configures the threshold value more than 5, + the usage limit will be checked only for 5 units. Otherwise, usage will be checked for the configured value. +- If the user purchases Additional 10 units along with standard license, the usage limit will be checked for configured value. ### References - [Cortex XDR](https://docs.paloaltonetworks.com/cortex/cortex-xdr.html) diff --git a/stix_shifter_modules/paloalto/configuration/config.json b/stix_shifter_modules/paloalto/configuration/config.json index b5b1a0681..b661c3df1 100644 --- a/stix_shifter_modules/paloalto/configuration/config.json +++ b/stix_shifter_modules/paloalto/configuration/config.json @@ -7,16 +7,11 @@ "type": "text" }, "quota_threshold": { - "default": 3, + "default": 5, "min": 1, - "max": 4, - "type": "number" - }, - "additional_unit_quota_threshold": { - "default": 12, - "min": 6, - "max": 12, - "type": "number" + "max": 15, + "type": "number", + "optional" : true }, "help": { "type": "link", diff --git a/stix_shifter_modules/paloalto/configuration/lang_en.json b/stix_shifter_modules/paloalto/configuration/lang_en.json index 5d7df0cf3..be3f57075 100644 --- a/stix_shifter_modules/paloalto/configuration/lang_en.json +++ b/stix_shifter_modules/paloalto/configuration/lang_en.json @@ -6,11 +6,7 @@ }, "quota_threshold": { "label": "The quota limit for the API", - "description": "Prevents the connector from exceeding the entire daily units(5) allocated for the Standard License. The threshold range is from 1 to 4 units" - }, - "additional_unit_quota_threshold": { - "label": "The quota limit for the API for the additionally purchased units", - "description": "Prevents the connector from exceeding the quota threshold limit of the 15 units ( 5 unit for standard license and additional 10 units ). The Threshold range is from 6 to 12 units" + "description": "Prevents the connector from exceeding the entire daily units allocated for the License. The Daily units for Standard license alone is 5 and it will be 15 if additional 10 units are purchased." }, "help": { "label": "Need additional help?", diff --git a/stix_shifter_modules/paloalto/stix_transmission/api_client.py b/stix_shifter_modules/paloalto/stix_transmission/api_client.py index 742479e0e..0306b1c9c 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/api_client.py +++ b/stix_shifter_modules/paloalto/stix_transmission/api_client.py @@ -43,7 +43,6 @@ def __init__(self, connection, configuration): self.result_limit = connection['options'].get('result_limit') self.timeout = connection['options']['timeout'] self.quota_threshold = connection['quota_threshold'] - self.additional_quota_threshold = connection['additional_unit_quota_threshold'] self.connector = __name__.split('.')[1] def ping_data_source(self): @@ -80,10 +79,13 @@ def get_remaining_quota(self): # The daily quota unit for standard license is 5. additional units up to 10 can be added. if quota_response_text['reply']['license_quota'] == 5 and \ quota_response_text['reply']['additional_purchased_quota'] == 0.0: + # For a Standard license,if the configured quota threshold is greater than 5, + # the threshold quota value is reset to 5. + self.quota_threshold = 5 if self.quota_threshold > 5 else self.quota_threshold if quota_response_text['reply']['used_quota'] >= self.quota_threshold: raise MaxDailyQuotaException else: - if quota_response_text['reply']['used_quota'] >= self.additional_quota_threshold: + if quota_response_text['reply']['used_quota'] >= self.quota_threshold: raise MaxDailyQuotaException return_obj['success'] = True else: diff --git a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py index 73ad115cc..eb272ca6e 100644 --- a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py +++ b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py @@ -462,7 +462,7 @@ def test_status_403_exception(self, get_search_results): def test_max_quota_exception(self, mock_ping): """test maximum quota threshold exception""" response = { - 'reply': {'license_quota': 5, 'additional_purchased_quota': 0.0, 'used_quota': 4, 'eval_quota': 0.0}} + 'reply': {'license_quota': 5, 'additional_purchased_quota': 0.0, 'used_quota': 5.01, 'eval_quota': 0.0}} mock_ping.side_effect = [MockResponse(MockStatusObj(200), json.dumps(response))] transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) query_response = transmission.query({}) From 245b17ff027e70a7904b5f2d44c000b02d4d25ca Mon Sep 17 00:00:00 2001 From: lakshmi51974368 <92726459+lakshmi51974368@users.noreply.github.com> Date: Wed, 6 Apr 2022 08:00:39 +0000 Subject: [PATCH 7/7] Updated the mappings of x-oca-event and x-oca-asset fields updated the mappings of x-oca-event and x-oca-asset fields --- stix_shifter_modules/paloalto/README.md | 781 +++++++++--------- .../stix_translation/json/config_map.json | 48 +- .../stix_translation/json/fields_map.json | 2 +- .../json/stix_2_1/to_stix_map.json | 476 ++++++++--- .../json/stix_2_1/xdr_data_from_stix_map.json | 106 ++- .../stix_translation/json/to_stix_map.json | 476 ++++++++--- .../json/xdr_data_from_stix_map.json | 110 ++- .../paloalto/stix_translation/transformers.py | 16 + .../stix_transmission/results_connector.py | 8 +- .../test_paloalto_json_to_stix.py | 26 +- .../test_paloalto_stix_to_query.py | 7 +- 11 files changed, 1358 insertions(+), 698 deletions(-) create mode 100644 stix_shifter_modules/paloalto/stix_translation/transformers.py diff --git a/stix_shifter_modules/paloalto/README.md b/stix_shifter_modules/paloalto/README.md index 62b1fc5d3..6bec5c518 100644 --- a/stix_shifter_modules/paloalto/README.md +++ b/stix_shifter_modules/paloalto/README.md @@ -36,7 +36,7 @@ translate paloalto query '{}' "[ipv4-addr:value = '1.1.0.0'] START t'2022-03-01T "queries": [ { "xdr_data": { - "query": "dataset = xdr_data | filter ((action_local_ip = \"1.1.0.0\" or action_remote_ip = \"1.1.0.0\") and (to_epoch(_time,\"millis\") >= 1646132400000 and to_epoch(_time,\"millis\") <= 1678273200003)) | alter dataset_name = \"xdr_data\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 1000 ", + "query": "dataset = xdr_data | filter ((action_local_ip = \"1.1.0.0\" or action_remote_ip = \"1.1.0.0\" or agent_ip_addresses = \"1.1.0.0\") and (to_epoch(_time,\"millis\") >= 1646132400000 and to_epoch(_time,\"millis\") <= 1678273200003)) | alter dataset_name = \"xdr_data\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 10000 ", "timeframe": { "from": 1646132400000, "to": 1678273200003 @@ -45,6 +45,7 @@ translate paloalto query '{}' "[ipv4-addr:value = '1.1.0.0'] START t'2022-03-01T } ] } + ``` #### STIX Transmit ping @@ -70,14 +71,14 @@ paloalto "{\"host\":\"xx.xx.xx\"}" "{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" query -"{ \"xdr_data\": { \"query\": \"dataset = xdr_data | filter ((action_local_ip = \\"1.1.0.0\\" or action_remote_ip = \\"1.1.0.0\\") and (to_epoch(_time,\\"millis\\") >= 1646132400000 and to_epoch(_time,\\"millis\\") <= 1678273200003)) | alter dataset_name = \\"xdr_data\\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 1000 \", \"timeframe\": { \"from\": 1646132400000, \"to\": 1678273200003 } } }" +"{\"xdr_data\": {\"query\": \"dataset = xdr_data | filter ((action_local_ip = \\"1.1.0.0\\" or action_remote_ip = \\"1.1.0.0\\" or agent_ip_addresses = \\"1.1.0.0\\") and (to_epoch(_time,\\"millis\\") >= 1646132400000 and to_epoch(_time,\\"millis\\") <= 1678273200003)) | alter dataset_name = \\"xdr_data\\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 10000 \", \"timeframe\": {\"from\": 1646132400000, \"to\": 1678273200003}}}" ``` #### STIX Transmit query - output ```json { "success": true, - "search_id": "d84862b9b7254a_22989_inv" + "search_id": "a106f2f4614f40_34940_inv" } ``` #### STIX Transmit status @@ -88,14 +89,15 @@ paloalto "{\"host\":\"xx.xx.xx\"}" "{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" status -"d84862b9b7254a_22989_inv" +"a106f2f4614f40_34940_inv" ``` #### STIX Transmit status - output ```json { "success": true, - "status": "COMPLETED" + "status": "COMPLETED", + "progress": 100 } ``` #### STIX Transmit results @@ -106,7 +108,7 @@ paloalto "{\"host\":\"xx.xx.xx\"}" "{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" results -"d84862b9b7254a_22989_inv" 0 2 +"a106f2f4614f40_34940_inv" 0 2 ``` #### STIX Transmit results - output @@ -116,36 +118,36 @@ results "data": [ { "xdr_data": { + "action_local_ip": "94.232.41.158", "action_network_protocol": "TCP", - "action_local_ip": "193.93.62.47", - "action_remote_ip": "1.1.0.0", - "action_local_port": 31029, - "action_remote_port": 3389, - "agent_id": "193.93.62.47", - "event_id": "MTM4NTk0MDcwNDA4NDQ0MzIwNw==", - "event_timestamp": 1646799589433, - "action_network_creation_time": 1646799589433, - "action_network_connection_id": "AdgzbOqYRXkAAAAAAB+exg==", - "action_proxy": "FALSE", + "action_remote_ip": "172.31.90.48", + "action_local_port": "31545", + "action_remote_port": "3389", + "agent_id": "94.232.41.158", + "event_id": "NDQ3NzY1MTgzNjU1MjQxNTg0Ng==", + "event_timestamp": "1648884264296", "event_type": "STORY", - "event_sub_type": "event_sub_type_4" + "event_sub_type": "event_sub_type_4", + "action_network_creation_time": "1648884264296", + "action_network_connection_id": "AdhGYq2dw14AAAAAAAEOxg==", + "action_proxy": "FALSE" } }, { "xdr_data": { + "action_local_ip": "94.232.41.158", "action_network_protocol": "TCP", - "action_local_ip": "193.93.62.6", - "action_remote_ip": "1.1.0.0", - "action_local_port": 47275, - "action_remote_port": 3389, - "agent_id": "193.93.62.6", - "event_id": "Nzc2MzU5MjE1NjE4NTYzNDI4OA==", - "event_timestamp": 1646799556894, - "action_network_creation_time": 1646799556894, - "action_network_connection_id": "AdgzbNczNawAAAAAAB+euA==", - "action_proxy": "FALSE", + "action_remote_ip": "172.31.90.48", + "action_local_port": "6142", + "action_remote_port": "3389", + "agent_id": "94.232.41.158", + "event_id": "MjAyOTYyMzMyMTg3Njc5MDY4NA==", + "event_timestamp": "1648884468933", "event_type": "STORY", - "event_sub_type": "event_sub_type_4" + "event_sub_type": "event_sub_type_4", + "action_network_creation_time": "1648884468933", + "action_network_connection_id": "AdhGYyeW9LcAAAAAAAEO5g==", + "action_proxy": "FALSE" } } ] @@ -156,11 +158,11 @@ results #### STIX Translate results ```shell -translate -paloalto -results -"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"events\"}" -" [ { \"xdr_data\": { \"action_network_protocol\": \"TCP\", \"action_local_ip\": \"193.93.62.47\", \"action_remote_ip\": \"1.1.0.0\", \"action_local_port\": 31029, \"action_remote_port\": 3389, \"agent_id\": \"193.93.62.47\", \"event_id\": \"MTM4NTk0MDcwNDA4NDQ0MzIwNw==\", \"event_timestamp\": 1646799589433, \"action_network_creation_time\": 1646799589433, \"action_network_connection_id\": \"AdgzbOqYRXkAAAAAAB+exg==\", \"action_proxy\": \"FALSE\", \"event_type\": \"STORY\", \"event_sub_type\": \"event_sub_type_4\" } }, { \"xdr_data\": { \"action_network_protocol\": \"TCP\", \"action_local_ip\": \"193.93.62.6\", \"action_remote_ip\": \"1.1.0.0\", \"action_local_port\": 47275, \"action_remote_port\": 3389, \"agent_id\": \"193.93.62.6\", \"event_id\": \"Nzc2MzU5MjE1NjE4NTYzNDI4OA==\", \"event_timestamp\": 1646799556894, \"action_network_creation_time\": 1646799556894, \"action_network_connection_id\": \"AdgzbNczNawAAAAAAB+euA==\", \"action_proxy\": \"FALSE\", \"event_type\": \"STORY\", \"event_sub_type\": \"event_sub_type_4\" } } ] " +translate +paloalto +results +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"system\",\"created\":\"2022-03-16T13:22:50.336Z\",\"modified\":\"2022-03-16T13:22:50.336Z\"}" +"[ { \"xdr_data\": { \"action_local_ip\": \"94.232.41.158\", \"action_network_protocol\": \"TCP\", \"action_remote_ip\": \"1.1.0.0\", \"action_local_port\": \"31545\", \"action_remote_port\": \"3389\", \"agent_id\": \"94.232.41.158\", \"event_id\": \"NDQ3NzY1MTgzNjU1MjQxNTg0Ng==\", \"event_timestamp\": \"1648884264296\", \"event_type\": \"STORY\", \"event_sub_type\": \"event_sub_type_4\", \"action_network_creation_time\": \"1648884264296\", \"action_network_connection_id\": \"AdhGYq2dw14AAAAAAAEOxg==\", \"action_proxy\": \"FALSE\" } }, { \"xdr_data\": { \"action_local_ip\": \"94.232.41.158\", \"action_network_protocol\": \"TCP\", \"action_remote_ip\": \"1.1.0.0\", \"action_local_port\": \"6142\", \"action_remote_port\": \"3389\", \"agent_id\": \"94.232.41.158\", \"event_id\": \"MjAyOTYyMzMyMTg3Njc5MDY4NA==\", \"event_timestamp\": \"1648884468933\", \"event_type\": \"STORY\", \"event_sub_type\": \"event_sub_type_4\", \"action_network_creation_time\": \"1648884468933\", \"action_network_connection_id\": \"AdhGYyeW9LcAAAAAAAEO5g==\", \"action_proxy\": \"FALSE\" } } ] " "{\"stix_validator\": true}" ``` @@ -168,108 +170,126 @@ results ```json { "type": "bundle", - "id": "bundle--c7d5cb7b-bfc2-4964-be1a-596a4eb37f72", + "id": "bundle--318a05c9-47e5-4146-80ce-e6b95c71c8bd", "objects": [ { "type": "identity", "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", "name": "paloalto", - "identity_class": "events" + "identity_class": "system", + "created": "2022-03-16T13:22:50.336Z", + "modified": "2022-03-16T13:22:50.336Z" }, { - "id": "observed-data--875927c9-575f-4410-bdf6-9bc6060b204c", + "id": "observed-data--713d4380-de85-40f7-a6c1-c893331e9c99", "type": "observed-data", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2022-03-09T08:08:41.394Z", - "modified": "2022-03-09T08:08:41.394Z", + "created": "2022-04-05T12:43:28.560Z", + "modified": "2022-04-05T12:43:28.560Z", "objects": { "0": { + "type": "ipv4-addr", + "value": "94.232.41.158" + }, + "1": { "type": "network-traffic", + "src_ref": "0", "protocols": [ "tcp" ], - "src_ref": "1", - "dst_ref": "2", - "src_port": 31029, + "dst_ref": "4", + "src_port": 31545, "dst_port": 3389, "extensions": { "x-paloalto-network": { - "creation_time": "2022-03-09T04:19:49.433Z", - "connection_id": "AdgzbOqYRXkAAAAAAB+exg==", + "creation_time": "2022-04-02T07:24:24.296Z", + "connection_id": "AdhGYq2dw14AAAAAAAEOxg==", "is_proxy": false } } }, - "1": { - "type": "ipv4-addr", - "value": "193.93.62.47" - }, "2": { - "type": "ipv4-addr", - "value": "1.1.0.0" + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "asset_id": "94.232.41.158" + } + } }, "3": { - "type": "x-oca-asset", - "asset_id": "193.93.62.47" + "type": "x-oca-event", + "network_ref": "1", + "code": "NDQ3NzY1MTgzNjU1MjQxNTg0Ng==", + "created": "2022-04-02T07:24:24.296Z", + "category": ["story"], + "action": "event_sub_type_4" }, "4": { - "type": "x-oca-event", - "event_id": "MTM4NTk0MDcwNDA4NDQ0MzIwNw==", - "time": "2022-03-09T04:19:49.433Z", - "event_type": "STORY", - "sub_type": "event_sub_type_4" + "type": "ipv4-addr", + "value": "1.1.0.0" } }, - "first_observed": "2022-03-09T08:08:41.394Z", - "last_observed": "2022-03-09T08:08:41.394Z", + "first_observed": "2022-04-05T12:43:28.560Z", + "last_observed": "2022-04-05T12:43:28.560Z", "number_observed": 1 }, { - "id": "observed-data--3b1d6e6b-95ea-406f-aafb-aa5e4056640b", + "id": "observed-data--d05e4f7d-0866-4f4d-be32-a39832ad27c1", "type": "observed-data", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2022-03-09T08:08:51.467Z", - "modified": "2022-03-09T08:08:51.467Z", + "created": "2022-04-05T12:43:28.565Z", + "modified": "2022-04-05T12:43:28.565Z", "objects": { "0": { + "type": "ipv4-addr", + "value": "94.232.41.158" + }, + "1": { "type": "network-traffic", + "src_ref": "0", "protocols": [ "tcp" ], - "src_ref": "1", - "dst_ref": "2", - "src_port": 47275, + "dst_ref": "4", + "src_port": 6142, "dst_port": 3389, "extensions": { "x-paloalto-network": { - "creation_time": "2022-03-09T04:19:16.894Z", - "connection_id": "AdgzbNczNawAAAAAAB+euA==", + "creation_time": "2022-04-02T07:27:48.933Z", + "connection_id": "AdhGYyeW9LcAAAAAAAEO5g==", "is_proxy": false } } }, - "1": { - "type": "ipv4-addr", - "value": "193.93.62.6" - }, "2": { - "type": "ipv4-addr", - "value": "1.1.0.0" + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "asset_id": "94.232.41.158" + } + } }, "3": { - "type": "x-oca-asset", - "asset_id": "193.93.62.6" + "type": "x-oca-event", + "network_ref": "1", + "code": "MjAyOTYyMzMyMTg3Njc5MDY4NA==", + "created": "2022-04-02T07:27:48.933Z", + "category": ["story"], + "action": "event_sub_type_4" }, "4": { - "type": "x-oca-event", - "event_id": "Nzc2MzU5MjE1NjE4NTYzNDI4OA==", - "time": "2022-03-09T04:19:16.894Z", - "event_type": "STORY", - "sub_type": "event_sub_type_4" + "type": "ipv4-addr", + "value": "1.1.0.0" } }, - "first_observed": "2022-03-09T08:08:51.467Z", - "last_observed": "2022-03-09T08:08:51.467Z", + "first_observed": "2022-04-05T12:43:28.565Z", + "last_observed": "2022-04-05T12:43:28.565Z", "number_observed": 1 } ], @@ -281,7 +301,7 @@ results #### STIX Translate query ```shell -translate paloalto query {} "([x-oca-event:event_type = 'file'] AND [file:size > 3000] OR [x-oca-asset:agent_version = '7.6.1.46600'] )START t'2022-02-10T00:00:00.000000Z' STOP t'2022-02-15T00:00:00.000000Z'" +translate paloalto query {} "([x-oca-event:category[*] = 'file'] AND [file:size > 3000] OR [x-oca-asset:agent_version = '7.6.1.46600'] )START t'2022-02-10T00:00:00.000000Z' STOP t'2022-02-15T00:00:00.000000Z'" ``` #### STIX Translate query - output @@ -291,7 +311,7 @@ translate paloalto query {} "([x-oca-event:event_type = 'file'] AND [file:size > "queries": [ { "xdr_data": { - "query": "dataset = xdr_data | filter (event_type = ENUM.FILE and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or (action_file_size > 3000 and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or (agent_version = \"7.6.1.46600\" and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) | alter dataset_name = \"xdr_data\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 1000 ", + "query": "dataset = xdr_data | filter (event_type = ENUM.FILE and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or (action_file_size > 3000 and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) | alter dataset_name = \"xdr_data\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 10000 ", "timeframe": { "from": 1644451200000, "to": 1644883200000 @@ -311,7 +331,7 @@ paloalto "{\"host\":\"xx.xx.xx\"}" "{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" query -" { \"xdr_data\": { \"query\": \"dataset = xdr_data | filter (event_type = ENUM.FILE and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) or (action_file_size > 3000 and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) or (agent_version = \\"7.6.1.46600\\" and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) | alter dataset_name = \\"xdr_data\\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 1000 \", \"timeframe\": { \"from\": 1644451200000, \"to\": 1644883200000 } } }" +"{ \"xdr_data\": { \"query\": \"dataset = xdr_data | filter (event_type = ENUM.FILE and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) or (action_file_size > 3000 and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) | alter dataset_name = \\"xdr_data\\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 10000 \", \"timeframe\": { \"from\": 1644451200000, \"to\": 1644883200000 } } }" ``` #### STIX Transmit query - output @@ -319,7 +339,7 @@ query ```json { "success": true, - "search_id": "8c1c048e556c4b_23040_inv" + "search_id": "3dc7d55520fe41_35090_inv" } ``` #### STIX Transmit status @@ -330,15 +350,17 @@ paloalto "{\"host\":\"xx.xx.xx\"}" "{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" status -"8c1c048e556c4b_23040_inv" +"3dc7d55520fe41_35090_inv" ``` #### STIX Transmit status - output ```json { "success": true, - "status": "COMPLETED" + "status": "COMPLETED", + "progress": 100 } + ``` #### STIX Transmit results @@ -348,7 +370,7 @@ paloalto "{\"host\":\"xx.xx.xx\"}" "{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" results -"8c1c048e556c4b_23040_inv" 0 2 +"3dc7d55520fe41_35090_inv" 0 2 ``` #### STIX Transmit results - output @@ -358,86 +380,93 @@ results "data": [ { "xdr_data": { - "action_file_name": "MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "agent_ip_addresses": [ + "172.31.31.67" + ], + "action_file_name": "DRIVERS", "actor_process_image_name": "System", "causality_actor_process_image_name": "System", "os_actor_process_image_name": "System", - "action_file_path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Support\\MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "action_file_size": "4255744", + "action_file_path": "C:\\Windows\\System32\\config\\DRIVERS", "actor_process_image_path": "System", "causality_actor_process_image_path": "System", "os_actor_process_image_path": "System", - "actor_process_os_pid": 4, - "causality_actor_process_os_pid": 4, - "os_actor_process_os_pid": 4, + "actor_process_os_pid": "4", + "causality_actor_process_os_pid": "4", + "os_actor_process_os_pid": "4", "actor_primary_user_sid": "S-1-5-18", "actor_primary_username": "NT AUTHORITY\\SYSTEM", "actor_process_logon_id": "1003", - "action_file_extension": "bin", - "action_file_type": 18, - "manifest_file_version": 5, + "action_file_attributes": "128", + "action_file_type": "18", + "manifest_file_version": "5", "actor_process_instance_id": "AdgdeB26mNQAAAAEAAAAAA==", "actor_process_causality_id": "AdgdeB26mNQAAAAEAAAAAA==", "actor_process_auth_id": "999", "actor_process_signature_vendor": "Microsoft Corporation", + "actor_process_signature_status": "SIGNED", "actor_process_signature_product": "Microsoft Windows", - "actor_process_execution_time": 1644385473948, + "actor_process_execution_time": "1644385473948", + "actor_process_is_native": "TRUE", "agent_version": "7.6.1.46600", "agent_hostname": "EC2AMAZ-IQFSLIL", "agent_content_version": "380-82571", - "agent_session_start_time": 1644385496543, + "agent_session_start_time": "1644385496543", "agent_id": "37a92aad549d41d184ec9fbdafbff55c", + "agent_os_type": "AGENT_OS_WINDOWS", "agent_os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", - "event_id": "AAABfvB4P2Qqn9UKABvBDw==", - "event_timestamp": 1644711919520, - "event_version": 25, - "actor_process_is_native": "TRUE", "agent_is_vdi": "FALSE", - "agent_os_type": "AGENT_OS_WINDOWS", + "event_id": "AAABfvQtkk8qn9UKACE2JA==", + "event_timestamp": "1644774134407", + "event_version": "25", "event_type": "FILE", - "event_sub_type": "FILE_WRITE", - "actor_process_signature_status": "SIGNED" + "event_sub_type": "FILE_OPEN" } }, { "xdr_data": { - "action_file_name": "MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "agent_ip_addresses": [ + "172.31.31.67" + ], + "action_file_name": "DRIVERS", "actor_process_image_name": "System", "causality_actor_process_image_name": "System", "os_actor_process_image_name": "System", - "action_file_path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Support\\MpWppTracing-20220210-055530-00000003-ffffffff.bin", + "action_file_size": "4255744", + "action_file_path": "C:\\Windows\\System32\\config\\DRIVERS", "actor_process_image_path": "System", "causality_actor_process_image_path": "System", "os_actor_process_image_path": "System", - "actor_process_os_pid": 4, - "causality_actor_process_os_pid": 4, - "os_actor_process_os_pid": 4, + "actor_process_os_pid": "4", + "causality_actor_process_os_pid": "4", + "os_actor_process_os_pid": "4", "actor_primary_user_sid": "S-1-5-18", "actor_primary_username": "NT AUTHORITY\\SYSTEM", "actor_process_logon_id": "1003", - "action_file_extension": "bin", - "action_file_type": 18, - "manifest_file_version": 5, + "action_file_type": "0", + "manifest_file_version": "5", "actor_process_instance_id": "AdgdeB26mNQAAAAEAAAAAA==", "actor_process_causality_id": "AdgdeB26mNQAAAAEAAAAAA==", "actor_process_auth_id": "999", "actor_process_signature_vendor": "Microsoft Corporation", + "actor_process_signature_status": "SIGNED", "actor_process_signature_product": "Microsoft Windows", - "actor_process_execution_time": 1644385473948, + "actor_process_execution_time": "1644385473948", + "actor_process_is_native": "TRUE", "agent_version": "7.6.1.46600", "agent_hostname": "EC2AMAZ-IQFSLIL", "agent_content_version": "380-82571", - "agent_session_start_time": 1644385496543, + "agent_session_start_time": "1644385496543", "agent_id": "37a92aad549d41d184ec9fbdafbff55c", + "agent_os_type": "AGENT_OS_WINDOWS", "agent_os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", - "event_id": "AAABfvB7dS4qn9UKABvHTA==", - "event_timestamp": 1644712129898, - "event_version": 25, - "actor_process_is_native": "TRUE", "agent_is_vdi": "FALSE", - "agent_os_type": "AGENT_OS_WINDOWS", + "event_id": "AAABfvWCZpMqn9UKACRWUw==", + "event_timestamp": "1644796471001", + "event_version": "25", "event_type": "FILE", - "event_sub_type": "FILE_WRITE", - "actor_process_signature_status": "SIGNED" + "event_sub_type": "FILE_WRITE" } } ] @@ -448,11 +477,11 @@ results #### STIX Translate results ```shell -translate -paloalto -results -"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"events\"}" -" [ { \"xdr_data\": { \"action_file_name\": \"MpWppTracing-20220210-055530-00000003-ffffffff.bin\", \"actor_process_image_name\": \"System\", \"causality_actor_process_image_name\": \"System\", \"os_actor_process_image_name\": \"System\", \"action_file_path\": \"C:\\ProgramData\\Microsoft\\Windows Defender\\Support\\MpWppTracing-20220210-055530-00000003-ffffffff.bin\", \"actor_process_image_path\": \"System\", \"causality_actor_process_image_path\": \"System\", \"os_actor_process_image_path\": \"System\", \"actor_process_os_pid\": 4, \"causality_actor_process_os_pid\": 4, \"os_actor_process_os_pid\": 4, \"actor_primary_user_sid\": \"S-1-5-18\", \"actor_primary_username\": \"NT AUTHORITY\\SYSTEM\", \"actor_process_logon_id\": \"1003\", \"action_file_extension\": \"bin\", \"action_file_type\": 18, \"manifest_file_version\": 5, \"actor_process_instance_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_causality_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_auth_id\": \"999\", \"actor_process_signature_vendor\": \"Microsoft Corporation\", \"actor_process_signature_product\": \"Microsoft Windows\", \"actor_process_execution_time\": 1644385473948, \"agent_version\": \"7.6.1.46600\", \"agent_hostname\": \"EC2AMAZ-IQFSLIL\", \"agent_content_version\": \"380-82571\", \"agent_session_start_time\": 1644385496543, \"agent_id\": \"37a92aad549d41d184ec9fbdafbff55c\", \"agent_os_sub_type\": \"Windows Server 2016 [10.0 (Build 17763)]\", \"event_id\": \"AAABfvB4P2Qqn9UKABvBDw==\", \"event_timestamp\": 1644711919520, \"event_version\": 25, \"actor_process_is_native\": \"TRUE\", \"agent_is_vdi\": \"FALSE\", \"agent_os_type\": \"AGENT_OS_WINDOWS\", \"event_type\": \"FILE\", \"event_sub_type\": \"FILE_WRITE\", \"actor_process_signature_status\": \"SIGNED\" } }, { \"xdr_data\": { \"action_file_name\": \"MpWppTracing-20220210-055530-00000003-ffffffff.bin\", \"actor_process_image_name\": \"System\", \"causality_actor_process_image_name\": \"System\", \"os_actor_process_image_name\": \"System\", \"action_file_path\": \"C:\\ProgramData\\Microsoft\\Windows Defender\\Support\\MpWppTracing-20220210-055530-00000003-ffffffff.bin\", \"actor_process_image_path\": \"System\", \"causality_actor_process_image_path\": \"System\", \"os_actor_process_image_path\": \"System\", \"actor_process_os_pid\": 4, \"causality_actor_process_os_pid\": 4, \"os_actor_process_os_pid\": 4, \"actor_primary_user_sid\": \"S-1-5-18\", \"actor_primary_username\": \"NT AUTHORITY\\SYSTEM\", \"actor_process_logon_id\": \"1003\", \"action_file_extension\": \"bin\", \"action_file_type\": 18, \"manifest_file_version\": 5, \"actor_process_instance_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_causality_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_auth_id\": \"999\", \"actor_process_signature_vendor\": \"Microsoft Corporation\", \"actor_process_signature_product\": \"Microsoft Windows\", \"actor_process_execution_time\": 1644385473948, \"agent_version\": \"7.6.1.46600\", \"agent_hostname\": \"EC2AMAZ-IQFSLIL\", \"agent_content_version\": \"380-82571\", \"agent_session_start_time\": 1644385496543, \"agent_id\": \"37a92aad549d41d184ec9fbdafbff55c\", \"agent_os_sub_type\": \"Windows Server 2016 [10.0 (Build 17763)]\", \"event_id\": \"AAABfvB7dS4qn9UKABvHTA==\", \"event_timestamp\": 1644712129898, \"event_version\": 25, \"actor_process_is_native\": \"TRUE\", \"agent_is_vdi\": \"FALSE\", \"agent_os_type\": \"AGENT_OS_WINDOWS\", \"event_type\": \"FILE\", \"event_sub_type\": \"FILE_WRITE\", \"actor_process_signature_status\": \"SIGNED\" } } ] " +translate +paloalto +results +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"system\",\"created\":\"2022-03-16T13:22:50.336Z\",\"modified\":\"2022-03-16T13:22:50.336Z\"}" +" [ { \"xdr_data\": { \"agent_ip_addresses\": [ \"172.31.31.67\" ], \"action_file_name\": \"DRIVERS\", \"actor_process_image_name\": \"System\", \"causality_actor_process_image_name\": \"System\", \"os_actor_process_image_name\": \"System\", \"action_file_size\": \"4255744\", \"action_file_path\": \"C:\\Windows\\System32\\config\\DRIVERS\", \"actor_process_image_path\": \"System\", \"causality_actor_process_image_path\": \"System\", \"os_actor_process_image_path\": \"System\", \"actor_process_os_pid\": \"4\", \"causality_actor_process_os_pid\": \"4\", \"os_actor_process_os_pid\": \"4\", \"actor_primary_user_sid\": \"S-1-5-18\", \"actor_primary_username\": \"NT AUTHORITY\\SYSTEM\", \"actor_process_logon_id\": \"1003\", \"action_file_attributes\": \"128\", \"action_file_type\": \"18\", \"manifest_file_version\": \"5\", \"actor_process_instance_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_causality_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_auth_id\": \"999\", \"actor_process_signature_vendor\": \"Microsoft Corporation\", \"actor_process_signature_status\": \"SIGNED\", \"actor_process_signature_product\": \"Microsoft Windows\", \"actor_process_execution_time\": \"1644385473948\", \"actor_process_is_native\": \"TRUE\", \"agent_version\": \"7.6.1.46600\", \"agent_hostname\": \"EC2AMAZ-IQFSLIL\", \"agent_content_version\": \"380-82571\", \"agent_session_start_time\": \"1644385496543\", \"agent_id\": \"37a92aad549d41d184ec9fbdafbff55c\", \"agent_os_type\": \"AGENT_OS_WINDOWS\", \"agent_os_sub_type\": \"Windows Server 2016 [10.0 (Build 17763)]\", \"agent_is_vdi\": \"FALSE\", \"event_id\": \"AAABfvQtkk8qn9UKACE2JA==\", \"event_timestamp\": \"1644774134407\", \"event_version\": \"25\", \"event_type\": \"FILE\", \"event_sub_type\": \"FILE_OPEN\" } }, { \"xdr_data\": { \"agent_ip_addresses\": [ \"172.31.31.67\" ], \"action_file_name\": \"DRIVERS\", \"actor_process_image_name\": \"System\", \"causality_actor_process_image_name\": \"System\", \"os_actor_process_image_name\": \"System\", \"action_file_size\": \"4255744\", \"action_file_path\": \"C:\\Windows\\System32\\config\\DRIVERS\", \"actor_process_image_path\": \"System\", \"causality_actor_process_image_path\": \"System\", \"os_actor_process_image_path\": \"System\", \"actor_process_os_pid\": \"4\", \"causality_actor_process_os_pid\": \"4\", \"os_actor_process_os_pid\": \"4\", \"actor_primary_user_sid\": \"S-1-5-18\", \"actor_primary_username\": \"NT AUTHORITY\\SYSTEM\", \"actor_process_logon_id\": \"1003\", \"action_file_type\": \"0\", \"manifest_file_version\": \"5\", \"actor_process_instance_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_causality_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_auth_id\": \"999\", \"actor_process_signature_vendor\": \"Microsoft Corporation\", \"actor_process_signature_status\": \"SIGNED\", \"actor_process_signature_product\": \"Microsoft Windows\", \"actor_process_execution_time\": \"1644385473948\", \"actor_process_is_native\": \"TRUE\", \"agent_version\": \"7.6.1.46600\", \"agent_hostname\": \"EC2AMAZ-IQFSLIL\", \"agent_content_version\": \"380-82571\", \"agent_session_start_time\": \"1644385496543\", \"agent_id\": \"37a92aad549d41d184ec9fbdafbff55c\", \"agent_os_type\": \"AGENT_OS_WINDOWS\", \"agent_os_sub_type\": \"Windows Server 2016 [10.0 (Build 17763)]\", \"agent_is_vdi\": \"FALSE\", \"event_id\": \"AAABfvWCZpMqn9UKACRWUw==\", \"event_timestamp\": \"1644796471001\", \"event_version\": \"25\", \"event_type\": \"FILE\", \"event_sub_type\": \"FILE_WRITE\" } } ]" "{\"stix_validator\": true}" ``` @@ -460,41 +489,84 @@ results ```json { "type": "bundle", - "id": "bundle--a1d2533a-11e1-4ee8-9c96-4c298f87a823", + "id": "bundle--de01a54d-dd8d-4005-b640-29e68c24d976", "objects": [ { "type": "identity", "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", "name": "paloalto", - "identity_class": "events" + "identity_class": "system", + "created": "2022-03-16T13:22:50.336Z", + "modified": "2022-03-16T13:22:50.336Z" }, { - "id": "observed-data--734f2fdb-7777-4a8f-8fb1-c08dceec94fa", + "id": "observed-data--ce1c1768-d2ad-41bd-a3d3-d11bf2ae3d94", "type": "observed-data", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2022-03-09T08:56:29.407Z", - "modified": "2022-03-09T08:56:29.407Z", + "created": "2022-04-05T13:01:57.723Z", + "modified": "2022-04-05T13:01:57.723Z", "objects": { "0": { + "type": "ipv4-addr", + "value": "172.31.31.67" + }, + "2": { + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "agent_version": "7.6.1.46600", + "content_version": "380-82571", + "start_time": "2022-02-09T05:44:56.543Z", + "asset_id": "37a92aad549d41d184ec9fbdafbff55c", + "os_type": "AGENT_OS_WINDOWS", + "os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "is_vdi": false + } + }, + "hostname": "EC2AMAZ-IQFSLIL" + }, + "3": { "type": "file", - "name": "MpWppTracing-20220210-055530-00000003-ffffffff.bin", - "parent_directory_ref": "9", + "name": "DRIVERS", + "size": 4255744, + "parent_directory_ref": "13", "extensions": { "x-paloalto-file": { - "extension": "bin", + "attributes": 128, "type": 18, "manifest_version": 5 } } }, - "1": { + "4": { + "type": "x-oca-event", + "file_ref": "3", + "process_ref": "6", + "parent_process_ref": "8", + "agent": "EC2AMAZ-IQFSLIL", + "code": "AAABfvQtkk8qn9UKACE2JA==", + "created": "2022-02-13T17:42:14.407Z", + "entensions": { + "x-paloalto-event": { + "version": 25 + } + }, + "category": [ + "file" + ], + "action": "FILE_OPEN" + }, + "5": { "type": "file", "name": "System" }, - "2": { + "6": { "type": "process", "name": "System", - "binary_ref": "1", + "binary_ref": "5", "pid": 4, "extensions": { "x-paloalto-process": { @@ -502,102 +574,123 @@ results "causality_id": "AdgdeB26mNQAAAAEAAAAAA==", "auth_id": "999", "signature_vendor": "Microsoft Corporation", + "signature_status": "SIGNED", "signature_product": "Microsoft Windows", "execution_time": "2022-02-09T05:44:33.948Z", - "is_native": true, - "signature_status": "SIGNED" + "is_native": true } } }, - "3": { + "7": { "type": "file", "name": "System" }, - "4": { + "8": { "type": "process", "name": "System" }, - "5": { + "9": { "type": "process", - "parent_ref": "4", - "binary_ref": "3", + "parent_ref": "8", + "binary_ref": "7", "pid": 4 }, - "6": { + "10": { "type": "file", "name": "System" }, - "7": { + "11": { "type": "process", "name": "System" }, - "8": { + "12": { "type": "process", - "parent_ref": "7", - "binary_ref": "6", + "parent_ref": "11", + "binary_ref": "10", "pid": 4 }, - "9": { + "13": { "type": "directory", - "path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Support" + "path": "C:\\Windows\\System32\\config" }, - "10": { + "14": { "type": "user-account", "user_id": "S-1-5-18", "display_name": "NT AUTHORITY\\SYSTEM", "account_login": "1003" - }, - "11": { - "type": "x-oca-asset", - "agent_version": "7.6.1.46600", - "hostname": "EC2AMAZ-IQFSLIL", - "content_version": "380-82571", - "start_time": "2022-02-09T05:44:56.543Z", - "asset_id": "37a92aad549d41d184ec9fbdafbff55c", - "os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", - "is_vdi": false, - "os_type": "AGENT_OS_WINDOWS" - }, - "12": { - "type": "x-oca-event", - "event_id": "AAABfvB4P2Qqn9UKABvBDw==", - "time": "2022-02-13T00:25:19.520Z", - "version": 25, - "event_type": "FILE", - "sub_type": "FILE_WRITE" } }, - "first_observed": "2022-03-09T08:56:29.407Z", - "last_observed": "2022-03-09T08:56:29.407Z", + "first_observed": "2022-04-05T16:17:58.820Z", + "last_observed": "2022-04-05T16:17:58.820Z", "number_observed": 1 }, { - "id": "observed-data--65f9e698-ecd2-41f9-b370-cf9dae876eeb", + "id": "observed-data--f3a29cc9-e4e1-431f-943c-edf91e42edc5", "type": "observed-data", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2022-03-09T08:56:29.746Z", - "modified": "2022-03-09T08:56:29.746Z", + "created": "2022-04-05T13:01:57.731Z", + "modified": "2022-04-05T13:01:57.731Z", "objects": { "0": { + "type": "ipv4-addr", + "value": "172.31.31.67" + }, + "2": { + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "agent_version": "7.6.1.46600", + "content_version": "380-82571", + "start_time": "2022-02-09T05:44:56.543Z", + "asset_id": "37a92aad549d41d184ec9fbdafbff55c", + "os_type": "AGENT_OS_WINDOWS", + "os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "is_vdi": false + } + }, + "hostname": "EC2AMAZ-IQFSLIL" + }, + "3": { "type": "file", - "name": "MpWppTracing-20220210-055530-00000003-ffffffff.bin", - "parent_directory_ref": "9", + "name": "DRIVERS", + "size": 4255744, + "parent_directory_ref": "13", "extensions": { "x-paloalto-file": { - "extension": "bin", - "type": 18, + "type": 0, "manifest_version": 5 } } }, - "1": { + "4": { + "type": "x-oca-event", + "file_ref": "3", + "process_ref": "6", + "parent_process_ref": "8", + "agent": "EC2AMAZ-IQFSLIL", + "code": "AAABfvWCZpMqn9UKACRWUw==", + "created": "2022-02-13T23:54:31.001Z", + "entensions": { + "x-paloalto-event": { + "version": 25 + } + }, + "category": [ + "file" + ], + "action": "FILE_WRITE" + }, + "5": { "type": "file", "name": "System" }, - "2": { + "6": { "type": "process", "name": "System", - "binary_ref": "1", + "binary_ref": "5", "pid": 4, "extensions": { "x-paloalto-process": { @@ -605,73 +698,54 @@ results "causality_id": "AdgdeB26mNQAAAAEAAAAAA==", "auth_id": "999", "signature_vendor": "Microsoft Corporation", + "signature_status": "SIGNED", "signature_product": "Microsoft Windows", "execution_time": "2022-02-09T05:44:33.948Z", - "is_native": true, - "signature_status": "SIGNED" + "is_native": true } } }, - "3": { + "7": { "type": "file", "name": "System" }, - "4": { + "8": { "type": "process", "name": "System" }, - "5": { + "9": { "type": "process", - "parent_ref": "4", - "binary_ref": "3", + "parent_ref": "8", + "binary_ref": "7", "pid": 4 }, - "6": { + "10": { "type": "file", "name": "System" }, - "7": { + "11": { "type": "process", "name": "System" }, - "8": { + "12": { "type": "process", - "parent_ref": "7", - "binary_ref": "6", + "parent_ref": "11", + "binary_ref": "10", "pid": 4 }, - "9": { + "13": { "type": "directory", - "path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Support" + "path": "C:\\Windows\\System32\\config" }, - "10": { + "14": { "type": "user-account", "user_id": "S-1-5-18", "display_name": "NT AUTHORITY\\SYSTEM", "account_login": "1003" - }, - "11": { - "type": "x-oca-asset", - "agent_version": "7.6.1.46600", - "hostname": "EC2AMAZ-IQFSLIL", - "content_version": "380-82571", - "start_time": "2022-02-09T05:44:56.543Z", - "asset_id": "37a92aad549d41d184ec9fbdafbff55c", - "os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", - "is_vdi": false, - "os_type": "AGENT_OS_WINDOWS" - }, - "12": { - "type": "x-oca-event", - "event_id": "AAABfvB7dS4qn9UKABvHTA==", - "time": "2022-02-13T00:28:49.898Z", - "version": 25, - "event_type": "FILE", - "sub_type": "FILE_WRITE" } }, - "first_observed": "2022-03-09T08:56:29.746Z", - "last_observed": "2022-03-09T08:56:29.746Z", + "first_observed": "2022-04-05T13:01:57.731Z", + "last_observed": "2022-04-05T13:01:57.731Z", "number_observed": 1 } ], @@ -683,7 +757,7 @@ results execute paloalto paloalto -"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class \":\"events\"}" +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"system\",\"created\":\"2022-03-16T13:22:50.336Z\",\"modified\":\"2022-03-16T13:22:50.336Z\"}" "{\"host\":\"xx.xx.xx\"}" "{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" "[file:name = 'chrome.exe' ] START t'2022-01-16T11:00:00.000Z' STOP t'2022-01-20T11:00:00.003Z'" @@ -692,272 +766,159 @@ paloalto ```json { "type": "bundle", - "id": "bundle--899d573b-3fcb-4561-bedd-6038fd4da192", + "id": "bundle--1683947d-9c7e-4d35-b70b-a0ccc90d5cba", "objects": [ { "type": "identity", "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", "name": "paloalto", - "identity_class ": "events" + "identity_class": "system", + "created": "2022-03-16T13:22:50.336Z", + "modified": "2022-03-16T13:22:50.336Z" }, { - "id": "observed-data--59220f58-f8ee-4d2b-a2df-367958e132b7", + "id": "observed-data--34da8902-db25-4388-8312-526347de345d", "type": "observed-data", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2022-03-09T09:04:32.272Z", - "modified": "2022-03-09T09:04:32.272Z", + "created": "2022-04-05T13:05:20.377Z", + "modified": "2022-04-05T13:05:20.377Z", "objects": { "0": { - "type": "file", - "name": "LICENSE", - "parent_directory_ref": "9", - "extensions": { - "x-paloalto-file": { - "type": 18, - "manifest_version": 5 - } - } - }, - "1": { - "type": "file", - "name": "chrome.exe", - "accessed": "2021-12-15T22:05:27.664Z", - "modified": "2021-12-12T08:19:30.068Z", - "parent_directory_ref": "10" + "type": "ipv4-addr", + "value": "172.31.31.236" }, "2": { - "type": "process", - "name": "chrome.exe", - "binary_ref": "1", - "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", - "created": "2021-11-10T06:59:32.948Z", - "pid": 4572, + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], "extensions": { - "x-paloalto-process": { - "instance_id": "AdgAs0qzIE0AABHcAAAAAA==", - "causality_id": "AdgAs0qzIE0AABHcAAAAAA==", - "auth_id": "999", - "signature_vendor": "Google LLC", - "signature_product": "Google LLC", - "extension": "exe", - "execution_time": "2022-01-03T15:05:05.983Z", - "is_native": true, - "signature_status": "SIGNED" + "x-paloalto-agent": { + "agent_version": "7.6.1.46600", + "content_version": "350-80787", + "start_time": "2022-01-19T10:02:28.366Z", + "asset_id": "f344796340f84d0ca7e0fdaedbcbd594", + "os_type": "AGENT_OS_WINDOWS", + "os_sub_type": "Windows Server 2016 [10.0 (Build 14393)]", + "is_vdi": false } - } + }, + "hostname": "EC2AMAZ-65BN1IK" }, "3": { "type": "file", - "name": "chrome.exe", - "parent_directory_ref": "11" - }, - "4": { - "type": "process", - "name": "chrome.exe" - }, - "5": { - "type": "process", - "parent_ref": "4", - "binary_ref": "3", - "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", - "created": "2021-11-10T06:59:32.948Z", - "pid": 4572 - }, - "6": { - "type": "file", - "name": "chrome.exe", - "accessed": "2021-12-15T22:05:27.664Z", - "modified": "2021-12-12T08:19:30.068Z", - "parent_directory_ref": "12" - }, - "7": { - "type": "process", - "name": "chrome.exe" - }, - "8": { - "type": "process", - "parent_ref": "7", - "binary_ref": "6", - "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", - "created": "2021-11-10T06:59:32.948Z", - "pid": 4572 - }, - "9": { - "type": "directory", - "path": "C:\\Users\\Administrator\\AppData\\Local\\Temp\\2\\4572_137263248" - }, - "10": { - "type": "directory", - "path": "C:\\Program Files\\Google\\Chrome\\Application" - }, - "11": { - "type": "directory", - "path": "C:\\Program Files\\Google\\Chrome\\Application" - }, - "12": { - "type": "directory", - "path": "C:\\Program Files\\Google\\Chrome\\Application" - }, - "13": { - "type": "user-account", - "user_id": "S-1-5-21-3039464837-300237904-2407637926-500", - "display_name": "EC2AMAZ-65BN1IK\\Administrator", - "account_login": "1003" - }, - "14": { - "type": "x-oca-asset", - "agent_version": "7.6.1.46600", - "hostname": "EC2AMAZ-65BN1IK", - "content_version": "350-80787", - "start_time": "2022-01-19T10:02:28.366Z", - "asset_id": "f344796340f84d0ca7e0fdaedbcbd594", - "os_sub_type": "Windows Server 2016 [10.0 (Build 14393)]", - "is_vdi": false, - "os_type": "AGENT_OS_WINDOWS" - }, - "15": { - "type": "x-oca-event", - "event_id": "AAABfnQpgUfunpI7AANI6Q==", - "time": "2022-01-19T21:06:24.409Z", - "version": 25, - "event_type": "FILE", - "sub_type": "FILE_WRITE" - } - }, - "x_actor_process_last_observed": "2021-12-15T22:05:27.664Z", - "x_os_actor_process_last_observed": "2021-12-15T22:05:27.664Z", - "x_process_actor_first_observed": "2021-11-10T06:59:32.948Z", - "x_process_causality_actor_first_observed": "2021-11-10T06:59:32.948Z", - "x_process_os_actor_first_observed": "2021-11-10T06:59:32.948Z", - "first_observed": "2022-03-09T09:04:32.272Z", - "last_observed": "2022-03-09T09:04:32.272Z", - "number_observed": 1 - }, - { - "id": "observed-data--1bd94f55-3b0d-4057-b281-f8c17cdc6cfa", - "type": "observed-data", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2022-03-09T09:04:32.272Z", - "modified": "2022-03-09T09:04:32.272Z", - "objects": { - "0": { - "type": "file", - "name": "22c34728-a90d-4fe4-8774-21563d2be0f3.tmp", - "size": 117576, - "parent_directory_ref": "9", + "name": "Network Persistent State", + "size": 4595, + "parent_directory_ref": "13", "extensions": { "x-paloalto-file": { - "extension": "tmp", - "writer": "AdgAs0qzIE0AABHcAAAAAA==", - "type": 0, + "attributes": 0, + "type": 18, "manifest_version": 5 } } }, - "1": { + "4": { + "type": "x-oca-event", + "file_ref": "3", + "process_ref": "6", + "parent_process_ref": "8", + "agent": "EC2AMAZ-65BN1IK", + "code": "AAABfnYIxcPunpI7AATXuQ==", + "created": "2022-01-20T05:49:53.598Z", + "entensions": { + "x-paloalto-event": { + "version": 25 + } + }, + "category": ["file"], + "action": "FILE_OPEN" + }, + "5": { "type": "file", "name": "chrome.exe", "accessed": "2021-12-15T22:05:27.664Z", "modified": "2021-12-12T08:19:30.068Z", - "parent_directory_ref": "10" + "parent_directory_ref": "14" }, - "2": { + "6": { "type": "process", "name": "chrome.exe", - "binary_ref": "1", - "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "binary_ref": "5", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --type=utility --utility-sub-type=network.mojom.NetworkService --field-trial-handle=1456,2805956566820005361,12799224617572967118,131072 --lang=en-US --service-sandbox-type=none --mojo-platform-channel-handle=1772 /prefetch:8", "created": "2021-11-10T06:59:32.948Z", - "pid": 4572, + "pid": 4724, "extensions": { "x-paloalto-process": { - "instance_id": "AdgAs0qzIE0AABHcAAAAAA==", + "instance_id": "AdgAs0xnjjUAABJ0AAAAAA==", "causality_id": "AdgAs0qzIE0AABHcAAAAAA==", "auth_id": "999", "signature_vendor": "Google LLC", + "signature_status": "SIGNED", "signature_product": "Google LLC", "extension": "exe", - "execution_time": "2022-01-03T15:05:05.983Z", - "is_native": true, - "signature_status": "SIGNED" + "execution_time": "2022-01-03T15:05:08.844Z", + "is_native": true } } }, - "3": { + "7": { "type": "file", "name": "chrome.exe", - "parent_directory_ref": "11" + "parent_directory_ref": "15" }, - "4": { + "8": { "type": "process", "name": "chrome.exe" }, - "5": { + "9": { "type": "process", - "parent_ref": "4", - "binary_ref": "3", + "parent_ref": "8", + "binary_ref": "7", "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", "created": "2021-11-10T06:59:32.948Z", "pid": 4572 }, - "6": { + "10": { "type": "file", "name": "chrome.exe", "accessed": "2021-12-15T22:05:27.664Z", "modified": "2021-12-12T08:19:30.068Z", - "parent_directory_ref": "12" + "parent_directory_ref": "16" }, - "7": { + "11": { "type": "process", "name": "chrome.exe" }, - "8": { + "12": { "type": "process", - "parent_ref": "7", - "binary_ref": "6", - "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "parent_ref": "11", + "binary_ref": "10", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --type=utility --utility-sub-type=network.mojom.NetworkService --field-trial-handle=1456,2805956566820005361,12799224617572967118,131072 --lang=en-US --service-sandbox-type=none --mojo-platform-channel-handle=1772 /prefetch:8", "created": "2021-11-10T06:59:32.948Z", - "pid": 4572 + "pid": 4724 }, - "9": { + "13": { "type": "directory", - "path": "C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data" + "path": "C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Network" }, - "10": { + "14": { "type": "directory", "path": "C:\\Program Files\\Google\\Chrome\\Application" }, - "11": { + "15": { "type": "directory", "path": "C:\\Program Files\\Google\\Chrome\\Application" }, - "12": { + "16": { "type": "directory", "path": "C:\\Program Files\\Google\\Chrome\\Application" }, - "13": { + "17": { "type": "user-account", "user_id": "S-1-5-21-3039464837-300237904-2407637926-500", "display_name": "EC2AMAZ-65BN1IK\\Administrator", "account_login": "1003" - }, - "14": { - "type": "x-oca-asset", - "agent_version": "7.6.1.46600", - "hostname": "EC2AMAZ-65BN1IK", - "content_version": "350-80787", - "start_time": "2022-01-19T10:02:28.366Z", - "asset_id": "f344796340f84d0ca7e0fdaedbcbd594", - "os_sub_type": "Windows Server 2016 [10.0 (Build 14393)]", - "is_vdi": false, - "os_type": "AGENT_OS_WINDOWS" - }, - "15": { - "type": "x-oca-event", - "event_id": "AAABfnRfbAPunpI7AANyKA==", - "time": "2022-01-19T22:05:17.974Z", - "version": 25, - "event_type": "FILE", - "sub_type": "FILE_WRITE" } }, "x_actor_process_last_observed": "2021-12-15T22:05:27.664Z", @@ -965,8 +926,8 @@ paloalto "x_process_actor_first_observed": "2021-11-10T06:59:32.948Z", "x_process_causality_actor_first_observed": "2021-11-10T06:59:32.948Z", "x_process_os_actor_first_observed": "2021-11-10T06:59:32.948Z", - "first_observed": "2022-03-09T09:04:32.272Z", - "last_observed": "2022-03-09T09:04:32.272Z", + "first_observed": "2022-04-05T13:05:20.377Z", + "last_observed": "2022-04-05T13:05:20.377Z", "number_observed": 1 } ], diff --git a/stix_shifter_modules/paloalto/stix_translation/json/config_map.json b/stix_shifter_modules/paloalto/stix_translation/json/config_map.json index 0e87bc6f8..f8add745e 100644 --- a/stix_shifter_modules/paloalto/stix_translation/json/config_map.json +++ b/stix_shifter_modules/paloalto/stix_translation/json/config_map.json @@ -28,7 +28,8 @@ "os_actor_process_file_create_time", "action_process_instance_execution_time", "actor_process_execution_time", - "action_network_creation_time" + "action_network_creation_time", + "event_timestamp" ], "mac_supported_fields": [ "mac", @@ -38,17 +39,54 @@ ], "enum_supported_fields": [ "action_network_protocol", - "agent_os_type", - "event_type" + "event_type", + "event_sub_type" ], "enum_supported_values": { "action_network_protocol": ["TCP", "UDP"], - "agent_os_type": ["AGENT_OS_LINUX", "AGENT_OS_MAC", "AGENT_OS_WINDOWS"], "event_type": [ "AGENT_STATUS","CLOUD_AUDIT_LOGS","DEVICE","EVENT_LOG","FILE","GAP","HOST_FIREWALL", "HOST_METADATA","HOST_STATUS","INJECTION","LOAD_IMAGE","LOGIN_EVENT","MOUNT","NAMESPACE", "NETWORK","PROCESS","PROCESS_HANDLE","REGISTRY","RPC_CALL","STORY","SYSTEM_CALL", - "THREAD","USER_STATUS_CHANGE","VPN_EVENT"] + "THREAD","USER_STATUS_CHANGE","VPN_EVENT"], + "event_sub_type": [ + "AGENT_STATUS_AGENT_BOOT" ,"AGENT_STATUS_AGENT_INSTALLED", + "AGENT_STATUS_AGENT_LOW_LEVEL_CONFIG_UPDATE" ,"AGENT_STATUS_AGENT_POLICY_FILTER_UPDATE" , + "AGENT_STATUS_AGENT_REPLAY_ENDED" ,"AGENT_STATUS_AGENT_SHUTDOWN" , + "AGENT_STATUS_AGENT_UNINSTALLED" ,"DEVICE_PLUG" ,"DEVICE_UNPLUG" , + "EVENT_LOG_AGENT_EVENT_LOG" ,"EVENT_LOG_DOMAIN_CONTROLLER" ,"FILE_CHANGE_MODE" , + "FILE_CHANGE_OWNER" ,"FILE_CREATE_NEW" ,"FILE_DELETE_EXT_ATTRIBUTE" , + "FILE_DIR_CHANGE_MODE" ,"FILE_DIR_CHANGE_OWNER" ,"FILE_DIR_CREATE" ,"FILE_DIR_LINK" , + "FILE_DIR_OPEN" ,"FILE_DIR_QUERY" ,"FILE_DIR_REMOVE" ,"FILE_DIR_RENAME" , + "FILE_DIR_SET_ATTR" ,"FILE_DIR_WRITE" ,"FILE_LINK" ,"FILE_OPEN" ,"FILE_REMOVE" , + "FILE_RENAME" ,"FILE_REPARSE" ,"FILE_SET_ATTRIBUTE" ,"FILE_SET_SECURITY_DESCRIPTOR" , + "FILE_WRITE" ,"GAP_GAP_ENDED" ,"GAP_GAP_STARTED", "HOST_FIREWALL_HOST_FW_ALLOW" , + "HOST_FIREWALL_HOST_FW_BLOCK" ,"HOST_METADATA_CHANGE" ,"HOST_STATUS_BOOT" , + "HOST_STATUS_RESUME" ,"HOST_STATUS_SHUTDOWN" ,"HOST_STATUS_SUSPEND" , + "INJECTION_CREATE_REMOTE_THREAD" ,"INJECTION_PROCESS_HOLLOW" ,"INJECTION_QUEUE_APC" , + "INJECTION_SET_THREAD_CONTEXT" ,"LOAD_IMAGE_MODULE" ,"LOAD_IMAGE_MPROTECT" , + "LOAD_IMAGE_PRELOAD" ,"LOAD_IMAGE_SO_LOAD" ,"MOUNT_DRIVE_MOUNT" , + "MOUNT_DRIVE_UNMOUNT" ,"NAMESPACE_SETNS" ,"NAMESPACE_UNSHARE" , + "NETWORK_DATAGRAM_STATISTICS" ,"NETWORK_DNS_QUERY" ,"NETWORK_HTTP_HEADER" , + "NETWORK_OUTBOUND_ICMP" ,"NETWORK_PROXY_CONNECT" ,"NETWORK_RAW_DATA" , + "NETWORK_RAW_PCAP_DATA" ,"NETWORK_STREAM_ACCEPT" ,"NETWORK_STREAM_CONNECT" , + "NETWORK_STREAM_CONNECT_FAILED" ,"NETWORK_STREAM_DISCONNECT" ,"NETWORK_STREAM_LISTEN" , + "NETWORK_STREAM_STATISTICS" ,"PROCESS_START" ,"PROCESS_STOP" ,"REGISTRY_CREATE_KEY" , + "REGISTRY_DELETE_KEY" ,"REGISTRY_DELETE_VALUE" ,"REGISTRY_LOAD" ,"REGISTRY_RENAME_KEY" , + "REGISTRY_RESTORE" ,"REGISTRY_SAVE" ,"REGISTRY_SET_VALUE" ,"REGISTRY_UNLOAD" , + "RPC_CALL_RPC_CALLED" ,"SYSTEM_CALL_CREATE_SYMBOLIC_LINK" ,"SYSTEM_CALL_GDI_BIT_BLT" , + "SYSTEM_CALL_GET_CLIPBOARD_DATA" ,"SYSTEM_CALL_NT_ALLOCATE_VIRTUAL_MEMORY_REMOTE" , + "SYSTEM_CALL_NT_CREATE_MUTANT" ,"SYSTEM_CALL_NT_DELAY_EXECUTION" , + "SYSTEM_CALL_NT_MAP_VIEW_OF_SECTION_REMOTE" ,"SYSTEM_CALL_NT_OPEN_FILE" , + "SYSTEM_CALL_NT_PROTECT_VIRTUAL_MEMORY_REMOTE" ,"SYSTEM_CALL_NT_QUEUE_APC_THREAD_EX_REMOTE" , + "SYSTEM_CALL_NT_READ_VIRTUAL_MEMORY_REMOTE" ,"SYSTEM_CALL_NT_SET_CONTEXT_THREAD_REMOTE" , + "SYSTEM_CALL_NT_TERMINATE_PROCESS_REMOTE","SYSTEM_CALL_NT_UNMAP_VIEW_OF_SECTION" , + "SYSTEM_CALL_NT_WRITE_VIRTUAL_MEMORY_REMOTE" ,"SYSTEM_CALL_OPEN_REMOTE_TOKEN" , + "SYSTEM_CALL_QUERY_PROCESS_LIST" ,"SYSTEM_CALL_QUERY_USER_TOKEN" , + "SYSTEM_CALL_REGISTER_RAW_INPUT_DEVICES" ,"SYSTEM_CALL_SET_INFO_VIRTUAL_MEMORY" , + "SYSTEM_CALL_SET_WINDOWS_HOOK_EX" ,"SYSTEM_CALL_SET_WIN_EVENT_HOOK" , + "SYSTEM_CALL_THREAD_IMPERSONATING" ,"USER_STATUS_CHANGE_LOGOFF" ,"USER_STATUS_CHANGE_LOGON" + ] }, "timestamp_supported_dataset": { "xdr_data": "_time" diff --git a/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json b/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json index 6e43b80e1..f0f2cdc5a 100644 --- a/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json +++ b/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json @@ -1,6 +1,6 @@ { "all_fields": [ - "action_local_ip","action_remote_ip","agent_ip_addresses_v6","dst_agent_ip_addresses_v6", + "action_local_ip","action_remote_ip","agent_ip_addresses","agent_ip_addresses_v6","dst_agent_ip_addresses_v6", "action_local_port","action_remote_port","action_network_protocol","action_pkts_sent", "action_pkts_received","action_file_name","action_process_image_name","actor_process_image_name", "causality_actor_process_image_name","os_actor_process_image_name","action_file_size","action_file_md5", diff --git a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json index 50c08b380..c423b00ce 100644 --- a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json @@ -9,6 +9,14 @@ "key": "network-traffic.src_ref", "object": "nt", "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ], + "group": true } ], "action_remote_ip": [ @@ -20,16 +28,67 @@ "key": "network-traffic.dst_ref", "object": "nt", "references": "dst_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "dst_ip" + ], + "group": true } ], - "agent_ip_addresses_v6": { - "key": "ipv6-addr.value", - "object": "src_ip" - }, - "dst_agent_ip_addresses_v6": { + "agent_ip_addresses": [ + { + "key": "ipv4-addr.value", + "object": "agent_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "agent_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "agent_ip" + ], + "group": true + } + ], + "agent_ip_addresses_v6": [ + { + "key": "ipv6-addr.value", + "object": "src_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ], + "group": true + } + ], + "dst_agent_ip_addresses_v6": [ + { "key": "ipv6-addr.value", - "object": "dst_ip" - }, + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": ["dst_ip"], + "group": true + } + ], "action_local_port": { "key": "network-traffic.src_port", "object": "nt", @@ -40,11 +99,18 @@ "object": "nt", "transformer": "ToInteger" }, - "action_network_protocol": { + "action_network_protocol": [ + { "key": "network-traffic.protocols", "object": "nt", "transformer": "ToLowercaseArray" - }, + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], "action_pkts_sent": { "key": "network-traffic.src_packets", "object": "nt", @@ -55,10 +121,17 @@ "object": "nt", "transformer": "ToInteger" }, - "action_file_name": { - "key": "file.name", - "object": "file" - }, + "action_file_name": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + } + ], "action_process_image_name": [ { "key": "file.name", @@ -68,6 +141,11 @@ "key": "process.image_ref", "object": "process_action", "references": "file_action_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" } ], "actor_process_image_name": [ @@ -79,6 +157,11 @@ "key": "process.image_ref", "object": "process_actor", "references": "file_actor_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" } ], "causality_actor_process_image_name": [ @@ -89,6 +172,11 @@ { "key": "process.parent_ref", "object": "process_causality_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_causality_actor" } ], "os_actor_process_image_name": [ @@ -99,6 +187,11 @@ { "key": "process.parent_ref", "object": "process_os_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_os_actor" } ], "action_file_size": { @@ -313,22 +406,50 @@ "references": "file_os_actor" } ], - "action_process_image_command_line": { - "key": "process.command_line", - "object": "process_action" - }, - "actor_process_command_line": { - "key": "process.command_line", - "object": "process_actor" - }, - "causality_actor_process_command_line": { - "key": "process.command_line", - "object": "process_causality_actor" - }, - "os_actor_process_command_line": { - "key": "process.command_line", - "object": "process_os_actor" - }, + "action_process_image_command_line": [ + { + "key": "process.command_line", + "object": "process_action" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_actor" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_causality_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_os_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_os_actor" + } + ], "action_process_file_create_time": [ { "key": "process.created_time", @@ -377,31 +498,66 @@ "transformer": "EpochToTimestamp" } ], - "action_module_process_os_pid": { - "key": "process.pid", - "object": "process_action_module", - "transformer": "ToInteger" - }, - "action_process_os_pid": { - "key": "process.pid", - "object": "process_action", - "transformer": "ToInteger" - }, - "actor_process_os_pid": { - "key": "process.pid", - "object": "process_actor", - "transformer": "ToInteger" - }, - "causality_actor_process_os_pid": { - "key": "process.pid", - "object": "process_causality_actor", - "transformer": "ToInteger" - }, - "os_actor_process_os_pid": { - "key": "process.pid", - "object": "process_os_actor", - "transformer": "ToInteger" - }, + "action_module_process_os_pid": [ + { + "key": "process.pid", + "object": "process_action_module", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action_module" + } + ], + "action_process_os_pid": [ + { + "key": "process.pid", + "object": "process_action", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_causality_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_os_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_os_actor" + } + ], "action_process_requested_parent_pid": [ { "key": "process.pid", @@ -412,6 +568,11 @@ "key": "process.parent_ref", "object": "process_action", "references": "action_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "action_parent_process" } ], "action_thread_parent_pid": [ @@ -424,6 +585,11 @@ "key": "process.parent_ref", "object": "process_action_thread", "references": "action_parent_thread" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "action_parent_thread" } ], "action_thread_child_pid": [ @@ -438,46 +604,117 @@ "references": "action_child_thread" } ], - "auth_domain": { - "key": "domain-name.value", - "object": "domain" - }, - "dst_host_metadata_domain": { - "key": "domain-name.value", - "object": "domain_dst" - }, - "host_metadata_domain": { - "key": "domain-name.value", - "object": "domain_host" - }, - "dst_action_url_category": { - "key": "url.value", - "object": "url" - }, - "action_registry_key_name": { - "key": "windows-registry-key.key", - "object": "windowsregistry" - }, - "action_registry_value_name": { - "key": "windows-registry-key.values", - "object": "windowsregistry" - }, - "mac": { - "key": "mac-addr.value", - "object": "mac" - }, - "associated_mac": { - "key": "mac-addr.value", - "object": "ass_mac" - }, - "dst_associated_mac": { - "key": "mac-addr.value", - "object": "dst_ass_mac" - }, - "dst_mac": { - "key": "mac-addr.value", - "object": "dst_mac" - }, + "auth_domain": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain" + } + ], + "dst_host_metadata_domain": [ + { + "key": "domain-name.value", + "object": "domain_dst" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain_dst" + } + ], + "host_metadata_domain": [ + { + "key": "domain-name.value", + "object": "domain_host" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain_host" + } + ], + "dst_action_url_category": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "action_registry_key_name": [ + { + "key": "windows-registry-key.key", + "object": "windowsregistry" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "windowsregistry" + } + ], + "action_registry_value_name": [ + { + "key": "windows-registry-key.values", + "object": "windowsregistry", + "transformer": "FormatToStixRegistryValue" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "windowsregistry" + } + ], + "mac": [ + { + "key": "mac-addr.value", + "object": "mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["mac"] + } + ], + "associated_mac": [ + { + "key": "mac-addr.value", + "object": "ass_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["ass_mac"] + } + ], + "dst_associated_mac": [ + { + "key": "mac-addr.value", + "object": "dst_ass_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["dst_ass_mac"] + } + ], + "dst_mac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["dst_mac"] + } + ], "actor_primary_user_sid": { "key": "user-account.user_id", "object": "user" @@ -670,45 +907,51 @@ "transformer": "StringToBool" }, "agent_version": { - "key": "x-oca-asset.agent_version", - "object": "asset" - }, - "agent_hostname": { - "key": "x-oca-asset.hostname", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.agent_version", "object": "asset" }, + "agent_hostname": [ + { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + { + "key": "x-oca-event.agent", + "object": "event" + } + ], "agent_content_version": { - "key": "x-oca-asset.content_version", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.content_version", "object": "asset" }, "agent_session_start_time": { - "key": "x-oca-asset.start_time", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.start_time", "object": "asset", "transformer": "EpochToTimestamp" }, "agent_id": { - "key": "x-oca-asset.asset_id", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.asset_id", "object": "asset" }, "agent_os_type": { - "key": "x-oca-asset.os_type", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.os_type", "object": "asset" }, "agent_os_sub_type": { - "key": "x-oca-asset.os_sub_type", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.os_sub_type", "object": "asset" }, "agent_is_vdi": { - "key": "x-oca-asset.is_vdi", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.is_vdi", "object": "asset", "transformer": "StringToBool" }, "action_user_agent": { - "key": "x-oca-asset.user_agent", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.user_agent", "object": "asset" }, "http_req_user_agent_header": { - "key": "x-oca-asset.agent_header", + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.agent_header", "object": "asset" }, "action_evtlog_data_fields": { @@ -757,37 +1000,38 @@ "transformer": "ToInteger" }, "event_id": { - "key": "x-oca-event.event_id", + "key": "x-oca-event.code", "object": "event" }, "vpn_event_description": { - "key": "x-oca-event.event_description", + "key": "x-oca-event.entensions.x-paloalto-event-ext.event_description", "object": "event" }, "event_timestamp": { - "key": "x-oca-event.time", + "key": "x-oca-event.created", "object": "event", "transformer": "EpochToTimestamp" }, "event_version": { - "key": "x-oca-event.version", + "key": "x-oca-event.entensions.x-paloalto-event-ext.version", "object": "event", "transformer": "ToInteger" }, "event_rpc_interface_uuid": { - "key": "x-oca-event.uuid", + "key": "x-oca-event.entensions.x-paloalto-event-ext.uuid", "object": "event" }, "event_address_mapped_image_path": { - "key": "x-oca-event.path", + "key": "x-oca-event.entensions.x-paloalto-event-ext.path", "object": "event" }, "event_type": { - "key": "x-oca-event.event_type", - "object": "event" + "key": "x-oca-event.category", + "object": "event", + "transformer": "ToLowercaseArray" }, "event_sub_type": { - "key": "x-oca-event.sub_type", + "key": "x-oca-event.action", "object": "event" }, "action_network_creation_time": { diff --git a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json index a59ebf0eb..4d3500cff 100644 --- a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json +++ b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json @@ -3,7 +3,8 @@ "fields": { "value": [ "action_local_ip", - "action_remote_ip" + "action_remote_ip", + "agent_ip_addresses" ] } }, @@ -27,7 +28,8 @@ "action_network_protocol" ], "src_ref.value": [ - "action_local_ip" + "action_local_ip", + "agent_ip_addresses" ], "dst_ref.value": [ "action_remote_ip" @@ -236,14 +238,18 @@ "hostname": [ "agent_hostname" ], - "asset_id": [ - "agent_id" - ], - "os_type": [ - "agent_os_type" + "ip_refs[*].value": [ + "action_local_ip", + "action_remote_ip", + "agent_ip_addresses_v6", + "agent_ip_addresses", + "dst_agent_ip_addresses_v6" ], - "user_agent": [ - "action_user_agent" + "mac_refs[*].value": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" ] } }, @@ -259,17 +265,85 @@ }, "x-oca-event": { "fields": { - "event_id": [ + "code": [ "event_id" ], - "event_description": [ - "vpn_event_description" + "category[*]": [ + "event_type" ], - "path": [ - "event_address_mapped_image_path" + "action": [ + "event_sub_type" ], - "event_type": [ - "event_type" + "created": [ + "event_timestamp" + ], + "agent": [ + "agent_hostname" + ], + "url_ref.value": [ + "dst_action_url_category" + ], + "file_ref.name": [ + "action_file_name" + ], + "process_ref.pid": [ + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid" + ], + "process_ref.command_line": [ + "action_process_image_command_line", + "actor_process_command_line" + ], + "process_ref.image_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "process_ref.parent_ref.name": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "process_ref.parent_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "process_ref.parent_ref.command_line": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "parent_process_ref.name": [ + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "parent_process_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "parent_process_ref.command_line": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "process_ref.creator_user_ref.user_id": [ + "action_process_username" + ], + "process_ref.image_ref.hashes.MD5": [ + "action_process_image_md5" + ], + "process_ref.image_ref.hashes.'SHA-256'": [ + "action_process_image_sha256" + ], + "domain_ref.value": [ + "auth_domain", + "dst_host_metadata_domain", + "host_metadata_domain" + ], + "registry_ref.key": [ + "action_registry_key_name" + ], + "registry_ref.values[*]": [ + "action_registry_value_name" ] } }, diff --git a/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json index 39d17fb7f..e780d4cc5 100644 --- a/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json @@ -9,6 +9,14 @@ "key": "network-traffic.src_ref", "object": "nt", "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ], + "group": true } ], "action_remote_ip": [ @@ -20,16 +28,65 @@ "key": "network-traffic.dst_ref", "object": "nt", "references": "dst_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "dst_ip" + ], + "group": true } ], - "agent_ip_addresses_v6": { - "key": "ipv6-addr.value", - "object": "src_ip" - }, - "dst_agent_ip_addresses_v6": { + "agent_ip_addresses": [ + { + "key": "ipv4-addr.value", + "object": "agent_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "agent_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": ["agent_ip"], + "group": true + } + ], + "agent_ip_addresses_v6": [ + { + "key": "ipv6-addr.value", + "object": "src_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ], + "group": true + } + ], + "dst_agent_ip_addresses_v6": [ + { "key": "ipv6-addr.value", - "object": "dst_ip" - }, + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": ["dst_ip"], + "group": true + } + ], "action_local_port": { "key": "network-traffic.src_port", "object": "nt", @@ -40,11 +97,18 @@ "object": "nt", "transformer": "ToInteger" }, - "action_network_protocol": { + "action_network_protocol": [ + { "key": "network-traffic.protocols", "object": "nt", "transformer": "ToLowercaseArray" - }, + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], "action_pkts_sent": { "key": "network-traffic.src_packets", "object": "nt", @@ -55,10 +119,17 @@ "object": "nt", "transformer": "ToInteger" }, - "action_file_name": { - "key": "file.name", - "object": "file" - }, + "action_file_name": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + } + ], "action_process_image_name": [ { "key": "file.name", @@ -72,6 +143,11 @@ "key": "process.binary_ref", "object": "process_action", "references": "file_action_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" } ], "actor_process_image_name": [ @@ -87,6 +163,11 @@ "key": "process.binary_ref", "object": "process_actor", "references": "file_actor_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" } ], "causality_actor_process_image_name": [ @@ -102,6 +183,11 @@ "key": "process.parent_ref", "object": "process_causality_actor", "references": "causality_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "causality_parent_process" } ], "os_actor_process_image_name": [ @@ -117,6 +203,11 @@ "key": "process.parent_ref", "object": "process_os_actor", "references": "os_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "os_parent_process" } ], "action_file_size": { @@ -331,22 +422,50 @@ "references": "file_os_actor" } ], - "action_process_image_command_line": { - "key": "process.command_line", - "object": "process_action" - }, - "actor_process_command_line": { - "key": "process.command_line", - "object": "process_actor" - }, - "causality_actor_process_command_line": { - "key": "process.command_line", - "object": "process_causality_actor" - }, - "os_actor_process_command_line": { - "key": "process.command_line", - "object": "process_os_actor" - }, + "action_process_image_command_line": [ + { + "key": "process.command_line", + "object": "process_action" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_actor" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_causality_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_os_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_os_actor" + } + ], "action_process_file_create_time": [ { "key": "process.created", @@ -395,31 +514,66 @@ "transformer": "EpochToTimestamp" } ], - "action_module_process_os_pid": { - "key": "process.pid", - "object": "process_action_module", - "transformer": "ToInteger" - }, - "action_process_os_pid": { - "key": "process.pid", - "object": "process_action", - "transformer": "ToInteger" - }, - "actor_process_os_pid": { - "key": "process.pid", - "object": "process_actor", - "transformer": "ToInteger" - }, - "causality_actor_process_os_pid": { - "key": "process.pid", - "object": "process_causality_actor", - "transformer": "ToInteger" - }, - "os_actor_process_os_pid": { - "key": "process.pid", - "object": "process_os_actor", - "transformer": "ToInteger" - }, + "action_module_process_os_pid": [ + { + "key": "process.pid", + "object": "process_action_module", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action_module" + } + ], + "action_process_os_pid": [ + { + "key": "process.pid", + "object": "process_action", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_causality_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_os_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_os_actor" + } + ], "action_process_requested_parent_pid": [ { "key": "process.pid", @@ -430,6 +584,11 @@ "key": "process.parent_ref", "object": "process_action", "references": "action_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "action_parent_process" } ], "action_thread_parent_pid": [ @@ -442,6 +601,11 @@ "key": "process.parent_ref", "object": "process_action_thread", "references": "action_parent_thread" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "action_parent_thread" } ], "action_thread_child_pid": [ @@ -456,46 +620,117 @@ "references": "action_child_thread" } ], - "auth_domain": { - "key": "domain-name.value", - "object": "domain" - }, - "dst_host_metadata_domain": { - "key": "domain-name.value", - "object": "domain_dst" - }, - "host_metadata_domain": { - "key": "domain-name.value", - "object": "domain_host" - }, - "dst_action_url_category": { - "key": "url.value", - "object": "url" - }, - "action_registry_key_name": { - "key": "windows-registry-key.key", - "object": "windowsregistry" - }, - "action_registry_value_name": { - "key": "windows-registry-key.values", - "object": "windowsregistry" - }, - "mac": { - "key": "mac-addr.value", - "object": "mac" - }, - "associated_mac": { - "key": "mac-addr.value", - "object": "ass_mac" - }, - "dst_associated_mac": { - "key": "mac-addr.value", - "object": "dst_ass_mac" - }, - "dst_mac": { - "key": "mac-addr.value", - "object": "dst_mac" - }, + "auth_domain": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain" + } + ], + "dst_host_metadata_domain": [ + { + "key": "domain-name.value", + "object": "domain_dst" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain_dst" + } + ], + "host_metadata_domain": [ + { + "key": "domain-name.value", + "object": "domain_host" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain_host" + } + ], + "dst_action_url_category": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "action_registry_key_name": [ + { + "key": "windows-registry-key.key", + "object": "windowsregistry" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "windowsregistry" + } + ], + "action_registry_value_name": [ + { + "key": "windows-registry-key.values", + "object": "windowsregistry", + "transformer": "FormatToStixRegistryValue" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "windowsregistry" + } + ], + "mac": [ + { + "key": "mac-addr.value", + "object": "mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["mac"] + } + ], + "associated_mac": [ + { + "key": "mac-addr.value", + "object": "ass_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["ass_mac"] + } + ], + "dst_associated_mac": [ + { + "key": "mac-addr.value", + "object": "dst_ass_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["dst_ass_mac"] + } + ], + "dst_mac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["dst_mac"] + } + ], "actor_primary_user_sid": { "key": "user-account.user_id", "object": "user" @@ -688,45 +923,51 @@ "transformer": "StringToBool" }, "agent_version": { - "key": "x-oca-asset.agent_version", - "object": "asset" - }, - "agent_hostname": { - "key": "x-oca-asset.hostname", + "key": "x-oca-asset.extensions.x-paloalto-agent.agent_version", "object": "asset" }, + "agent_hostname": [ + { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + { + "key": "x-oca-event.agent", + "object": "event" + } + ], "agent_content_version": { - "key": "x-oca-asset.content_version", + "key": "x-oca-asset.extensions.x-paloalto-agent.content_version", "object": "asset" }, "agent_session_start_time": { - "key": "x-oca-asset.start_time", + "key": "x-oca-asset.extensions.x-paloalto-agent.start_time", "object": "asset", "transformer": "EpochToTimestamp" }, "agent_id": { - "key": "x-oca-asset.asset_id", + "key": "x-oca-asset.extensions.x-paloalto-agent.asset_id", "object": "asset" }, "agent_os_type": { - "key": "x-oca-asset.os_type", + "key": "x-oca-asset.extensions.x-paloalto-agent.os_type", "object": "asset" }, "agent_os_sub_type": { - "key": "x-oca-asset.os_sub_type", + "key": "x-oca-asset.extensions.x-paloalto-agent.os_sub_type", "object": "asset" }, "agent_is_vdi": { - "key": "x-oca-asset.is_vdi", + "key": "x-oca-asset.extensions.x-paloalto-agent.is_vdi", "object": "asset", "transformer": "StringToBool" }, "action_user_agent": { - "key": "x-oca-asset.user_agent", + "key": "x-oca-asset.extensions.x-paloalto-agent.user_agent", "object": "asset" }, "http_req_user_agent_header": { - "key": "x-oca-asset.agent_header", + "key": "x-oca-asset.extensions.x-paloalto-agent.agent_header", "object": "asset" }, "action_evtlog_data_fields": { @@ -774,38 +1015,39 @@ "object": "evtlog", "transformer": "ToInteger" }, - "event_id" :{ - "key": "x-oca-event.event_id", + "event_id": { + "key": "x-oca-event.code", "object": "event" }, "vpn_event_description": { - "key": "x-oca-event.event_description", + "key": "x-oca-event.entensions.x-paloalto-event.event_description", "object": "event" }, "event_timestamp": { - "key": "x-oca-event.time", + "key": "x-oca-event.created", "object": "event", "transformer": "EpochToTimestamp" }, "event_version": { - "key": "x-oca-event.version", + "key": "x-oca-event.entensions.x-paloalto-event.version", "object": "event", "transformer": "ToInteger" }, "event_rpc_interface_uuid": { - "key": "x-oca-event.uuid", + "key": "x-oca-event.entensions.x-paloalto-event.uuid", "object": "event" }, "event_address_mapped_image_path": { - "key": "x-oca-event.path", + "key": "x-oca-event.entensions.x-paloalto-event.path", "object": "event" }, "event_type": { - "key": "x-oca-event.event_type", - "object": "event" + "key": "x-oca-event.category", + "object": "event", + "transformer": "ToLowercaseArray" }, "event_sub_type": { - "key": "x-oca-event.sub_type", + "key": "x-oca-event.action", "object": "event" }, "action_network_creation_time": { diff --git a/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json index 089e7c897..9147b6418 100644 --- a/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json +++ b/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json @@ -3,7 +3,8 @@ "fields": { "value": [ "action_local_ip", - "action_remote_ip" + "action_remote_ip", + "agent_ip_addresses" ] } }, @@ -27,7 +28,8 @@ "action_network_protocol" ], "src_ref.value": [ - "action_local_ip" + "action_local_ip", + "agent_ip_addresses" ], "dst_ref.value": [ "action_remote_ip" @@ -242,14 +244,18 @@ "hostname": [ "agent_hostname" ], - "asset_id": [ - "agent_id" - ], - "os_type": [ - "agent_os_type" + "ip_refs[*].value": [ + "action_local_ip", + "action_remote_ip", + "agent_ip_addresses_v6", + "agent_ip_addresses", + "dst_agent_ip_addresses_v6" ], - "user_agent": [ - "action_user_agent" + "mac_refs[*].value": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" ] } }, @@ -265,17 +271,89 @@ }, "x-oca-event": { "fields": { - "event_id": [ + "code": [ "event_id" ], - "event_description": [ - "vpn_event_description" + "category[*]": [ + "event_type" ], - "path": [ - "event_address_mapped_image_path" + "action": [ + "event_sub_type" ], - "event_type": [ - "event_type" + "created": [ + "event_timestamp" + ], + "agent": [ + "agent_hostname" + ], + "url_ref.value": [ + "dst_action_url_category" + ], + "file_ref.name": [ + "action_file_name" + ], + "process_ref.pid": [ + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid" + ], + "process_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "process_ref.command_line": [ + "action_process_image_command_line", + "actor_process_command_line" + ], + "process_ref.binary_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "process_ref.parent_ref.name": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "process_ref.parent_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "process_ref.parent_ref.command_line": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "parent_process_ref.name": [ + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "parent_process_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "parent_process_ref.command_line": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "process_ref.creator_user_ref.user_id": [ + "action_process_username" + ], + "process_ref.binary_ref.hashes.MD5": [ + "action_process_image_md5" + ], + "process_ref.binary_ref.hashes.'SHA-256'": [ + "action_process_image_sha256" + ], + "domain_ref.value": [ + "auth_domain", + "dst_host_metadata_domain", + "host_metadata_domain" + ], + "registry_ref.key": [ + "action_registry_key_name" + ], + "registry_ref.values[*]": [ + "action_registry_value_name" ] } }, diff --git a/stix_shifter_modules/paloalto/stix_translation/transformers.py b/stix_shifter_modules/paloalto/stix_translation/transformers.py new file mode 100644 index 000000000..8b1e728d9 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/transformers.py @@ -0,0 +1,16 @@ +from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer +from stix_shifter_utils.utils import logger + +LOGGER = logger.set_logger(__name__) + + +class FormatToStixRegistryValue(ValueTransformer): + """A value transformer to convert paloalto Registry value to windows-registry-key.value STIX""" + + @staticmethod + def transform(registry): + + try: + return [{'name': registry}] + except ValueError: + LOGGER.error("Cannot convert root value to Stix formatted windows registry value") diff --git a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py index ae1d6557a..533614be8 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py +++ b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py @@ -190,13 +190,19 @@ def check_object(stix_data_map, mandatory_map, data, log, field, value): if obj in mandatory_map.keys(): data = ResultsConnector.check_mandatory_map(mandatory_map, obj, log, data, field, value) else: - data[field] = value + # Since these ip addresses may receive a single ip or list of ips, they are sent as list value + # for the to_stix conversion + data[field] = value.split(",") if field in ["agent_ip_addresses", "agent_ip_addresses_v6", + "dst_agent_ip_addresses_v6"] else value + elif isinstance(stix_data_map, dict): if stix_data_map["object"] in mandatory_map.keys(): data = ResultsConnector.check_mandatory_map(mandatory_map, stix_data_map["object"], log, data, field, value) else: + data[field] = value + except (KeyError, IndexError, TypeError) as e: raise e return data diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py index 5f68d5758..8181c6cc3 100644 --- a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py +++ b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py @@ -146,10 +146,10 @@ def test_network_traffic_json_to_stix(self): assert network_obj is not None assert network_obj["type"] == 'network-traffic' assert network_obj["src_port"] == 54083 - assert network_obj['src_ref'] == '1' + assert network_obj['src_ref'] == '2' assert 'tcp' in network_obj["protocols"] assert network_obj["dst_port"] == 80 - assert network_obj['dst_ref'] == '2' + assert network_obj['dst_ref'] == '4' def test_mac_addr_json_to_stix(self): """ @@ -251,7 +251,7 @@ def test_windows_registry_key_json_to_stix(self): assert windows_obj is not None assert windows_obj['type'] == 'windows-registry-key' assert windows_obj['key'] == 'HKEY_LOCAL_MACHINE\\SYSTEM\\ControlSet001\\Services\\terminpt\\Enum' - assert windows_obj['values'] == 'Start' + assert windows_obj['values'] == [{'name':'Start'}] def test_url_json_to_stix(self): """to test url stix object properties""" @@ -355,12 +355,12 @@ def test_asset_json_to_stix(self): asset_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'x-oca-asset') assert asset_obj is not None assert asset_obj['type'] == 'x-oca-asset' - assert asset_obj['agent_version'] == "7.6.1.46600" + assert asset_obj['extensions']['x-paloalto-agent']['agent_version'] == "7.6.1.46600" assert asset_obj['hostname'] == "EC2AMAZ-IQFSLIL" - assert asset_obj['content_version'] == "350-80787" - assert asset_obj['start_time'] == "2022-01-20T07:04:01.933Z" - assert asset_obj['os_sub_type'] == "Windows Server 2016" - assert asset_obj['is_vdi'] == False + assert asset_obj['extensions']['x-paloalto-agent']['content_version'] == "350-80787" + assert asset_obj['extensions']['x-paloalto-agent']['start_time'] == "2022-01-20T07:04:01.933Z" + assert asset_obj['extensions']['x-paloalto-agent']['os_sub_type'] == "Windows Server 2016" + assert asset_obj['extensions']['x-paloalto-agent']['is_vdi'] == False def test_evtlog_json_to_stix(self): """to test custom evtlog stix object properties""" @@ -411,11 +411,11 @@ def test_event_json_to_stix(self): objects = observed_data['objects'] event_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') assert event_obj is not None - assert event_obj['event_id'] == "OTE0MTk5MTg2MDI1NzUyODc0NQ==" - assert event_obj['time'] == '1975-03-21T11:12:13.729Z' - assert event_obj['version'] == 25 - assert event_obj['uuid'] == '{00000136-0000-0000-C000-000000000046}' - assert event_obj['event_type'] == "EVENT_LOG" + assert event_obj['code'] == "OTE0MTk5MTg2MDI1NzUyODc0NQ==" + assert event_obj['created'] == '1975-03-21T11:12:13.729Z' + assert event_obj['entensions']['x-paloalto-event']['version'] == 25 + assert event_obj['entensions']['x-paloalto-event']['uuid'] == '{00000136-0000-0000-C000-000000000046}' + assert event_obj['category'] == ["event_log"] def test_custom_network_json_to_stix(self): """to test custom network stix object properties""" diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py index 737266c74..68e7a80d9 100644 --- a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py +++ b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py @@ -21,7 +21,8 @@ def _remove_timestamp_from_query(queries): return re.sub(pattern2, '{}', replace_pat1) -all_fields = "dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses_v6,dst_agent_ip_addresses_v6," \ +all_fields = "dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6," \ + "dst_agent_ip_addresses_v6," \ "action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received," \ "action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name," \ "os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5," \ @@ -80,7 +81,7 @@ def test_ipv4_query(self): query = translation.translate('paloalto', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_local_ip = \"172.31.90.48\" or " - "action_remote_ip = \"172.31.90.48\") and " + "action_remote_ip = \"172.31.90.48\" or agent_ip_addresses = \"172.31.90.48\") and " "(to_epoch(_time,\"millis\") >= 1645615464114 and to_epoch(_time,\"millis\") <= 1645615764114)) " "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" "'from': 1645615464114, 'to': " @@ -445,7 +446,7 @@ def test_combined_observation_AND(self): "action_thread_child_pid = 868) and " "(to_epoch(_time,\"millis\") >= 1642590000000 and to_epoch(_time,\"millis\") <= 1644231600003)) " "or ((action_local_ip = \"172.31.31.67\" or action_remote_ip " - "= \"172.31.31.67\") and action_remote_port = 53996 and " + "= \"172.31.31.67\" or agent_ip_addresses = \"172.31.31.67\") and action_remote_port = 53996 and " "(to_epoch(_time,\"millis\") >= 1642590000000 and to_epoch(_time,\"millis\") <= 1644231600003)) " "| alter dataset_name = " "\"xdr_data\" | fields " + all_fields + " | limit 10000 ', "