diff --git a/stix_shifter_modules/paloalto/README.md b/stix_shifter_modules/paloalto/README.md new file mode 100644 index 000000000..6bec5c518 --- /dev/null +++ b/stix_shifter_modules/paloalto/README.md @@ -0,0 +1,953 @@ +# PaloAlto Cortex XDR Connector + +**Table of Contents** + +- [PaloAlto Cortex XDR API Endpoints](#paloalto-cortex-xdr-api-endpoints) +- [Pattern expression with STIX attributes - Single Observation](#single-observation) +- [Pattern expression with STIX attributes - Multiple Observation](#multiple-observation) +- [Pattern expression with STIX attributes - Execute Query](#stix-execute-query) +- [Limitations](#limitations) +- [References](#references) + +### PaloAlto Cortex XDR API Endpoints + + |Connector Method|PaloAlto Cortex XDR API Endpoint| Method + | ---- | ------ | -----| + |Query Endpoint |https:///public_api/v1/xql/start_xql_query/|POST + |Result Endpoint |https:///public_api/v1/xql/get_query_results/|POST + |Stream Endpoint |https:///public_api/v1/xql/get_query_results_stream/|POST + |Quota Endpoint |https:///public_api/v1/xql/get_quota/|POST + +### Format for calling stix-shifter from the command line +``` +$ stix-shifter translate query "" "" "" +``` +### Pattern expression with STIX attributes + +### Single Observation + +#### STIX Translate query +```shell +translate paloalto query '{}' "[ipv4-addr:value = '1.1.0.0'] START t'2022-03-01T11:00:00.000Z' STOP t'2023-03-08T11:00:00.003Z'" +``` +#### STIX Translate query - output +```json +{ + "queries": [ + { + "xdr_data": { + "query": "dataset = xdr_data | filter ((action_local_ip = \"1.1.0.0\" or action_remote_ip = \"1.1.0.0\" or agent_ip_addresses = \"1.1.0.0\") and (to_epoch(_time,\"millis\") >= 1646132400000 and to_epoch(_time,\"millis\") <= 1678273200003)) | alter dataset_name = \"xdr_data\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 10000 ", + "timeframe": { + "from": 1646132400000, + "to": 1678273200003 + } + } + } + ] +} + +``` +#### STIX Transmit ping + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +ping +``` + +#### STIX Transmit ping - output +```json +{ + "success": true +} +``` +#### STIX Transmit query + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +query +"{\"xdr_data\": {\"query\": \"dataset = xdr_data | filter ((action_local_ip = \\"1.1.0.0\\" or action_remote_ip = \\"1.1.0.0\\" or agent_ip_addresses = \\"1.1.0.0\\") and (to_epoch(_time,\\"millis\\") >= 1646132400000 and to_epoch(_time,\\"millis\\") <= 1678273200003)) | alter dataset_name = \\"xdr_data\\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 10000 \", \"timeframe\": {\"from\": 1646132400000, \"to\": 1678273200003}}}" +``` + +#### STIX Transmit query - output +```json +{ + "success": true, + "search_id": "a106f2f4614f40_34940_inv" +} +``` +#### STIX Transmit status + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +status +"a106f2f4614f40_34940_inv" +``` + +#### STIX Transmit status - output +```json +{ + "success": true, + "status": "COMPLETED", + "progress": 100 +} +``` +#### STIX Transmit results + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +results +"a106f2f4614f40_34940_inv" 0 2 +``` + +#### STIX Transmit results - output +```json +{ + "success": true, + "data": [ + { + "xdr_data": { + "action_local_ip": "94.232.41.158", + "action_network_protocol": "TCP", + "action_remote_ip": "172.31.90.48", + "action_local_port": "31545", + "action_remote_port": "3389", + "agent_id": "94.232.41.158", + "event_id": "NDQ3NzY1MTgzNjU1MjQxNTg0Ng==", + "event_timestamp": "1648884264296", + "event_type": "STORY", + "event_sub_type": "event_sub_type_4", + "action_network_creation_time": "1648884264296", + "action_network_connection_id": "AdhGYq2dw14AAAAAAAEOxg==", + "action_proxy": "FALSE" + } + }, + { + "xdr_data": { + "action_local_ip": "94.232.41.158", + "action_network_protocol": "TCP", + "action_remote_ip": "172.31.90.48", + "action_local_port": "6142", + "action_remote_port": "3389", + "agent_id": "94.232.41.158", + "event_id": "MjAyOTYyMzMyMTg3Njc5MDY4NA==", + "event_timestamp": "1648884468933", + "event_type": "STORY", + "event_sub_type": "event_sub_type_4", + "action_network_creation_time": "1648884468933", + "action_network_connection_id": "AdhGYyeW9LcAAAAAAAEO5g==", + "action_proxy": "FALSE" + } + } + ] +} +``` + + +#### STIX Translate results + +```shell +translate +paloalto +results +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"system\",\"created\":\"2022-03-16T13:22:50.336Z\",\"modified\":\"2022-03-16T13:22:50.336Z\"}" +"[ { \"xdr_data\": { \"action_local_ip\": \"94.232.41.158\", \"action_network_protocol\": \"TCP\", \"action_remote_ip\": \"1.1.0.0\", \"action_local_port\": \"31545\", \"action_remote_port\": \"3389\", \"agent_id\": \"94.232.41.158\", \"event_id\": \"NDQ3NzY1MTgzNjU1MjQxNTg0Ng==\", \"event_timestamp\": \"1648884264296\", \"event_type\": \"STORY\", \"event_sub_type\": \"event_sub_type_4\", \"action_network_creation_time\": \"1648884264296\", \"action_network_connection_id\": \"AdhGYq2dw14AAAAAAAEOxg==\", \"action_proxy\": \"FALSE\" } }, { \"xdr_data\": { \"action_local_ip\": \"94.232.41.158\", \"action_network_protocol\": \"TCP\", \"action_remote_ip\": \"1.1.0.0\", \"action_local_port\": \"6142\", \"action_remote_port\": \"3389\", \"agent_id\": \"94.232.41.158\", \"event_id\": \"MjAyOTYyMzMyMTg3Njc5MDY4NA==\", \"event_timestamp\": \"1648884468933\", \"event_type\": \"STORY\", \"event_sub_type\": \"event_sub_type_4\", \"action_network_creation_time\": \"1648884468933\", \"action_network_connection_id\": \"AdhGYyeW9LcAAAAAAAEO5g==\", \"action_proxy\": \"FALSE\" } } ] " +"{\"stix_validator\": true}" +``` + +#### STIX Translate results - output +```json +{ + "type": "bundle", + "id": "bundle--318a05c9-47e5-4146-80ce-e6b95c71c8bd", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "paloalto", + "identity_class": "system", + "created": "2022-03-16T13:22:50.336Z", + "modified": "2022-03-16T13:22:50.336Z" + }, + { + "id": "observed-data--713d4380-de85-40f7-a6c1-c893331e9c99", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-04-05T12:43:28.560Z", + "modified": "2022-04-05T12:43:28.560Z", + "objects": { + "0": { + "type": "ipv4-addr", + "value": "94.232.41.158" + }, + "1": { + "type": "network-traffic", + "src_ref": "0", + "protocols": [ + "tcp" + ], + "dst_ref": "4", + "src_port": 31545, + "dst_port": 3389, + "extensions": { + "x-paloalto-network": { + "creation_time": "2022-04-02T07:24:24.296Z", + "connection_id": "AdhGYq2dw14AAAAAAAEOxg==", + "is_proxy": false + } + } + }, + "2": { + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "asset_id": "94.232.41.158" + } + } + }, + "3": { + "type": "x-oca-event", + "network_ref": "1", + "code": "NDQ3NzY1MTgzNjU1MjQxNTg0Ng==", + "created": "2022-04-02T07:24:24.296Z", + "category": ["story"], + "action": "event_sub_type_4" + }, + "4": { + "type": "ipv4-addr", + "value": "1.1.0.0" + } + }, + "first_observed": "2022-04-05T12:43:28.560Z", + "last_observed": "2022-04-05T12:43:28.560Z", + "number_observed": 1 + }, + { + "id": "observed-data--d05e4f7d-0866-4f4d-be32-a39832ad27c1", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-04-05T12:43:28.565Z", + "modified": "2022-04-05T12:43:28.565Z", + "objects": { + "0": { + "type": "ipv4-addr", + "value": "94.232.41.158" + }, + "1": { + "type": "network-traffic", + "src_ref": "0", + "protocols": [ + "tcp" + ], + "dst_ref": "4", + "src_port": 6142, + "dst_port": 3389, + "extensions": { + "x-paloalto-network": { + "creation_time": "2022-04-02T07:27:48.933Z", + "connection_id": "AdhGYyeW9LcAAAAAAAEO5g==", + "is_proxy": false + } + } + }, + "2": { + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "asset_id": "94.232.41.158" + } + } + }, + "3": { + "type": "x-oca-event", + "network_ref": "1", + "code": "MjAyOTYyMzMyMTg3Njc5MDY4NA==", + "created": "2022-04-02T07:27:48.933Z", + "category": ["story"], + "action": "event_sub_type_4" + }, + "4": { + "type": "ipv4-addr", + "value": "1.1.0.0" + } + }, + "first_observed": "2022-04-05T12:43:28.565Z", + "last_observed": "2022-04-05T12:43:28.565Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +### Multiple Observation + +#### STIX Translate query +```shell +translate paloalto query {} "([x-oca-event:category[*] = 'file'] AND [file:size > 3000] OR [x-oca-asset:agent_version = '7.6.1.46600'] )START t'2022-02-10T00:00:00.000000Z' STOP t'2022-02-15T00:00:00.000000Z'" +``` + +#### STIX Translate query - output + +```json +{ + "queries": [ + { + "xdr_data": { + "query": "dataset = xdr_data | filter (event_type = ENUM.FILE and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) or (action_file_size > 3000 and (to_epoch(_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= 1644883200000)) | alter dataset_name = \"xdr_data\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 10000 ", + "timeframe": { + "from": 1644451200000, + "to": 1644883200000 + } + } + } + ] +} +``` + + +#### STIX Transmit query + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +query +"{ \"xdr_data\": { \"query\": \"dataset = xdr_data | filter (event_type = ENUM.FILE and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) or (action_file_size > 3000 and (to_epoch(_time,\\"millis\\") >= 1644451200000 and to_epoch(_time,\\"millis\\") <= 1644883200000)) | alter dataset_name = \\"xdr_data\\" | fields dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received,action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name,os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5,action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2,action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time,actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time,actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path,action_process_image_path,action_registry_file_path,actor_process_image_path,causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line,actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line,action_process_file_create_time,actor_process_file_create_time,causality_actor_process_file_create_time,os_actor_process_file_create_time,action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid,os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid,action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain,host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac,associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid,actor_primary_username,actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes,action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status,action_file_signature_vendor,action_file_signature_product,action_file_info_description,action_file_group,action_file_group_name,action_file_type,action_file_info_file_version,manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name,action_file_info_product_name,action_file_id,action_file_wildfire_verdict,action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id,actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor,actor_process_signature_status,actor_process_signature_product,actor_process_image_extension,action_process_termination_code,action_process_termination_date,action_remote_process_thread_id,action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel,action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname,agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi,action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description,action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid,action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description,event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type,event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data,action_proxy,host_metadata_hostname,action_external_hostname | limit 10000 \", \"timeframe\": { \"from\": 1644451200000, \"to\": 1644883200000 } } }" +``` + +#### STIX Transmit query - output + +```json +{ + "success": true, + "search_id": "3dc7d55520fe41_35090_inv" +} +``` +#### STIX Transmit status + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +status +"3dc7d55520fe41_35090_inv" +``` + +#### STIX Transmit status - output +```json +{ + "success": true, + "status": "COMPLETED", + "progress": 100 +} + +``` +#### STIX Transmit results + +```shell +transmit +paloalto +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +results +"3dc7d55520fe41_35090_inv" 0 2 +``` + +#### STIX Transmit results - output +```json +{ + "success": true, + "data": [ + { + "xdr_data": { + "agent_ip_addresses": [ + "172.31.31.67" + ], + "action_file_name": "DRIVERS", + "actor_process_image_name": "System", + "causality_actor_process_image_name": "System", + "os_actor_process_image_name": "System", + "action_file_size": "4255744", + "action_file_path": "C:\\Windows\\System32\\config\\DRIVERS", + "actor_process_image_path": "System", + "causality_actor_process_image_path": "System", + "os_actor_process_image_path": "System", + "actor_process_os_pid": "4", + "causality_actor_process_os_pid": "4", + "os_actor_process_os_pid": "4", + "actor_primary_user_sid": "S-1-5-18", + "actor_primary_username": "NT AUTHORITY\\SYSTEM", + "actor_process_logon_id": "1003", + "action_file_attributes": "128", + "action_file_type": "18", + "manifest_file_version": "5", + "actor_process_instance_id": "AdgdeB26mNQAAAAEAAAAAA==", + "actor_process_causality_id": "AdgdeB26mNQAAAAEAAAAAA==", + "actor_process_auth_id": "999", + "actor_process_signature_vendor": "Microsoft Corporation", + "actor_process_signature_status": "SIGNED", + "actor_process_signature_product": "Microsoft Windows", + "actor_process_execution_time": "1644385473948", + "actor_process_is_native": "TRUE", + "agent_version": "7.6.1.46600", + "agent_hostname": "EC2AMAZ-IQFSLIL", + "agent_content_version": "380-82571", + "agent_session_start_time": "1644385496543", + "agent_id": "37a92aad549d41d184ec9fbdafbff55c", + "agent_os_type": "AGENT_OS_WINDOWS", + "agent_os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "agent_is_vdi": "FALSE", + "event_id": "AAABfvQtkk8qn9UKACE2JA==", + "event_timestamp": "1644774134407", + "event_version": "25", + "event_type": "FILE", + "event_sub_type": "FILE_OPEN" + } + }, + { + "xdr_data": { + "agent_ip_addresses": [ + "172.31.31.67" + ], + "action_file_name": "DRIVERS", + "actor_process_image_name": "System", + "causality_actor_process_image_name": "System", + "os_actor_process_image_name": "System", + "action_file_size": "4255744", + "action_file_path": "C:\\Windows\\System32\\config\\DRIVERS", + "actor_process_image_path": "System", + "causality_actor_process_image_path": "System", + "os_actor_process_image_path": "System", + "actor_process_os_pid": "4", + "causality_actor_process_os_pid": "4", + "os_actor_process_os_pid": "4", + "actor_primary_user_sid": "S-1-5-18", + "actor_primary_username": "NT AUTHORITY\\SYSTEM", + "actor_process_logon_id": "1003", + "action_file_type": "0", + "manifest_file_version": "5", + "actor_process_instance_id": "AdgdeB26mNQAAAAEAAAAAA==", + "actor_process_causality_id": "AdgdeB26mNQAAAAEAAAAAA==", + "actor_process_auth_id": "999", + "actor_process_signature_vendor": "Microsoft Corporation", + "actor_process_signature_status": "SIGNED", + "actor_process_signature_product": "Microsoft Windows", + "actor_process_execution_time": "1644385473948", + "actor_process_is_native": "TRUE", + "agent_version": "7.6.1.46600", + "agent_hostname": "EC2AMAZ-IQFSLIL", + "agent_content_version": "380-82571", + "agent_session_start_time": "1644385496543", + "agent_id": "37a92aad549d41d184ec9fbdafbff55c", + "agent_os_type": "AGENT_OS_WINDOWS", + "agent_os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "agent_is_vdi": "FALSE", + "event_id": "AAABfvWCZpMqn9UKACRWUw==", + "event_timestamp": "1644796471001", + "event_version": "25", + "event_type": "FILE", + "event_sub_type": "FILE_WRITE" + } + } + ] +} +``` + + +#### STIX Translate results + +```shell +translate +paloalto +results +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"system\",\"created\":\"2022-03-16T13:22:50.336Z\",\"modified\":\"2022-03-16T13:22:50.336Z\"}" +" [ { \"xdr_data\": { \"agent_ip_addresses\": [ \"172.31.31.67\" ], \"action_file_name\": \"DRIVERS\", \"actor_process_image_name\": \"System\", \"causality_actor_process_image_name\": \"System\", \"os_actor_process_image_name\": \"System\", \"action_file_size\": \"4255744\", \"action_file_path\": \"C:\\Windows\\System32\\config\\DRIVERS\", \"actor_process_image_path\": \"System\", \"causality_actor_process_image_path\": \"System\", \"os_actor_process_image_path\": \"System\", \"actor_process_os_pid\": \"4\", \"causality_actor_process_os_pid\": \"4\", \"os_actor_process_os_pid\": \"4\", \"actor_primary_user_sid\": \"S-1-5-18\", \"actor_primary_username\": \"NT AUTHORITY\\SYSTEM\", \"actor_process_logon_id\": \"1003\", \"action_file_attributes\": \"128\", \"action_file_type\": \"18\", \"manifest_file_version\": \"5\", \"actor_process_instance_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_causality_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_auth_id\": \"999\", \"actor_process_signature_vendor\": \"Microsoft Corporation\", \"actor_process_signature_status\": \"SIGNED\", \"actor_process_signature_product\": \"Microsoft Windows\", \"actor_process_execution_time\": \"1644385473948\", \"actor_process_is_native\": \"TRUE\", \"agent_version\": \"7.6.1.46600\", \"agent_hostname\": \"EC2AMAZ-IQFSLIL\", \"agent_content_version\": \"380-82571\", \"agent_session_start_time\": \"1644385496543\", \"agent_id\": \"37a92aad549d41d184ec9fbdafbff55c\", \"agent_os_type\": \"AGENT_OS_WINDOWS\", \"agent_os_sub_type\": \"Windows Server 2016 [10.0 (Build 17763)]\", \"agent_is_vdi\": \"FALSE\", \"event_id\": \"AAABfvQtkk8qn9UKACE2JA==\", \"event_timestamp\": \"1644774134407\", \"event_version\": \"25\", \"event_type\": \"FILE\", \"event_sub_type\": \"FILE_OPEN\" } }, { \"xdr_data\": { \"agent_ip_addresses\": [ \"172.31.31.67\" ], \"action_file_name\": \"DRIVERS\", \"actor_process_image_name\": \"System\", \"causality_actor_process_image_name\": \"System\", \"os_actor_process_image_name\": \"System\", \"action_file_size\": \"4255744\", \"action_file_path\": \"C:\\Windows\\System32\\config\\DRIVERS\", \"actor_process_image_path\": \"System\", \"causality_actor_process_image_path\": \"System\", \"os_actor_process_image_path\": \"System\", \"actor_process_os_pid\": \"4\", \"causality_actor_process_os_pid\": \"4\", \"os_actor_process_os_pid\": \"4\", \"actor_primary_user_sid\": \"S-1-5-18\", \"actor_primary_username\": \"NT AUTHORITY\\SYSTEM\", \"actor_process_logon_id\": \"1003\", \"action_file_type\": \"0\", \"manifest_file_version\": \"5\", \"actor_process_instance_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_causality_id\": \"AdgdeB26mNQAAAAEAAAAAA==\", \"actor_process_auth_id\": \"999\", \"actor_process_signature_vendor\": \"Microsoft Corporation\", \"actor_process_signature_status\": \"SIGNED\", \"actor_process_signature_product\": \"Microsoft Windows\", \"actor_process_execution_time\": \"1644385473948\", \"actor_process_is_native\": \"TRUE\", \"agent_version\": \"7.6.1.46600\", \"agent_hostname\": \"EC2AMAZ-IQFSLIL\", \"agent_content_version\": \"380-82571\", \"agent_session_start_time\": \"1644385496543\", \"agent_id\": \"37a92aad549d41d184ec9fbdafbff55c\", \"agent_os_type\": \"AGENT_OS_WINDOWS\", \"agent_os_sub_type\": \"Windows Server 2016 [10.0 (Build 17763)]\", \"agent_is_vdi\": \"FALSE\", \"event_id\": \"AAABfvWCZpMqn9UKACRWUw==\", \"event_timestamp\": \"1644796471001\", \"event_version\": \"25\", \"event_type\": \"FILE\", \"event_sub_type\": \"FILE_WRITE\" } } ]" +"{\"stix_validator\": true}" +``` + +#### STIX Translate results - output +```json +{ + "type": "bundle", + "id": "bundle--de01a54d-dd8d-4005-b640-29e68c24d976", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "paloalto", + "identity_class": "system", + "created": "2022-03-16T13:22:50.336Z", + "modified": "2022-03-16T13:22:50.336Z" + }, + { + "id": "observed-data--ce1c1768-d2ad-41bd-a3d3-d11bf2ae3d94", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-04-05T13:01:57.723Z", + "modified": "2022-04-05T13:01:57.723Z", + "objects": { + "0": { + "type": "ipv4-addr", + "value": "172.31.31.67" + }, + "2": { + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "agent_version": "7.6.1.46600", + "content_version": "380-82571", + "start_time": "2022-02-09T05:44:56.543Z", + "asset_id": "37a92aad549d41d184ec9fbdafbff55c", + "os_type": "AGENT_OS_WINDOWS", + "os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "is_vdi": false + } + }, + "hostname": "EC2AMAZ-IQFSLIL" + }, + "3": { + "type": "file", + "name": "DRIVERS", + "size": 4255744, + "parent_directory_ref": "13", + "extensions": { + "x-paloalto-file": { + "attributes": 128, + "type": 18, + "manifest_version": 5 + } + } + }, + "4": { + "type": "x-oca-event", + "file_ref": "3", + "process_ref": "6", + "parent_process_ref": "8", + "agent": "EC2AMAZ-IQFSLIL", + "code": "AAABfvQtkk8qn9UKACE2JA==", + "created": "2022-02-13T17:42:14.407Z", + "entensions": { + "x-paloalto-event": { + "version": 25 + } + }, + "category": [ + "file" + ], + "action": "FILE_OPEN" + }, + "5": { + "type": "file", + "name": "System" + }, + "6": { + "type": "process", + "name": "System", + "binary_ref": "5", + "pid": 4, + "extensions": { + "x-paloalto-process": { + "instance_id": "AdgdeB26mNQAAAAEAAAAAA==", + "causality_id": "AdgdeB26mNQAAAAEAAAAAA==", + "auth_id": "999", + "signature_vendor": "Microsoft Corporation", + "signature_status": "SIGNED", + "signature_product": "Microsoft Windows", + "execution_time": "2022-02-09T05:44:33.948Z", + "is_native": true + } + } + }, + "7": { + "type": "file", + "name": "System" + }, + "8": { + "type": "process", + "name": "System" + }, + "9": { + "type": "process", + "parent_ref": "8", + "binary_ref": "7", + "pid": 4 + }, + "10": { + "type": "file", + "name": "System" + }, + "11": { + "type": "process", + "name": "System" + }, + "12": { + "type": "process", + "parent_ref": "11", + "binary_ref": "10", + "pid": 4 + }, + "13": { + "type": "directory", + "path": "C:\\Windows\\System32\\config" + }, + "14": { + "type": "user-account", + "user_id": "S-1-5-18", + "display_name": "NT AUTHORITY\\SYSTEM", + "account_login": "1003" + } + }, + "first_observed": "2022-04-05T16:17:58.820Z", + "last_observed": "2022-04-05T16:17:58.820Z", + "number_observed": 1 + }, + { + "id": "observed-data--f3a29cc9-e4e1-431f-943c-edf91e42edc5", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-04-05T13:01:57.731Z", + "modified": "2022-04-05T13:01:57.731Z", + "objects": { + "0": { + "type": "ipv4-addr", + "value": "172.31.31.67" + }, + "2": { + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "agent_version": "7.6.1.46600", + "content_version": "380-82571", + "start_time": "2022-02-09T05:44:56.543Z", + "asset_id": "37a92aad549d41d184ec9fbdafbff55c", + "os_type": "AGENT_OS_WINDOWS", + "os_sub_type": "Windows Server 2016 [10.0 (Build 17763)]", + "is_vdi": false + } + }, + "hostname": "EC2AMAZ-IQFSLIL" + }, + "3": { + "type": "file", + "name": "DRIVERS", + "size": 4255744, + "parent_directory_ref": "13", + "extensions": { + "x-paloalto-file": { + "type": 0, + "manifest_version": 5 + } + } + }, + "4": { + "type": "x-oca-event", + "file_ref": "3", + "process_ref": "6", + "parent_process_ref": "8", + "agent": "EC2AMAZ-IQFSLIL", + "code": "AAABfvWCZpMqn9UKACRWUw==", + "created": "2022-02-13T23:54:31.001Z", + "entensions": { + "x-paloalto-event": { + "version": 25 + } + }, + "category": [ + "file" + ], + "action": "FILE_WRITE" + }, + "5": { + "type": "file", + "name": "System" + }, + "6": { + "type": "process", + "name": "System", + "binary_ref": "5", + "pid": 4, + "extensions": { + "x-paloalto-process": { + "instance_id": "AdgdeB26mNQAAAAEAAAAAA==", + "causality_id": "AdgdeB26mNQAAAAEAAAAAA==", + "auth_id": "999", + "signature_vendor": "Microsoft Corporation", + "signature_status": "SIGNED", + "signature_product": "Microsoft Windows", + "execution_time": "2022-02-09T05:44:33.948Z", + "is_native": true + } + } + }, + "7": { + "type": "file", + "name": "System" + }, + "8": { + "type": "process", + "name": "System" + }, + "9": { + "type": "process", + "parent_ref": "8", + "binary_ref": "7", + "pid": 4 + }, + "10": { + "type": "file", + "name": "System" + }, + "11": { + "type": "process", + "name": "System" + }, + "12": { + "type": "process", + "parent_ref": "11", + "binary_ref": "10", + "pid": 4 + }, + "13": { + "type": "directory", + "path": "C:\\Windows\\System32\\config" + }, + "14": { + "type": "user-account", + "user_id": "S-1-5-18", + "display_name": "NT AUTHORITY\\SYSTEM", + "account_login": "1003" + } + }, + "first_observed": "2022-04-05T13:01:57.731Z", + "last_observed": "2022-04-05T13:01:57.731Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` +#### STIX Execute query +```shell +execute +paloalto +paloalto +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"paloalto\",\"identity_class\":\"system\",\"created\":\"2022-03-16T13:22:50.336Z\",\"modified\":\"2022-03-16T13:22:50.336Z\"}" +"{\"host\":\"xx.xx.xx\"}" +"{\"auth\":{\"api_key\": \"xxxx\", \"api_key_id\": \"xx\",\"tenant\":\"xxxx\"}}" +"[file:name = 'chrome.exe' ] START t'2022-01-16T11:00:00.000Z' STOP t'2022-01-20T11:00:00.003Z'" +``` +#### STIX Execute query - output +```json +{ + "type": "bundle", + "id": "bundle--1683947d-9c7e-4d35-b70b-a0ccc90d5cba", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "paloalto", + "identity_class": "system", + "created": "2022-03-16T13:22:50.336Z", + "modified": "2022-03-16T13:22:50.336Z" + }, + { + "id": "observed-data--34da8902-db25-4388-8312-526347de345d", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2022-04-05T13:05:20.377Z", + "modified": "2022-04-05T13:05:20.377Z", + "objects": { + "0": { + "type": "ipv4-addr", + "value": "172.31.31.236" + }, + "2": { + "type": "x-oca-asset", + "ip_refs": [ + "0" + ], + "extensions": { + "x-paloalto-agent": { + "agent_version": "7.6.1.46600", + "content_version": "350-80787", + "start_time": "2022-01-19T10:02:28.366Z", + "asset_id": "f344796340f84d0ca7e0fdaedbcbd594", + "os_type": "AGENT_OS_WINDOWS", + "os_sub_type": "Windows Server 2016 [10.0 (Build 14393)]", + "is_vdi": false + } + }, + "hostname": "EC2AMAZ-65BN1IK" + }, + "3": { + "type": "file", + "name": "Network Persistent State", + "size": 4595, + "parent_directory_ref": "13", + "extensions": { + "x-paloalto-file": { + "attributes": 0, + "type": 18, + "manifest_version": 5 + } + } + }, + "4": { + "type": "x-oca-event", + "file_ref": "3", + "process_ref": "6", + "parent_process_ref": "8", + "agent": "EC2AMAZ-65BN1IK", + "code": "AAABfnYIxcPunpI7AATXuQ==", + "created": "2022-01-20T05:49:53.598Z", + "entensions": { + "x-paloalto-event": { + "version": 25 + } + }, + "category": ["file"], + "action": "FILE_OPEN" + }, + "5": { + "type": "file", + "name": "chrome.exe", + "accessed": "2021-12-15T22:05:27.664Z", + "modified": "2021-12-12T08:19:30.068Z", + "parent_directory_ref": "14" + }, + "6": { + "type": "process", + "name": "chrome.exe", + "binary_ref": "5", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --type=utility --utility-sub-type=network.mojom.NetworkService --field-trial-handle=1456,2805956566820005361,12799224617572967118,131072 --lang=en-US --service-sandbox-type=none --mojo-platform-channel-handle=1772 /prefetch:8", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4724, + "extensions": { + "x-paloalto-process": { + "instance_id": "AdgAs0xnjjUAABJ0AAAAAA==", + "causality_id": "AdgAs0qzIE0AABHcAAAAAA==", + "auth_id": "999", + "signature_vendor": "Google LLC", + "signature_status": "SIGNED", + "signature_product": "Google LLC", + "extension": "exe", + "execution_time": "2022-01-03T15:05:08.844Z", + "is_native": true + } + } + }, + "7": { + "type": "file", + "name": "chrome.exe", + "parent_directory_ref": "15" + }, + "8": { + "type": "process", + "name": "chrome.exe" + }, + "9": { + "type": "process", + "parent_ref": "8", + "binary_ref": "7", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --flag-switches-begin --flag-switches-end --origin-trial-disabled-features=CaptureHandle", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4572 + }, + "10": { + "type": "file", + "name": "chrome.exe", + "accessed": "2021-12-15T22:05:27.664Z", + "modified": "2021-12-12T08:19:30.068Z", + "parent_directory_ref": "16" + }, + "11": { + "type": "process", + "name": "chrome.exe" + }, + "12": { + "type": "process", + "parent_ref": "11", + "binary_ref": "10", + "command_line": "\"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\" --type=utility --utility-sub-type=network.mojom.NetworkService --field-trial-handle=1456,2805956566820005361,12799224617572967118,131072 --lang=en-US --service-sandbox-type=none --mojo-platform-channel-handle=1772 /prefetch:8", + "created": "2021-11-10T06:59:32.948Z", + "pid": 4724 + }, + "13": { + "type": "directory", + "path": "C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Network" + }, + "14": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "15": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "16": { + "type": "directory", + "path": "C:\\Program Files\\Google\\Chrome\\Application" + }, + "17": { + "type": "user-account", + "user_id": "S-1-5-21-3039464837-300237904-2407637926-500", + "display_name": "EC2AMAZ-65BN1IK\\Administrator", + "account_login": "1003" + } + }, + "x_actor_process_last_observed": "2021-12-15T22:05:27.664Z", + "x_os_actor_process_last_observed": "2021-12-15T22:05:27.664Z", + "x_process_actor_first_observed": "2021-11-10T06:59:32.948Z", + "x_process_causality_actor_first_observed": "2021-11-10T06:59:32.948Z", + "x_process_os_actor_first_observed": "2021-11-10T06:59:32.948Z", + "first_observed": "2022-04-05T13:05:20.377Z", + "last_observed": "2022-04-05T13:05:20.377Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +### Limitations +- Each XQL API query entails a cost of query units calculated according to the complexity and number of search results. +- Queries called without enough quota will fail. + +### Observations +- The quota limit range for the API call would be from 1 to 15 (Standard 5 + 10 Additional ) units. + The default value of quota thershold limit in CP4S UI is 5. +- If the user purchases standard license only(5 units), and configures the threshold value more than 5, + the usage limit will be checked only for 5 units. Otherwise, usage will be checked for the configured value. +- If the user purchases Additional 10 units along with standard license, the usage limit will be checked for configured value. + +### References +- [Cortex XDR](https://docs.paloaltonetworks.com/cortex/cortex-xdr.html) +- [Get Started with XQL](https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-xql-language-reference/get-started-with-xql.html) +- [Cortex XDR™ XQL Schema Reference](https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-xql-schema-reference.html) +- [Cortex XDR API Overview](https://docs.paloaltonetworks.com/cortex/cortex-xdr/cortex-xdr-api/cortex-xdr-api-overview.html) diff --git a/stix_shifter_modules/paloalto/__init__.py b/stix_shifter_modules/paloalto/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/paloalto/configuration/config.json b/stix_shifter_modules/paloalto/configuration/config.json new file mode 100644 index 000000000..b661c3df1 --- /dev/null +++ b/stix_shifter_modules/paloalto/configuration/config.json @@ -0,0 +1,35 @@ +{ + "connection": { + "type": { + "displayName": "Palo Alto Cortex XDR" + }, + "host": { + "type": "text" + }, + "quota_threshold": { + "default": 5, + "min": 1, + "max": 15, + "type": "number", + "optional" : true + }, + "help": { + "type": "link", + "default": "data-sources.html" + } + }, + "configuration": { + "auth": { + "type" : "fields", + "tenant": { + "type": "password" + }, + "api_key": { + "type": "password" + }, + "api_key_id": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/configuration/lang_en.json b/stix_shifter_modules/paloalto/configuration/lang_en.json new file mode 100644 index 000000000..be3f57075 --- /dev/null +++ b/stix_shifter_modules/paloalto/configuration/lang_en.json @@ -0,0 +1,32 @@ +{ + "connection": { + "host": { + "label": "Management IP address or Hostname", + "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" + }, + "quota_threshold": { + "label": "The quota limit for the API", + "description": "Prevents the connector from exceeding the entire daily units allocated for the License. The Daily units for Standard license alone is 5 and it will be 15 if additional 10 units are purchased." + }, + "help": { + "label": "Need additional help?", + "description": "More details on the data source setting can be found in the specified link" + } + }, + "configuration": { + "auth": { + "tenant": { + "label": "Tenant", + "description": "Tenant Id of Palo Alto Cortex XDR Application" + }, + "api_key": { + "label": "API Key", + "description": "The API Key is a unique identifier required for authenticating API calls." + }, + "api_key_id": { + "label": "API Key Id", + "description": "The API Key ID is a unique token used to authenticate the API Key" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/entry_point.py b/stix_shifter_modules/paloalto/entry_point.py new file mode 100644 index 000000000..8c8fedd99 --- /dev/null +++ b/stix_shifter_modules/paloalto/entry_point.py @@ -0,0 +1,14 @@ +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint + + +class EntryPoint(BaseEntryPoint): + + # python main.py translate paloalto query '{}' "[ipv4-addr:value = '127.0.0.1']" + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + self.set_async(True) + if connection: + self.setup_transmission_simple(connection, configuration) + + self.setup_translation_simple(dialect_default='xdr_data') diff --git a/stix_shifter_modules/paloalto/stix_translation/__init__.py b/stix_shifter_modules/paloalto/stix_translation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/paloalto/stix_translation/json/config_map.json b/stix_shifter_modules/paloalto/stix_translation/json/config_map.json new file mode 100644 index 000000000..f8add745e --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/config_map.json @@ -0,0 +1,103 @@ +{ + "int_supported_fields": [ + "action_local_port", + "action_remote_port", + "action_pkts_sent", + "action_pkts_received", + "action_file_size", + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid", + "action_process_requested_parent_pid", + "action_thread_parent_pid", + "action_thread_child_pid" + ], + "timestamp_supported_fields": [ + "action_file_access_time", + "actor_process_file_access_time", + "os_actor_process_file_access_time", + "action_file_mod_time", + "actor_process_file_mod_time", + "os_actor_process_file_mod_time", + "action_file_create_time", + "action_process_file_create_time", + "actor_process_file_create_time", + "causality_actor_process_file_create_time", + "os_actor_process_file_create_time", + "action_process_instance_execution_time", + "actor_process_execution_time", + "action_network_creation_time", + "event_timestamp" + ], + "mac_supported_fields": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" + ], + "enum_supported_fields": [ + "action_network_protocol", + "event_type", + "event_sub_type" + ], + "enum_supported_values": { + "action_network_protocol": ["TCP", "UDP"], + "event_type": [ + "AGENT_STATUS","CLOUD_AUDIT_LOGS","DEVICE","EVENT_LOG","FILE","GAP","HOST_FIREWALL", + "HOST_METADATA","HOST_STATUS","INJECTION","LOAD_IMAGE","LOGIN_EVENT","MOUNT","NAMESPACE", + "NETWORK","PROCESS","PROCESS_HANDLE","REGISTRY","RPC_CALL","STORY","SYSTEM_CALL", + "THREAD","USER_STATUS_CHANGE","VPN_EVENT"], + "event_sub_type": [ + "AGENT_STATUS_AGENT_BOOT" ,"AGENT_STATUS_AGENT_INSTALLED", + "AGENT_STATUS_AGENT_LOW_LEVEL_CONFIG_UPDATE" ,"AGENT_STATUS_AGENT_POLICY_FILTER_UPDATE" , + "AGENT_STATUS_AGENT_REPLAY_ENDED" ,"AGENT_STATUS_AGENT_SHUTDOWN" , + "AGENT_STATUS_AGENT_UNINSTALLED" ,"DEVICE_PLUG" ,"DEVICE_UNPLUG" , + "EVENT_LOG_AGENT_EVENT_LOG" ,"EVENT_LOG_DOMAIN_CONTROLLER" ,"FILE_CHANGE_MODE" , + "FILE_CHANGE_OWNER" ,"FILE_CREATE_NEW" ,"FILE_DELETE_EXT_ATTRIBUTE" , + "FILE_DIR_CHANGE_MODE" ,"FILE_DIR_CHANGE_OWNER" ,"FILE_DIR_CREATE" ,"FILE_DIR_LINK" , + "FILE_DIR_OPEN" ,"FILE_DIR_QUERY" ,"FILE_DIR_REMOVE" ,"FILE_DIR_RENAME" , + "FILE_DIR_SET_ATTR" ,"FILE_DIR_WRITE" ,"FILE_LINK" ,"FILE_OPEN" ,"FILE_REMOVE" , + "FILE_RENAME" ,"FILE_REPARSE" ,"FILE_SET_ATTRIBUTE" ,"FILE_SET_SECURITY_DESCRIPTOR" , + "FILE_WRITE" ,"GAP_GAP_ENDED" ,"GAP_GAP_STARTED", "HOST_FIREWALL_HOST_FW_ALLOW" , + "HOST_FIREWALL_HOST_FW_BLOCK" ,"HOST_METADATA_CHANGE" ,"HOST_STATUS_BOOT" , + "HOST_STATUS_RESUME" ,"HOST_STATUS_SHUTDOWN" ,"HOST_STATUS_SUSPEND" , + "INJECTION_CREATE_REMOTE_THREAD" ,"INJECTION_PROCESS_HOLLOW" ,"INJECTION_QUEUE_APC" , + "INJECTION_SET_THREAD_CONTEXT" ,"LOAD_IMAGE_MODULE" ,"LOAD_IMAGE_MPROTECT" , + "LOAD_IMAGE_PRELOAD" ,"LOAD_IMAGE_SO_LOAD" ,"MOUNT_DRIVE_MOUNT" , + "MOUNT_DRIVE_UNMOUNT" ,"NAMESPACE_SETNS" ,"NAMESPACE_UNSHARE" , + "NETWORK_DATAGRAM_STATISTICS" ,"NETWORK_DNS_QUERY" ,"NETWORK_HTTP_HEADER" , + "NETWORK_OUTBOUND_ICMP" ,"NETWORK_PROXY_CONNECT" ,"NETWORK_RAW_DATA" , + "NETWORK_RAW_PCAP_DATA" ,"NETWORK_STREAM_ACCEPT" ,"NETWORK_STREAM_CONNECT" , + "NETWORK_STREAM_CONNECT_FAILED" ,"NETWORK_STREAM_DISCONNECT" ,"NETWORK_STREAM_LISTEN" , + "NETWORK_STREAM_STATISTICS" ,"PROCESS_START" ,"PROCESS_STOP" ,"REGISTRY_CREATE_KEY" , + "REGISTRY_DELETE_KEY" ,"REGISTRY_DELETE_VALUE" ,"REGISTRY_LOAD" ,"REGISTRY_RENAME_KEY" , + "REGISTRY_RESTORE" ,"REGISTRY_SAVE" ,"REGISTRY_SET_VALUE" ,"REGISTRY_UNLOAD" , + "RPC_CALL_RPC_CALLED" ,"SYSTEM_CALL_CREATE_SYMBOLIC_LINK" ,"SYSTEM_CALL_GDI_BIT_BLT" , + "SYSTEM_CALL_GET_CLIPBOARD_DATA" ,"SYSTEM_CALL_NT_ALLOCATE_VIRTUAL_MEMORY_REMOTE" , + "SYSTEM_CALL_NT_CREATE_MUTANT" ,"SYSTEM_CALL_NT_DELAY_EXECUTION" , + "SYSTEM_CALL_NT_MAP_VIEW_OF_SECTION_REMOTE" ,"SYSTEM_CALL_NT_OPEN_FILE" , + "SYSTEM_CALL_NT_PROTECT_VIRTUAL_MEMORY_REMOTE" ,"SYSTEM_CALL_NT_QUEUE_APC_THREAD_EX_REMOTE" , + "SYSTEM_CALL_NT_READ_VIRTUAL_MEMORY_REMOTE" ,"SYSTEM_CALL_NT_SET_CONTEXT_THREAD_REMOTE" , + "SYSTEM_CALL_NT_TERMINATE_PROCESS_REMOTE","SYSTEM_CALL_NT_UNMAP_VIEW_OF_SECTION" , + "SYSTEM_CALL_NT_WRITE_VIRTUAL_MEMORY_REMOTE" ,"SYSTEM_CALL_OPEN_REMOTE_TOKEN" , + "SYSTEM_CALL_QUERY_PROCESS_LIST" ,"SYSTEM_CALL_QUERY_USER_TOKEN" , + "SYSTEM_CALL_REGISTER_RAW_INPUT_DEVICES" ,"SYSTEM_CALL_SET_INFO_VIRTUAL_MEMORY" , + "SYSTEM_CALL_SET_WINDOWS_HOOK_EX" ,"SYSTEM_CALL_SET_WIN_EVENT_HOOK" , + "SYSTEM_CALL_THREAD_IMPERSONATING" ,"USER_STATUS_CHANGE_LOGOFF" ,"USER_STATUS_CHANGE_LOGON" + ] + }, + "timestamp_supported_dataset": { + "xdr_data": "_time" + }, + "mandatory_properties_to_stix": { + "user": ["actor_primary_user_sid","action_process_user_sid"], + "file": ["action_file_name"], + "file_action_process" : ["action_process_image_name"], + "file_actor_process": ["actor_process_image_name"], + "file_causality_process": ["causality_actor_process_image_name"], + "file_os_actor": ["os_actor_process_image_name"], + "nt": [["action_local_ip","action_remote_ip"],"action_network_protocol"] + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json b/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json new file mode 100644 index 000000000..f0f2cdc5a --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/fields_map.json @@ -0,0 +1,45 @@ +{ + "all_fields": [ + "action_local_ip","action_remote_ip","agent_ip_addresses","agent_ip_addresses_v6","dst_agent_ip_addresses_v6", + "action_local_port","action_remote_port","action_network_protocol","action_pkts_sent", + "action_pkts_received","action_file_name","action_process_image_name","actor_process_image_name", + "causality_actor_process_image_name","os_actor_process_image_name","action_file_size","action_file_md5", + "action_module_md5","action_process_image_md5","action_file_authenticode_sha1", + "action_file_authenticode_sha2","action_file_sha256","action_module_sha256", + "action_process_image_sha256","action_file_access_time","actor_process_file_access_time", + "os_actor_process_file_access_time","action_file_mod_time","actor_process_file_mod_time", + "os_actor_process_file_mod_time","action_file_create_time","action_file_path", + "action_process_image_path","action_registry_file_path","actor_process_image_path", + "causality_actor_process_image_path","os_actor_process_image_path","action_process_image_command_line", + "actor_process_command_line","causality_actor_process_command_line","os_actor_process_command_line", + "action_process_file_create_time","actor_process_file_create_time", + "causality_actor_process_file_create_time","os_actor_process_file_create_time", + "action_module_process_os_pid","action_process_os_pid","actor_process_os_pid", + "causality_actor_process_os_pid","os_actor_process_os_pid","action_process_requested_parent_pid", + "action_thread_parent_pid","action_thread_child_pid","action_process_username","auth_domain", + "dst_host_metadata_domain","host_metadata_domain","dst_action_url_category","action_registry_key_name", + "action_registry_value_name","mac","associated_mac","dst_associated_mac","dst_mac", + "actor_primary_user_sid","action_process_user_sid","actor_primary_username","actor_process_logon_id", + "action_file_info_company","action_file_extension","action_file_attributes", + "action_file_internal_zipped_files","action_file_last_writer_actor","action_file_signature_status", + "action_file_signature_vendor","action_file_signature_product","action_file_info_description", + "action_file_group","action_file_group_name","action_file_type","action_file_info_file_version", + "manifest_file_version","action_file_info_product_version","action_file_owner", + "action_file_owner_name","action_file_info_product_name","action_file_id", + "action_file_wildfire_verdict","action_file_hash_control_verdict","actor_process_instance_id", + "actor_process_causality_id","actor_process_auth_id","actor_process_container_id", + "actor_process_signature_vendor","actor_process_signature_status","actor_process_signature_product", + "actor_process_image_extension","action_process_termination_code","action_process_termination_date", + "action_remote_process_thread_id","action_process_instance_execution_time", + "actor_process_execution_time","action_process_handle_is_kernel","action_process_is_container_root", + "actor_process_is_native","agent_version","agent_hostname","agent_content_version", + "agent_session_start_time","agent_id","agent_os_type","agent_os_sub_type", + "agent_is_vdi","action_user_agent","http_req_user_agent_header","action_evtlog_data_fields", + "action_evtlog_description","action_evtlog_source","action_evtlog_event_id","action_evtlog_level", + "action_evtlog_tid","action_evtlog_uid","action_evtlog_pid","action_evtlog_message", + "action_evtlog_version","event_id","vpn_event_description","event_timestamp","event_version", + "event_rpc_interface_uuid","event_address_mapped_image_path","event_type","event_sub_type", + "action_network_creation_time","action_network_connection_id","action_network_packet_data", + "action_proxy","host_metadata_hostname","action_external_hostname" + ] +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/operators.json b/stix_shifter_modules/paloalto/stix_translation/json/operators.json new file mode 100644 index 000000000..c4048192e --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/operators.json @@ -0,0 +1,15 @@ +{ + "ComparisonExpressionOperators.And": "and", + "ComparisonExpressionOperators.Or": "or", + "ComparisonComparators.Equal": "=", + "ComparisonComparators.NotEqual": "!=", + "ComparisonComparators.Like": "contains", + "ComparisonComparators.Matches": "~=", + "ComparisonComparators.GreaterThan": ">", + "ComparisonComparators.GreaterThanOrEqual": ">=", + "ComparisonComparators.LessThan": "<", + "ComparisonComparators.LessThanOrEqual": "<=", + "ComparisonComparators.In": "in", + "ObservationOperators.Or": "or", + "ObservationOperators.And": "or" + } \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..c423b00ce --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,1064 @@ +{ + "xdr_data": { + "action_local_ip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ], + "group": true + } + ], + "action_remote_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "dst_ip" + ], + "group": true + } + ], + "agent_ip_addresses": [ + { + "key": "ipv4-addr.value", + "object": "agent_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "agent_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "agent_ip" + ], + "group": true + } + ], + "agent_ip_addresses_v6": [ + { + "key": "ipv6-addr.value", + "object": "src_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ], + "group": true + } + ], + "dst_agent_ip_addresses_v6": [ + { + "key": "ipv6-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": ["dst_ip"], + "group": true + } + ], + "action_local_port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "action_remote_port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "action_network_protocol": [ + { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "action_pkts_sent": { + "key": "network-traffic.src_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "action_pkts_received": { + "key": "network-traffic.dst_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "action_file_name": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + } + ], + "action_process_image_name": [ + { + "key": "file.name", + "object": "file_action_process" + }, + { + "key": "process.image_ref", + "object": "process_action", + "references": "file_action_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_image_name": [ + { + "key": "file.name", + "object": "file_actor_process" + }, + { + "key": "process.image_ref", + "object": "process_actor", + "references": "file_actor_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_image_name": [ + { + "key": "file.name", + "object": "file_causality_process" + }, + { + "key": "process.parent_ref", + "object": "process_causality_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_image_name": [ + { + "key": "file.name", + "object": "file_os_actor" + }, + { + "key": "process.parent_ref", + "object": "process_os_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_os_actor" + } + ], + "action_file_size": { + "key": "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_authenticode_sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "action_file_authenticode_sha2": { + "key": "file.hashes.SHA-2", + "object": "file" + }, + "action_file_md5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "action_module_md5": { + "key": "file.hashes.MD5", + "object": "file_action_module" + }, + "action_process_image_md5": [ + { + "key": "file.hashes.MD5", + "object": "file_action_process" + }, + { + "key": "process.image_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "action_file_sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "action_module_sha256": { + "key": "file.hashes.SHA-256", + "object": "file_action_module" + }, + "action_process_image_sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "file_action_process" + }, + { + "key": "process.image_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "action_file_access_time": [ + { + "key": "file.atime", + "object": "file", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_action_file_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_access_time": [ + { + "key": "file.atime", + "object": "file_actor_process", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_actor_process_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_access_time": [ + { + "key": "file.atime", + "object": "file_os_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_os_actor_process_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_file_mod_time": [ + { + "key": "file.mtime", + "object": "file", + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_mod_time": [ + { + "key": "file.mtime", + "object": "file_actor_process", + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_mod_time": [ + { + "key": "file.mtime", + "object": "file_os_actor", + "transformer": "EpochToTimestamp" + } + ], + "action_file_create_time": [ + { + "key": "file.ctime", + "object": "file", + "transformer": "EpochToTimestamp" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_file_path": [ + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "action_registry_file_path": [ + { + "key": "directory.path", + "object": "directory_action_registry", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_action_registry", + "references": "directory_action_registry" + } + ], + "action_process_image_path": [ + { + "key": "directory.path", + "object": "directory_action_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_action_process", + "references": "directory_action_process" + }, + { + "key": "process.image_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_actor_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_actor_process", + "references": "directory_actor_process" + }, + { + "key": "process.image_ref", + "object": "process_actor", + "references": "file_actor_process" + } + ], + "causality_actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_causality_actor", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_causality_process", + "references": "directory_causality_actor" + }, + { + "key": "process.image_ref", + "object": "process_causality_actor", + "references": "file_causality_process" + } + ], + "os_actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_os_actor", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_os_actor", + "references": "directory_os_actor" + }, + { + "key": "process.image_ref", + "object": "process_os_actor", + "references": "file_os_actor" + } + ], + "action_process_image_command_line": [ + { + "key": "process.command_line", + "object": "process_action" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_actor" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_causality_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_os_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_os_actor" + } + ], + "action_process_file_create_time": [ + { + "key": "process.created_time", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_create_time": [ + { + "key": "process.created_time", + "object": "process_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "causality_actor_process_file_create_time": [ + { + "key": "process.created_time", + "object": "process_causality_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_causality_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_create_time": [ + { + "key": "process.created_time", + "object": "process_os_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_os_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_module_process_os_pid": [ + { + "key": "process.pid", + "object": "process_action_module", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action_module" + } + ], + "action_process_os_pid": [ + { + "key": "process.pid", + "object": "process_action", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_causality_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_os_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_os_actor" + } + ], + "action_process_requested_parent_pid": [ + { + "key": "process.pid", + "object": "action_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process_action", + "references": "action_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "action_parent_process" + } + ], + "action_thread_parent_pid": [ + { + "key": "process.pid", + "object": "action_parent_thread", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process_action_thread", + "references": "action_parent_thread" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "action_parent_thread" + } + ], + "action_thread_child_pid": [ + { + "key": "process.pid", + "object": "action_child_thread", + "transformer": "ToInteger" + }, + { + "key": "process.child_refs", + "object": "process_action", + "references": "action_child_thread" + } + ], + "auth_domain": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain" + } + ], + "dst_host_metadata_domain": [ + { + "key": "domain-name.value", + "object": "domain_dst" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain_dst" + } + ], + "host_metadata_domain": [ + { + "key": "domain-name.value", + "object": "domain_host" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain_host" + } + ], + "dst_action_url_category": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "action_registry_key_name": [ + { + "key": "windows-registry-key.key", + "object": "windowsregistry" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "windowsregistry" + } + ], + "action_registry_value_name": [ + { + "key": "windows-registry-key.values", + "object": "windowsregistry", + "transformer": "FormatToStixRegistryValue" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "windowsregistry" + } + ], + "mac": [ + { + "key": "mac-addr.value", + "object": "mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["mac"] + } + ], + "associated_mac": [ + { + "key": "mac-addr.value", + "object": "ass_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["ass_mac"] + } + ], + "dst_associated_mac": [ + { + "key": "mac-addr.value", + "object": "dst_ass_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["dst_ass_mac"] + } + ], + "dst_mac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["dst_mac"] + } + ], + "actor_primary_user_sid": { + "key": "user-account.user_id", + "object": "user" + }, + "action_process_user_sid": { + "key": "user-account.extensions.x-paloalto-user-ext.process_user_id", + "object": "user" + }, + "actor_primary_username": { + "key": "user-account.display_name", + "object": "user" + }, + "action_process_username": [ + { + "key": "user-account.extensions.x-paloalto-user-ext.process_user_name", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process_action", + "references": "user" + } + ], + "actor_process_logon_id": { + "key": "user-account.account_login", + "object": "user" + }, + "action_file_info_company": { + "key": "file.extensions.x-paloalto-file-ext.company", + "object": "file" + }, + "action_file_extension": { + "key": "file.extensions.x-paloalto-file-ext.extension", + "object": "file" + }, + "action_file_attributes": { + "key": "file.extensions.x-paloalto-file-ext.attributes", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_internal_zipped_files": { + "key": "file.extensions.x-paloalto-file-ext.zipped_files", + "object": "file" + }, + "action_file_last_writer_actor": { + "key": "file.extensions.x-paloalto-file-ext.writer", + "object": "file" + }, + "action_file_mode": { + "key": "file.extensions.x-paloalto-file-ext.mode", + "object": "file" + }, + "action_file_signature_status": { + "key": "file.extensions.x-paloalto-file-ext.signature_status", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_signature_vendor": { + "key": "file.extensions.x-paloalto-file-ext.signature_vendor", + "object": "file" + }, + "action_file_signature_product": { + "key": "file.extensions.x-paloalto-file-ext.signature_product", + "object": "file" + }, + "action_file_info_description": { + "key": "file.extensions.x-paloalto-file-ext.file_description", + "object": "file" + }, + "action_file_group": { + "key": "file.extensions.x-paloalto-file-ext.group", + "object": "file" + }, + "action_file_group_name": { + "key": "file.extensions.x-paloalto-file-ext.group_name", + "object": "file" + }, + "action_file_type": { + "key": "file.extensions.x-paloalto-file-ext.type", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_info_file_version": { + "key": "file.extensions.x-paloalto-file-ext.version", + "object": "file" + }, + "manifest_file_version": { + "key": "file.extensions.x-paloalto-file-ext.manifest_version", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_info_product_version": { + "key": "file.extensions.x-paloalto-file-ext.product_version", + "object": "file" + }, + "action_file_owner": { + "key": "file.extensions.x-paloalto-file-ext.owner", + "object": "file" + }, + "action_file_owner_name": { + "key": "file.extensions.x-paloalto-file-ext.owner_name", + "object": "file" + }, + "action_file_info_product_name": { + "key": "file.extensions.x-paloalto-file-ext.product_name", + "object": "file" + }, + "action_file_id": { + "key": "file.extensions.x-paloalto-file-ext.id", + "object": "file" + }, + "action_file_wildfire_verdict": { + "key": "file.extensions.x-paloalto-file-ext.wildfire_verdict", + "object": "file" + }, + "action_file_hash_control_verdict": { + "key": "file.extensions.x-paloalto-process-ext.control_verdict", + "object": "file" + }, + "actor_process_instance_id": { + "key": "process.extensions.x-paloalto-process-ext.instance_id", + "object": "process_actor" + }, + "actor_process_causality_id": { + "key": "process.extensions.x-paloalto-process-ext.causality_id", + "object": "process_actor" + }, + "actor_process_auth_id": { + "key": "process.extensions.x-paloalto-process-ext.auth_id", + "object": "process_actor" + }, + "actor_process_container_id": { + "key": "process.extensions.x-paloalto-process-ext.container_id", + "object": "process_actor" + }, + "actor_process_signature_vendor": { + "key": "process.extensions.x-paloalto-process-ext.signature_vendor", + "object": "process_actor" + }, + "actor_process_signature_status": { + "key": "process.extensions.x-paloalto-process-ext.signature_status", + "object": "process_actor" + }, + "actor_process_signature_product": { + "key": "process.extensions.x-paloalto-process-ext.signature_product", + "object": "process_actor" + }, + "actor_process_image_extension": { + "key": "process.extensions.x-paloalto-process-ext.extension", + "object": "process_actor" + }, + "action_process_termination_code": { + "key": "process.extensions.x-paloalto-process-ext.termination_code", + "object": "process_action", + "transformer": "ToInteger" + }, + "action_process_termination_date": { + "key": "process.extensions.x-paloalto-process-ext.termination_date", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + "action_remote_process_thread_id": { + "key": "process.extensions.x-paloalto-process-ext.tid", + "object": "process_action", + "transformer": "ToInteger" + }, + "action_process_instance_execution_time": { + "key": "process.extensions.x-paloalto-process-ext.instance_exec_time", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + "actor_process_execution_time": { + "key": "process.extensions.x-paloalto-process-ext.execution_time", + "object": "process_actor", + "transformer": "EpochToTimestamp" + }, + "action_process_handle_is_kernel": { + "key": "process.extensions.x-paloalto-process-ext.is_kernel", + "object": "process_action", + "transformer": "StringToBool" + }, + "action_process_is_container_root": { + "key": "process.extensions.x-paloalto-process-ext.is_root", + "object": "process_action", + "transformer": "StringToBool" + }, + "actor_process_is_native": { + "key": "process.extensions.x-paloalto-process-ext.is_native", + "object": "process_actor", + "transformer": "StringToBool" + }, + "agent_version": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.agent_version", + "object": "asset" + }, + "agent_hostname": [ + { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + { + "key": "x-oca-event.agent", + "object": "event" + } + ], + "agent_content_version": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.content_version", + "object": "asset" + }, + "agent_session_start_time": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.start_time", + "object": "asset", + "transformer": "EpochToTimestamp" + }, + "agent_id": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.asset_id", + "object": "asset" + }, + "agent_os_type": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.os_type", + "object": "asset" + }, + "agent_os_sub_type": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.os_sub_type", + "object": "asset" + }, + "agent_is_vdi": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.is_vdi", + "object": "asset", + "transformer": "StringToBool" + }, + "action_user_agent": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.user_agent", + "object": "asset" + }, + "http_req_user_agent_header": { + "key": "x-oca-asset.extensions.x-paloalto-agent-ext.agent_header", + "object": "asset" + }, + "action_evtlog_data_fields": { + "key": "x-paloalto-evtlog.data_fields", + "object": "evtlog" + }, + "action_evtlog_description": { + "key": "x-paloalto-evtlog.description", + "object": "evtlog" + }, + "action_evtlog_source": { + "key": "x-paloalto-evtlog.source", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_event_id": { + "key": "x-paloalto-evtlog.evtlog_id", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_level": { + "key": "x-paloalto-evtlog.level", + "object": "evtlog" + }, + "action_evtlog_tid": { + "key": "x-paloalto-evtlog.tid", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_uid": { + "key": "x-paloalto-evtlog.uid", + "object": "evtlog" + }, + "action_evtlog_pid": { + "key": "x-paloalto-evtlog.pid", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_message": { + "key": "x-paloalto-evtlog.message", + "object": "evtlog" + }, + "action_evtlog_version": { + "key": "x-paloalto-evtlog.version", + "object": "evtlog", + "transformer": "ToInteger" + }, + "event_id": { + "key": "x-oca-event.code", + "object": "event" + }, + "vpn_event_description": { + "key": "x-oca-event.entensions.x-paloalto-event-ext.event_description", + "object": "event" + }, + "event_timestamp": { + "key": "x-oca-event.created", + "object": "event", + "transformer": "EpochToTimestamp" + }, + "event_version": { + "key": "x-oca-event.entensions.x-paloalto-event-ext.version", + "object": "event", + "transformer": "ToInteger" + }, + "event_rpc_interface_uuid": { + "key": "x-oca-event.entensions.x-paloalto-event-ext.uuid", + "object": "event" + }, + "event_address_mapped_image_path": { + "key": "x-oca-event.entensions.x-paloalto-event-ext.path", + "object": "event" + }, + "event_type": { + "key": "x-oca-event.category", + "object": "event", + "transformer": "ToLowercaseArray" + }, + "event_sub_type": { + "key": "x-oca-event.action", + "object": "event" + }, + "action_network_creation_time": { + "key": "network-traffic.extensions.x-paloalto-network-ext.creation_time", + "object": "nt", + "transformer": "EpochToTimestamp" + }, + "action_network_connection_id": { + "key": "network-traffic.extensions.x-paloalto-network-ext.connection_id", + "object": "nt" + }, + "action_network_packet_data": { + "key": "network-traffic.extensions.x-paloalto-network-ext.packet_data", + "object": "nt" + }, + "action_proxy": { + "key": "network-traffic.extensions.x-paloalto-network-ext.is_proxy", + "object": "nt", + "transformer": "StringToBool" + }, + "host_metadata_hostname": { + "key": "network-traffic.extensions.x-paloalto-network-ext.metadata_hostname", + "object": "nt" + }, + "action_external_hostname": { + "key": "network-traffic.extensions.x-paloalto-network-ext.external_hostname", + "object": "nt" + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json new file mode 100644 index 000000000..4d3500cff --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/stix_2_1/xdr_data_from_stix_map.json @@ -0,0 +1,361 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "action_local_ip", + "action_remote_ip", + "agent_ip_addresses" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "agent_ip_addresses_v6", + "dst_agent_ip_addresses_v6" + ] + } + }, + "network-traffic": { + "fields": { + "src_port": [ + "action_local_port" + ], + "dst_port": [ + "action_remote_port" + ], + "protocols[*]": [ + "action_network_protocol" + ], + "src_ref.value": [ + "action_local_ip", + "agent_ip_addresses" + ], + "dst_ref.value": [ + "action_remote_ip" + ], + "src_packets": [ + "action_pkts_sent" + ], + "dst_packets": [ + "action_pkts_received" + ] + } + }, + "file": { + "fields": { + "name": [ + "action_file_name", + "action_process_image_name", + "actor_process_image_name", + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "size": [ + "action_file_size" + ], + "hashes.MD5": [ + "action_file_md5", + "action_module_md5", + "action_process_image_md5" + ], + "hashes.'SHA-1'": [ + "action_file_authenticode_sha1" + ], + "hashes.'SHA-2'": [ + "action_file_authenticode_sha2" + ], + "hashes.'SHA-256'": [ + "action_file_sha256", + "action_module_sha256", + "action_process_image_sha256" + ], + "atime": [ + "action_file_access_time", + "actor_process_file_access_time", + "os_actor_process_file_access_time" + ], + "mtime": [ + "action_file_mod_time", + "actor_process_file_mod_time", + "os_actor_process_file_mod_time" + ], + "ctime": [ + "action_file_create_time" + ], + "parent_directory_ref.path": [ + "action_file_path", + "action_process_image_path", + "action_registry_file_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "directory": { + "fields": { + "path": [ + "action_file_path", + "action_process_image_path", + "action_registry_file_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "process": { + "fields": { + "command_line": [ + "action_process_image_command_line", + "actor_process_command_line", + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "created_time": [ + "action_process_file_create_time", + "actor_process_file_create_time", + "causality_actor_process_file_create_time", + "os_actor_process_file_create_time" + ], + "pid": [ + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid", + "action_process_requested_parent_pid", + "action_thread_parent_pid", + "action_thread_child_pid" + ], + "parent_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "child_refs.pid": [ + "action_thread_child_pid" + ], + "creator_user_ref.user_id": [ + "action_process_username" + ], + "parent_ref.name": [ + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "image_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "image_ref.hashes.MD5": [ + "action_process_image_md5" + ], + "image_ref.hashes.'SHA-256'": [ + "action_process_image_sha256" + ], + "image_ref.parent_directory_ref.path": [ + "action_process_image_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "auth_domain", + "dst_host_metadata_domain", + "host_metadata_domain" + ] + } + }, + "url": { + "fields": { + "value": [ + "dst_action_url_category" + ] + } + }, + "windows-registry-key": { + "fields": { + "key": [ + "action_registry_key_name" + ], + "values[*]": [ + "action_registry_value_name" + ] + } + }, + "mac-addr": { + "fields": { + "value": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" + ] + } + }, + "user-account": { + "fields": { + "user_id": [ + "actor_primary_user_sid", + "action_process_user_sid" + ], + "display_name": [ + "actor_primary_username", + "action_process_username" + ], + "account_login": [ + "actor_process_logon_id" + ] + } + }, + "x-paloalto-file": { + "fields": { + "extension": [ + "action_file_extension" + ], + "file_description": [ + "action_file_info_description" + ] + } + }, + "x-paloalto-process": { + "fields": { + "extension": [ + "actor_process_image_extension" + ], + "execution_time": [ + "action_process_instance_execution_time", + "actor_process_execution_time" + ] + } + }, + "x-oca-asset": { + "fields": { + "hostname": [ + "agent_hostname" + ], + "ip_refs[*].value": [ + "action_local_ip", + "action_remote_ip", + "agent_ip_addresses_v6", + "agent_ip_addresses", + "dst_agent_ip_addresses_v6" + ], + "mac_refs[*].value": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" + ] + } + }, + "x-paloalto-evtlog": { + "fields": { + "description": [ + "action_evtlog_description" + ], + "message": [ + "action_evtlog_message" + ] + } + }, + "x-oca-event": { + "fields": { + "code": [ + "event_id" + ], + "category[*]": [ + "event_type" + ], + "action": [ + "event_sub_type" + ], + "created": [ + "event_timestamp" + ], + "agent": [ + "agent_hostname" + ], + "url_ref.value": [ + "dst_action_url_category" + ], + "file_ref.name": [ + "action_file_name" + ], + "process_ref.pid": [ + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid" + ], + "process_ref.command_line": [ + "action_process_image_command_line", + "actor_process_command_line" + ], + "process_ref.image_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "process_ref.parent_ref.name": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "process_ref.parent_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "process_ref.parent_ref.command_line": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "parent_process_ref.name": [ + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "parent_process_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "parent_process_ref.command_line": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "process_ref.creator_user_ref.user_id": [ + "action_process_username" + ], + "process_ref.image_ref.hashes.MD5": [ + "action_process_image_md5" + ], + "process_ref.image_ref.hashes.'SHA-256'": [ + "action_process_image_sha256" + ], + "domain_ref.value": [ + "auth_domain", + "dst_host_metadata_domain", + "host_metadata_domain" + ], + "registry_ref.key": [ + "action_registry_key_name" + ], + "registry_ref.values[*]": [ + "action_registry_value_name" + ] + } + }, + "x-paloalto-network": { + "fields": { + "creation_time": [ + "action_network_creation_time" + ], + "hostname": [ + "host_metadata_hostname", + "action_external_hostname" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json new file mode 100644 index 000000000..e780d4cc5 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/to_stix_map.json @@ -0,0 +1,1080 @@ +{ + "xdr_data": { + "action_local_ip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ], + "group": true + } + ], + "action_remote_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "dst_ip" + ], + "group": true + } + ], + "agent_ip_addresses": [ + { + "key": "ipv4-addr.value", + "object": "agent_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "agent_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": ["agent_ip"], + "group": true + } + ], + "agent_ip_addresses_v6": [ + { + "key": "ipv6-addr.value", + "object": "src_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "src_ip" + ], + "group": true + } + ], + "dst_agent_ip_addresses_v6": [ + { + "key": "ipv6-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": ["dst_ip"], + "group": true + } + ], + "action_local_port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "action_remote_port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "action_network_protocol": [ + { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "action_pkts_sent": { + "key": "network-traffic.src_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "action_pkts_received": { + "key": "network-traffic.dst_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "action_file_name": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + } + ], + "action_process_image_name": [ + { + "key": "file.name", + "object": "file_action_process" + }, + { + "key": "process.name", + "object": "process_action" + }, + { + "key": "process.binary_ref", + "object": "process_action", + "references": "file_action_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_image_name": [ + { + "key": "file.name", + "object": "file_actor_process" + }, + { + "key": "process.name", + "object": "process_actor" + }, + { + "key": "process.binary_ref", + "object": "process_actor", + "references": "file_actor_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_image_name": [ + { + "key": "file.name", + "object": "file_causality_process" + }, + { + "key": "process.name", + "object": "causality_parent_process" + }, + { + "key": "process.parent_ref", + "object": "process_causality_actor", + "references": "causality_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "causality_parent_process" + } + ], + "os_actor_process_image_name": [ + { + "key": "file.name", + "object": "file_os_actor" + }, + { + "key": "process.name", + "object": "os_parent_process" + }, + { + "key": "process.parent_ref", + "object": "process_os_actor", + "references": "os_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "os_parent_process" + } + ], + "action_file_size": { + "key": "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_authenticode_sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "action_file_authenticode_sha2": { + "key": "file.hashes.SHA-2", + "object": "file" + }, + "action_file_md5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "action_module_md5": { + "key": "file.hashes.MD5", + "object": "file_action_module" + }, + "action_process_image_md5": [ + { + "key": "file.hashes.MD5", + "object": "file_action_process" + }, + { + "key": "process.binary_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "action_file_sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "action_module_sha256": { + "key": "file.hashes.SHA-256", + "object": "file_action_module" + }, + "action_process_image_sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "file_action_process" + }, + { + "key": "process.binary_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "action_file_access_time": [ + { + "key": "file.accessed", + "object": "file", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_action_file_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_access_time": [ + { + "key": "file.accessed", + "object": "file_actor_process", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_actor_process_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_access_time": [ + { + "key": "file.accessed", + "object": "file_os_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_os_actor_process_last_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_file_mod_time": [ + { + "key": "file.modified", + "object": "file", + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_mod_time": [ + { + "key": "file.modified", + "object": "file_actor_process", + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_mod_time": [ + { + "key": "file.modified", + "object": "file_os_actor", + "transformer": "EpochToTimestamp" + } + ], + "action_file_create_time": [ + { + "key": "file.created", + "object": "file", + "transformer": "EpochToTimestamp" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_file_path": [ + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "action_registry_file_path": [ + { + "key": "directory.path", + "object": "directory_action_registry", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_action_registry", + "references": "directory_action_registry" + } + ], + "action_process_image_path": [ + { + "key": "directory.path", + "object": "directory_action_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_action_process", + "references": "directory_action_process" + }, + { + "key": "process.binary_ref", + "object": "process_action", + "references": "file_action_process" + } + ], + "actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_actor_process", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_actor_process", + "references": "directory_actor_process" + }, + { + "key": "process.binary_ref", + "object": "process_actor", + "references": "file_actor_process" + } + ], + "causality_actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_causality_actor", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_causality_process", + "references": "directory_causality_actor" + }, + { + "key": "process.binary_ref", + "object": "process_causality_actor", + "references": "file_causality_process" + } + ], + "os_actor_process_image_path": [ + { + "key": "directory.path", + "object": "directory_os_actor", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_os_actor", + "references": "directory_os_actor" + }, + { + "key": "process.binary_ref", + "object": "process_os_actor", + "references": "file_os_actor" + } + ], + "action_process_image_command_line": [ + { + "key": "process.command_line", + "object": "process_action" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_actor" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_causality_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_command_line": [ + { + "key": "process.command_line", + "object": "process_os_actor" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_os_actor" + } + ], + "action_process_file_create_time": [ + { + "key": "process.created", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "actor_process_file_create_time": [ + { + "key": "process.created", + "object": "process_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "causality_actor_process_file_create_time": [ + { + "key": "process.created", + "object": "process_causality_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_causality_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "os_actor_process_file_create_time": [ + { + "key": "process.created", + "object": "process_os_actor", + "transformer": "EpochToTimestamp" + }, + { + "key": "x_process_os_actor_first_observed", + "cybox": false, + "transformer": "EpochToTimestamp" + } + ], + "action_module_process_os_pid": [ + { + "key": "process.pid", + "object": "process_action_module", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action_module" + } + ], + "action_process_os_pid": [ + { + "key": "process.pid", + "object": "process_action", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_action" + } + ], + "actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_actor" + } + ], + "causality_actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_causality_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_causality_actor" + } + ], + "os_actor_process_os_pid": [ + { + "key": "process.pid", + "object": "process_os_actor", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process_os_actor" + } + ], + "action_process_requested_parent_pid": [ + { + "key": "process.pid", + "object": "action_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process_action", + "references": "action_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "action_parent_process" + } + ], + "action_thread_parent_pid": [ + { + "key": "process.pid", + "object": "action_parent_thread", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process_action_thread", + "references": "action_parent_thread" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "action_parent_thread" + } + ], + "action_thread_child_pid": [ + { + "key": "process.pid", + "object": "action_child_thread", + "transformer": "ToInteger" + }, + { + "key": "process.child_refs", + "object": "process_action", + "references": "action_child_thread" + } + ], + "auth_domain": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain" + } + ], + "dst_host_metadata_domain": [ + { + "key": "domain-name.value", + "object": "domain_dst" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain_dst" + } + ], + "host_metadata_domain": [ + { + "key": "domain-name.value", + "object": "domain_host" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain_host" + } + ], + "dst_action_url_category": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "action_registry_key_name": [ + { + "key": "windows-registry-key.key", + "object": "windowsregistry" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "windowsregistry" + } + ], + "action_registry_value_name": [ + { + "key": "windows-registry-key.values", + "object": "windowsregistry", + "transformer": "FormatToStixRegistryValue" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "windowsregistry" + } + ], + "mac": [ + { + "key": "mac-addr.value", + "object": "mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["mac"] + } + ], + "associated_mac": [ + { + "key": "mac-addr.value", + "object": "ass_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["ass_mac"] + } + ], + "dst_associated_mac": [ + { + "key": "mac-addr.value", + "object": "dst_ass_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["dst_ass_mac"] + } + ], + "dst_mac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": ["dst_mac"] + } + ], + "actor_primary_user_sid": { + "key": "user-account.user_id", + "object": "user" + }, + "action_process_user_sid": { + "key": "user-account.extensions.x-paloalto-user.process_user_id", + "object": "user" + }, + "actor_primary_username": { + "key": "user-account.display_name", + "object": "user" + }, + "action_process_username": [ + { + "key": "user-account.extensions.x-paloalto-user.process_user_name", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process_action", + "references": "user" + } + ], + "actor_process_logon_id": { + "key": "user-account.account_login", + "object": "user" + }, + "action_file_info_company": { + "key": "file.extensions.x-paloalto-file.company", + "object": "file" + }, + "action_file_extension": { + "key": "file.extensions.x-paloalto-file.extension", + "object": "file" + }, + "action_file_attributes": { + "key": "file.extensions.x-paloalto-file.attributes", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_internal_zipped_files": { + "key": "file.extensions.x-paloalto-file.zipped_files", + "object": "file" + }, + "action_file_last_writer_actor": { + "key": "file.extensions.x-paloalto-file.writer", + "object": "file" + }, + "action_file_mode": { + "key": "file.extensions.x-paloalto-file.mode", + "object": "file" + }, + "action_file_signature_status": { + "key": "file.extensions.x-paloalto-file.signature_status", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_signature_vendor": { + "key": "file.extensions.x-paloalto-file.signature_vendor", + "object": "file" + }, + "action_file_signature_product": { + "key": "file.extensions.x-paloalto-file.signature_product", + "object": "file" + }, + "action_file_info_description": { + "key": "file.extensions.x-paloalto-file.file_description", + "object": "file" + }, + "action_file_group": { + "key": "file.extensions.x-paloalto-file.group", + "object": "file" + }, + "action_file_group_name": { + "key": "file.extensions.x-paloalto-file.group_name", + "object": "file" + }, + "action_file_type": { + "key": "file.extensions.x-paloalto-file.type", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_info_file_version": { + "key": "file.extensions.x-paloalto-file.version", + "object": "file" + }, + "manifest_file_version": { + "key": "file.extensions.x-paloalto-file.manifest_version", + "object": "file", + "transformer": "ToInteger" + }, + "action_file_info_product_version": { + "key": "file.extensions.x-paloalto-file.product_version", + "object": "file" + }, + "action_file_owner": { + "key": "file.extensions.x-paloalto-file.owner", + "object": "file" + }, + "action_file_owner_name": { + "key": "file.extensions.x-paloalto-file.owner_name", + "object": "file" + }, + "action_file_info_product_name": { + "key": "file.extensions.x-paloalto-file.product_name", + "object": "file" + }, + "action_file_id": { + "key": "file.extensions.x-paloalto-file.id", + "object": "file" + }, + "action_file_wildfire_verdict": { + "key": "file.extensions.x-paloalto-file.wildfire_verdict", + "object": "file" + }, + "action_file_hash_control_verdict": { + "key": "file.extensions.x-paloalto-process.control_verdict", + "object": "file" + }, + "actor_process_instance_id": { + "key": "process.extensions.x-paloalto-process.instance_id", + "object": "process_actor" + }, + "actor_process_causality_id": { + "key": "process.extensions.x-paloalto-process.causality_id", + "object": "process_actor" + }, + "actor_process_auth_id": { + "key": "process.extensions.x-paloalto-process.auth_id", + "object": "process_actor" + }, + "actor_process_container_id": { + "key": "process.extensions.x-paloalto-process.container_id", + "object": "process_actor" + }, + "actor_process_signature_vendor": { + "key": "process.extensions.x-paloalto-process.signature_vendor", + "object": "process_actor" + }, + "actor_process_signature_status": { + "key": "process.extensions.x-paloalto-process.signature_status", + "object": "process_actor" + }, + "actor_process_signature_product": { + "key": "process.extensions.x-paloalto-process.signature_product", + "object": "process_actor" + }, + "actor_process_image_extension": { + "key": "process.extensions.x-paloalto-process.extension", + "object": "process_actor" + }, + "action_process_termination_code": { + "key": "process.extensions.x-paloalto-process.termination_code", + "object": "process_action", + "transformer": "ToInteger" + }, + "action_process_termination_date": { + "key": "process.extensions.x-paloalto-process.termination_date", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + "action_remote_process_thread_id": { + "key": "process.extensions.x-paloalto-process.tid", + "object": "process_action", + "transformer": "ToInteger" + }, + "action_process_instance_execution_time": { + "key": "process.extensions.x-paloalto-process.instance_exec_time", + "object": "process_action", + "transformer": "EpochToTimestamp" + }, + "actor_process_execution_time": { + "key": "process.extensions.x-paloalto-process.execution_time", + "object": "process_actor", + "transformer": "EpochToTimestamp" + }, + "action_process_handle_is_kernel": { + "key": "process.extensions.x-paloalto-process.is_kernel", + "object": "process_action", + "transformer": "StringToBool" + }, + "action_process_is_container_root": { + "key": "process.extensions.x-paloalto-process.is_root", + "object": "process_action", + "transformer": "StringToBool" + }, + "actor_process_is_native": { + "key": "process.extensions.x-paloalto-process.is_native", + "object": "process_actor", + "transformer": "StringToBool" + }, + "agent_version": { + "key": "x-oca-asset.extensions.x-paloalto-agent.agent_version", + "object": "asset" + }, + "agent_hostname": [ + { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + { + "key": "x-oca-event.agent", + "object": "event" + } + ], + "agent_content_version": { + "key": "x-oca-asset.extensions.x-paloalto-agent.content_version", + "object": "asset" + }, + "agent_session_start_time": { + "key": "x-oca-asset.extensions.x-paloalto-agent.start_time", + "object": "asset", + "transformer": "EpochToTimestamp" + }, + "agent_id": { + "key": "x-oca-asset.extensions.x-paloalto-agent.asset_id", + "object": "asset" + }, + "agent_os_type": { + "key": "x-oca-asset.extensions.x-paloalto-agent.os_type", + "object": "asset" + }, + "agent_os_sub_type": { + "key": "x-oca-asset.extensions.x-paloalto-agent.os_sub_type", + "object": "asset" + }, + "agent_is_vdi": { + "key": "x-oca-asset.extensions.x-paloalto-agent.is_vdi", + "object": "asset", + "transformer": "StringToBool" + }, + "action_user_agent": { + "key": "x-oca-asset.extensions.x-paloalto-agent.user_agent", + "object": "asset" + }, + "http_req_user_agent_header": { + "key": "x-oca-asset.extensions.x-paloalto-agent.agent_header", + "object": "asset" + }, + "action_evtlog_data_fields": { + "key": "x-paloalto-evtlog.data_fields", + "object": "evtlog" + }, + "action_evtlog_description": { + "key": "x-paloalto-evtlog.description", + "object": "evtlog" + }, + "action_evtlog_source": { + "key": "x-paloalto-evtlog.source", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_event_id": { + "key": "x-paloalto-evtlog.evtlog_id", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_level": { + "key": "x-paloalto-evtlog.level", + "object": "evtlog" + }, + "action_evtlog_tid": { + "key": "x-paloalto-evtlog.tid", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_uid": { + "key": "x-paloalto-evtlog.uid", + "object": "evtlog" + }, + "action_evtlog_pid": { + "key": "x-paloalto-evtlog.pid", + "object": "evtlog", + "transformer": "ToInteger" + }, + "action_evtlog_message": { + "key": "x-paloalto-evtlog.message", + "object": "evtlog" + }, + "action_evtlog_version": { + "key": "x-paloalto-evtlog.version", + "object": "evtlog", + "transformer": "ToInteger" + }, + "event_id": { + "key": "x-oca-event.code", + "object": "event" + }, + "vpn_event_description": { + "key": "x-oca-event.entensions.x-paloalto-event.event_description", + "object": "event" + }, + "event_timestamp": { + "key": "x-oca-event.created", + "object": "event", + "transformer": "EpochToTimestamp" + }, + "event_version": { + "key": "x-oca-event.entensions.x-paloalto-event.version", + "object": "event", + "transformer": "ToInteger" + }, + "event_rpc_interface_uuid": { + "key": "x-oca-event.entensions.x-paloalto-event.uuid", + "object": "event" + }, + "event_address_mapped_image_path": { + "key": "x-oca-event.entensions.x-paloalto-event.path", + "object": "event" + }, + "event_type": { + "key": "x-oca-event.category", + "object": "event", + "transformer": "ToLowercaseArray" + }, + "event_sub_type": { + "key": "x-oca-event.action", + "object": "event" + }, + "action_network_creation_time": { + "key": "network-traffic.extensions.x-paloalto-network.creation_time", + "object": "nt", + "transformer": "EpochToTimestamp" + }, + "action_network_connection_id": { + "key": "network-traffic.extensions.x-paloalto-network.connection_id", + "object": "nt" + }, + "action_network_packet_data": { + "key": "network-traffic.extensions.x-paloalto-network.packet_data", + "object": "nt" + }, + "action_proxy": { + "key": "network-traffic.extensions.x-paloalto-network.is_proxy", + "object": "nt", + "transformer": "StringToBool" + }, + "host_metadata_hostname": { + "key": "network-traffic.extensions.x-paloalto-network.metadata_hostname", + "object": "nt" + }, + "action_external_hostname": { + "key": "network-traffic.extensions.x-paloalto-network.external_hostname", + "object": "nt" + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json b/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json new file mode 100644 index 000000000..9147b6418 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/json/xdr_data_from_stix_map.json @@ -0,0 +1,371 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "action_local_ip", + "action_remote_ip", + "agent_ip_addresses" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "agent_ip_addresses_v6", + "dst_agent_ip_addresses_v6" + ] + } + }, + "network-traffic": { + "fields": { + "src_port": [ + "action_local_port" + ], + "dst_port": [ + "action_remote_port" + ], + "protocols[*]": [ + "action_network_protocol" + ], + "src_ref.value": [ + "action_local_ip", + "agent_ip_addresses" + ], + "dst_ref.value": [ + "action_remote_ip" + ], + "src_packets": [ + "action_pkts_sent" + ], + "dst_packets": [ + "action_pkts_received" + ] + } + }, + "file": { + "fields": { + "name": [ + "action_file_name", + "action_process_image_name", + "actor_process_image_name", + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "size": [ + "action_file_size" + ], + "hashes.MD5": [ + "action_file_md5", + "action_module_md5", + "action_process_image_md5" + ], + "hashes.'SHA-1'": [ + "action_file_authenticode_sha1" + ], + "hashes.'SHA-2'": [ + "action_file_authenticode_sha2" + ], + "hashes.'SHA-256'": [ + "action_file_sha256", + "action_module_sha256", + "action_process_image_sha256" + ], + "accessed": [ + "action_file_access_time", + "actor_process_file_access_time", + "os_actor_process_file_access_time" + ], + "modified": [ + "action_file_mod_time", + "actor_process_file_mod_time", + "os_actor_process_file_mod_time" + ], + "created": [ + "action_file_create_time" + ], + "parent_directory_ref.path": [ + "action_file_path", + "action_process_image_path", + "action_registry_file_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "directory": { + "fields": { + "path": [ + "action_file_path", + "action_process_image_path", + "action_registry_file_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "process": { + "fields": { + "command_line": [ + "action_process_image_command_line", + "actor_process_command_line", + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "created": [ + "action_process_file_create_time", + "actor_process_file_create_time", + "causality_actor_process_file_create_time", + "os_actor_process_file_create_time" + ], + "name": [ + "action_process_image_name", + "actor_process_image_name", + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "pid": [ + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid", + "action_process_requested_parent_pid", + "action_thread_parent_pid", + "action_thread_child_pid" + ], + "parent_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "child_refs.pid": [ + "action_thread_child_pid" + ], + "creator_user_ref.user_id": [ + "action_process_username" + ], + "parent_ref.name": [ + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "binary_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "binary_ref.hashes.MD5": [ + "action_process_image_md5" + ], + "binary_ref.hashes.'SHA-256'": [ + "action_process_image_sha256" + ], + "binary_ref.parent_directory_ref.path": [ + "action_process_image_path", + "actor_process_image_path", + "causality_actor_process_image_path", + "os_actor_process_image_path" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "auth_domain", + "dst_host_metadata_domain", + "host_metadata_domain" + ] + } + }, + "url": { + "fields": { + "value": [ + "dst_action_url_category" + ] + } + }, + "windows-registry-key": { + "fields": { + "key": [ + "action_registry_key_name" + ], + "values[*]": [ + "action_registry_value_name" + ] + } + }, + "mac-addr": { + "fields": { + "value": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" + ] + } + }, + "user-account": { + "fields": { + "user_id": [ + "actor_primary_user_sid", + "action_process_user_sid" + ], + "display_name": [ + "actor_primary_username", + "action_process_username" + ], + "account_login": [ + "actor_process_logon_id" + ] + } + }, + "x-paloalto-file": { + "fields": { + "extension": [ + "action_file_extension" + ], + "file_description": [ + "action_file_info_description" + ] + } + }, + "x-paloalto-process": { + "fields": { + "extension": [ + "actor_process_image_extension" + ], + "execution_time": [ + "action_process_instance_execution_time", + "actor_process_execution_time" + ] + } + }, + "x-oca-asset": { + "fields": { + "hostname": [ + "agent_hostname" + ], + "ip_refs[*].value": [ + "action_local_ip", + "action_remote_ip", + "agent_ip_addresses_v6", + "agent_ip_addresses", + "dst_agent_ip_addresses_v6" + ], + "mac_refs[*].value": [ + "mac", + "associated_mac", + "dst_associated_mac", + "dst_mac" + ] + } + }, + "x-paloalto-evtlog": { + "fields": { + "description": [ + "action_evtlog_description" + ], + "message": [ + "action_evtlog_message" + ] + } + }, + "x-oca-event": { + "fields": { + "code": [ + "event_id" + ], + "category[*]": [ + "event_type" + ], + "action": [ + "event_sub_type" + ], + "created": [ + "event_timestamp" + ], + "agent": [ + "agent_hostname" + ], + "url_ref.value": [ + "dst_action_url_category" + ], + "file_ref.name": [ + "action_file_name" + ], + "process_ref.pid": [ + "action_module_process_os_pid", + "action_process_os_pid", + "actor_process_os_pid", + "causality_actor_process_os_pid", + "os_actor_process_os_pid" + ], + "process_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "process_ref.command_line": [ + "action_process_image_command_line", + "actor_process_command_line" + ], + "process_ref.binary_ref.name": [ + "action_process_image_name", + "actor_process_image_name" + ], + "process_ref.parent_ref.name": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "process_ref.parent_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "process_ref.parent_ref.command_line": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "parent_process_ref.name": [ + "causality_actor_process_image_name", + "os_actor_process_image_name" + ], + "parent_process_ref.pid": [ + "action_process_requested_parent_pid", + "action_thread_parent_pid" + ], + "parent_process_ref.command_line": [ + "causality_actor_process_command_line", + "os_actor_process_command_line" + ], + "process_ref.creator_user_ref.user_id": [ + "action_process_username" + ], + "process_ref.binary_ref.hashes.MD5": [ + "action_process_image_md5" + ], + "process_ref.binary_ref.hashes.'SHA-256'": [ + "action_process_image_sha256" + ], + "domain_ref.value": [ + "auth_domain", + "dst_host_metadata_domain", + "host_metadata_domain" + ], + "registry_ref.key": [ + "action_registry_key_name" + ], + "registry_ref.values[*]": [ + "action_registry_value_name" + ] + } + }, + "x-paloalto-network": { + "fields": { + "creation_time": [ + "action_network_creation_time" + ], + "hostname": [ + "host_metadata_hostname", + "action_external_hostname" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/paloalto/stix_translation/query_constructor.py b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py new file mode 100644 index 000000000..3ff5fae8d --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py @@ -0,0 +1,485 @@ +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, \ + ComparisonExpression, ComparisonComparators, Pattern, CombinedComparisonExpression, CombinedObservationExpression +import logging +import re +from datetime import datetime, timedelta +from os import path +import json + +logger = logging.getLogger(__name__) + +START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)" +MAC = '^(([0-9a-fA-F]{2}[:-]){5}([0-9a-fA-F]{2}))$' +CONFIG_MAP_PATH = "json/config_map.json" +FIELDS_MAP_PATH = "json/fields_map.json" + +STOP_TIME = datetime.utcnow() + + +class StartStopQualifierValueException(Exception): + pass + + +class FileNotFoundException(Exception): + pass + + +class QueryStringPatternTranslator: + """ + translate stix pattern to native data source query language + """ + + def __init__(self, pattern: Pattern, data_model_mapper, options): + logger.info("Palo Alto Cortex XDR Connector") + self.dmm = data_model_mapper + self.options = options + self.timeframe = [] + self.comparator_lookup = self.dmm.map_comparator() + self.config_map = self.load_json(CONFIG_MAP_PATH) + self.all_fields_map = self.load_json(FIELDS_MAP_PATH) + self.translated_query = self.parse_expression(pattern) + self.qualified_query = self._create_formatted_query(self.dmm, self.translated_query, + self.timeframe, self.all_fields_map, options) + + @staticmethod + def load_json(rel_path_of_file): + """ + Consumes a json file and returns a dictionary + :param rel_path_of_file: str + :return: dict + """ + + _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file + try: + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundException + except FileNotFoundException as e: + raise FileNotFoundError(f'{rel_path_of_file} not found') from e + + @staticmethod + def _create_formatted_query(dmm, translated_query, timeframe, all_fields_map, options): + """ + Formation of Palo Alto - native query language structure + :param dmm + :param translated_query:str + :param timeframe:list + :param all_fields_map:dict + :param options:dict + :return: formatted_query :list + """ + limit = options["result_limit"] + dataset_name = dmm.dialect + all_fields = all_fields_map["all_fields"] # all_fields included in to_stix_mapping + fields = ','.join(all_fields) + query = f'dataset = {dataset_name} | filter {translated_query} | alter dataset_name = \"{dataset_name}\" ' \ + f'| fields dataset_name,{fields} | limit {limit} ' # adding custom field 'dataset_name' to query, + # since this 'dataset_name' field will be used in translate results + formatted_query = { + dataset_name: {"query": query, "timeframe": {"from": min(timeframe), "to": max(timeframe)}}} + return [formatted_query] + + def _format_set(self, values, mapped_field_type, mapped_fields_array) -> str: + """ + Formats value in the event of set operation + :param values + :param mapped_field_type: str + :param mapped_fields_array: list + :return formatted value + """ + gen = values.element_iterator() + formatted_value = ','.join(QueryStringPatternTranslator._escape_value( + self._format_value_type(value, mapped_field_type, mapped_fields_array), mapped_field_type) + for value in gen) + return f'({formatted_value})' + + @staticmethod + def _format_match(value, mapped_field_type) -> str: + """ + Formats value in the event of match operation + :param value + :param mapped_field_type:str + :return formatted string type value + """ + if mapped_field_type != "string": + raise NotImplementedError('MATCHES operators is supported only for string type input') + if '^' in value and value.index('^') != 0: + raise NotImplementedError('^ symbol should be at the starting position of the expression') + if '$' in value and value.index('$') != len(value) - 1: + raise NotImplementedError('$ symbol should be at the ending position of the expression') + value = '{}'.format(value.replace('\\', '\\\\').replace('\"', '\\"').replace('(', '\\(').replace(')', '\\)')) + return f'\"{value}\"' + + @staticmethod + def _format_equality(value, mapped_field_type) -> str: + """ + Formats value in the event of equality operation + :param value + :param mapped_field_type: str + :return formatted value + """ + return QueryStringPatternTranslator._escape_value(value, mapped_field_type) + + @staticmethod + def _format_like(value, mapped_field_type) -> str: + """ + Formats value in the event of LIKE operation + :param value + :param mapped_field_type: str + :return formatted value + """ + if mapped_field_type != "string": + raise NotImplementedError("LIKE operator is supported only for string type input") + return QueryStringPatternTranslator._escape_value(value, mapped_field_type) + + @staticmethod + def _escape_value(value, mapped_field_type) -> str: + """ + adds escape characters to string type value + :param value + :param mapped_field_type: str + :return formatted value + """ + if isinstance(value, str) and mapped_field_type != "enum": + value = f'\"{value}\"' + return str(value) + + def _check_enum_supported_values(self, converted_value, mapped_fields_array): + """ + checks for enum supported values + :param mapped_fields_array: list + :param converted_value:str + :return enum formatted value :str + """ + try: + formatted_values = converted_value.upper() + if formatted_values not in self.config_map["enum_supported_values"][mapped_fields_array[0]]: + raise NotImplementedError(f'Unsupported ENUM values provided. Possible supported enum values are' + f'{self.config_map["enum_supported_values"][mapped_fields_array[0]]}') + return f'ENUM.{formatted_values}' + except (KeyError, IndexError, TypeError) as e: + raise KeyError(f'{mapped_fields_array[0]} is not found in enum_supported_values') from e + + def _format_value_type(self, value, mapped_field_type, mapped_fields_array): + """ + check input value format that matches with the mapped field value type + :param value + :param mapped_field_type: str + :param mapped_fields_array: list + :return formatted value + """ + converted_value = str(value) + if mapped_field_type == "enum": + converted_value = self._check_enum_supported_values(converted_value, mapped_fields_array) + elif mapped_field_type == "timestamp": + converted_value = QueryStringPatternTranslator._format_datetime(converted_value) + elif mapped_field_type == "mac": + compile_mac_regex = re.compile(MAC) + if not compile_mac_regex.search(converted_value): + raise NotImplementedError(f'Invalid mac address - {converted_value} provided') + elif mapped_field_type == "int": + if not converted_value.isdigit(): + raise NotImplementedError(f'string type input - {converted_value} is not supported for ' + f'integer type fields') + converted_value = int(value) + elif mapped_field_type == "boolean": + converted_value = QueryStringPatternTranslator._check_boolean_value(converted_value) + return converted_value + + def _check_value_comparator_support(self, value, comparator, mapped_field_type): + """ + checks the comparator and value support + :param value + :param comparator + :param mapped_field_type: str + """ + operator = self.comparator_lookup[str(comparator)] + if mapped_field_type == "enum" and (comparator not in [ComparisonComparators.Equal, ComparisonComparators.In, + ComparisonComparators.NotEqual]): + raise NotImplementedError(f'{operator} operator is not supported for Enum type input. Possible supported ' + f'operator are [=,!=,in,not in]') + if isinstance(value, str) and comparator not in [ComparisonComparators.Equal, ComparisonComparators.NotEqual, + ComparisonComparators.Like, ComparisonComparators.Matches]: + raise NotImplementedError(f'{operator} operator is not supported for string type input') + if isinstance(value, bool) and comparator not in [ComparisonComparators.Equal, ComparisonComparators.NotEqual]: + raise NotImplementedError(f'{operator} operator is not supported for Boolean type input') + + @staticmethod + def _check_boolean_value(converted_value): + """ + returns boolean value of input + :param converted_value:str + :return bool + """ + if converted_value.lower() == "true" or (converted_value.isdigit() and converted_value == "1"): + boolean_value = bool(True) + elif converted_value.lower() == "false" or (converted_value.isdigit() and converted_value == "0"): + boolean_value = bool(False) + else: + raise NotImplementedError('Invalid boolean type input') + return boolean_value + + @staticmethod + def _format_negate(comparator): + """ + returns negation of input operator + :param comparator:str + :return str + """ + negate_comparator = { + ">": "<=", + ">=": "<", + "<": ">=", + "<=": ">", + "=": "!=", + "!=": "=", + "contains": "not contains", + "in": "not in", + "~=": "!~=" + } + return negate_comparator[comparator] + + @staticmethod + def _format_datetime(value): + """ + Converts timestamp to milliseconds + :param value + :return: int, converted epoch value + """ + try: + time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ' + epoch = datetime(1970, 1, 1) + converted_time = int(((datetime.strptime(value, + time_pattern) - epoch).total_seconds()) * 1000) + return converted_time + except ValueError: + pass + raise NotImplementedError(f'cannot convert the timestamp {value} to milliseconds') + + @staticmethod + def _check_time_range_values(converted_timestamp): + """ + checks for valid start and stop time + :param converted_timestamp: list + """ + if converted_timestamp[0] > converted_timestamp[1]: + raise StartStopQualifierValueException('Start time should be lesser than Stop time') + + @staticmethod + def _parse_time_range(qualifier, time_range): + """ + Converts qualifier timestamp to epoch + :param qualifier: str + :param time_range: int + return: list of converted epoch values + """ + try: + compile_timestamp_regex = re.compile(START_STOP_PATTERN) + converted_timestamp = [] + if qualifier and compile_timestamp_regex.search(qualifier): + time_range_iterator = compile_timestamp_regex.finditer(qualifier) + time_range_list = [each.group() for each in time_range_iterator] + else: + start_time = STOP_TIME - timedelta(minutes=time_range) + converted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + # limit 3 digit value for millisecond + converted_stop_time = STOP_TIME.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + time_range_list = [converted_start_time, converted_stop_time] + for timestamp in time_range_list: + converted_time = QueryStringPatternTranslator._format_datetime(timestamp) + converted_timestamp.append(converted_time) + return converted_timestamp + except (KeyError, IndexError, TypeError) as e: + raise e + + def _add_timestamp_to_query(self, query, qualifier): + """ + adds timestamp filter to Palo Alto Cortex XDR query + :param query: str + :param qualifier + :return str + """ + converted_timestamp = QueryStringPatternTranslator._parse_time_range(qualifier, self.options["time_range"]) + QueryStringPatternTranslator._check_time_range_values(converted_timestamp) # check timestamp value range + self.timeframe += converted_timestamp + if self.dmm.dialect in self.config_map["timestamp_supported_dataset"].keys(): + timestamp_field_name = self.config_map["timestamp_supported_dataset"][self.dmm.dialect] + timestamp_field = f' and (to_epoch({timestamp_field_name},\"millis\") >= {converted_timestamp[0]} and ' \ + f'to_epoch({timestamp_field_name},\"millis\") <= {converted_timestamp[1]})' + if timestamp_field not in query: + query = f'({query} {timestamp_field})' + else: + query = f'({query})' + return query + + def _check_mapped_field_type(self, mapped_field_array): + """ + Returns the type of mapped field array + :param mapped_field_array: list + :return: str + """ + mapped_field = mapped_field_array[0] + mapped_field_type = "string" + for key, value in self.config_map.items(): + if mapped_field in value and key in ["int_supported_fields", "enum_supported_fields", + "boolean_supported_fields", "timestamp_supported_fields", + "mac_supported_fields"]: + mapped_field_type = key.split('_')[0] + break + return mapped_field_type + + @staticmethod + def _parse_mapped_fields(value, comparator, mapped_fields_array): + """ + parse mapped fields into boolean expression + :param value: str + :param comparator: str + :param mapped_fields_array: list + :return: str + """ + comparison_string = "" + mapped_fields_count = len(mapped_fields_array) + for field_name in mapped_fields_array: + comparison_string += f'{field_name} {comparator} {value}' + if mapped_fields_count > 1: + comparison_string += " or " + mapped_fields_count -= 1 + if len(mapped_fields_array) > 1: + # More than one data source field maps to the STIX attribute, so group comparisons together. + grouped_comparison_string = "(" + comparison_string + ")" + comparison_string = grouped_comparison_string + return comparison_string + + def _lookup_comparison_operator(self, expression_operator): + """ + lookup operators support in Palo Alto Cortex XDR + :param expression_operator:enum object + :return str + """ + if str(expression_operator) not in self.comparator_lookup: + raise NotImplementedError( + f'Comparison operator {expression_operator.name} unsupported for Palo Alto Cortex XDR connector') + + return self.comparator_lookup[str(expression_operator)] + + def _eval_comparison_value(self, expression, mapped_field_type, mapped_fields_array): + """ + Function for parsing comparison expression value + :param expression: expression object + :param mapped_field_type:str + :param mapped_fields_array:list + :return: formatted expression value + """ + if expression.comparator == ComparisonComparators.Matches: + value = QueryStringPatternTranslator._format_match(expression.value, mapped_field_type) + elif expression.comparator == ComparisonComparators.In: + value = self._format_set(expression.value, mapped_field_type, mapped_fields_array) + elif expression.comparator in [ComparisonComparators.GreaterThan, ComparisonComparators.GreaterThanOrEqual, + ComparisonComparators.LessThan, ComparisonComparators.LessThanOrEqual, + ComparisonComparators.Equal, ComparisonComparators.NotEqual]: + value = self._format_value_type(expression.value, mapped_field_type, mapped_fields_array) + self._check_value_comparator_support(value, expression.comparator, mapped_field_type) + value = self._format_equality(value, mapped_field_type) + elif expression.comparator == ComparisonComparators.Like: + value = self._format_value_type(expression.value, mapped_field_type, mapped_fields_array) + value = self._format_like(value, mapped_field_type) + else: + raise NotImplementedError('Unknown comparator expression operator') + return value + + def _eval_combined_comparison_exp(self, expression): + """ + Function for parsing combined comparison expression + :param expression: expression object + """ + operator = self._lookup_comparison_operator(expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + if not expression_01 or not expression_02: + return '' + if isinstance(expression.expr1, CombinedComparisonExpression): + expression_01 = f'{expression_01}' + if isinstance(expression.expr2, CombinedComparisonExpression): + expression_02 = f'{expression_02}' + + query_string = f'{expression_01} {operator} {expression_02}' + return f'{query_string}' + + def _eval_combined_observation_exp(self, expression, qualifier=None): + """ + Function for parsing combined observation expression + :param expression: expression object + :param qualifier: qualifier + """ + operator = self._lookup_comparison_operator(expression.operator) + expression_01 = self._parse_expression(expression.expr1, qualifier) + expression_02 = self._parse_expression(expression.expr2, qualifier) + query = '' + if expression_01 and expression_02: + query = f'{expression_01} {operator} {expression_02}' + elif expression_01: + query = f'{expression_01}' + elif expression_02: + query = f'{expression_02}' + return query + + def _parse_expression(self, expression, qualifier=None) -> str: + """ + parse ANTLR pattern to Palo Alto Cortex XDR query + :param expression: expression object, ANTLR parsed expression object + :param qualifier: str, default in None + :return str + """ + if isinstance(expression, ComparisonExpression): # Base Case + stix_objects = expression.object_path.split(':') + mapped_fields_array = self.dmm.map_field(stix_objects[0], stix_objects[1]) + comparator = self._lookup_comparison_operator(expression.comparator) + if expression.negated: + comparator = QueryStringPatternTranslator._format_negate(comparator) + mapped_field_type = self._check_mapped_field_type(mapped_fields_array) + value = self._eval_comparison_value(expression, mapped_field_type, mapped_fields_array) + + comparison_string = self._parse_mapped_fields(value, comparator, mapped_fields_array) + return comparison_string + + elif isinstance(expression, CombinedComparisonExpression): + return self._eval_combined_comparison_exp(expression) + elif isinstance(expression, ObservationExpression): + query_string = self._parse_expression(expression.comparison_expression) + return self._add_timestamp_to_query(query_string, qualifier) + + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + if isinstance(expression.observation_expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(expression.observation_expression.operator) + expression_01 = self._parse_expression(expression.observation_expression.expr1, expression.qualifier) + expression_02 = self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + query_string = f'{expression_01} {operator} {expression_02}' + else: + query_string = self._parse_expression(expression.observation_expression, expression.qualifier) + if qualifier is not None: + query_string = self._add_timestamp_to_query(query_string, qualifier) + return query_string + elif isinstance(expression, CombinedObservationExpression): + return self._eval_combined_observation_exp(expression, qualifier) + elif isinstance(expression, Pattern): + return f'{self._parse_expression(expression.expression)}' + else: + raise RuntimeError(f'Unknown Recursion Case for expression={expression},' + f' type(expression)={type(expression)}') + + def parse_expression(self, pattern: Pattern): + return self._parse_expression(pattern) + + +def translate_pattern(pattern: Pattern, data_model_mapping, options): + """ + Conversion of ANTLR pattern to Palo ALTO CORTEX XDR query + :param pattern: expression object, ANTLR parsed expression object + :param data_model_mapping: DataMapper object, mapping object obtained by parsing json + :param options: dict, time_range defaults to 5 + :return: list, PALO ALTO queries + """ + + query = QueryStringPatternTranslator(pattern, data_model_mapping, options).qualified_query + return query diff --git a/stix_shifter_modules/paloalto/stix_translation/query_translator.py b/stix_shifter_modules/paloalto/stix_translation/query_translator.py new file mode 100644 index 000000000..845ad4f11 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/query_translator.py @@ -0,0 +1,27 @@ +import logging + +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator +from . import query_constructor + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file + :param antlr_parsing_object: Antlr parsing objects for the STIX pattern + :type antlr_parsing_object: object + :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into + another format. This should default to something if one isn't passed in + :type mapping: str (filepath) + :return: transformed query string + :rtype: str + """ + + logger.info("Converting STIX2 Pattern to data source query") + + query_string = query_constructor.translate_pattern( + antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/paloalto/stix_translation/results_translator.py b/stix_shifter_modules/paloalto/stix_translation/results_translator.py new file mode 100644 index 000000000..746b8300c --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/results_translator.py @@ -0,0 +1,5 @@ +from stix_shifter_utils.stix_translation.src.json_to_stix.json_to_stix import JSONToStix + + +class ResultsTranslator(JSONToStix): + pass diff --git a/stix_shifter_modules/paloalto/stix_translation/transformers.py b/stix_shifter_modules/paloalto/stix_translation/transformers.py new file mode 100644 index 000000000..8b1e728d9 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_translation/transformers.py @@ -0,0 +1,16 @@ +from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer +from stix_shifter_utils.utils import logger + +LOGGER = logger.set_logger(__name__) + + +class FormatToStixRegistryValue(ValueTransformer): + """A value transformer to convert paloalto Registry value to windows-registry-key.value STIX""" + + @staticmethod + def transform(registry): + + try: + return [{'name': registry}] + except ValueError: + LOGGER.error("Cannot convert root value to Stix formatted windows registry value") diff --git a/stix_shifter_modules/paloalto/stix_transmission/__init__.py b/stix_shifter_modules/paloalto/stix_transmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/paloalto/stix_transmission/api_client.py b/stix_shifter_modules/paloalto/stix_transmission/api_client.py new file mode 100644 index 000000000..0306b1c9c --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/api_client.py @@ -0,0 +1,191 @@ +from stix_shifter_utils.stix_transmission.utils.RestApiClient import RestApiClient +from stix_shifter_utils.utils import logger +from stix_shifter_utils.utils.error_response import ErrorResponder +from .response_mapper import ResponseMapper +from datetime import datetime, timezone +import secrets +import string +import hashlib +import json +from requests.exceptions import ConnectionError + + +class MaxDailyQuotaException(Exception): + pass + + +class APIClient: + QUERY_ENDPOINT = 'public_api/v1/xql/start_xql_query/' + RESULT_ENDPOINT = 'public_api/v1/xql/get_query_results/' + STREAM_ENDPOINT = 'public_api/v1/xql/get_query_results_stream/' + QUOTA_ENDPOINT = '/public_api/v1/xql/get_quota/' + + def __init__(self, connection, configuration): + self.auth = configuration.get('auth') + self.logger = logger.set_logger(__name__) + nonce = "".join([secrets.choice(string.ascii_letters + string.digits) for _ in range(64)]) + timestamp = int(datetime.now(timezone.utc).timestamp()) * 1000 + self.auth = configuration.get('auth') + auth_key = f"{self.auth['api_key']}{nonce}{timestamp}" + auth_key = auth_key.encode("utf-8") + api_key_hash = hashlib.sha256(auth_key).hexdigest() + headers = { + "x-xdr-timestamp": str(timestamp), + "x-xdr-nonce": nonce, + "x-xdr-auth-id": str(self.auth['api_key_id']), + "Authorization": api_key_hash + } + self.client = RestApiClient(connection.get('host'), + connection.get('port', None), + headers, + url_modifier_function=None, + ) + self.result_limit = connection['options'].get('result_limit') + self.timeout = connection['options']['timeout'] + self.quota_threshold = connection['quota_threshold'] + self.connector = __name__.split('.')[1] + + def ping_data_source(self): + """ + Ping the Data Source + :return: Response object + """ + data = { + "request_data": {} + } + + return self.client.call_api(self.QUOTA_ENDPOINT, 'POST', headers=self.client.headers, + data=json.dumps(data), timeout=self.timeout) + + def get_remaining_quota(self): + """ + Pings the quota endpoint to fetch the remaining quota + :return: Response object + """ + return_obj = {} + response_dict = {} + data = { + "request_data": {} + } + quota_wrapper = None + + try: + quota_wrapper = self.client.call_api(self.QUOTA_ENDPOINT, 'POST', headers=self.client.headers, + data=json.dumps(data), timeout=self.timeout) + quota_response_code = quota_wrapper.response.status_code + quota_response_text = json.loads(quota_wrapper.read().decode('utf-8')) + if quota_response_code == 200: + if 'reply' in quota_response_text.keys(): + # The daily quota unit for standard license is 5. additional units up to 10 can be added. + if quota_response_text['reply']['license_quota'] == 5 and \ + quota_response_text['reply']['additional_purchased_quota'] == 0.0: + # For a Standard license,if the configured quota threshold is greater than 5, + # the threshold quota value is reset to 5. + self.quota_threshold = 5 if self.quota_threshold > 5 else self.quota_threshold + if quota_response_text['reply']['used_quota'] >= self.quota_threshold: + raise MaxDailyQuotaException + else: + if quota_response_text['reply']['used_quota'] >= self.quota_threshold: + raise MaxDailyQuotaException + return_obj['success'] = True + else: + return_obj = ResponseMapper().status_code_mapping(quota_response_code, quota_response_text) + + except ValueError as ex: + if quota_wrapper is not None: + self.logger.debug(quota_wrapper.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + except MaxDailyQuotaException: + response_dict['type'] = "MaxDailyQuotaException" + response_dict['message'] = "query usage exceeded max daily quota" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + except Exception as ex: + if 'timeout_error' in str(ex): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = ex.__class__.__name__ + response_dict['message'] = ex + self.logger.error('error when getting search results: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + + return return_obj + + def create_search(self, query): + """ + Queries the data source + :return: Response object + """ + return_obj = self.get_remaining_quota() + if return_obj['success']: + if not isinstance(query, dict): + query = json.loads(query) + + for dataset in query.keys(): + query[dataset]["tenants"] = self.auth['tenant'].split(",") + data = { + "request_data": + query[dataset] + } + return self.client.call_api(self.QUERY_ENDPOINT, 'POST', headers=self.client.headers, + data=json.dumps(data), timeout=self.timeout) + return return_obj + + def get_search_status(self, search_id): + """ + Queries the data source to fetch the status of api call + :return: Response object + """ + data = { + "request_data": { + "query_id": search_id, + "pending_flag": True, + "limit": self.result_limit, + "format": "json" + } + } + return self.client.call_api(self.RESULT_ENDPOINT, 'POST', headers=self.client.headers, data=json.dumps(data), + timeout=self.timeout) + + def get_search_results(self, search_id): + """ + Return the search results + :param search_id:str + :return: Response object + """ + data = { + "request_data": { + "query_id": search_id, + "pending_flag": False, + "limit": self.result_limit, + "format": "json" + } + } + return self.client.call_api(self.RESULT_ENDPOINT, 'POST', headers=self.client.headers, data=json.dumps(data), + timeout=self.timeout) + + @staticmethod + def delete_search(): + """ + Delete operation of a search id is not supported in Palo Alto Cortex XDR + :return dict + """ + return {"code": 200, "success": True} + + def get_stream_results(self, stream_id): + """ + Return the stream results + :param stream_id: string + :return: Raw Json data + """ + data = { + "request_data": + {"stream_id": stream_id, + "is_gzip_compressed": False + } + } + return self.client.call_api(self.STREAM_ENDPOINT, 'POST', headers=self.client.headers, data=json.dumps(data), + timeout=self.timeout) diff --git a/stix_shifter_modules/paloalto/stix_transmission/delete_connector.py b/stix_shifter_modules/paloalto/stix_transmission/delete_connector.py new file mode 100644 index 000000000..3baf0d4a5 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/delete_connector.py @@ -0,0 +1,30 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + + +class DeleteConnector(BaseDeleteConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def delete_query_connection(self, search_id): + """ + Delete operation of a search id is not supported in Palo Alto Cortex XDR + :param search_id:str + :return dict + """ + try: + response_dict = self.api_client.delete_search() + response_code = response_dict['code'] + # Construct a response object + return_obj = {} + if response_code == 200: + return_obj['success'] = response_dict['success'] + return_obj['message'] = 'Delete operation of a search id is not supported in Palo Alto Cortex XDR' + else: + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj + except Exception as err: + self.logger.error('error when deleting search %s:', err) + raise NotImplementedError('Error in delete search id operation') from err diff --git a/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py b/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py new file mode 100644 index 000000000..b1fcff8cc --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/error_mapper.py @@ -0,0 +1,42 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + "ConnectionError": ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + "AttributeError": ErrorCode.TRANSMISSION_INVALID_PARAMETER, + "AuthenticationError": ErrorCode.TRANSMISSION_AUTH_CREDENTIALS, + "SyntaxError": ErrorCode.TRANSMISSION_QUERY_LOGICAL_ERROR, + "EmptyResultException": ErrorCode.TRANSMISSION_RESPONSE_EMPTY_RESULT, + "APIPermissionException": ErrorCode.TRANSMISSION_FORBIDDEN, + "InvalidLicense": ErrorCode.TRANSMISSION_CONNECT, + "InvalidJsonException": ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + "MaxDailyQuotaException": ErrorCode.TRANSMISSION_CONNECT, + "TimeoutError": ErrorCode.TRANSMISSION_CONNECT +} + + +class ErrorMapper: + """ + Set Error Code + """ + logger = logger.set_logger(__name__) + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + + @staticmethod + def set_error_code(json_data, return_obj): + err_type = None + try: + err_type = json_data['type'] + except KeyError: + pass + + error_type = ErrorMapper.DEFAULT_ERROR + + if err_type in error_mapping: + error_type = error_mapping.get(err_type) + + if error_type == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: %s", str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_type) diff --git a/stix_shifter_modules/paloalto/stix_transmission/ping_connector.py b/stix_shifter_modules/paloalto/stix_transmission/ping_connector.py new file mode 100644 index 000000000..adac8f908 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/ping_connector.py @@ -0,0 +1,49 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +from requests.exceptions import ConnectionError +from .response_mapper import ResponseMapper + + +class PingConnector(BasePingConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def ping_connection(self): + """ + Ping the endpoint + :return: dict + """ + return_obj = {} + response_dict = {} + response_wrapper = None + try: + response_wrapper = self.api_client.ping_data_source() + response_code = response_wrapper.response.status_code + response_text = json.loads(response_wrapper.read().decode('utf-8')) + if response_code == 200 and 'reply' in response_text.keys(): + return_obj['success'] = True + else: + return_obj = ResponseMapper().status_code_mapping(response_code, response_text) + + except ValueError as ex: + if response_wrapper is not None: + self.logger.debug(response_wrapper.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + + except Exception as exe: + if 'timeout_error' in str(exe): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = exe.__class__.__name__ + response_dict['message'] = exe + self.logger.error('error when getting search results: %s', exe) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj diff --git a/stix_shifter_modules/paloalto/stix_transmission/query_connector.py b/stix_shifter_modules/paloalto/stix_transmission/query_connector.py new file mode 100644 index 000000000..f8a3a1e43 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/query_connector.py @@ -0,0 +1,52 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_query_connector import BaseQueryConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +from requests.exceptions import ConnectionError +from .response_mapper import ResponseMapper + + +class QueryConnector(BaseQueryConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def create_query_connection(self, query): + """ + Function to create query connection + :param query: dict, Query + :return: dict + """ + return_obj = {} + response_dict = {} + response_wrapper = None + try: + response_wrapper = self.api_client.create_search(query) + if isinstance(response_wrapper, dict): + return response_wrapper + response_code = response_wrapper.response.status_code + response_text = json.loads(response_wrapper.read().decode('utf-8')) + if response_code == 200 and 'reply' in response_text.keys(): + return_obj['success'] = True + return_obj['search_id'] = response_text['reply'] + else: + return_obj = ResponseMapper().status_code_mapping(response_code, response_text) + except ValueError as ex: + if response_wrapper is not None and not isinstance(response_wrapper, dict): + self.logger.debug(response_wrapper.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + + except Exception as exep: + if 'timeout_error' in str(exep): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = exep.__class__.__name__ + response_dict['message'] = exep + self.logger.error('error when getting search results: %s', exep) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj diff --git a/stix_shifter_modules/paloalto/stix_transmission/response_mapper.py b/stix_shifter_modules/paloalto/stix_transmission/response_mapper.py new file mode 100644 index 000000000..850b52649 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/response_mapper.py @@ -0,0 +1,104 @@ +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + + +class InvalidAuthenticationException(Exception): + pass + + +class InternalServerErrorException(Exception): + pass + + +class InvalidLicenseException(Exception): + pass + + +class APIPermissionException(Exception): + pass + + +class InvalidJsonException(Exception): + pass + + +class InvalidQueryException(Exception): + pass + + +class ResponseMapper: + logger = logger.set_logger(__name__) + connector = __name__.split('.')[1] + + @staticmethod + def status_code_mapping(response_code, response_text): + """ + raise exceptions for the error response code + :return dict + """ + return_obj = {} + response_dict = {} + + try: + if response_code == 500: + return_obj = ResponseMapper.handle_response_500(response_text, return_obj, response_dict) + else: + response_codes_match = { + 400: InvalidJsonException(response_text['reply']['err_msg']), + 401: InvalidAuthenticationException, + 402: InvalidLicenseException, + 403: APIPermissionException + } + raise response_codes_match[response_code] + except InvalidAuthenticationException: + response_dict['type'] = "AuthenticationError" + response_dict['message'] = "Invalid api_key" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except InvalidLicenseException: + response_dict['type'] = "InvalidLicense" + response_dict['message'] = "User does not have the required license type to run this API" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except APIPermissionException: + response_dict['type'] = "APIPermissionException" + response_dict['message'] = "The provided API Key does not have the required RBAC permissions to run " \ + "this API" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except InvalidJsonException as ex: + response_dict['type'] = "InvalidJsonException" + response_dict['message'] = ex + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except Exception as ex: + response_dict['type'] = ex.__class__.__name__ + response_dict['message'] = ex + ResponseMapper.logger.error('error when getting search results: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + return return_obj + + @staticmethod + def handle_response_500(response_text, return_obj, response_dict): + """ + Handle 500 error code response + :param response_text: dict + :param response_dict: dict + :param return_obj: dict + """ + try: + if isinstance(response_text['reply']['err_extra'], dict) and \ + 'parse_err' in response_text['reply']['err_extra'].keys(): + raise InvalidQueryException + raise InternalServerErrorException(response_text['reply']['err_msg']) + + except InvalidQueryException: + response_dict['type'] = "SyntaxError" + response_dict['message'] = 'Bad query Syntax' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except InternalServerErrorException as ex: + response_dict['type'] = "AttributeError" + response_dict['message'] = ex + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + except Exception as exp: + response_dict['type'] = exp.__class__.__name__ + response_dict['message'] = exp + ResponseMapper.logger.error('error when getting search results: %s', exp) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=ResponseMapper.connector) + return return_obj diff --git a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py new file mode 100644 index 000000000..533614be8 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py @@ -0,0 +1,264 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +from requests.exceptions import ConnectionError +from .response_mapper import ResponseMapper +from os import path + +TO_STIX_PATH = "../stix_translation/json/to_stix_map.json" +CONFIG_MAP_PATH = "../stix_translation/json/config_map.json" + + +class FileNotFoundException(Exception): + pass + + +class InvalidQueryException(Exception): + pass + + +class ResultsConnector(BaseResultsConnector): + + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + @staticmethod + def load_json(rel_path_of_file): + """ + Consumes a json file and returns a dictionary + :param rel_path_of_file: str + :return: dict + """ + _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file + try: + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundException + except FileNotFoundException as e: + raise FileNotFoundError(f'{rel_path_of_file} not found') from e + + def create_results_connection(self, search_id, offset, length): + """ + Fetching the results using search id, offset and length + :param search_id: str, search id generated in transmit query + :param offset: str, offset value + :param length: str, length value + :return: dict + """ + response_dict = {} + return_obj = {} + results = [] + response_wrapper = None + to_stix_mapping = ResultsConnector.load_json(TO_STIX_PATH) + mandatory_map = ResultsConnector.load_json(CONFIG_MAP_PATH)["mandatory_properties_to_stix"] + try: + min_range = int(offset) + max_range = int(offset) + int(length) + # Grab the response, extract the response code, and convert it to readable json + response_wrapper = self.api_client.get_search_results(search_id) + response_text = json.loads(response_wrapper.read().decode('utf-8')) + if response_wrapper.code != 200: + return_obj = ResponseMapper().status_code_mapping(response_wrapper.code, response_text) + else: + if 'status' in response_text['reply'].keys() and response_text['reply']['status'] \ + in ('SUCCESS', 'PARTIAL_SUCCESS'): + if 'data' in response_text['reply']['results'].keys() and \ + response_text['reply']['number_of_results'] > 0: + results = ResultsConnector.format_results_data( + response_text['reply']['results']['data'], to_stix_mapping, mandatory_map) + elif 'stream_id' in response_text['reply']['results'].keys(): + stream_wrapper = self.api_client.get_stream_results( + response_text['reply']['results']['stream_id']) + if stream_wrapper.code != 200: + return_obj = ResponseMapper().status_code_mapping(stream_wrapper.code, + stream_wrapper.read().decode('utf-8')) + else: + results = ResultsConnector.format_stream_data(stream_wrapper.read().decode('utf-8'), + to_stix_mapping, mandatory_map) + return_obj['success'] = True + return_obj['data'] = results[min_range:max_range] if results else [] + + elif 'status' in response_text['reply'].keys() and response_text['reply']['status'] == 'FAIL': + raise InvalidQueryException + + except ValueError as ex: + if response_wrapper is not None: + self.logger.debug(response_wrapper.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + except InvalidQueryException: + response_dict['type'] = "SyntaxError" + response_dict['message'] = 'Tenant Query Failed' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + except Exception as ex: + if 'timeout_error' in str(ex): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = ex.__class__.__name__ + response_dict['message'] = ex + self.logger.error('error when getting search results: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj + + @staticmethod + def format_results_data(result_data, to_stix_mapping, mandatory_map): + """ + Format the results in json format + :param result_data: list of dictionary items + :param to_stix_mapping: to stix mapping dictionary + :param mandatory_map: dictionary + :return: list + """ + results = [] + dataset_map = result_data[0]['dataset_name'] + try: + for log in result_data: + data = {} + results_dict = {} + dataset = "" + for field, value in log.items(): + if value is not None and value != "NULL" and value != '' and field != 'dataset_name' \ + and (field in to_stix_mapping[dataset_map].keys()): + stix_data_map = to_stix_mapping[dataset_map][field] + data = ResultsConnector.check_object(stix_data_map, mandatory_map, data, log, + field, value) + elif field == 'dataset_name': + dataset = value + + results_dict[dataset] = data + results.append(results_dict) + except (KeyError, IndexError, TypeError) as e: + raise e + return results + + @staticmethod + def format_stream_data(stream_data, to_stix_mapping, mandatory_map): + """ + Format the stream data into json format + :param stream_data: string + :param to_stix_mapping: to stix mapping dictionary + :param mandatory_map: dictionary + :return: list + """ + results = [] + try: + temp_data = stream_data.split("\n") + dataset_map = json.loads(temp_data[0])['dataset_name'] + for log in temp_data: + if log > " ": + data = {} + results_dict = {} + dataset = "" + log_dict = json.loads(log) + for field, value in log_dict.items(): + if value is not None and value != "NULL" and value != '' and field != 'dataset_name' \ + and (field in to_stix_mapping[dataset_map].keys()): + stix_data_map = to_stix_mapping[dataset_map][field] + data = ResultsConnector.check_object(stix_data_map, mandatory_map, + data, log_dict, field, value) + elif field == 'dataset_name': + dataset = value + results_dict[dataset] = data + results.append(results_dict) + except (KeyError, IndexError, TypeError) as e: + raise e + return results + + @staticmethod + def check_object(stix_data_map, mandatory_map, data, log, field, value): + """ + The mandatory stix fields are removed, if no value is received from data source + :param stix_data_map: list of dictionary items + :param mandatory_map : list of dictionary items + :param data : dictionary + :param log: dictionary + :param field : string + :param value: string/integer + :return: list of dictionary items + """ + try: + + if isinstance(stix_data_map, list): + obj_list = [obj["object"] for obj in stix_data_map if "object" in obj.keys()] + for obj in obj_list: + if obj in mandatory_map.keys(): + data = ResultsConnector.check_mandatory_map(mandatory_map, obj, log, data, field, value) + else: + # Since these ip addresses may receive a single ip or list of ips, they are sent as list value + # for the to_stix conversion + data[field] = value.split(",") if field in ["agent_ip_addresses", "agent_ip_addresses_v6", + "dst_agent_ip_addresses_v6"] else value + + elif isinstance(stix_data_map, dict): + if stix_data_map["object"] in mandatory_map.keys(): + data = ResultsConnector.check_mandatory_map(mandatory_map, stix_data_map["object"], + log, data, field, value) + else: + + data[field] = value + + except (KeyError, IndexError, TypeError) as e: + raise e + return data + + @staticmethod + def check_mandatory_map(mandatory_map, obj, log, data, field, value): + """ + Donot add the the fields which belongs to the object of mandatory field , + if data source doesnt return any value if the mandatory field + :param mandatory_map: list of dictionary items + :param obj : string + :param log : dictionary + :param data : dictionary + :param field : string + :param value : string/integer + """ + try: + if "file" in obj: + if mandatory_map[obj][0] in log.keys() and log[mandatory_map[obj][0]] != "NULL" and \ + log[mandatory_map[obj][0]] != '' and \ + log[mandatory_map[obj][0]] is not None: + data[field] = value + elif obj in ["user", "nt"]: + if obj == "nt": + data = ResultsConnector.process_nt_obj(mandatory_map, obj, data, log, value, field) + else: + if any((item in log.keys() and log[item] != "NULL" and log[item] != '' and log[item] is not None) + for item in mandatory_map[obj]): + data[field] = value + except (KeyError, IndexError, TypeError) as e: + raise e + return data + + @staticmethod + def process_nt_obj(mandatory_map, obj, data, log, value, field): + """ + check if any of the fields in the list is not null + :param mandatory_map: list of dictionary items + :param obj : string + :param log : dictionary + :param data : dictionary + :param field : string + :param value : string/integer + """ + nt_obj = False + try: + for i in mandatory_map[obj]: + if isinstance(i, list): + if any((item in log.keys() and log[item] != "NULL" and log[item] != '' and log[item] is not None) + for item in i): + nt_obj = True + else: + if nt_obj: + data[i] = log[i] if (log[i] != "NULL" and log[i] != '' and log[i] is not None) else "NULL" + data[field] = value + + except (KeyError, IndexError, TypeError) as e: + raise e + return data diff --git a/stix_shifter_modules/paloalto/stix_transmission/status_connector.py b/stix_shifter_modules/paloalto/stix_transmission/status_connector.py new file mode 100644 index 000000000..b6b19ebc6 --- /dev/null +++ b/stix_shifter_modules/paloalto/stix_transmission/status_connector.py @@ -0,0 +1,124 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import BaseStatusConnector +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from enum import Enum +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +from requests.exceptions import ConnectionError +from .response_mapper import ResponseMapper + + +class InvalidResponseException(Exception): + pass + + +class InvalidQueryException(Exception): + pass + + +class PaloAltoStatus(Enum): + PENDING = 'PENDING' + COMPLETED = 'COMPLETED' + FAIL = 'FAIL' + + +class StatusConnector(BaseStatusConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + # Map data source status to connector status + @staticmethod + def get_status(status): + """ + Return the status of the search id + :param status: str, + :return: str + """ + switcher = { + PaloAltoStatus.PENDING.value: Status.RUNNING, + PaloAltoStatus.COMPLETED.value: Status.COMPLETED, + PaloAltoStatus.FAIL.value: Status.ERROR + } + return switcher.get(status).value + + def create_status_connection(self, search_id): + """ + Fetching the progress and the status of the search id + :param search_id: str, search id + :return: dict + """ + return_obj = {} + response_dict = {} + response = None + try: + response = self.api_client.get_search_status(search_id) + # Based on the response + response_code = response.code + response_read = response.read() + response_text = json.loads(response_read) + + if response_code == 200: + if 'status' in response_text['reply'].keys(): + # Since PaloAlto API doesnt return the numerical value of progress, the value for Pending + # status is set to 50. If the status is completed, it is set to 100. + if response_text['reply']['status'] == "PENDING": + return_obj['success'] = True + return_obj['status'] = StatusConnector.get_status('PENDING') + return_obj['progress'] = 50 + elif response_text['reply']['status'] == "SUCCESS": + if 'number_of_results' in response_text['reply'].keys(): + return_obj['success'] = True + return_obj['status'] = StatusConnector.get_status('COMPLETED') + return_obj['progress'] = 100 + else: + raise InvalidResponseException + else: + return_obj = self.handle_fail_response(response_text, return_obj, response_dict) + else: + return_obj = ResponseMapper().status_code_mapping(response_code, response_text) + + except ValueError as ex: + if response is not None: + self.logger.debug(response.read()) + raise Exception(f'Cannot parse response: {ex}') from ex + + except InvalidResponseException: + response_dict['type'] = 'EmptyResultException' + response_dict['message'] = 'Empty results received from Tenant' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + + except ConnectionError: + response_dict['type'] = "ConnectionError" + response_dict['message'] = "Invalid Host" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + except Exception as err: + if 'timeout_error' in str(err): + response_dict['type'] = 'TimeoutError' + else: + response_dict['type'] = err.__class__.__name__ + response_dict['message'] = err + self.logger.error('error when getting search results: %s', err) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj + + def handle_fail_response(self, response_text, return_obj, response_dict): + """ + Handle partial success or failed status response + :param return_obj: dict + :param response_dict: dict + :param response_text: dict + """ + try: + if response_text['reply']['status'] == "PARTIAL_SUCCESS": + return_obj['success'] = True + return_obj['status'] = StatusConnector.get_status('COMPLETED') + return_obj['message'] = "Partial Success -At least one tenant failed to execute the query" + elif response_text['reply']['status'] == "FAIL": + raise InvalidQueryException + except InvalidQueryException: + + response_dict['type'] = "SyntaxError" + response_dict['message'] = 'Tenant Query Failed' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.api_client.connector) + return return_obj diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py new file mode 100644 index 000000000..8181c6cc3 --- /dev/null +++ b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_json_to_stix.py @@ -0,0 +1,443 @@ +import unittest +from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator +from stix_shifter_modules.paloalto.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers + +MODULE = "paloalto" +entry_point = EntryPoint() +map_data = entry_point.get_results_translator().map_data +data_source = { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "paloalto", + "identity_class": "events" +} +options = {} + + +class TestPaloaltoResultsToStix(unittest.TestCase): + """ + class to perform unit test case for paloalto translate results + """ + + @staticmethod + def get_first(itr, constraint): + """ + return the obj in the itr if constraint is true + """ + return next( + (obj for obj in itr if constraint(obj)), + None + ) + + @staticmethod + def get_first_of_type(itr, typ): + """ + to check whether the object belongs to respective stix object + """ + return TestPaloaltoResultsToStix.get_first(itr, lambda o: isinstance(o, dict) and o.get('type') == typ) + + def test_file_json_to_stix(self): + """to test File stix object properties""" + + data = {'xdr_data': {'action_file_name': 'arp_cache_periodic.py', + 'action_file_size': '7943', + 'action_file_md5': 'f9d1ba13674eaf75c129534aba30487e', + 'action_module_md5': 'a2f22af507acc1961cf9504491f3e4b0', + 'action_process_image_md5': '93f1eb1ed4475f58f6870a385021c92d', + 'action_file_sha256': 'f6c82fe48662c1e6fa1b4732c020672bd6a929d0e0102712a4064d5c1351bfaf', + 'action_module_sha256': '2e365914142b4c0f8340594e253e8fa25f452338238b87e7d303af89d0747d78', + 'action_process_image_sha256': 'b55e59f545cdb5866b538c46fc280e51b4955df6b3a\ + 76686a73e80333757f003', + 'actor_process_file_access_time': '1634092022616', + 'os_actor_process_file_access_time': '1634096052944', + 'actor_process_file_mod_time': '1640745638339', + 'os_actor_process_file_mod_time': '1634096052944', + 'actor_process_file_create_time': '1631299512000', + 'os_actor_process_file_create_time': '1644385345812', + 'action_file_path': 'C:\\Program Files (x86)\\Google\\Policies', + 'action_process_image_path': 'C:\\Windows\\System32\\wevtutil.exe', + 'action_registry_file_path': 'C:\\Windows\\AppCompat\\Programs\\Amcache.hve', + 'actor_process_image_path': 'C:\\Program Files\\Amazon\\SSM\\amazon-ssm-agent.exe', + 'causality_actor_process_image_path': 'C:\\Windows\\System32\\lsass.exe', + 'os_actor_process_image_path': 'C:\\Program Files\\Amazon\\SSM\\amazon-ssm-agent.exe' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + file_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'file') + assert file_obj is not None + assert file_obj['type'] == 'file' + assert file_obj['name'] == 'arp_cache_periodic.py' + assert file_obj['size'] == 7943 + + def test_process_json_to_stix(self): + """ to test process stix object properties """ + data = {'xdr_data': {'action_process_image_command_line': 'C:\Windows\system32\lsass.exe', + 'actor_process_command_line': 'C:\Windows\system32\lsass.exe', + 'causality_actor_process_command_line': 'C:\Windows\system32\lsass.exe', + 'os_actor_process_command_line': 'C:\Windows\system32\lsass.exe', + 'actor_process_file_create_time': '1631299512000', + 'causality_actor_process_file_create_time': '1536995564723', + 'os_actor_process_file_create_time': '1637334466002', + 'action_process_image_name': 'wevtutil.exe', + 'actor_process_image_name': 'amazon-ssm-agent.exe', + 'causality_actor_process_image_name': 'pycharm64.exe', + 'os_actor_process_image_name': 'lsass.exe', + 'action_process_os_pid': '6228', + 'actor_process_os_pid': '4300', + 'causality_actor_process_os_pid': '5144', + 'os_actor_process_os_pid': '2744', + 'action_process_requested_parent_pid': '5144', + 'action_process_username': 'NT AUTHORITY\\NETWORK SERVICE', + 'actor_effective_username': 'EC2AMAZ-4KPRAA7\\Administrator' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + process_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'process') + assert process_obj is not None + assert process_obj['type'] == 'process' + assert process_obj['name'] == 'wevtutil.exe' + assert process_obj['pid'] == 6228 + assert process_obj['command_line'] == 'C:\Windows\system32\lsass.exe' + + def test_network_traffic_json_to_stix(self): + """to test network-traffic stix object properties""" + + data = {'xdr_data': {'action_local_port': '54083', + 'action_remote_port': '80', + 'action_network_protocol': 'TCP', + 'action_local_ip': '172.28.32.1', + 'action_remote_ip': '23.205.106.166' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + network_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'network-traffic') + + assert network_obj is not None + assert network_obj["type"] == 'network-traffic' + assert network_obj["src_port"] == 54083 + assert network_obj['src_ref'] == '2' + assert 'tcp' in network_obj["protocols"] + assert network_obj["dst_port"] == 80 + assert network_obj['dst_ref'] == '4' + + def test_mac_addr_json_to_stix(self): + """ + to test mac stix object properties + """ + data = {'xdr_data': {'associated_mac': '00:15:5d:6a:79:52'}} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + mac_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'mac-addr') + assert mac_obj is not None, 'mac-aadr object type not found' + assert mac_obj.keys() == {'type', 'value'} + assert mac_obj['type'] == 'mac-addr' + assert mac_obj['value'] == '00:15:5d:6a:79:52' + + def test_user_account_json_to_stix(self): + """to test user-account stix object properties""" + + data = {'xdr_data': {'actor_process_logon_id': '794419589', + 'action_process_username': 'EC2AMAZ-IQFSLIL\\Administrator', + 'actor_primary_username': 'NT AUTHORITY\\SYSTEM', + 'actor_primary_user_sid': 'S-1-5-18' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + user_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'user-account') + + assert user_obj is not None + assert user_obj['type'] == 'user-account' + assert user_obj['user_id'] == "S-1-5-18" + assert user_obj['extensions']['x-paloalto-user']['process_user_name'] == 'EC2AMAZ-IQFSLIL\\Administrator' + + def test_domain_name_json_to_stix(self): + """to test domain-name stix object properties""" + + data = {'xdr_data': {'auth_domain': 'dl.delivery.mp.microsoft.com', + 'dst_host_metadata_domain': '8.tlu.dl.delivery.mp.microsoft.com', + 'host_metadata_domain': '7.tlu.dl.delivery.mp.microsoft.com' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + domain_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'domain-name') + assert domain_obj is not None + assert domain_obj['type'] == 'domain-name' + assert domain_obj['value'] == 'dl.delivery.mp.microsoft.com' + + def test_windows_registry_key_json_to_stix(self): + """to test windows registry stix object properties""" + + data = {'xdr_data': {'action_registry_key_name': 'HKEY_LOCAL_MACHINE\\SYSTEM\\ControlSet001\\Services\\terminpt' + '\\Enum', + 'action_registry_value_name': 'Start' + + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + windows_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'windows-registry-key') + assert windows_obj is not None + assert windows_obj['type'] == 'windows-registry-key' + assert windows_obj['key'] == 'HKEY_LOCAL_MACHINE\\SYSTEM\\ControlSet001\\Services\\terminpt\\Enum' + assert windows_obj['values'] == [{'name':'Start'}] + + def test_url_json_to_stix(self): + """to test url stix object properties""" + + data = {'xdr_data': {'dst_action_url_category': 'https://paloalto/index.com' + + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + + url_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'url') + assert url_obj is not None + assert url_obj['type'] == 'url' + assert url_obj['value'] == 'https://paloalto/index.com' + + def test_custom_file_json_to_stix(self): + """to test custom file stix object properties""" + data = {'xdr_data': { + 'action_file_extension': 'json', + 'action_file_attributes': 128, + 'action_file_last_writer_actor': 'AdgAsdUgVlUAAAbYAAAAAA==', + 'action_file_signature_status': 3, + 'action_file_type': 18, + 'manifest_file_version': 5 + }} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + custom_file_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'file') + assert custom_file_obj is not None + assert custom_file_obj['type'] == 'file' + assert custom_file_obj['extensions']['x-paloalto-file']['extension'] == "json" + assert custom_file_obj['extensions']['x-paloalto-file']['attributes'] == 128 + assert custom_file_obj['extensions']['x-paloalto-file']['writer'] == "AdgAsdUgVlUAAAbYAAAAAA==" + assert custom_file_obj['extensions']['x-paloalto-file']['manifest_version'] == 5 + + def test_custom_process_json_to_stix(self): + """to test custom process stix object properties""" + data = {'xdr_data': { + 'actor_process_instance_id': 'AdgAsdUgVlUAAAbYAAAAAA==', + 'actor_process_causality_id': 'AdgdeB9glMcAAAOYAAAAAA==', + 'actor_process_signature_vendor': 'Microsoft Corporation', + 'actor_process_signature_status': 'SIGNED', + 'actor_process_signature_product': 'Microsoft Windows', + 'action_process_termination_date': 1646315246849, + 'actor_process_execution_time': 1641280255000, + 'actor_process_is_native': 'FALSE' + }} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + custom_process_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'process') + assert custom_process_obj is not None + assert custom_process_obj['type'] == 'process' + assert custom_process_obj['extensions']['x-paloalto-process']['instance_id'] == "AdgAsdUgVlUAAAbYAAAAAA==" + assert custom_process_obj['extensions']['x-paloalto-process']['signature_vendor'] == "Microsoft Corporation" + assert custom_process_obj['extensions']['x-paloalto-process']['signature_status'] == "SIGNED" + assert custom_process_obj['extensions']['x-paloalto-process']['execution_time'] == "2022-01-04T07:10:55.000Z" + assert custom_process_obj['extensions']['x-paloalto-process']['is_native'] == False + + def test_asset_json_to_stix(self): + """to test custom oca-asset stix object properties""" + data = {'xdr_data': { + 'agent_version': '7.6.1.46600', + 'agent_hostname': 'EC2AMAZ-IQFSLIL', + 'agent_content_version': '350-80787', + 'agent_session_start_time': 1642662241933, + 'agent_id': '45.9.20.38', + 'agent_os_sub_type': 'Windows Server 2016', + 'agent_is_vdi': 'FALSE' + }} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + asset_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'x-oca-asset') + assert asset_obj is not None + assert asset_obj['type'] == 'x-oca-asset' + assert asset_obj['extensions']['x-paloalto-agent']['agent_version'] == "7.6.1.46600" + assert asset_obj['hostname'] == "EC2AMAZ-IQFSLIL" + assert asset_obj['extensions']['x-paloalto-agent']['content_version'] == "350-80787" + assert asset_obj['extensions']['x-paloalto-agent']['start_time'] == "2022-01-20T07:04:01.933Z" + assert asset_obj['extensions']['x-paloalto-agent']['os_sub_type'] == "Windows Server 2016" + assert asset_obj['extensions']['x-paloalto-agent']['is_vdi'] == False + + def test_evtlog_json_to_stix(self): + """to test custom evtlog stix object properties""" + data = {'xdr_data': { + 'action_evtlog_description': 'An account was logged off', + 'action_evtlog_source': 3, + 'action_evtlog_event_id': 4625, + 'action_evtlog_uid': 'S-1-5-19', + 'action_evtlog_level': 'INFO', + 'action_evtlog_version': 2 + }} + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + evtlog_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'x-paloalto-evtlog') + assert evtlog_obj is not None + assert evtlog_obj['type'] == 'x-paloalto-evtlog' + assert evtlog_obj['description'] == "An account was logged off" + assert evtlog_obj['source'] == 3 + assert evtlog_obj['evtlog_id'] == 4625 + assert evtlog_obj['uid'] == "S-1-5-19" + assert evtlog_obj['level'] == "INFO" + + def test_event_json_to_stix(self): + """to test custom event stix object properties""" + data = {'xdr_data': { + 'event_id': 'OTE0MTk5MTg2MDI1NzUyODc0NQ==', + 'event_timestamp': 164632333729, + 'event_version': 25, + 'event_rpc_interface_uuid': '{00000136-0000-0000-C000-000000000046}', + 'event_type': 'EVENT_LOG' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + event_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert event_obj is not None + assert event_obj['code'] == "OTE0MTk5MTg2MDI1NzUyODc0NQ==" + assert event_obj['created'] == '1975-03-21T11:12:13.729Z' + assert event_obj['entensions']['x-paloalto-event']['version'] == 25 + assert event_obj['entensions']['x-paloalto-event']['uuid'] == '{00000136-0000-0000-C000-000000000046}' + assert event_obj['category'] == ["event_log"] + + def test_custom_network_json_to_stix(self): + """to test custom network stix object properties""" + data = {'xdr_data': { + 'action_network_creation_time': 164632333729, + 'action_network_connection_id': 'AdgAsdUgVlUAAAbYAAAAAA==', + 'action_proxy': 'FALSE', + 'action_external_hostname': 'Windows 8' + }} + + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + objects = observed_data['objects'] + network_obj = TestPaloaltoResultsToStix.get_first_of_type(objects.values(), 'network-traffic') + assert network_obj is not None + assert network_obj['extensions']['x-paloalto-network']['creation_time'] == '1975-03-21T11:12:13.729Z' + assert network_obj['extensions']['x-paloalto-network']['connection_id'] == "AdgAsdUgVlUAAAbYAAAAAA==" + assert network_obj['extensions']['x-paloalto-network']['is_proxy'] == False + assert network_obj['extensions']['x-paloalto-network']['external_hostname'] == 'Windows 8' diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py new file mode 100644 index 000000000..68e7a80d9 --- /dev/null +++ b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py @@ -0,0 +1,561 @@ +from stix_shifter.stix_translation import stix_translation +from stix_shifter_utils.utils.error_response import ErrorCode +import unittest +import re + +translation = stix_translation.StixTranslation() + + +def _remove_timestamp_from_query(queries): + pattern1 = r"to_epoch\(_time,\"millis\"\)\s*.=\s*\d{0,13}\s*and\s*to_epoch\(_time,\"millis\"\)\s*.=\s*\d{0,13}" + pattern2 = r"\{'from':\s*\d{0,13},\s*'to':\s*\d{0,13}\}" + if isinstance(queries, list): + modified_queries = [] + for query in queries: + replace_pat1 = re.sub(pattern1, '', str(query)) + replace_pat2 = re.sub(pattern2, '{}', replace_pat1) + modified_queries.append(replace_pat2) + return modified_queries + elif isinstance(queries, str): + replace_pat1 = re.sub(pattern1, '', queries) + return re.sub(pattern2, '{}', replace_pat1) + + +all_fields = "dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6," \ + "dst_agent_ip_addresses_v6," \ + "action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received," \ + "action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name," \ + "os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5," \ + "action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2," \ + "action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time," \ + "actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time," \ + "actor_process_file_mod_time,os_actor_process_file_mod_time,action_file_create_time,action_file_path," \ + "action_process_image_path,action_registry_file_path,actor_process_image_path," \ + "causality_actor_process_image_path,os_actor_process_image_path,action_process_image_command_line," \ + "actor_process_command_line,causality_actor_process_command_line,os_actor_process_command_line," \ + "action_process_file_create_time,actor_process_file_create_time," \ + "causality_actor_process_file_create_time,os_actor_process_file_create_time," \ + "action_module_process_os_pid,action_process_os_pid,actor_process_os_pid,causality_actor_process_os_pid," \ + "os_actor_process_os_pid,action_process_requested_parent_pid,action_thread_parent_pid," \ + "action_thread_child_pid,action_process_username,auth_domain,dst_host_metadata_domain," \ + "host_metadata_domain,dst_action_url_category,action_registry_key_name,action_registry_value_name,mac," \ + "associated_mac,dst_associated_mac,dst_mac,actor_primary_user_sid,action_process_user_sid," \ + "actor_primary_username," \ + "actor_process_logon_id,action_file_info_company,action_file_extension,action_file_attributes," \ + "action_file_internal_zipped_files,action_file_last_writer_actor,action_file_signature_status," \ + "action_file_signature_vendor,action_file_signature_product,action_file_info_description," \ + "action_file_group,action_file_group_name,action_file_type,action_file_info_file_version," \ + "manifest_file_version,action_file_info_product_version,action_file_owner,action_file_owner_name," \ + "action_file_info_product_name,action_file_id,action_file_wildfire_verdict," \ + "action_file_hash_control_verdict,actor_process_instance_id,actor_process_causality_id," \ + "actor_process_auth_id,actor_process_container_id,actor_process_signature_vendor," \ + "actor_process_signature_status,actor_process_signature_product,actor_process_image_extension," \ + "action_process_termination_code,action_process_termination_date,action_remote_process_thread_id," \ + "action_process_instance_execution_time,actor_process_execution_time,action_process_handle_is_kernel," \ + "action_process_is_container_root,actor_process_is_native,agent_version,agent_hostname," \ + "agent_content_version,agent_session_start_time,agent_id,agent_os_type,agent_os_sub_type,agent_is_vdi," \ + "action_user_agent,http_req_user_agent_header,action_evtlog_data_fields,action_evtlog_description," \ + "action_evtlog_source,action_evtlog_event_id,action_evtlog_level,action_evtlog_tid,action_evtlog_uid," \ + "action_evtlog_pid,action_evtlog_message,action_evtlog_version,event_id,vpn_event_description," \ + "event_timestamp,event_version,event_rpc_interface_uuid,event_address_mapped_image_path,event_type," \ + "event_sub_type,action_network_creation_time,action_network_connection_id,action_network_packet_data," \ + "action_proxy,host_metadata_hostname,action_external_hostname" + + +class TestQueryTranslator(unittest.TestCase): + if __name__ == "__main__": + unittest.main() + + def _test_query_assertions(self, query, queries): + """ + to assert the each query in the list against expected result + """ + self.assertIsInstance(queries, list) + self.assertIsInstance(query, dict) + self.assertIsInstance(query['queries'], list) + for index, each_query in enumerate(query.get('queries'), start=0): + self.assertEqual(each_query, queries[index]) + + def test_ipv4_query(self): + stix_pattern = "[ipv4-addr:value = '172.31.90.48']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_local_ip = \"172.31.90.48\" or " + "action_remote_ip = \"172.31.90.48\" or agent_ip_addresses = \"172.31.90.48\") and " + "(to_epoch(_time,\"millis\") >= 1645615464114 and to_epoch(_time,\"millis\") <= 1645615764114)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" + "'from': 1645615464114, 'to': " + "1645615764114}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_traffic_query(self): + stix_pattern = "[network-traffic:dst_port=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port = 53996 and " + "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" + " | alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" + "'from': 1645615637334, 'to': " + "1645615937334}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_greater_than(self): + stix_pattern = "[network-traffic:dst_port>53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port > 53996 and " + "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" + " | alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" + "'from': 1645615637334, 'to': " + "1645615937334}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_not_equals(self): + stix_pattern = "[network-traffic:dst_port!=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port != 53996 and " + "(to_epoch(_time,\"millis\") >= 1645615637334 and to_epoch(_time,\"millis\") <= 1645615937334))" + " | alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', 'timeframe': {" + "'from': 1645615637334, 'to': " + "1645615937334}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_enum_type(self): + stix_pattern = "[network-traffic:protocols[*] IN ('TCP','udp')]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_network_protocol in (ENUM.TCP," + "ENUM.UDP) and " + "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645635857746, " + "'to': 1645636157746}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_not_greater_than(self): + stix_pattern = "[network-traffic:dst_port NOT > 53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port <= 53996 and " + "(to_epoch(_time,\"millis\") >= 1645636616556 and to_epoch(_time,\"millis\") <= 1645636916556)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1645636616556, 'to': " + "1645636916556}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_less_than(self): + stix_pattern = "[network-traffic:dst_port<53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port < 53996 and " + "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1645636370847, 'to': " + "1645636670847}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_lessthan_or_equals(self): + stix_pattern = "[network-traffic:dst_port<=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port <= 53996 and " + "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1645636370847, 'to': " + "1645636670847}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_network_query_greaterthan_or_equals(self): + stix_pattern = "[network-traffic:dst_port>=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port >= 53996 and " + "(to_epoch(_time,\"millis\") >= 1645636370847 and to_epoch(_time,\"millis\") <= 1645636670847)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1645636370847, 'to': " + "1645636670847}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_file_like_operator(self): + stix_pattern = "[file:name LIKE 'edr-2022-02-09_08-56-58-474-checksum.txt']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or action_process_image_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or actor_process_image_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or causality_actor_process_image_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\" or os_actor_process_image_name contains " + "\"edr-2022-02-09_08-56-58-474-checksum.txt\") and " + "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645635327346, " + "'to': 1645635627346}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_file_IN_operator(self): + stix_pattern = "[file:name IN ('SentinelOne_1.binlog','edr-2022-02-09_08-56-58-47-checksum.txt')]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name in (" + "\"SentinelOne_1.binlog\",\"edr-2022-02-09_08-56-58-47-checksum.txt\") or " + "action_process_image_name in (\"SentinelOne_1.binlog\"," + "\"edr-2022-02-09_08-56-58-47-checksum.txt\") or actor_process_image_name in (" + "\"SentinelOne_1.binlog\",\"edr-2022-02-09_08-56-58-47-checksum.txt\") or " + "causality_actor_process_image_name in (\"SentinelOne_1.binlog\"," + "\"edr-2022-02-09_08-56-58-47-checksum.txt\") or os_actor_process_image_name in (" + "\"SentinelOne_1.binlog\",\"edr-2022-02-09_08-56-58-47-checksum.txt\")) and " + "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1645635327346, 'to': " + "1645635627346}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_file_matches_operator(self): + stix_pattern = "[file:name MATCHES '^g.{2}.exe']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name ~= \"^g.{2}.exe\" or " + "action_process_image_name ~= \"^g.{2}.exe\" or actor_process_image_name ~= \"^g.{2}.exe\" or " + "causality_actor_process_image_name ~= \"^g.{2}.exe\" or os_actor_process_image_name ~= \"^g.{" + "2}.exe\") and " + "(to_epoch(_time,\"millis\") >= 1645635327346 and to_epoch(_time,\"millis\") <= 1645635627346)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645635327346, " + "'to': 1645635627346}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_mac_address_query(self): + stix_pattern = "[mac-addr:value ='12:83:0e:be:f3:1d']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((mac = \"12:83:0e:be:f3:1d\" or " + "associated_mac = \"12:83:0e:be:f3:1d\" or dst_associated_mac = \"12:83:0e:be:f3:1d\" or dst_mac " + "= \"12:83:0e:be:f3:1d\") and " + "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645635857746, " + "'to': 1645636157746}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_process_not_matches_operator(self): + stix_pattern = "[file:name NOT MATCHES '^g.{2}.exe']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name !~= \"^g.{2}.exe\" or " + "action_process_image_name !~= \"^g.{2}.exe\" or actor_process_image_name !~= \"^g.{2}.exe\" or " + "causality_actor_process_image_name !~= \"^g.{2}.exe\" or os_actor_process_image_name !~= \"^g.{" + "2}.exe\") and " + "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645636692740, " + "'to': 1645636992740}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_process_not_in_operator(self): + stix_pattern = "[process:name NOT IN('conhost.exe','AtBroker.exe')]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\") or actor_process_image_name not in (\"conhost.exe\"," + "\"AtBroker.exe\") or causality_actor_process_image_name not in (\"conhost.exe\",\"AtBroker.exe\") " + "or os_actor_process_image_name not in (\"conhost.exe\",\"AtBroker.exe\")) and " + "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) " + "| alter " + "dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1645636692740, 'to': " + "1645636992740}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_windows_regkey_not_like_operator(self): + stix_pattern = "[windows-registry-key:values[*] NOT LIKE 'DeltaUpdateFailure']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_registry_value_name not contains " + "\"DeltaUpdateFailure\" and " + "(to_epoch(_time,\"millis\") >= 1645638324427 and to_epoch(_time,\"millis\") <= 1645638624427)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645638324427, " + "'to': 1645638624427}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_from_multiple_comparison_expressions_joined_by_AND(self): + stix_pattern = "[file:name LIKE 'arp_cache.py' AND windows-registry-key:values[*] NOT LIKE 'SlotPerRow']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_registry_value_name not contains " + "\"SlotPerRow\" and (action_file_name contains \"arp_cache.py\" or action_process_image_name " + "contains \"arp_cache.py\" or actor_process_image_name contains \"arp_cache.py\" or " + "causality_actor_process_image_name contains \"arp_cache.py\" or os_actor_process_image_name " + "contains \"arp_cache.py\") and " + "(to_epoch(_time,\"millis\") >= 1645638324427 and to_epoch(_time,\"millis\") <= 1645638624427)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645638324427, " + "'to': 1645638624427}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_from_multiple_comparison_expressions_joined_by_OR(self): + stix_pattern = "[network-traffic:dst_port!=53996 OR process:name NOT MATCHES '^G.{5}U.{5}.exe']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_process_image_name !~= \"^G.{5}U.{" + "5}.exe\" or actor_process_image_name !~= \"^G.{5}U.{5}.exe\" or " + "causality_actor_process_image_name !~= \"^G.{5}U.{5}.exe\" or os_actor_process_image_name !~= " + "\"^G.{5}U.{5}.exe\") or action_remote_port != 53996 and " + "(to_epoch(_time,\"millis\") >= 1645635761271 and to_epoch(_time,\"millis\") <= 1645636061271)) " + "| alter dataset_name = " + "\"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1645635761271," + " 'to': 1645636061271}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_query_for_morethan_two_comparison_expressions_joined_by_AND(self): + stix_pattern = "[file:name NOT LIKE 'arp_cache.py' AND windows-registry-key:values[*] IN" \ + "('SlotPerRow','DeltaUpdateFailure') AND network-traffic:dst_port!=53996]" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port != 53996 and " + "action_registry_value_name in (\"SlotPerRow\",\"DeltaUpdateFailure\") and (action_file_name not " + "contains \"arp_cache.py\" or action_process_image_name not contains \"arp_cache.py\" or " + "actor_process_image_name not contains \"arp_cache.py\" or causality_actor_process_image_name not " + "contains \"arp_cache.py\" or os_actor_process_image_name not contains \"arp_cache.py\") and " + "(to_epoch(_time,\"millis\") >= 1645636692740 and to_epoch(_time,\"millis\") <= 1645636992740)) | " + "alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1645636692740, " + "'to': 1645636992740}}}"] + queries = _remove_timestamp_from_query(queries) + + self._test_query_assertions(query, queries) + + def test_query_for_morethan_two_comparison_expressions_joined_by_OR(self): + stix_pattern = "[network-traffic:dst_port!=53996 OR process:name NOT MATCHES '^G.{5}U.{5}.exe' OR " \ + "mac-addr:value ='12:83:0e:be:f3:1d']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((mac = \"12:83:0e:be:f3:1d\" or " + "associated_mac = \"12:83:0e:be:f3:1d\" or dst_associated_mac = \"12:83:0e:be:f3:1d\" or dst_mac " + "= \"12:83:0e:be:f3:1d\") or (action_process_image_name !~= \"^G.{5}U.{5}.exe\" or " + "actor_process_image_name !~= \"^G.{5}U.{5}.exe\" or causality_actor_process_image_name !~= \"^G.{" + "5}U.{5}.exe\" or os_actor_process_image_name !~= \"^G.{5}U.{5}.exe\") or action_remote_port != " + "53996 and " + "(to_epoch(_time,\"millis\") >= 1645636966650 and to_epoch(_time,\"millis\") <= 1645637266650)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645636966650, " + "'to': 1645637266650}}}"] + + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_and_without_qualifier_query(self): + stix_pattern = "[file:name LIKE 'metadata']START t'2022-01-01T00:00:00.030Z' STOP t'2022-02-07T00:00:00.030Z'" \ + " AND[windows-registry-key:values[*] NOT LIKE 'SlotPerRow'OR " \ + "process:name MATCHES '^G.{5}U.{5}.exe']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_file_name contains \"metadata\" or " + "action_process_image_name contains \"metadata\" or actor_process_image_name contains \"metadata\" " + "or causality_actor_process_image_name contains \"metadata\" or os_actor_process_image_name " + "contains \"metadata\") and (to_epoch(_time,\"millis\") >= 1640995200030 and to_epoch(_time," + "\"millis\") <= 1644192000030)) or ((action_process_image_name ~= \"^G.{5}U.{5}.exe\" or " + "actor_process_image_name ~= \"^G.{5}U.{5}.exe\" or causality_actor_process_image_name ~= \"^G.{" + "5}U.{5}.exe\" or os_actor_process_image_name ~= \"^G.{5}U.{5}.exe\") or " + "action_registry_value_name not contains \"SlotPerRow\" and " + "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " + "| alter dataset_name = " + "\"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1640995200030, " + "'to': 1645636157746}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_observation_with_invalid_operator(self): + stix_pattern = "[network-traffic:dst_port LIKE '53996']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'LIKE operator is supported only for string type input' in result['error'] + + def test_invalid_mac_address(self): + stix_pattern = "[mac-addr:value = '00:00:00']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'Invalid mac address' in result['error'] + + def test_invalid_stix_pattern(self): + stix_pattern = "[not_a_valid_pattern]" + result = translation.translate('paloalto', 'query', '{}', stix_pattern, {'validate_pattern': 'true'}) + assert result['success'] is False + assert ErrorCode.TRANSLATION_STIX_VALIDATION.value == result['code'] + assert stix_pattern[1:-1] in result['error'] + + def test_combined_observation_AND(self): + stix_pattern = "([process:pid = '868' OR process:name = 'svchost.exe'] AND " \ + "[network-traffic:dst_port = '53996' AND ipv4-addr:value = '172.31.31.67']) " \ + "START t'2022-01-19T11:00:00.000Z' STOP t'2022-02-07T11:00:00.003Z'" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_process_image_name = \"svchost.exe\" " + "or actor_process_image_name = \"svchost.exe\" or causality_actor_process_image_name = " + "\"svchost.exe\" or os_actor_process_image_name = \"svchost.exe\") or (" + "action_module_process_os_pid = 868 or action_process_os_pid = 868 or actor_process_os_pid = 868 " + "or causality_actor_process_os_pid = 868 or os_actor_process_os_pid = 868 or " + "action_process_requested_parent_pid = 868 or action_thread_parent_pid = 868 or " + "action_thread_child_pid = 868) and " + "(to_epoch(_time,\"millis\") >= 1642590000000 and to_epoch(_time,\"millis\") <= 1644231600003)) " + "or ((action_local_ip = \"172.31.31.67\" or action_remote_ip " + "= \"172.31.31.67\" or agent_ip_addresses = \"172.31.31.67\") and action_remote_port = 53996 and " + "(to_epoch(_time,\"millis\") >= 1642590000000 and to_epoch(_time,\"millis\") <= 1644231600003)) " + "| alter dataset_name = " + "\"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': 1642590000000, " + "'to': 1644231600003}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combined_observation_OR(self): + stix_pattern = "([file:name NOT MATCHES '^g.{2}.exe' AND network-traffic:dst_port <= 137]OR " \ + "[mac-addr:value = '12:83:0e:be:f3:1d' OR process:name LIKE 'svchost.exe']) " \ + "START t'2022-01-10T11:00:00.000Z' STOP t'2022-02-05T11:00:00.003Z'" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter (action_remote_port <= 137 and (" + "action_file_name !~= \"^g.{2}.exe\" or action_process_image_name !~= \"^g.{2}.exe\" or " + "actor_process_image_name !~= \"^g.{2}.exe\" or causality_actor_process_image_name !~= \"^g.{" + "2}.exe\" or os_actor_process_image_name !~= \"^g.{2}.exe\") and " + "(to_epoch(_time,\"millis\") >= 1641812400000 and to_epoch(_time,\"millis\") <= 1644058800003)) " + "or ((action_process_image_name " + "contains \"svchost.exe\" or actor_process_image_name contains \"svchost.exe\" or " + "causality_actor_process_image_name contains \"svchost.exe\" or os_actor_process_image_name " + "contains \"svchost.exe\") or (mac = \"12:83:0e:be:f3:1d\" or associated_mac = " + "\"12:83:0e:be:f3:1d\" or dst_associated_mac = \"12:83:0e:be:f3:1d\" or dst_mac = " + "\"12:83:0e:be:f3:1d\") and " + "(to_epoch(_time,\"millis\") >= 1641812400000 and to_epoch(_time,\"millis\") <= 1644058800003)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1641812400000, " + "'to': 1644058800003}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_invalid_data_for_matches(self): + stix_pattern = "[process:pid MATCHES '53996']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'MATCHES operators is supported only for string type input' in result['error'] + + def test_invalid_regex_for_matches(self): + stix_pattern = "[process:name MATCHES 'wild^fire$']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert '^ symbol should be at the starting position of the expression' in result['error'] + + def test_invalid_enum_values(self): + stix_pattern = "[network-traffic:protocols[*] = 'idp']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'Unsupported ENUM values provided' in result['error'] + + def test_matches_operator_for_enum(self): + stix_pattern = "[network-traffic:protocols[*] MATCHES 'tcp']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'MATCHES operators is supported only for string type input' in result['error'] + + def test_invalid_operator_for_enum(self): + stix_pattern = "[network-traffic:protocols[*] < 'tcp']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'operator is not supported for Enum type input' in result['error'] + + def test_invalid_operator_for_string_input(self): + stix_pattern = "[process:name < 'conhost.exe' ]" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'operator is not supported for string type input' in result['error'] + + def test_invalid_qualifier(self): + stix_pattern = "[process:pid < '53996'] START " \ + "t'2022-02-01T08:43:10.003Z' STOP t'2022-01-07T10:43:10.005Z' " + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert "invalid_parameter" == result['code'] + assert 'Start time should be lesser than Stop time' in result['error'] + + def test_invalid_dollar_regexp(self): + stix_pattern = "[process:name MATCHES '^wildfire$s']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert '$ symbol should be at the ending position of the expression' in result['error'] + + def test_invalid_value_for_timestamp_field(self): + stix_pattern = "[process:created = '2022-02-0108:43:10.003Z']" + result = translation.translate('paloalto', 'query', '{}', stix_pattern) + assert result['success'] is False + assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] + assert 'cannot convert the timestamp' in result['error'] + + def test_format_timestamp_fields(self): + stix_pattern = "[process:created = '2022-02-01T08:43:10.003Z']" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'xdr_data': {'query': 'dataset = xdr_data | filter ((action_process_file_create_time = " + "1643704990003 or actor_process_file_create_time = 1643704990003 or " + "causality_actor_process_file_create_time = 1643704990003 or os_actor_process_file_create_time = " + "1643704990003) and " + "(to_epoch(_time,\"millis\") >= 1645635857746 and to_epoch(_time,\"millis\") <= 1645636157746)) " + "| alter dataset_name = \"xdr_data\" | fields " + all_fields + " | limit 10000 ', " + "'timeframe': {'from': " + "1645635857746, " + "'to': 1645636157746}}}"] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) diff --git a/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py new file mode 100644 index 000000000..eb272ca6e --- /dev/null +++ b/stix_shifter_modules/paloalto/tests/stix_transmission/test_paloalto.py @@ -0,0 +1,589 @@ +import unittest +from unittest.mock import patch +from stix_shifter.stix_transmission import stix_transmission +from stix_shifter_modules.paloalto.entry_point import EntryPoint +import json +from requests.exceptions import ConnectionError + + +class MockStatusObj: + def __init__(self, code): + self.status_code = code + + +class MockResponse: + def __init__(self, response_code, obj): + self.response = response_code + self.object = obj + + def read(self): + return bytearray(self.object, 'utf-8') + + +class PaloaltoMockResponse: + """ class for Palo Alto mock response""" + + def __init__(self, response_code, txt): + self.status_code = response_code + self.content = txt + + def read(self): + """ to read contents of results returned by api""" + return bytearray(self.content, 'utf-8') + + +class StatusResponse: + """ class for status response""" + + def __init__(self, code, txt): + self.code = code + self.content = txt + + def read(self): + return bytearray(self.content, 'utf-8') + + +class PingResponse: + """ class for ping outer response""" + + def __init__(self, response_object): + self.response = response_object + + def read(self): + return bytearray(self.response.content, 'utf-8') + + +class TestPaloaltoConnection(unittest.TestCase, object): + + @staticmethod + def connection(): + return { + "host": "hostbla" + } + + @staticmethod + def configuration(): + return { + "auth": { + "api_key": "bla", + "api_key_id": "bla", + "tenant": "bla" + } + } + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping(self, mock_ping_response): + """test ping connection""" + mocked_return_value = '{"reply":{"used_quota": 0.08015277777777775}}' + mock_ping = PaloaltoMockResponse(200, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is True + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') + def test_query_response(self, mock_search_response): + """test create search query""" + mocked_return_value = '{"reply": {"search_id": "07f63c733f5946_15006_inv"}}' + mock_search = PaloaltoMockResponse(200, mocked_return_value) + search_response = PingResponse(mock_search) + mock_search_response.return_value = search_response + + query = json.dumps({"xdr_data": {"query": "dataset = xdr_data | filter ((action_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\") or actor_process_image_name not " + "in (\"conhost.exe\",\"AtBroker.exe\") or " + "causality_actor_process_image_name not in (\"conhost.exe\"," + "\"AtBroker.exe\") or os_actor_process_image_name not in (" + "\"conhost.exe\",\"AtBroker.exe\")) and (to_epoch(_time," + "\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") <= " + "1644883200000)) or ((action_process_file_create_time = " + "1643704990003 or actor_process_file_create_time = 1643704990003 or " + "causality_actor_process_file_create_time = 1643704990003 or " + "os_actor_process_file_create_time = 1643704990003) and (to_epoch(" + "_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") " + "<= 1644883200000)) or ((action_process_image_name ~= \"wildfire$\" " + "or actor_process_image_name ~= \"wildfire$\" or " + "causality_actor_process_image_name ~= \"wildfire$\" or " + "os_actor_process_image_name ~= \"wildfire$\") and (to_epoch(" + "_time,\"millis\") >= 1644451200000 and to_epoch(_time,\"millis\") " + "<= 1644883200000)) | alter dataset_name = \"xdr_data\" | fields " + "dataset_name,action_local_ip,action_remote_ip," + "agent_ip_addresses_v6,dst_agent_ip_addresses_v6,action_local_port," + "action_remote_port,action_network_protocol,action_file_name," + "action_file_size,action_file_md5,action_module_md5," + "action_process_image_md5,action_file_authenticode_sha1," + "action_file_authenticode_sha2,action_file_sha256," + "action_module_sha256,action_process_image_sha256," + "action_file_access_time,actor_process_file_access_time," + "os_actor_process_file_access_time,action_file_mod_time," + "actor_process_file_mod_time,os_actor_process_file_mod_time," + "action_file_create_time,action_file_path," + "action_process_image_path,action_registry_file_path," + "actor_process_image_path,causality_actor_process_image_path," + "os_actor_process_image_path,action_process_image_command_line," + "actor_process_command_line,causality_actor_process_command_line," + "os_actor_process_command_line,action_process_file_create_time," + "actor_process_file_create_time," + "causality_actor_process_file_create_time," + "os_actor_process_file_create_time,action_process_image_name," + "actor_process_image_name,causality_actor_process_image_name," + "os_actor_process_image_name,action_module_process_os_pid ," + "action_process_os_pid,actor_process_os_pid," + "causality_actor_process_os_pid,os_actor_process_os_pid," + "action_process_requested_parent_pid,action_thread_parent_pid," + "action_thread_child_pid,action_process_username,auth_domain," + "dst_host_metadata_domain,host_metadata_domain," + "dst_action_url_category ,action_registry_key_name," + "action_registry_value_name,mac,associated_mac,dst_associated_mac ," + "dst_mac,dst_user_id,user_id,action_username," + "actor_primary_username,actor_process_logon_id | limit 10000 ", + "timeframe": {"from": 1644451200000, "to": 1644883200000}}}) + + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query(query) + assert query_response is not None + assert 'success' in query_response + assert query_response['success'] is True + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_response(self, mock_status_response): + """test status response""" + mocked_return_value = '{"reply" : {"status": "SUCCESS","number_of_results":100}}' + mock_status_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is True + assert status_response['status'] == "COMPLETED" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_result_response(self, mock_result_response): + """test for valid result response""" + mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", + "number_of_results": 1, + "results": {"data": [{"dataset_name": "xdr_data", + "causality_actor_process_image_name": + "taskhostw.exe"}]}}}) + mock_result_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'] == [{'xdr_data': {'causality_actor_process_image_name': 'taskhostw.exe'}}] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_result_with_empty_nt_response(self, mock_result_response): + """test for valid result response""" + mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", + "number_of_results": 1, + "results": {"data": [{"dataset_name": "xdr_data", + 'action_local_ip': '', 'action_remote_ip': '', + 'agent_ip_addresses_v6': None, + 'dst_agent_ip_addresses_v6': None, + 'action_local_port': 0, + 'action_remote_port': 0, + 'action_pkts_sent': None, + 'action_pkts_received': None, + 'action_network_protocol': "NULL", + 'actor_process_image_name': "lsass.exe"}]}}}) + mock_result_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "62428d95420f47_24655_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'] == [{'xdr_data': {'actor_process_image_name': 'lsass.exe'}}] + + def test_delete_response(self): + """test delete response""" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + search_id = "e1d1b56ca81845_15180_inv" + delete_response = transmission.delete(search_id) + assert delete_response is not None + assert delete_response['success'] is True + assert delete_response['message'] == "Delete operation of a search id is not supported in Palo Alto Cortex XDR" + + def test_is_async(self): + """check for synchronous or asynchronous""" + entry_point = EntryPoint(self.connection(), self.configuration()) + check_async = entry_point.is_async() + assert check_async is True + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_400_exception(self, mock_ping_response): + """Test ping response with 400 exception""" + mocked_return_value = '{"reply":{"err_msg": "InvalidJson"}}' + mock_ping = PaloaltoMockResponse(400, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert 'error' in ping_response + assert ping_response['code'] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_401_auth_exception(self, mock_ping_response): + """test 401 authentication error exception""" + mocked_return_value = '{"reply": { "err_msg" : "auth Error"}}' + mock_ping = PaloaltoMockResponse(401, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert ping_response['code'] == "authentication_fail" + assert "Invalid api_key" in ping_response['error'] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_500_exception(self, mock_ping_response): + """check 500 bad query exception""" + mocked_return_value = '{"reply": {"err_extra":{"parse_err" :"Internal server error" }}}' + mock_ping = PaloaltoMockResponse(500, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert "Bad query Syntax" in ping_response["error"] + assert ping_response["code"] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_internal_server_exception(self, mock_ping_response): + """check 500 bad query exception""" + mocked_return_value = '{"reply": {"err_msg":"Internal server error","err_extra":"server error"}}' + mock_ping = PaloaltoMockResponse(500, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert "Internal server error" in ping_response["error"] + assert ping_response["code"] == "invalid_parameter" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_402_license_exception(self, mock_ping_response): + """Check 402 invalid license exception""" + mocked_return_value = '{"reply": { "err_msg" : "Invalid license"}}' + mock_ping = PaloaltoMockResponse(402, mocked_return_value) + mock_ping_response.return_value = PingResponse(mock_ping) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert 'error' in ping_response + assert ping_response['code'] == "service_unavailable" + assert "User does not have the required license type to run this API" in ping_response['error'] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_empty_result_exception(self, mock_status_response): + """Test empty results exception""" + mocked_return_value = '{"reply" : {"status": "SUCCESS"}}' + mock_status_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is False + assert 'error' in status_response + assert "Empty results received from Tenant" in status_response['error'] + assert status_response['code'] == "no_results" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_partial_success_exception(self, mock_status_response): + """Test partial success status response exception""" + mocked_return_value = '{"reply" : {"status": "PARTIAL_SUCCESS"}}' + mock_status_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is True + assert status_response['message'] == "Partial Success -At least one tenant failed to execute the query" + assert status_response['status'] == "COMPLETED" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_fail_exception(self, mock_search_response): + """Test status fail exception""" + mocked_return_value = '{"reply" : {"status": "FAIL"}}' + mock_search_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is False + assert "Tenant Query Failed" in status_response['error'] + assert status_response['code'] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_failed_query_response(self, mock_result_response): + """Test failed query result response""" + mocked_return_value = json.dumps({"reply": {"status": "FAIL"}}) + mock_result_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert "Tenant Query Failed" in result_response['error'] + assert result_response['code'] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') + def test_create_query_value_error_exception(self, mock_search_response): + """test create query value error with invalid json""" + mocked_return_value = "Invalid_json" + mock_search = PaloaltoMockResponse(200, mocked_return_value) + search_response = PingResponse(mock_search) + mock_search_response.return_value = search_response + query = "" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query(query) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert "Cannot parse response" in query_response["error"] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_value_error_exception(self, mock_search_response): + """test ping connector value error with invalid json""" + mocked_return_value = "Invalid_json" + mock_search = PaloaltoMockResponse(200, mocked_return_value) + search_response = PingResponse(mock_search) + mock_search_response.return_value = search_response + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.ping() + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert "Cannot parse response" in query_response["error"] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.create_search') + def test_create_query_exception(self, mock_search_response): + """Test search response with 400 exception""" + mocked_return_value = '{"reply": { "err_msg" : "400 error"}}' + mock_ping = PaloaltoMockResponse(400, mocked_return_value) + mock_search_response.return_value = PingResponse(mock_ping) + query = "{}" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.query(query) + assert ping_response is not None + assert ping_response['success'] is False + assert 'error' in ping_response + assert ping_response['code'] == "invalid_query" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_results_value_error_exception(self, get_search_results): + """test results connector value error with invalid json""" + mocked_return_value = "Invalid json" + get_search_results.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert "Cannot parse response" in result_response["error"] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_results_401_exception(self, get_search_results): + """test results with 401 exception""" + mocked_return_value = '{"reply": { "err_msg" : "auth Error"}}' + get_search_results.return_value = StatusResponse(401, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert result_response['code'] == "authentication_fail" + assert "Invalid api_key" in result_response['error'] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_pending_response(self, mock_search_response): + """Test status pending exception""" + mocked_return_value = '{"reply" : {"status": "PENDING"}}' + mock_search_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert 'success' in status_response + assert status_response['success'] is True + assert status_response['status'] == "RUNNING" + assert status_response['progress'] == 50 + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_value_error_exception(self, mock_search_status): + """test status connector value error with invalid json""" + mocked_return_value = "Invalid json" + mock_search_status.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert status_response['success'] is False + assert 'error' in status_response + assert "Cannot parse response" in status_response["error"] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_status') + def test_status_403_exception(self, get_search_results): + """test results with 403 exception""" + mocked_return_value = '{"reply": { "err_msg" : "api permission exception"}}' + get_search_results.return_value = StatusResponse(403, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status(search_id) + assert status_response is not None + assert status_response['success'] is False + assert status_response['code'] == "forbidden" + assert "The provided API Key does not have the required RBAC permissions to run this API" in \ + status_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_max_quota_exception(self, mock_ping): + """test maximum quota threshold exception""" + response = { + 'reply': {'license_quota': 5, 'additional_purchased_quota': 0.0, 'used_quota': 5.01, 'eval_quota': 0.0}} + mock_ping.side_effect = [MockResponse(MockStatusObj(200), json.dumps(response))] + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query({}) + assert query_response is not None + assert query_response['success'] is False + assert query_response['code'] == "service_unavailable" + assert "query usage exceeded max daily quota" in query_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_max_additional_quota_exception(self, mock_ping): + """test maximum additional quota threshold exception""" + response = { + 'reply': {'license_quota': 5, 'additional_purchased_quota': 10.0, 'used_quota': 12, 'eval_quota': 0.0}} + mock_ping.side_effect = [MockResponse(MockStatusObj(200), json.dumps(response))] + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query({}) + assert query_response is not None + assert query_response['success'] is False + assert query_response['code'] == "service_unavailable" + assert "query usage exceeded max daily quota" in query_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_quota_invalid_json_exception(self, mock_ping): + """test quota invalid json exception""" + response = "invalid json" + mock_ping.side_effect = [MockResponse(MockStatusObj(200), response)] + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + query_response = transmission.query({}) + assert query_response is not None + assert query_response['success'] is False + assert 'error' in query_response + assert "Cannot parse response" in query_response["error"] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_invalid_host(self, mock_ping): + """Test Invalid host""" + mock_ping.side_effect = ConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert "Invalid Host" in ping_response['error'] + assert ping_response['code'] == "service_unavailable" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_invalid_host_for_status(self, mock_query): + """Test Invalid host for Status API""" + mock_query.side_effect = ConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + status_response = transmission.status("123_inv") + assert status_response is not None + assert status_response['success'] is False + assert "Invalid Host" in status_response['error'] + assert status_response['code'] == "service_unavailable" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_invalid_host_for_results(self, mock_query): + """Test Invalid host for Results API""" + mock_query.side_effect = ConnectionError("Invalid Host") + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + results_response = transmission.results("123_inv", 0, 2) + assert results_response is not None + assert results_response['success'] is False + assert "Invalid Host" in results_response['error'] + assert results_response['code'] == "service_unavailable" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClient.RestApiClient.call_api') + def test_timeout_error(self, mock_ping): + """Test Timeout Error""" + mock_ping.side_effect = TimeoutError("timeout_error (30 sec)") + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert "timeout_error (30 sec)" in ping_response['error'] + assert ping_response['code'] == "service_unavailable" + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_stream_results') + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_format_stream_data(self, mock_search_response, mock_stream_response): + stream_id = "123" + mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", + "number_of_results": 10001, + "results": {"stream_id": stream_id}}}) + mock_search_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "e1d1b56ca81845_15180_inv" + stream_return_value = '{"dataset_name":"xdr_data","action_local_ip":"65.0.202.35","action_remote_ip":' \ + '"172.31.90.48","action_local_port":"50893","action_remote_port":"3389",' \ + '"action_network_protocol":"TCP"}' + mock_stream_response.return_value = StatusResponse(200, stream_return_value) + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'] == [{'xdr_data': {'action_local_ip': '65.0.202.35', + 'action_network_protocol': 'TCP', + 'action_remote_ip': '172.31.90.48', + 'action_local_port': '50893', + 'action_remote_port': '3389'}}] + + @patch('stix_shifter_modules.paloalto.stix_transmission.api_client.APIClient.get_search_results') + def test_result_with_empty_user_response(self, mock_result_response): + """test for valid result response""" + mocked_return_value = json.dumps({"reply": {"status": "SUCCESS", + "number_of_results": 1, + "results": {"data": [{"dataset_name": "xdr_data", + "actor_primary_user_sid": "S123", + "actor_primary_username": "username", + "actor_process_logon_id": "id12"}]}}}) + mock_result_response.return_value = StatusResponse(200, mocked_return_value) + search_id = "62428d95420f47_24655_inv" + transmission = stix_transmission.StixTransmission('paloalto', self.connection(), self.configuration()) + offset = 0 + length = 3 + result_response = transmission.results(search_id, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'] == [{'xdr_data': {'actor_primary_user_sid': 'S123', + 'actor_primary_username': 'username', + 'actor_process_logon_id': 'id12'}}]