diff --git a/data/cybox/crowdstrike_logscale/crowdstrike_edr_events_04042024.json b/data/cybox/crowdstrike_logscale/crowdstrike_edr_events_04042024.json new file mode 100644 index 000000000..6be9c9bd0 --- /dev/null +++ b/data/cybox/crowdstrike_logscale/crowdstrike_edr_events_04042024.json @@ -0,0 +1,1021 @@ +{ + "type": "bundle", + "id": "bundle--58009c81-55e4-433e-b704-3f6ad1ca6adc", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "crowdstrike_logscale", + "identity_class": "system", + "created": "2023-12-24T13:22:50.336Z", + "modified": "2023-12-24T13:22:50.336Z" + }, + { + "id": "observed-data--054fed68-c9c8-4dd8-910a-33e72e8aa8dc", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-04-03T16:57:35.123Z", + "modified": "2024-04-03T16:57:35.123Z", + "objects": { + "0": { + "type": "file", + "x_extension": "exe", + "name": "ruby.exe", + "x_path": "\\Device\\ruby.exe", + "parent_directory_ref": "3", + "hashes": { + "MD5": "10101010101010101010101001011001", + "SHA-256": "1010101010101010101010100101100101010101010101010101010101010100" + } + }, + "1": { + "type": "x-crowdstrike-detection-behavior", + "behavior_id": "41002", + "confidence": 100, + "control_graph_id": "ctg:7adb:123", + "description": "A process triggered a medium severity custom rule.", + "display_name": "CustomIOAWinMedium", + "process_ref": "2", + "ioc_description": "\\Device\\ruby.exe", + "ioc_source": "library_load", + "ioc_type": "hash_sha256", + "ioc_value": "123", + "objective": "Falcon Detection Method", + "pattern_disposition": 2048, + "pattern_disposition_details": { + "blocking_unsupported_or_disabled": "false", + "bootup_safeguard_enabled": "false", + "critical_process_disabled": "false", + "detect": "false", + "fs_operation_blocked": "false", + "handle_operation_downgraded": "false", + "inddet_mask": "false", + "indicator": "false", + "kill_action_failed": "false", + "kill_parent": "false", + "kill_process": "false", + "kill_subprocess": "false", + "operation_blocked": "false", + "policy_disabled": "false", + "process_blocked": "true", + "quarantine_file": "false", + "quarantine_machine": "false", + "registry_operation_blocked": "false", + "rooting": "false", + "sensor_only": "false", + "suspend_parent": "false", + "suspend_process": "false" + }, + "rule_instance_id": 3, + "rule_instance_version": "3", + "scenario": "suspicious_activity", + "severity": 50, + "ttp_tagging_ref": "6", + "template_instance_id": "3", + "created_time": "2024-03-14T13:41:22Z", + "user_ref": "7" + }, + "2": { + "type": "process", + "command_line": "\"C:\\Ruby25-x64\\bin\\ruby\" -x \"C:\\Ruby25-x64\\bin\\ridk.cmd\" install", + "name": "ruby.exe", + "binary_ref": "0", + "parent_ref": "4", + "x_process_graph_id": "pid:7adb:456", + "creator_user_ref": "7" + }, + "3": { + "type": "directory", + "path": "\\Device\\Ruby25-x64\\bin" + }, + "4": { + "type": "process", + "command_line": "C:\\cmd.exe /c \"\"C:\\Ruby25-x64\\bin\\ridk.cmd\" install\"", + "binary_ref": "5", + "x_process_graph_id": "pid:7adb:xyz" + }, + "5": { + "type": "file", + "hashes": { + "MD5": "10101010101010101010101001011001", + "SHA-256": "1010101010101010101010100101100101010101010101010101010101010100" + } + }, + "6": { + "type": "x-ibm-ttp-tagging", + "name": "Custom Intelligence", + "extensions": { + "mitre-attack-ext": { + "tactic_id": "CSTA0005", + "technique_name": "Indicator of Attack", + "technique_id": "CST0004" + } + } + }, + "7": { + "type": "user-account", + "user_id": "S-1-5-21", + "display_name": "Administrator" + }, + "8": { + "type": "x-ibm-finding", + "x_logscale_repository": "TestRepository", + "x_logscale_event_id": "xpC1sUz8JCF1oyTpI7dSVld5_363_49_1712140849", + "x_behavior_refs": [ + "1" + ], + "ttp_tagging_refs": [ + "6" + ], + "x_behaviors_processed": [ + "pid:7adb:456:41002" + ], + "time_observed": "2024-03-14T13:41:29.903Z", + "x_last_updated": "2024-04-03T10:40:49.668767Z", + "name": "ldt:7adb:123", + "src_ip_ref": "11", + "src_os_ref": "14", + "x_is_email_sent": "false", + "x_first_behavior_observed": "2024-03-14T13:41:22Z", + "x_last_behavior_observed": "2024-03-14T13:41:22Z", + "confidence": 100, + "severity": 50, + "x_severity_name": "Medium", + "x_seconds_to_resolved": "0", + "x_seconds_to_triaged": "0", + "x_status": "new", + "finding_type": "alert" + }, + "9": { + "type": "x-oca-asset", + "x_cid": "cid123", + "x_agent_ref": "10", + "x_bios_manufacturer": "Xen", + "x_bios_version": "4.2.amazon", + "device_id": "did123", + "ip_refs": [ + "11", + "12" + ], + "x_first_seen": "2023-05-16T05:10:55Z", + "x_device_groups": [ + "97350feebe4541e8a615c0d3f18acdf3", + "bb1e1190b46348e69e10785030e8b23d" + ], + "hostname": "host", + "x_instance_id": "i-123", + "x_last_seen": "2024-03-14T13:28:09Z", + "mac_refs": [ + "13" + ], + "x_last_modified": "2024-03-14T13:37:17Z", + "os_ref": "14", + "x_host_type_number": "3", + "host_type": "Server", + "x_service_provider": "AWS_EC2_V2", + "x_service_account_id": "9876", + "x_status": "normal", + "x_system_manufacturer": "Xen", + "x_system_product_name": "HVM domU" + }, + "10": { + "type": "x-crowdstrike-edr-agent", + "load_flags": "0", + "local_time": "2024-03-14T12:34:02.565Z", + "version": "7.11.18110.0", + "config_id_base": "65994763", + "config_id_build": "18110", + "config_id_platform": "3" + }, + "11": { + "type": "ipv4-addr", + "value": "2.3.4.5" + }, + "12": { + "type": "ipv4-addr", + "value": "2.3.4.5", + "resolves_to_refs": [ + "13" + ] + }, + "13": { + "type": "mac-addr", + "value": "01:01:01:01:ab:cd" + }, + "14": { + "type": "software", + "x_major_version": "10", + "x_minor_version": "0", + "version": "Windows Server 2022", + "x_id": "0", + "name": "Windows" + } + }, + "last_observed": "2024-04-03T10:40:49.668Z", + "first_observed": "2024-03-14T13:41:29.903Z", + "number_observed": 1 + }, + { + "id": "observed-data--3768a4d3-74a5-4b45-8394-4d01722e2de1", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-04-03T17:01:37.394Z", + "modified": "2024-04-03T17:01:37.394Z", + "objects": { + "0": { + "type": "file", + "x_extension": "exe", + "name": "WMIC.exe", + "x_path": "\\Device\\wbem\\WMIC.exe", + "parent_directory_ref": "3", + "hashes": { + "MD5": "10101010101010101010101001011001", + "SHA-256": "1010101010101010101010100101100101010101010101010101010101010100" + } + }, + "1": { + "type": "x-crowdstrike-detection-behavior", + "behavior_id": "41002", + "confidence": 100, + "control_graph_id": "ctg:did123:4455", + "description": "A process triggered a medium severity custom rule.", + "display_name": "CustomIOAWinMedium", + "process_ref": "2", + "ioc_description": "\\Device\\wbem\\WMIC.exe", + "ioc_source": "library_load", + "ioc_type": "hash_sha256", + "ioc_value": "bd8", + "objective": "Falcon Detection Method", + "pattern_disposition": 2048, + "pattern_disposition_details": { + "blocking_unsupported_or_disabled": "false", + "bootup_safeguard_enabled": "false", + "critical_process_disabled": "false", + "detect": "false", + "fs_operation_blocked": "false", + "handle_operation_downgraded": "false", + "inddet_mask": "false", + "indicator": "false", + "kill_action_failed": "false", + "kill_parent": "false", + "kill_process": "false", + "kill_subprocess": "false", + "operation_blocked": "false", + "policy_disabled": "false", + "process_blocked": "true", + "quarantine_file": "false", + "quarantine_machine": "false", + "registry_operation_blocked": "false", + "rooting": "false", + "sensor_only": "false", + "suspend_parent": "false", + "suspend_process": "false" + }, + "rule_instance_id": 3, + "rule_instance_version": "3", + "scenario": "suspicious_activity", + "severity": 50, + "ttp_tagging_ref": "6", + "template_instance_id": "3", + "created_time": "2024-01-18T06:09:26Z", + "user_ref": "7" + }, + "2": { + "type": "process", + "command_line": "C:\\wbem\\wmic.exe /namespace:\\\\root\\inventorylogging path msftsil_managementtasks call setloggingstate ", + "name": "WMIC.exe", + "binary_ref": "0", + "parent_ref": "4", + "x_process_graph_id": "pid:did123:1122", + "creator_user_ref": "7" + }, + "3": { + "type": "directory", + "path": "\\Device\\wbem" + }, + "4": { + "type": "process", + "command_line": "\"C:\\cmd.exe\" /d /c C:\\silcollector.cmd configure", + "binary_ref": "5", + "x_process_graph_id": "pid:did123:214919408229" + }, + "5": { + "type": "file", + "hashes": { + "MD5": "10101010101010101010101001011001", + "SHA-256": "1010101010101010101010100101100101010101010101010101010101010100" + } + }, + "6": { + "type": "x-ibm-ttp-tagging", + "name": "Custom Intelligence", + "extensions": { + "mitre-attack-ext": { + "tactic_id": "CSTA0005", + "technique_name": "Indicator of Attack", + "technique_id": "CST0004" + } + } + }, + "7": { + "type": "user-account", + "user_id": "S-1-5-18", + "display_name": "user1" + }, + "8": { + "type": "x-ibm-finding", + "x_logscale_repository": "TestRepository", + "x_logscale_event_id": "xpC1sUz8JCF1oyTpI7dSVld5_363_116_1712140849", + "x_behavior_refs": [ + "1" + ], + "ttp_tagging_refs": [ + "6" + ], + "x_behaviors_processed": [ + "pid:did123:1122:41002" + ], + "time_observed": "2024-01-18T06:09:33.081Z", + "x_last_updated": "2024-04-03T10:40:49.669334Z", + "name": "ldt:did123:4455", + "src_ip_ref": "11", + "src_os_ref": "14", + "x_is_email_sent": "false", + "x_first_behavior_observed": "2024-01-18T06:09:26Z", + "x_last_behavior_observed": "2024-01-18T06:09:26Z", + "confidence": 100, + "severity": 50, + "x_severity_name": "Medium", + "x_seconds_to_resolved": "0", + "x_seconds_to_triaged": "0", + "x_status": "new", + "finding_type": "alert" + }, + "9": { + "type": "x-oca-asset", + "x_cid": "cid123", + "x_agent_ref": "10", + "x_bios_manufacturer": "Xen", + "x_bios_version": "4.11.amazon", + "device_id": "did123", + "ip_refs": [ + "11", + "12" + ], + "x_first_seen": "2023-05-16T05:10:55Z", + "x_device_groups": [ + "97350feebe4541e8a615c0d3f18acdf3", + "bb1e1190b46348e69e10785030e8b23d" + ], + "hostname": "host", + "x_instance_id": "i-123", + "x_last_seen": "2024-01-18T06:08:34Z", + "mac_refs": [ + "13" + ], + "x_last_modified": "2024-01-18T06:08:40Z", + "os_ref": "14", + "x_host_type_number": "3", + "host_type": "Server", + "x_service_provider": "AWS_EC2_V2", + "x_service_account_id": "9876", + "x_status": "normal", + "x_system_manufacturer": "Xen", + "x_system_product_name": "HVM domU" + }, + "10": { + "type": "x-crowdstrike-edr-agent", + "load_flags": "1", + "local_time": "2024-01-18T06:08:24.864Z", + "version": "7.05.17706.0", + "config_id_base": "65994763", + "config_id_build": "17706", + "config_id_platform": "3" + }, + "11": { + "type": "ipv4-addr", + "value": "44.202.166.182" + }, + "12": { + "type": "ipv4-addr", + "value": "2.3.4.5", + "resolves_to_refs": [ + "13" + ] + }, + "13": { + "type": "mac-addr", + "value": "01:01:01:01:ab:cd" + }, + "14": { + "type": "software", + "x_major_version": "10", + "x_minor_version": "0", + "version": "Windows Server 2022", + "x_id": "0", + "name": "Windows" + } + }, + "last_observed": "2024-04-03T10:40:49.669Z", + "first_observed": "2024-01-18T06:09:33.081Z", + "number_observed": 1 + }, + { + "id": "observed-data--a70eddf0-2388-4663-b7e0-83a3c2d5c95b", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-04-03T17:09:04.816Z", + "modified": "2024-04-03T17:09:04.816Z", + "objects": { + "0": { + "type": "file", + "x_extension": "exe", + "name": "wusa.exe", + "x_path": "\\Device\\wusa.exe", + "parent_directory_ref": "3", + "hashes": { + "MD5": "22333333344444455555", + "SHA-256": "887777322334454111111111111111111" + } + }, + "1": { + "type": "x-crowdstrike-detection-behavior", + "behavior_id": "41002", + "confidence": 100, + "control_graph_id": "ctg:did123:214759104287", + "description": "A process triggered a medium severity custom rule.", + "display_name": "CustomIOAWinMedium", + "process_ref": "2", + "ioc_description": "\\Device\\wusa.exe", + "ioc_source": "library_load", + "ioc_type": "hash_sha256", + "ioc_value": "1abcd", + "objective": "Falcon Detection Method", + "pattern_disposition": 2048, + "pattern_disposition_details": { + "blocking_unsupported_or_disabled": "false", + "bootup_safeguard_enabled": "false", + "critical_process_disabled": "false", + "detect": "false", + "fs_operation_blocked": "false", + "handle_operation_downgraded": "false", + "inddet_mask": "false", + "indicator": "false", + "kill_action_failed": "false", + "kill_parent": "false", + "kill_process": "false", + "kill_subprocess": "false", + "operation_blocked": "false", + "policy_disabled": "false", + "process_blocked": "true", + "quarantine_file": "false", + "quarantine_machine": "false", + "registry_operation_blocked": "false", + "rooting": "false", + "sensor_only": "false", + "suspend_parent": "false", + "suspend_process": "false" + }, + "rule_instance_id": 3, + "rule_instance_version": "3", + "scenario": "suspicious_activity", + "severity": 50, + "ttp_tagging_ref": "6", + "template_instance_id": "3", + "created_time": "2024-01-24T14:17:11Z", + "user_ref": "7" + }, + "2": { + "type": "process", + "command_line": "\"C:\\wusa.exe\" \"C:\\windows10.0-f.msu\" ", + "name": "wusa.exe", + "binary_ref": "0", + "parent_ref": "4", + "x_process_graph_id": "pid:did123:222674754477", + "creator_user_ref": "7" + }, + "3": { + "type": "directory", + "path": "\\Device" + }, + "4": { + "type": "process", + "command_line": "\"C:\\cmd.exe\" ", + "binary_ref": "5", + "x_process_graph_id": "pid:did123:222673021986" + }, + "5": { + "type": "file", + "hashes": { + "MD5": "10101010101010101010101010110", + "SHA-256": "101011010101010010101010101010101010101010101010101001" + } + }, + "6": { + "type": "x-ibm-ttp-tagging", + "name": "Custom Intelligence", + "extensions": { + "mitre-attack-ext": { + "tactic_id": "CSTA0005", + "technique_name": "Indicator of Attack", + "technique_id": "CST0004" + } + } + }, + "7": { + "type": "user-account", + "user_id": "S-1-5-21", + "display_name": "Administrator" + }, + "8": { + "type": "x-ibm-finding", + "x_logscale_repository": "TestRepository", + "x_logscale_event_id": "xpC1sUz8JCF1oyTpI7dSVld5_363_87_1712140849", + "x_behavior_refs": [ + "1" + ], + "ttp_tagging_refs": [ + "6" + ], + "x_behaviors_processed": [ + "pid:did123:222674754477:41002" + ], + "time_observed": "2024-01-24T14:17:19.087Z", + "x_last_updated": "2024-04-03T10:40:28.680870Z", + "name": "ldt:did123:214759104287", + "src_ip_ref": "11", + "src_os_ref": "14", + "x_is_email_sent": "false", + "x_first_behavior_observed": "2024-01-24T14:17:11Z", + "x_last_behavior_observed": "2024-01-24T14:17:11Z", + "confidence": 100, + "severity": 50, + "x_severity_name": "Medium", + "x_seconds_to_resolved": "0", + "x_seconds_to_triaged": "0", + "x_status": "new", + "finding_type": "alert" + }, + "9": { + "type": "x-oca-asset", + "x_cid": "cid123", + "x_agent_ref": "10", + "x_bios_manufacturer": "Xen", + "x_bios_version": "4.11.amazon", + "device_id": "did123", + "ip_refs": [ + "11", + "12" + ], + "x_first_seen": "2023-05-16T05:10:55Z", + "x_device_groups": [ + "97350feebe4541e8a615c0d3f18acdf3", + "bb1e1190b46348e69e10785030e8b23d" + ], + "hostname": "host", + "x_instance_id": "i-123", + "x_last_seen": "2024-01-24T13:47:13Z", + "mac_refs": [ + "13" + ], + "x_last_modified": "2024-01-24T14:12:41Z", + "os_ref": "14", + "x_host_type_number": "3", + "host_type": "Server", + "x_service_provider": "AWS_EC2_V2", + "x_service_account_id": "9876", + "x_status": "normal", + "x_system_manufacturer": "Xen", + "x_system_product_name": "HVM domU" + }, + "10": { + "type": "x-crowdstrike-edr-agent", + "load_flags": "1", + "local_time": "2024-01-23T12:32:59.287Z", + "version": "7.05.17706.0", + "config_id_base": "65994763", + "config_id_build": "17706", + "config_id_platform": "3" + }, + "11": { + "type": "ipv4-addr", + "value": "9.9.9.9" + }, + "12": { + "type": "ipv4-addr", + "value": "2.3.4.5", + "resolves_to_refs": [ + "13" + ] + }, + "13": { + "type": "mac-addr", + "value": "01:01:01:01:ab:cd" + }, + "14": { + "type": "software", + "x_major_version": "10", + "x_minor_version": "0", + "version": "Windows Server 2022", + "x_id": "0", + "name": "Windows" + } + }, + "last_observed": "2024-04-03T10:40:28.680Z", + "first_observed": "2024-01-24T14:17:19.087Z", + "number_observed": 1 + },{ + "id": "observed-data--892c29eb-7938-4a08-bd80-ab29e58ac3f5", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-04-04T02:51:17.680Z", + "modified": "2024-04-04T02:51:17.680Z", + "objects": { + "0": { + "type": "file", + "x_extension": "exe", + "name": "cmd.exe", + "x_path": "\\Device\\cmd.exe", + "parent_directory_ref": "3", + "hashes": { + "MD5": "111111100000000000000000111", + "SHA-256": "111111111111111111111100000000000000" + } + }, + "1": { + "type": "x-crowdstrike-detection-behavior", + "behavior_id": "41002", + "confidence": 100, + "control_graph_id": "ctg:did123:214754619929", + "description": "A process triggered a medium severity custom rule.", + "display_name": "CustomIOAWinMedium", + "process_ref": "2", + "ioc_description": "\\Device\\cmd.exe", + "ioc_source": "library_load", + "ioc_type": "hash_sha256", + "ioc_value": "eb71ea69dd19f728ab9240565e8c7efb59821e19e3788e289301e1e74940c208", + "objective": "Falcon Detection Method", + "pattern_disposition": 2048, + "pattern_disposition_details": { + "blocking_unsupported_or_disabled": "false", + "bootup_safeguard_enabled": "false", + "critical_process_disabled": "false", + "detect": "false", + "fs_operation_blocked": "false", + "handle_operation_downgraded": "false", + "inddet_mask": "false", + "indicator": "false", + "kill_action_failed": "false", + "kill_parent": "false", + "kill_process": "false", + "kill_subprocess": "false", + "operation_blocked": "false", + "policy_disabled": "false", + "process_blocked": "true", + "quarantine_file": "false", + "quarantine_machine": "false", + "registry_operation_blocked": "false", + "rooting": "false", + "sensor_only": "false", + "suspend_parent": "false", + "suspend_process": "false" + }, + "rule_instance_id": 3, + "rule_instance_version": "3", + "scenario": "suspicious_activity", + "severity": 50, + "ttp_tagging_ref": "6", + "template_instance_id": "3", + "created_time": "2024-01-23T12:34:00Z", + "user_ref": "7" + }, + "2": { + "type": "process", + "command_line": "C:\\cmd.exe /c C:\\reg.exe query hklm\\software\\microsoft\\windows\\softwareinventorylogging /v collectiontime /reg:64", + "name": "cmd.exe", + "binary_ref": "0", + "parent_ref": "4", + "x_process_graph_id": "pid:did123:219231592680", + "creator_user_ref": "7" + }, + "3": { + "type": "directory", + "path": "\\Device" + }, + "4": { + "type": "process", + "command_line": "\"C:\\cmd.exe\" /d /c C:\\silcollector.cmd configure", + "binary_ref": "5", + "x_process_graph_id": "pid:did123:219225422769" + }, + "5": { + "type": "file", + "hashes": { + "MD5": "111111111111111111111", + "SHA-256": "a111111111111aaaaaaaaaaaaaaa1111111" + } + }, + "6": { + "type": "x-ibm-ttp-tagging", + "name": "Custom Intelligence", + "extensions": { + "mitre-attack-ext": { + "tactic_id": "CSTA0005", + "technique_name": "Indicator of Attack", + "technique_id": "CST0004" + } + } + }, + "7": { + "type": "user-account", + "user_id": "S-1-5-18", + "display_name": "user" + }, + "8": { + "type": "x-ibm-finding", + "x_logscale_repository": "TestRepository", + "x_logscale_event_id": "xpC1sUz8JCF1oyTpI7dSVld5_363_119_1712140849", + "x_behavior_refs": [ + "1" + ], + "ttp_tagging_refs": [ + "6" + ], + "x_behaviors_processed": [ + "pid:did123:219231592680:41002" + ], + "time_observed": "2024-01-23T12:34:07.127Z", + "x_last_updated": "2024-04-03T10:40:49.669334Z", + "name": "ldt:did123:214754619929", + "src_ip_ref": "11", + "src_os_ref": "14", + "x_is_email_sent": "false", + "x_first_behavior_observed": "2024-01-23T12:34:00Z", + "x_last_behavior_observed": "2024-01-23T12:34:00Z", + "confidence": 100, + "severity": 50, + "x_severity_name": "Medium", + "x_seconds_to_resolved": "0", + "x_seconds_to_triaged": "0", + "x_status": "new", + "finding_type": "alert" + }, + "9": { + "type": "x-oca-asset", + "x_cid": "cid123", + "x_agent_ref": "10", + "x_bios_manufacturer": "Xen", + "x_bios_version": "4.11.amazon", + "device_id": "did123", + "ip_refs": [ + "11", + "12" + ], + "x_first_seen": "2023-05-16T05:10:55Z", + "x_device_groups": [ + "97350feebe4541e8a615c0d3f18acdf3", + "bb1e1190b46348e69e10785030e8b23d" + ], + "hostname": "host", + "x_instance_id": "i-123", + "x_last_seen": "2024-01-23T12:33:12Z", + "mac_refs": [ + "13" + ], + "x_last_modified": "2024-01-23T12:33:16Z", + "os_ref": "14", + "x_host_type_number": "3", + "host_type": "Server", + "x_service_provider": "AWS_EC2_V2", + "x_service_account_id": "9876", + "x_status": "normal", + "x_system_manufacturer": "Xen", + "x_system_product_name": "HVM domU" + }, + "10": { + "type": "x-crowdstrike-edr-agent", + "load_flags": "1", + "local_time": "2024-01-23T12:32:59.287Z", + "version": "7.05.17706.0", + "config_id_base": "65994763", + "config_id_build": "17706", + "config_id_platform": "3" + }, + "11": { + "type": "ipv4-addr", + "value": "9.9.9.9" + }, + "12": { + "type": "ipv4-addr", + "value": "2.3.4.5", + "resolves_to_refs": [ + "13" + ] + }, + "13": { + "type": "mac-addr", + "value": "01:01:01:01:ab:cd" + }, + "14": { + "type": "software", + "x_major_version": "10", + "x_minor_version": "0", + "version": "Windows Server 2022", + "x_id": "0", + "name": "Windows" + } + }, + "last_observed": "2024-04-03T10:40:49.669Z", + "first_observed": "2024-01-23T12:34:07.127Z", + "number_observed": 1 + }, + { + "id": "observed-data--ef14dda2-c17c-4d9a-b031-019e27d0fa63", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-04-04T02:52:44.790Z", + "modified": "2024-04-04T02:52:44.790Z", + "objects": { + "0": { + "type": "file", + "x_extension": "exe", + "name": "conhost.exe", + "x_path": "\\Device\\conhost.exe", + "parent_directory_ref": "3", + "hashes": { + "MD5": "222222222222222222222222222", + "SHA-256": "33333333333333333333333333333333333" + } + }, + "1": { + "type": "x-crowdstrike-detection-behavior", + "behavior_id": "41002", + "confidence": 100, + "control_graph_id": "ctg:did123:240518818424", + "description": "A process triggered a medium severity custom rule.", + "display_name": "CustomIOAWinMedium", + "process_ref": "2", + "objective": "Falcon Detection Method", + "pattern_disposition": 10240, + "pattern_disposition_details": { + "blocking_unsupported_or_disabled": "false", + "bootup_safeguard_enabled": "false", + "critical_process_disabled": "true", + "detect": "false", + "fs_operation_blocked": "false", + "handle_operation_downgraded": "false", + "inddet_mask": "false", + "indicator": "false", + "kill_action_failed": "false", + "kill_parent": "false", + "kill_process": "false", + "kill_subprocess": "false", + "operation_blocked": "false", + "policy_disabled": "false", + "process_blocked": "true", + "quarantine_file": "false", + "quarantine_machine": "false", + "registry_operation_blocked": "false", + "rooting": "false", + "sensor_only": "false", + "suspend_parent": "false", + "suspend_process": "false" + }, + "rule_instance_id": 3, + "rule_instance_version": "3", + "scenario": "suspicious_activity", + "severity": 50, + "ttp_tagging_ref": "4", + "template_instance_id": "3", + "created_time": "2024-02-19T04:59:15Z", + "user_ref": "5" + }, + "2": { + "type": "process", + "command_line": "C:\\Windows\\system32\\conhost.exe 0xffffffff -ForceV1", + "name": "conhost.exe", + "binary_ref": "0", + "x_process_graph_id": "pid:did123:0011", + "creator_user_ref": "5" + }, + "3": { + "type": "directory", + "path": "\\Device" + }, + "4": { + "type": "x-ibm-ttp-tagging", + "name": "Custom Intelligence", + "extensions": { + "mitre-attack-ext": { + "tactic_id": "CSTA0005", + "technique_name": "Indicator of Attack", + "technique_id": "CST0004" + } + } + }, + "5": { + "type": "user-account", + "user_id": "S-1-5-18", + "display_name": "user2" + }, + "6": { + "type": "x-ibm-finding", + "x_logscale_repository": "TestRepository", + "x_logscale_event_id": "xpC1sUz8JCF1oyTpI7dSVld5_363_125_1712140849", + "x_behavior_refs": [ + "1" + ], + "ttp_tagging_refs": [ + "4" + ], + "x_behaviors_processed": [ + "pid:did123:0011:41002" + ], + "time_observed": "2024-02-19T04:59:22.678Z", + "x_last_updated": "2024-04-03T10:40:49.669334Z", + "name": "ldt:did123:240518818424", + "src_ip_ref": "9", + "src_os_ref": "12", + "x_is_email_sent": "false", + "x_first_behavior_observed": "2024-02-19T04:59:15Z", + "x_last_behavior_observed": "2024-02-19T04:59:15Z", + "confidence": 100, + "severity": 50, + "x_severity_name": "Medium", + "x_seconds_to_resolved": "0", + "x_seconds_to_triaged": "0", + "x_status": "new", + "finding_type": "alert" + }, + "7": { + "type": "x-oca-asset", + "x_cid": "cid123", + "x_agent_ref": "8", + "x_bios_manufacturer": "Xen", + "x_bios_version": "4.11.amazon", + "device_id": "did123", + "ip_refs": [ + "9", + "10" + ], + "x_first_seen": "2023-05-16T05:10:55Z", + "x_device_groups": [ + "97350feebe4541e8a615c0d3f18acdf3", + "bb1e1190b46348e69e10785030e8b23d" + ], + "hostname": "host", + "x_instance_id": "i-123", + "x_last_seen": "2024-02-19T04:59:18Z", + "mac_refs": [ + "11" + ], + "x_last_modified": "2024-02-19T04:59:19Z", + "os_ref": "12", + "x_host_type_number": "3", + "host_type": "Server", + "x_service_provider": "AWS_EC2_V2", + "x_service_account_id": "9876", + "x_status": "normal", + "x_system_manufacturer": "Xen", + "x_system_product_name": "HVM domU" + }, + "8": { + "type": "x-crowdstrike-edr-agent", + "load_flags": "1", + "local_time": "2024-02-19T04:59:08.771Z", + "version": "7.06.17807.0", + "config_id_base": "65994763", + "config_id_build": "17807", + "config_id_platform": "3" + }, + "9": { + "type": "ipv4-addr", + "value": "9.3.4.5" + }, + "10": { + "type": "ipv4-addr", + "value": "2.3.4.5", + "resolves_to_refs": [ + "11" + ] + }, + "11": { + "type": "mac-addr", + "value": "01:01:01:01:ab:cd" + }, + "12": { + "type": "software", + "x_major_version": "10", + "x_minor_version": "0", + "version": "Windows Server 2022", + "x_id": "0", + "name": "Windows" + } + }, + "last_observed": "2024-04-03T10:40:49.669Z", + "first_observed": "2024-02-19T04:59:22.678Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/README.md b/stix_shifter_modules/crowdstrike_logscale/README.md new file mode 100644 index 000000000..7be340eb3 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/README.md @@ -0,0 +1,897 @@ +# Crowdstrike Logscale + +## Supported STIX Mappings + +See the [table of mappings](crowdstrike_logscale_supported_stix.md) for the STIX objects and operators supported by this connector. + +**Table of Contents** +- [Crowdstrike Logscale API Endpoints](#crowdstrike-logscale-api-endpoints) +- [Curl Command to test the API Endpoints](#curl-command-to-test-api-endpoints) +- [Format of calling Stix shifter from Command Line](#format-for-calling-stix-shifter-from-the-command-line) +- [Pattern expression with STIX attributes and CUSTOM attributes - Single Observation](#single-observation) +- [Pattern expression with STIX and CUSTOM attributes - Multiple Observation](#multiple-observation) +- [STIX Execute Query](#stix-execute-query) +- [Types of Attributes](#type-of-attributes) +- [Current Connector Features](#current-connector-features) +- [Connector Extension](#connector-extension) +- [Recommendations](#recommendations) +- [Limitations](#limitations) +- [References](#references) +- [Appendix](#appendix) + +### Crowdstrike Logscale API Endpoints + + | Connector Method | Crowdstrike Logscale API Endpoint | Method | + |------------------|--------------------------------------------------------------------------------|--------| + | Ping Endpoint | Status API - api/v1/status | GET | + | Query Endpoint | Query Job API - api/v1/repositories/{respository}/queryjobs | POST | + | Status Endpoint | Query Job Poll API - api/v1/repositories/{respository}/queryjobs/{Query Job id} | GET | + | Results Endpoint | Query Job Poll API - api/v1/repositories/{respository}/queryjobs/{Query Job id} | GET | + | Delete Endpoint | Query Job Poll API - api/v1/repositories/{respository}/queryjobs/{Query Job id} | DELETE | + + +### CURL command to test API Endpoints +#### Ping +``` +curl --location 'https://{hostname}/api/v1/status' \ +--header 'Accept: application/json' +``` +#### Query +``` +curl --location 'https://{hostname}/api/v1/repositories/{repository_name}/queryjobs' \ +--header 'Accept: application/json' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer {Repository API token}' \ +--data '{ + "queryString": "device.local_ip =~ cidr(subnet=\"1.1.1.1/32\") | tail(10)", + "start": 1702598400000, + "end": 1707461040000 +}' +``` + +#### Results +``` +curl --location 'https://{hostname}/api/v1/repositories/{repository_name}/queryjobs/{id}' \ +--header 'Accept: application/json' \ +--header 'Authorization: Bearer {Repository API token}' +``` + +### Format for calling stix-shifter from the command line +``` +python main.py `` `` `` `` + +``` + +### Pattern expression with STIX and CUSTOM attributes + +#### Single Observation + +#### STIX Translate query +```shell +translate crowdstrike_logscale query "{}" "[process:name IN ('cmd.exe','calc.exe') AND x-oca-asset:hostname LIKE 'EC2' OR ipv4-addr:value ISSUBSET '1.1.1.1/32'] START t'2024-03-20T00:00:00.000Z' STOP t'2024-04-01T00:00:00.000Z'" +``` +#### STIX Translate query - Output +```json +{ + "queries": [ + { + "source": "crowdstrikeedr", + "queryString": "device.local_ip =~ cidr(subnet=\"1.1.1.1/32\") | tail(10000)", + "start": 1710892800000, + "end": 1711929600000 + }, + { + "source": "crowdstrikeedr", + "queryString": "device.external_ip =~ cidr(subnet=\"1.1.1.1/32\") | tail(10000)", + "start": 1710892800000, + "end": 1711929600000 + }, + { + "source": "crowdstrikeedr", + "queryString": "(device.hostname = /EC2/i and @rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"filename\"\\s*:\\s*(\"cmd\\.exe\"|\"calc\\.exe\")/) | tail(10000)", + "start": 1710892800000, + "end": 1711929600000 + } + ] +} +``` +#### STIX Transmit Query +```shell +transmit +crowdstrike_logscale +"{\"host\":\"xxx\",\"repository\":\"TestRepository\"}" +"{\"auth\":{\"api_token\": \"123\"}}" +results +"{ \"source\": \"crowdstrikeedr\", \"queryString\": \"device.external_ip =~ cidr(subnet=\\"1.1.1.1/32\\") | tail(10000)\", \"start\": 1710892800000, \"end\": 1711929600000 }" +0 +1 +``` +#### STIX Transmit Query - Output + +```json +{ + "success": true, + "search_id": "P7-xxxxxxxxx:crowdstrikeedr" +} +``` +#### STIX Transmit Status +```shell +transmit +crowdstrike_logscale +"{\"host\":\"xxx\",\"repository\":\"TestRepository\"}" +"{\"auth\":{\"api_token\": \"123\"}}" +status "P7-xxxxxxxxx:crowdstrikeedr" +``` + +#### STIX Transmit Status - Output + +```json +{ + "success": true, + "status": "COMPLETED", + "progress": 100 +} +``` +#### STIX Transmit Results +```shell +transmit +crowdstrike_logscale +"{\"host\":\"xxx\",\"repository\":\"TestRepository\"}" +"{\"auth\":{\"api_token\": \"123\"}}" +results "P7-xxxxxxxxx:crowdstrikeedr" 0 1 +``` + +#### STIX Transmit Results - Output +```json +{ + "success": true, + "metadata": { + "input_query_string": "cidr(subnet=\"1.1.1.1/32\",field=device.external_ip)", + "start": 1710892800000, + "last_event_id": "ATzrtyg4xCKOqQnD9NodpvsY_363_125_1711549348", + "last_event_timestamp": 1711549348062 + }, + "data": [ + { + "crowdstrikeedr": { + "@timestamp": 1711549348062, + "@timestamp.nanos": "194000", + "#repo": "TestRepository", + "#type": "CrowdStrike_Spotlight", + "@id": "ATzrtyg4xCKOqQnD9NodpvsY_363_125_1711549348", + "@ingesttimestamp": "1711549348586", + "@rawstring": "{\"cid\": \"123\", \"created_timestamp\": \"2024-01-23T12:33:15.170758259Z\", \"detection_id\": \"ldt:xyz:123\", \"device\": {\"device_id\": \"7adb123\", \"cid\": \"123\", \"agent_load_flags\": \"1\", \"agent_local_time\": \"2024-01-23T12:32:59.287Z\", \"agent_version\": \"7.05.17706.0\", \"bios_manufacturer\": \"Xen\", \"bios_version\": \"4.11.amazon\", \"config_id_base\": \"65994763\", \"config_id_build\": \"17706\", \"config_id_platform\": \"3\", \"external_ip\": \"1.1.1.1\", \"hostname\": \"host\", \"first_seen\": \"2023-05-16T05:10:55Z\", \"last_login_timestamp\": \"2024-01-04T06:12:09Z\", \"last_login_user\": \"test user\", \"last_seen\": \"2024-01-23T12:33:10Z\", \"local_ip\": \"2.2.2.2\", \"mac_address\": \"01-01-01-01-01-01\", \"major_version\": \"10\", \"minor_version\": \"0\", \"os_version\": \"Windows Server 2022\", \"platform_id\": \"0\", \"platform_name\": \"Windows\", \"product_type\": \"3\", \"product_type_desc\": \"Server\", \"status\": \"normal\", \"system_manufacturer\": \"Xen\", \"system_product_name\": \"HVM domU\", \"groups\": [\"97350feebe4541e8a615c0d3f18acdf3\", \"bb1e1190b46348e69e10785030e8b23d\"], \"modified_timestamp\": \"2024-01-23T12:33:13Z\", \"instance_id\": \"i-123\", \"service_provider\": \"AWS_EC2_V2\", \"service_provider_account_id\": \"98765\"}, \"behaviors\": [{\"device_id\": \"7adb123\", \"timestamp\": \"2024-01-23T12:33:07Z\", \"template_instance_id\": \"3\", \"behavior_id\": \"41002\", \"filename\": \"conhost.exe\", \"filepath\": \"\\\\Device\\\\conhost.exe\", \"alleged_filetype\": \"exe\", \"cmdline\": \"C:\\\\conhost.exe 0xffffffff -ForceV1\", \"scenario\": \"suspicious_activity\", \"objective\": \"Falcon Detection Method\", \"tactic\": \"Custom Intelligence\", \"tactic_id\": \"CSTA0005\", \"technique\": \"Indicator of Attack\", \"technique_id\": \"CST0004\", \"display_name\": \"CustomIOAWinMedium\", \"description\": \"A process triggered a medium severity custom rule.\", \"severity\": 50, \"confidence\": 100, \"ioc_type\": \"\", \"ioc_value\": \"\", \"ioc_source\": \"\", \"ioc_description\": \"\", \"user_name\": \"user1\", \"user_id\": \"S-1-5-18\", \"control_graph_id\": \"ctg:xyz:123\", \"triggering_process_graph_id\": \"pid:7adb:123\", \"sha256\": \"1010101010101010101010100101100101010101010101010101010101010100\", \"md5\": \"11111111111111111111111111111111\", \"parent_details\": {\"parent_sha256\": \"\", \"parent_md5\": \"\", \"parent_cmdline\": \"\", \"parent_process_graph_id\": \"pid:xyz:123\"}, \"pattern_disposition\": 10240, \"pattern_disposition_details\": {\"indicator\": false, \"detect\": false, \"inddet_mask\": false, \"sensor_only\": false, \"rooting\": false, \"kill_process\": false, \"kill_subprocess\": false, \"quarantine_machine\": false, \"quarantine_file\": false, \"policy_disabled\": false, \"kill_parent\": false, \"operation_blocked\": false, \"process_blocked\": true, \"registry_operation_blocked\": false, \"critical_process_disabled\": true, \"bootup_safeguard_enabled\": false, \"fs_operation_blocked\": false, \"handle_operation_downgraded\": false, \"kill_action_failed\": false, \"blocking_unsupported_or_disabled\": false, \"suspend_process\": false, \"suspend_parent\": false}, \"rule_instance_id\": \"3\", \"rule_instance_version\": 3}], \"email_sent\": false, \"first_behavior\": \"2024-01-23T12:33:07Z\", \"last_behavior\": \"2024-01-23T12:33:07Z\", \"max_confidence\": 100, \"max_severity\": 50, \"max_severity_displayname\": \"Medium\", \"show_in_ui\": true, \"status\": \"new\", \"hostinfo\": {\"domain\": \"\"}, \"seconds_to_triaged\": 0, \"seconds_to_resolved\": 0, \"behaviors_processed\": [\"pid:7adb:123:41002\"], \"date_updated\": \"2024-03-27T14:22:28.062194Z\"}", + "@timezone": "Z", + "behaviors": [ + { + "alleged_filetype": "exe", + "behavior_id": "41002", + "cmdline": "C:\\conhost.exe 0xffffffff -ForceV1", + "confidence": "100", + "control_graph_id": "ctg:xyz:123", + "description": "A process triggered a medium severity custom rule.", + "device_id": "7adb123", + "display_name": "CustomIOAWinMedium", + "filename": "conhost.exe", + "filepath": "\\Device\\conhost.exe", + "md5": "11111111111111111111111111111111", + "objective": "Falcon Detection Method", + "parent_details": { + "parent_process_graph_id": "pid:xyz:123" + }, + "pattern_disposition": "10240", + "pattern_disposition_details": { + "blocking_unsupported_or_disabled": "false", + "bootup_safeguard_enabled": "false", + "critical_process_disabled": "true", + "detect": "false", + "fs_operation_blocked": "false", + "handle_operation_downgraded": "false", + "inddet_mask": "false", + "indicator": "false", + "kill_action_failed": "false", + "kill_parent": "false", + "kill_process": "false", + "kill_subprocess": "false", + "operation_blocked": "false", + "policy_disabled": "false", + "process_blocked": "true", + "quarantine_file": "false", + "quarantine_machine": "false", + "registry_operation_blocked": "false", + "rooting": "false", + "sensor_only": "false", + "suspend_parent": "false", + "suspend_process": "false" + }, + "rule_instance_id": "3", + "rule_instance_version": "3", + "scenario": "suspicious_activity", + "severity": "50", + "sha256": "1010101010101010101010100101100101010101010101010101010101010100", + "tactic": "Custom Intelligence", + "tactic_id": "CSTA0005", + "technique": "Indicator of Attack", + "technique_id": "CST0004", + "template_instance_id": "3", + "timestamp": "2024-01-23T12:33:07Z", + "triggering_process_graph_id": "pid:7adb:123", + "user_id": "S-1-5-18", + "user_name": "user1" + } + ], + "behaviors_processed": [ + "pid:7adb:123:41002" + ], + "cid": "123", + "created_timestamp": "2024-01-23T12:33:15.170758259Z", + "date_updated": "2024-03-27T14:22:28.062194Z", + "detection_id": "ldt:xyz:123", + "device": { + "agent_load_flags": "1", + "agent_local_time": "2024-01-23T12:32:59.287Z", + "agent_version": "7.05.17706.0", + "bios_manufacturer": "Xen", + "bios_version": "4.11.amazon", + "cid": "123", + "config_id_base": "65994763", + "config_id_build": "17706", + "config_id_platform": "3", + "device_id": "7adb123", + "external_ip": "1.1.1.1", + "first_seen": "2023-05-16T05:10:55Z", + "groups": [ + "97350feebe4541e8a615c0d3f18acdf3", + "bb1e1190b46348e69e10785030e8b23d" + ], + "hostname": "host", + "instance_id": "i-123", + "last_login_timestamp": "2024-01-04T06:12:09Z", + "last_login_user": "test user", + "last_seen": "2024-01-23T12:33:10Z", + "local_ip": "2.2.2.2", + "mac_address": "01-01-01-01-01-01", + "major_version": "10", + "minor_version": "0", + "modified_timestamp": "2024-01-23T12:33:13Z", + "os_version": "Windows Server 2022", + "platform_id": "0", + "platform_name": "Windows", + "product_type": "3", + "product_type_desc": "Server", + "service_provider": "AWS_EC2_V2", + "service_provider_account_id": "98765", + "status": "normal", + "system_manufacturer": "Xen", + "system_product_name": "HVM domU" + }, + "email_sent": "false", + "first_behavior": "2024-01-23T12:33:07Z", + "last_behavior": "2024-01-23T12:33:07Z", + "max_confidence": "100", + "max_severity": "50", + "max_severity_displayname": "Medium", + "seconds_to_resolved": "0", + "seconds_to_triaged": "0", + "show_in_ui": "true", + "status": "new", + "finding_type": "alert" + } + } + ] +} +``` + +#### STIX Translate results +```json +{ + "type": "bundle", + "id": "bundle--fc54a014-447c-4cde-88ee-129a4083ca24", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "crowdstrike_logscale", + "identity_class": "events", + "created": "2024-04-02T13:22:50.336Z", + "modified": "2024-04-02T13:22:50.336Z" + }, + { + "id": "observed-data--3a204585-59bc-481b-898b-a065965f82e5", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-04-03T13:24:07.549Z", + "modified": "2024-04-03T13:24:07.549Z", + "objects": { + "0": { + "type": "file", + "x_extension": "exe", + "name": "conhost.exe", + "x_path": "\\Device\\conhost.exe", + "parent_directory_ref": "3", + "hashes": { + "MD5": "11111111111111111111111111111111", + "SHA-256": "1010101010101010101010100101100101010101010101010101010101010100" + } + }, + "1": { + "type": "x-crowdstrike-detection-behavior", + "behavior_id": "41002", + "confidence": 100, + "control_graph_id": "ctg:xyz:123", + "description": "A process triggered a medium severity custom rule.", + "display_name": "CustomIOAWinMedium", + "process_ref": "2", + "objective": "Falcon Detection Method", + "pattern_disposition": 10240, + "pattern_disposition_details": { + "blocking_unsupported_or_disabled": "false", + "bootup_safeguard_enabled": "false", + "critical_process_disabled": "true", + "detect": "false", + "fs_operation_blocked": "false", + "handle_operation_downgraded": "false", + "inddet_mask": "false", + "indicator": "false", + "kill_action_failed": "false", + "kill_parent": "false", + "kill_process": "false", + "kill_subprocess": "false", + "operation_blocked": "false", + "policy_disabled": "false", + "process_blocked": "true", + "quarantine_file": "false", + "quarantine_machine": "false", + "registry_operation_blocked": "false", + "rooting": "false", + "sensor_only": "false", + "suspend_parent": "false", + "suspend_process": "false" + }, + "rule_instance_id": 3, + "rule_instance_version": "3", + "scenario": "suspicious_activity", + "severity": 50, + "ttp_tagging_ref": "5", + "template_instance_id": "3", + "created_time": "2024-01-23T12:33:07Z", + "user_ref": "6" + }, + "2": { + "type": "process", + "command_line": "C:\\conhost.exe 0xffffffff -ForceV1", + "name": "conhost.exe", + "binary_ref": "0", + "parent_ref": "4", + "x_process_graph_id": "pid:7adb:123", + "creator_user_ref": "6" + }, + "3": { + "type": "directory", + "path": "\\Device" + }, + "4": { + "type": "process", + "x_process_graph_id": "pid:xyz:123" + }, + "5": { + "type": "x-ibm-ttp-tagging", + "name": "Custom Intelligence", + "extensions": { + "mitre-attack-ext": { + "tactic_id": "CSTA0005", + "technique_name": "Indicator of Attack", + "technique_id": "CST0004" + } + } + }, + "6": { + "type": "user-account", + "user_id": "S-1-5-18", + "display_name": "user1" + }, + "7": { + "type": "x-ibm-finding", + "x_logscale_repository": "TestRepository", + "x_logscale_event_id": "ATzrtyg4xCKOqQnD9NodpvsY_363_125_1711549348", + "x_behavior_refs": [ + "1" + ], + "ttp_tagging_refs": [ + "5" + ], + "x_behaviors_processed": [ + "pid:7adb:123:41002" + ], + "time_observed": "2024-01-23T12:33:15.170Z", + "x_last_updated": "2024-03-27T14:22:28.062194Z", + "name": "ldt:xyz:123", + "src_ip_ref": "10", + "src_os_ref": "13", + "x_is_email_sent": "false", + "x_first_behavior_observed": "2024-01-23T12:33:07Z", + "x_last_behavior_observed": "2024-01-23T12:33:07Z", + "confidence": 100, + "severity": 50, + "x_severity_name": "Medium", + "x_seconds_to_resolved": "0", + "x_seconds_to_triaged": "0", + "x_status": "new", + "finding_type": "alert" + }, + "8": { + "type": "x-oca-asset", + "x_cid": "123", + "x_agent_ref": "9", + "x_bios_manufacturer": "Xen", + "x_bios_version": "4.11.amazon", + "device_id": "7adb123", + "ip_refs": [ + "10", + "11" + ], + "x_first_seen": "2023-05-16T05:10:55Z", + "x_device_groups": [ + "97350feebe4541e8a615c0d3f18acdf3", + "bb1e1190b46348e69e10785030e8b23d" + ], + "hostname": "host", + "x_instance_id": "i-123", + "x_last_seen": "2024-01-23T12:33:10Z", + "mac_refs": [ + "12" + ], + "x_last_modified": "2024-01-23T12:33:13Z", + "os_ref": "13", + "x_host_type_number": "3", + "host_type": "Server", + "x_service_provider": "AWS_EC2_V2", + "x_service_account_id": "98765", + "x_status": "normal", + "x_system_manufacturer": "Xen", + "x_system_product_name": "HVM domU" + }, + "9": { + "type": "x-crowdstrike-edr-agent", + "load_flags": "1", + "local_time": "2024-01-23T12:32:59.287Z", + "version": "7.05.17706.0", + "config_id_base": "65994763", + "config_id_build": "17706", + "config_id_platform": "3" + }, + "10": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "11": { + "type": "ipv4-addr", + "value": "2.2.2.2", + "resolves_to_refs": [ + "12" + ] + }, + "12": { + "type": "mac-addr", + "value": "01:01:01:01:01:01" + }, + "13": { + "type": "software", + "x_major_version": "10", + "x_minor_version": "0", + "version": "Windows Server 2022", + "x_id": "0", + "name": "Windows" + } + }, + "last_observed": "2024-03-27T14:22:28.062Z", + "first_observed": "2024-01-23T12:33:15.170Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` +#### Multiple Observation +```shell +translate crowdstrike_logscale query {} +"([x-ibm-ttp-tagging:extensions.'mitre-attack-ext'.technique_name = 'Indicator of Attack' OR ipv4-addr:value = '4.4.4.4'] AND [x-oca-asset:x_instance_id = 'i-0123' AND x-ibm-finding:severity > 30 AND file:hashes.MD5 = 'e7a6babc90f4'])START t'2023-12-19T16:43:26.000Z' STOP t'2023-12-24T05:22:26.003Z'" + +``` +#### STIX Multiple observation - Output +```json +{ + "queries": [ + { + "source": "crowdstrikeedr", + "queryString": "((device.local_ip = \"4.4.4.4\" or device.external_ip = \"4.4.4.4\") or @rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"technique\"\\s*:\\s*\"Indicator\\ of\\ Attack\"/) or ((@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"parent_details\"\\s*:\\s*\\{.*\"parent_md5\"\\s*:\\s*\"e7a6babc90f4\"/ or @rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"md5\"\\s*:\\s*\"e7a6babc90f4\"/) and (max_severity > 30 and device.instance_id = \"i-0123\")) | tail(10000)", + "start": 1703004206000, + "end": 1703395346003 + } + ] +} +``` + +### STIX Execute query +```shell +execute +crowdstrike_logscale +crowdstrike_logscale +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"crowdstrike_logscale\",\"identity_class\":\"system\",\"created\":\"2023-12-24T13:22:50.336Z\",\"modified\":\"2022-12-24T13:22:50.336Z\"}" +"{\"host\":\"xyz\",\"repository\":\"TestRepository\"}" +"{\"auth\":{\"api_token\": \"123\"}}" +"[ipv4-addr:value = '3.4.5.6' AND software:version = 'Windows Server 2022' OR x-oca-asset:host_type = 'Server' OR x-crowdstrike-detection-behavior:control_graph_id IN ('ctg:654','ctg:123')] START t'2024-04-01T00:00:00.000Z' STOP t'2024-04-03T11:00:00.000Z'" +``` + +#### STIX Execute query - Output +```json +{ + "type": "bundle", + "id": "bundle--44b07b95-fd6f-43b3-86a6-b98e8b9c1e01", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "crowdstrike_logscale", + "identity_class": "system", + "created": "2023-12-24T13:22:50.336Z", + "modified": "2023-12-24T13:22:50.336Z" + }, + { + "id": "observed-data--5f3a0294-527a-4a68-adee-a7d93799a801", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2024-04-03T14:11:53.817Z", + "modified": "2024-04-03T14:11:53.817Z", + "objects": { + "0": { + "type": "file", + "x_extension": "exe", + "name": "conhost.exe", + "x_path": "\\Device\\conhost.exe", + "parent_directory_ref": "3", + "hashes": { + "MD5": "1010101010101101010100101101010", + "SHA-256": "1111111111111111111111111111111111111111111111111111111111111111" + } + }, + "1": { + "type": "x-crowdstrike-detection-behavior", + "behavior_id": "41002", + "confidence": 100, + "control_graph_id": "ctg:123", + "description": "A process triggered a medium severity custom rule.", + "display_name": "CustomIOAWinMedium", + "process_ref": "2", + "objective": "Falcon Detection Method", + "pattern_disposition": 10240, + "pattern_disposition_details": { + "blocking_unsupported_or_disabled": "false", + "bootup_safeguard_enabled": "false", + "critical_process_disabled": "true", + "detect": "false", + "fs_operation_blocked": "false", + "handle_operation_downgraded": "false", + "inddet_mask": "false", + "indicator": "false", + "kill_action_failed": "false", + "kill_parent": "false", + "kill_process": "false", + "kill_subprocess": "false", + "operation_blocked": "false", + "policy_disabled": "false", + "process_blocked": "true", + "quarantine_file": "false", + "quarantine_machine": "false", + "registry_operation_blocked": "false", + "rooting": "false", + "sensor_only": "false", + "suspend_parent": "false", + "suspend_process": "false" + }, + "rule_instance_id": 3, + "rule_instance_version": "3", + "scenario": "suspicious_activity", + "severity": 50, + "ttp_tagging_ref": "6", + "template_instance_id": "3", + "created_time": "2024-01-25T07:06:28Z", + "user_ref": "7" + }, + "2": { + "type": "process", + "command_line": "C:\\Windows\\system32\\conhost.exe 0xffffffff -ForceV1", + "name": "conhost.exe", + "binary_ref": "0", + "parent_ref": "4", + "x_process_graph_id": "pid:7adb:232202950915", + "creator_user_ref": "7" + }, + "3": { + "type": "directory", + "path": "\\Device\\HarddiskVolume1\\Windows\\System32" + }, + "4": { + "type": "process", + "command_line": "\"C:\\Windows\\System32\\cmd.exe\" /c \"\"C:\\Program Files\\Amazon\\EC2Launch\\EC2Launch.exe\" wallpaper --path=\"C:\\ProgramData\\Amazon\\EC2Launch\\wallpaper\\Ec2Wallpaper.jpg\" --attributes=\"hostName,instanceId,privateIpAddress,publicIpAddress,instanceSize,availabilityZone,architecture,memory,network\" \"", + "binary_ref": "5", + "x_process_graph_id": "pid:7adb:232200947972" + }, + "5": { + "type": "file", + "hashes": { + "MD5": "111111111111111111111111111111111111", + "SHA-256": "1212121212122121212121212121212121212121212212121121211212" + } + }, + "6": { + "type": "x-ibm-ttp-tagging", + "name": "Custom Intelligence", + "extensions": { + "mitre-attack-ext": { + "tactic_id": "CSTA0005", + "technique_name": "Indicator of Attack", + "technique_id": "CST0004" + } + } + }, + "7": { + "type": "user-account", + "user_id": "S-1-5-21", + "display_name": "test user" + }, + "8": { + "type": "x-ibm-finding", + "x_logscale_repository": "TestRepository", + "x_logscale_event_id": "x456", + "x_behavior_refs": [ + "1" + ], + "ttp_tagging_refs": [ + "6" + ], + "x_behaviors_processed": [ + "pid:7adb:232202950915:41002" + ], + "time_observed": "2024-01-25T07:06:36.831Z", + "x_last_updated": "2024-04-03T10:40:49.669334Z", + "name": "ldt:123", + "src_ip_ref": "11", + "src_os_ref": "14", + "x_is_email_sent": "false", + "x_first_behavior_observed": "2024-01-25T07:06:28Z", + "x_last_behavior_observed": "2024-01-25T07:06:28Z", + "confidence": 100, + "severity": 50, + "x_severity_name": "Medium", + "x_seconds_to_resolved": "0", + "x_seconds_to_triaged": "0", + "x_status": "new", + "finding_type": "alert" + }, + "9": { + "type": "x-oca-asset", + "x_cid": "id123", + "x_agent_ref": "10", + "x_bios_manufacturer": "Xen", + "x_bios_version": "4.2.amazon", + "device_id": "7adb", + "ip_refs": [ + "11", + "12" + ], + "x_first_seen": "2023-05-16T05:10:55Z", + "x_device_groups": [ + "97350feebe4541e8a615c0d3f18acdf3", + "bb1e1190b46348e69e10785030e8b23d" + ], + "hostname": "host", + "x_instance_id": "i-123", + "x_last_seen": "2024-01-25T06:51:54Z", + "mac_refs": [ + "13" + ], + "x_last_modified": "2024-01-25T07:06:13Z", + "os_ref": "14", + "x_host_type_number": "3", + "host_type": "Server", + "x_service_provider": "AWS_EC2_V2", + "x_service_account_id": "978657", + "x_status": "normal", + "x_system_manufacturer": "Xen", + "x_system_product_name": "HVM domU" + }, + "10": { + "type": "x-crowdstrike-edr-agent", + "load_flags": "1", + "local_time": "2024-01-25T06:51:42.661Z", + "version": "7.05.17706.0", + "config_id_base": "65994763", + "config_id_build": "17706", + "config_id_platform": "3" + }, + "11": { + "type": "ipv4-addr", + "value": "5.6.7.8" + }, + "12": { + "type": "ipv4-addr", + "value": "2.3.4.5", + "resolves_to_refs": [ + "13" + ] + }, + "13": { + "type": "mac-addr", + "value": "10:10:10:10:10:10" + }, + "14": { + "type": "software", + "x_major_version": "10", + "x_minor_version": "0", + "version": "Windows Server 2022", + "x_id": "0", + "name": "Windows" + } + }, + "last_observed": "2024-04-03T10:40:49.669Z", + "first_observed": "2024-01-25T07:06:36.831Z", + "number_observed": 1 + }], + "spec_version": "2.0" +} +``` +### Type of Attributes + + | Type | Description | Example | + |---------------------------|---------------------------------|-----| + | List of dictionary fields | A list containing one or more dictionaries | "behaviors": [{"alleged_filetype": "exe"}, {"alleged_filetype": "txt"}] | + | List of values fields/Array fields | A list containing one or more values | "behaviors_processed": ["123","abc"] | + + +### Current connector Features +- It has mappings which supports only Crowdstrike Falcon EDR detection logs. +- The Input repository which is provided to the connector should contain only Crowdstrike Falcon EDR detection logs in JSON format. + +### Connector Extension +Recommendations to be followed to add new log source to the connector +- The structure of log source data which is ingested into logscale should be of type JSON . +- The JSON data which has been ingested into logscale should be inserted without new line and should be inserted as raw data. +- As Logscale doesn't have unified data schema, separate mapping files(from_stix_map.json, to_stix_map.json) needs to + be created for each log source that are newly added to this connector module. + Example: {custom}_from_stix_map.json, {custom}_to_stix_map.json +- The mapping of list (list of values/list of dictionary) fields in from_stix_map should be mentioned with [*] suffix. + Example, behaviors[*].id. Here 'behaviors' is a list of dictionaries with id as attribute key inside behaviors. + +### Recommendations + +- For connector usage, it is recommended to maintain logs from single log source per repository in Crowdstrike Logscale. + Example Crowdstrike Falcon EDR detection logs could be stored in repository_1 and Okta logs in repository_2. +- Make sure, there is no parsing error during ingestion of logs into Crowdstrike Logscale. + +### Limitations +- LIKE,MATCHES, <, >, <=, >= operators are not supported for list of dictionary fields. +- IN, <, >, <=, >= operators are not supported for array fields + + +### References +- [LogScale Documentation](https://library.humio.com/) +- [Query Langauage Syntax](https://library.humio.com/data-analysis/syntax.html) +- [Search API | Integrations](https://library.humio.com/integrations/api-search.html) +- [Health Check API | Integrations](https://library.humio.com/integrations/api-health-check.html) +- [FalconLogScaleCollector | Falcon LogScaleCollector 1.3.0-1.5.1](https://library.humio.com/falcon-logscale-collector/log-shippers-log-collector.html) +- [Crowdstrike EDR Log injestion through log shippers](https://github.com/CrowdStrike/HEC-Log-Shipper/tree/main) + +### Appendix +List of fields that available in schema of Crowdstrike Falcon EDR detection logs + + | Data source Field | Data Type | Mapped STIX field | + |------------------------------------------------------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------| + | behaviors.alleged_filetype | List of Dictionary field | file:x_extension | + | behaviors.behavior_id | List of Dictionary field | x-ibm-finding:x_behavior_refs.behavior_id | + | behaviors.cmdline | List of Dictionary field | process:command_line | + | behaviors.confidence | List of Dictionary field | x-crowdstrike-detection-behavior:confidence | + | behaviors.control_graph_id | List of Dictionary field | x-crowdstrike-detection-behavior:control_graph_id | + | behaviors.description | List of Dictionary field | x-crowdstrike-detection-behavior:description | + | behaviors.device_id | List of Dictionary field | | + | behaviors.display_name | List of Dictionary field | x-crowdstrike-detection-behavior:display_name | + | behaviors.filename | List of Dictionary field | file:name, process:name, process: binary_ref.name,x-crowdstrike-detection-behavior:process_ref.name | + | behaviors.filepath | List of Dictionary field | file:x_path, file:parent_directory_ref.path, directory:path | + | behaviors.ioc_description | List of Dictionary field | x-crowdstrike-detection-behavior:ioc_description | + | behaviors.ioc_source | List of Dictionary field | x-crowdstrike-detection-behavior:ioc_source | + | behaviors.ioc_type | List of Dictionary field | x-crowdstrike-detection-behavior:ioc_type | + | behaviors.ioc_value | List of Dictionary field | x-crowdstrike-detection-behavior:ioc_value | + | behaviors.md5 | List of Dictionary field | process: binary_ref.hashes.MD5 , file:hashes.MD5 | + | behaviors.objective | List of Dictionary field | x-crowdstrike-detection-behavior:objective | + | behaviors.parent_details.parent_cmdline | List of Dictionary field | process: parent_ref.command_line | + | behaviors.parent_details.parent_md5 | List of Dictionary field | file:hashes.MD5 | + | behaviors.parent_details.parent_process_graph_id | List of Dictionary field | process:parent_ref.x_process_graph_id | + | behaviors.parent_details.parent_sha256 | List of Dictionary field | file:hashes.'SHA-256' | + | behaviors.pattern_disposition | List of Dictionary field | x-crowdstrike-detection-behavior:pattern_disposition | + | behaviors.pattern_disposition_details.blocking_unsupported_or_disabled | List of Dictionary field | | + | behaviors.pattern_disposition_details.bootup_safeguard_enabled | List of Dictionary field | | + | behaviors.pattern_disposition_details.critical_process_disabled | List of Dictionary field | | + | behaviors.pattern_disposition_details.detect | List of Dictionary field | | + | behaviors.pattern_disposition_details.fs_operation_blocked | List of Dictionary field | | + | behaviors.pattern_disposition_details.handle_operation_downgraded | List of Dictionary field | | + | behaviors.pattern_disposition_details.inddet_mask | List of Dictionary field | | + | behaviors.pattern_disposition_details.indicator | List of Dictionary field | | + | behaviors.pattern_disposition_details.kill_action_failed | List of Dictionary field | | + | behaviors.pattern_disposition_details.kill_parent | List of Dictionary field | | + | behaviors.pattern_disposition_details.kill_process | List of Dictionary field | | + | behaviors.pattern_disposition_details.kill_subprocess | List of Dictionary field | | + | behaviors.pattern_disposition_details.operation_blocked | List of Dictionary field | | + | behaviors.pattern_disposition_details.policy_disabled | List of Dictionary field | | + | behaviors.pattern_disposition_details.process_blocked | List of Dictionary field | | + | behaviors.pattern_disposition_details.quarantine_file | List of Dictionary field | | + | behaviors.pattern_disposition_details.quarantine_machine | List of Dictionary field | | + | behaviors.pattern_disposition_details.registry_operation_blocked | List of Dictionary field | | + | behaviors.pattern_disposition_details.rooting | List of Dictionary field | | + | behaviors.pattern_disposition_details.sensor_only | List of Dictionary field | | + | behaviors.pattern_disposition_details.suspend_parent | List of Dictionary field | | + | behaviors.pattern_disposition_details.suspend_process | List of Dictionary field | | + | behaviors.rule_instance_id | List of Dictionary field | x-crowdstrike-detection-behavior:rule_instance_id | + | behaviors.rule_instance_version | List of Dictionary field | x-crowdstrike-detection-behavior:rule_instance_version | + | behaviors.scenario | List of Dictionary field | x-crowdstrike-detection-behavior:scenario | + | behaviors.severity | List of Dictionary field | x-crowdstrike-detection-behavior:severity | + | behaviors.sha256 | List of Dictionary field | process:binary_ref.hashes.'SHA-256', file:hashes.'SHA-256' | + | behaviors.tactic | List of Dictionary field | x-ibm-finding:ttp_tagging_refs.name,x-crowdstrike-detection-behavior:ttp_tagging_ref.name,x-ibm-ttp-tagging:name | + | behaviors.tactic_id | List of Dictionary field | x-ibm-ttp-tagging:extensions.'mitre-attack-ext'.tactic_id | + | behaviors.technique | List of Dictionary field | x-ibm-ttp-tagging:extensions.'mitre-attack-ext'.technique_name | + | behaviors.technique_id | List of Dictionary field | x-ibm-ttp-tagging:extensions.'mitre-attack-ext'.technique_id | + | behaviors.template_instance_id | List of Dictionary field | x-crowdstrike-detection-behavior:template_instance_id | + | behaviors.timestamp | List of Dictionary field | x-crowdstrike-detection-behavior:created_time | + | behaviors.triggering_process_graph_id | List of Dictionary field | process:x_process_graph_id | + | behaviors.user_id | List of Dictionary field | process:creator_user_ref.user_id, user-account:user_id,x-crowdstrike-detection-behavior:user_ref.user_id | + | behaviors.user_name | List of Dictionary field | user-account:display_name | + | behaviors_processed | List field | x-ibm-finding:x_behaviors_processed | + | cid | Dictionary field | x-oca-asset:x_cid | | + | created_timestamp | Dictionary field | x-ibm-finding:time_observed | + | date_updated | Dictionary field | x-ibm-finding:x_last_updated | + | detection_id | Dictionary field | x-ibm-finding:name | + | device.agent_load_flags | Dictionary field | | + | device.agent_local_time | Dictionary field | x-crowdstrike-edr-agent:local_time | + | device.agent_version | Dictionary field | x-oca-asset:x_agent_ref.version,x-crowdstrike-edr-agent:version | + | device.bios_manufacturer | Dictionary field | x-oca-asset:x_bios_manufacturer | + | device.bios_version | Dictionary field | x-oca-asset:x_bios_version | + | device.cid | Dictionary field | x-oca-asset:x_cid | + | device.config_id_base | Dictionary field | x-crowdstrike-edr-agent:config_id_base | + | device.config_id_build | Dictionary field | x-crowdstrike-edr-agent:config_id_build | + | device.config_id_platform | Dictionary field | x-crowdstrike-edr-agent:config_id_platform | + | device.device_id | Dictionary field | x-oca-asset:device_id | + | device.external_ip | Dictionary field | ipv4-addr:value, ipv6-addr:value,x-ibm-finding:src_ip_ref.value | | + | device.first_seen | Dictionary field | x-oca-asset:x_first_seen | + | device.groups | List Field | x-oca-asset:x_device_groups | + | device.hostname | Dictionary field | x-oca-asset:hostname | + | device.instance_id | Dictionary field | x-oca-asset:x_instance_id | + | device.last_login_timestamp | Dictionary field | | + | device.last_login_user | Dictionary field | | + | device.last_seen | Dictionary field | x-oca-asset:x_last_seen | + | device.local_ip | Dictionary field | ipv4-addr:value, ipv6-addr:value | | + | device.mac_address | Dictionary field | mac-addr:value | | + | device.major_version | Dictionary field | software:x_major_version | | + | device.minor_version | Dictionary field | software:x_minor_version | | + | device.modified_timestamp | Dictionary field | x-oca-asset:x_last_modified | + | device.os_version | Dictionary field | software:version | | + | device.platform_id | Dictionary field | software:x_id | | + | device.platform_name | Dictionary field | software:name,x-oca-asset:os_ref.name,x-ibm-finding:src_os_ref.name | | + | device.product_type | Dictionary field | x-oca-asset:x_host_type_number | + | device.product_type_desc | Dictionary field | x-oca-asset:host_type | + | device.service_provider | Dictionary field | x-oca-asset:x_service_provider | + | device.service_provider_account_id | Dictionary field | x-oca-asset:x_service_account_id | + | device.status | Dictionary field | x-oca-asset:x_status | + | device.system_manufacturer | Dictionary field | x-oca-asset:x_system_manufacturer | + | device.system_product_name | Dictionary field | x-oca-asset:x_system_product_name | + | email_sent | Dictionary field | | + | first_behavior | Dictionary field | x-ibm-finding:x_first_behavior_observed | + | hostinfo.domain | Dictionary field | domain-name:value | | + | last_behavior | Dictionary field | x-ibm-finding:x_last_behavior_observed | + | max_confidence | Dictionary field | x-ibm-finding:confidence | + | max_severity | Dictionary field | x-ibm-finding:severity | + | max_severity_displayname | Dictionary field | x-ibm-finding:x_severity_name | + | seconds_to_resolved | Dictionary field | | + | seconds_to_triaged | Dictionary field | | + | show_in_ui | Dictionary field | | + | status | Dictionary field | x-ibm-finding:x_status | + + diff --git a/stix_shifter_modules/crowdstrike_logscale/__init__.py b/stix_shifter_modules/crowdstrike_logscale/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/crowdstrike_logscale/configuration/config.json b/stix_shifter_modules/crowdstrike_logscale/configuration/config.json new file mode 100644 index 000000000..e3202d209 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/configuration/config.json @@ -0,0 +1,36 @@ +{ + "connection": { + "type": { + "displayName": "CrowdStrike Falcon LogScale", + "group": "crowdstrike" + }, + "host": { + "type": "text", + "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_:/\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_:/\\-]*[A-Za-z0-9])$" + }, + "help": { + "type": "link", + "default": "data-sources.html" + }, + "repository": { + "type": "text" + }, + "options": { + "api_page_size": { + "default": 2000, + "min": 1000, + "max": 10000, + "type": "number", + "optional": true + } + } + }, + "configuration": { + "auth": { + "type" : "fields", + "api_token": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/configuration/lang_en.json b/stix_shifter_modules/crowdstrike_logscale/configuration/lang_en.json new file mode 100644 index 000000000..e53cfe404 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/configuration/lang_en.json @@ -0,0 +1,30 @@ +{ + "connection": { + "host": { + "label": "Management IP address or hostname", + "description": "Specify the IP address or hostname of the data source" + }, + "help": { + "label": "Need additional help?", + "description": "More details on the data source setting can be found in the specified link" + }, + "repository": { + "label": "Repository", + "description": "A container storing log events collected from various data source." + }, + "options": { + "api_page_size": { + "label": "API Page Size", + "description": "Number of records per API call. Default value is above or equal to 2000" + } + } + }, + "configuration": { + "auth": { + "api_token": { + "label": "Repository API token", + "description": "The API token of a Repository to access the repository related API's" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/crowdstrike_logscale_supported_stix.md b/stix_shifter_modules/crowdstrike_logscale/crowdstrike_logscale_supported_stix.md new file mode 100644 index 000000000..6be8d4914 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/crowdstrike_logscale_supported_stix.md @@ -0,0 +1,247 @@ +##### Updated on 12/20/23 +## Crowdstirke Logscale +### Results STIX Domain Objects +* Identity +* Observed Data + +### Supported STIX Operators +*Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).* + +| STIX Operator | Data Source Operator | +|--|--| +| AND (Comparison) | and | +| OR (Comparison) | or | +| > | > | +| < | < | +| >= | >= | +| <= | <= | +| = | = | +| LIKE | = | +| IN | = | +| MATCHES | = | +| != | != | +| ISSUBSET | =~ | +| OR (Observation) | or | +| AND (Observation) | or | +|
| | +### Searchable STIX objects and properties +| STIX Object and Property | Mapped Data Source Fields | +|--|--| +| **ipv4-addr**:value | device.local_ip, device.external_ip | +| **ipv6-addr**:value | device.local_ip, device.external_ip | +| **mac-addr**:value | device.mac_address | +| **process**:name | behaviors[*].filename | +| **process**:command_line | behaviors[*].cmdline | +| **process**:x_process_graph_id | behaviors[*].triggering_process_graph_id | +| **process**:parent_ref.command_line | behaviors[*].parent_details.parent_cmdline | +| **process**:parent_ref.x_process_graph_id | behaviors[*].parent_details.parent_process_graph_id | +| **process**:binary_ref.name | behaviors[*].filename | +| **process**:binary_ref.hashes.MD5 | behaviors[*].md5 | +| **process**:binary_ref.hashes.'SHA-256' | behaviors[*].sha256 | +| **process**:creator_user_ref.user_id | behaviors[*].user_id | +| **file**:name | behaviors[*].filename | +| **file**:hashes.MD5 | behaviors[\*].parent_details.parent_md5, behaviors[\*].md5 | +| **file**:hashes.'SHA-256' | behaviors[\*].parent_details.parent_sha256, behaviors[\*].sha256 | +| **file**:x_extension | behaviors[*].alleged_filetype | +| **file**:x_path | behaviors[*].filepath | +| **file**:parent_directory_ref.path | behaviors[*].filepath | +| **directory**:path | behaviors[*].filepath | +| **user-account**:user_id | behaviors[*].user_id | +| **user-account**:display_name | behaviors[*].user_name | +| **software**:name | device.platform_name | +| **software**:version | device.os_version | +| **software**:x_id | device.platform_id | +| **software**:x_minor_version | device.minor_version | +| **software**:x_major_version | device.major_version | +| **x-ibm-finding**:name | detection_id | +| **x-ibm-finding**:severity | max_severity | +| **x-ibm-finding**:confidence | max_confidence | +| **x-ibm-finding**:time_observed | created_timestamp | +| **x-ibm-finding**:x_first_behavior_observed | first_behavior | +| **x-ibm-finding**:x_last_behavior_observed | last_behavior | +| **x-ibm-finding**:src_ip_ref.value | device.external_ip | +| **x-ibm-finding**:x_severity_name | max_severity_displayname | +| **x-ibm-finding**:x_status | status | +| **x-ibm-finding**x_logscale_event_id | @id | +| **x-ibm-finding**:x_last_updated | date_updated | +| **x-ibm-finding**:x_behaviors_processed[*] | behaviors_processed[*] | +| **x-ibm-finding**:x_behavior_refs[*].behavior_id | behaviors[*].behavior_id | +| **x-ibm-finding**:src_os_ref.name | device.platform_name | +| **x-ibm-finding**:ttp_tagging_refs.name | behaviors[*].tactic | +| **x-oca-asset**:hostname | device.hostname | +| **x-oca-asset**:device_id | device.device_id | +| **x-oca-asset**:ip_refs[*].value | device.local_ip, device.external_ip | +| **x-oca-asset**:mac_refs[*].value | device.mac_address | +| **x-oca-asset**:host_type | device.product_type_desc | +| **x-oca-asset**:x_instance_id | device.instance_id | +| **x-oca-asset**:x_host_type_number | device.product_type | +| **x-oca-asset**:x_first_seen | device.first_seen | +| **x-oca-asset**:x_last_seen | device.last_seen | +| **x-oca-asset**:x_bios_manufacturer | device.bios_manufacturer | +| **x-oca-asset**:x_bios_version | device.bios_version | +| **x-oca-asset**:x_last_modified | device.modified_timestamp | +| **x-oca-asset**:x_service_provider | device.service_provider | +| **x-oca-asset**:x_service_account_id | device.service_provider_account_id | +| **x-oca-asset**:x_status | device.status | +| **x-oca-asset**:x_system_product_name | device.system_product_name | +| **x-oca-asset**:x_system_manufacturer | device.system_manufacturer | +| **x-oca-asset**:x_agent_ref.version | device.agent_version | +| **x-oca-asset**:x_cid | cid | +| **x-oca-asset**:x_device_groups[*] | device.groups[*] | +| **x-oca-asset**:os_ref.name | device.platform_name | +| **x-ibm-ttp-tagging**:name | behaviors[*].tactic | +| **x-ibm-ttp-tagging**:extensions.'mitre-attack-ext'.technique_name | behaviors[*].technique | +| **x-ibm-ttp-tagging**:extensions.'mitre-attack-ext'.technique_id | behaviors[*].technique_id | +| **x-ibm-ttp-tagging**:extensions.'mitre-attack-ext'.tactic_id | behaviors[*].tactic_id | +| **x-crowdstrike-detection-behavior**:description | behaviors[*].description | +| **x-crowdstrike-detection-behavior**:objective | behaviors[*].objective | +| **x-crowdstrike-detection-behavior**:behavior_id | behaviors[*].behavior_id | +| **x-crowdstrike-detection-behavior**:display_name | behaviors[*].display_name | +| **x-crowdstrike-detection-behavior**:rule_instance_version | behaviors[*].rule_instance_version | +| **x-crowdstrike-detection-behavior**:pattern_disposition | behaviors[*].pattern_disposition | +| **x-crowdstrike-detection-behavior**:rule_instance_id | behaviors[*].rule_instance_id | +| **x-crowdstrike-detection-behavior**:created_time | behaviors[*].timestamp | +| **x-crowdstrike-detection-behavior**:control_graph_id | behaviors[*].control_graph_id | +| **x-crowdstrike-detection-behavior**:severity | behaviors[*].severity | +| **x-crowdstrike-detection-behavior**:confidence | behaviors[*].confidence | +| **x-crowdstrike-detection-behavior**:template_instance_id | behaviors[*].template_instance_id | +| **x-crowdstrike-detection-behavior**:scenario | behaviors[*].scenario | +| **x-crowdstrike-detection-behavior**:ioc_description | behaviors[*].ioc_description | +| **x-crowdstrike-detection-behavior**:ioc_source | behaviors[*].ioc_source | +| **x-crowdstrike-detection-behavior**:ioc_type | behaviors[*].ioc_type | +| **x-crowdstrike-detection-behavior**:ioc_value | behaviors[*].ioc_value | +| **x-crowdstrike-detection-behavior**:ttp_tagging_ref.name | behaviors[*].tactic | +| **x-crowdstrike-detection-behavior**:process_ref.name | behaviors[*].filename | +| **x-crowdstrike-detection-behavior**:user_ref.user_id | behaviors[*].user_id | +| **x-crowdstrike-edr-agent**:local_time | device.agent_local_time | +| **x-crowdstrike-edr-agent**:version | device.agent_version | +| **x-crowdstrike-edr-agent**:config_id_platform | device.config_id_platform | +| **x-crowdstrike-edr-agent**:config_id_base | device.config_id_base | +| **x-crowdstrike-edr-agent**:config_id_build | device.config_id_build | +|
| | +### Supported STIX Objects and Properties for Query Results +| STIX Object | STIX Property | Data Source Field | +|----------------------------------|--------------------------------------------|--| +| domain-name | value | domain | +|
| | | +| ipv4-addr | value | local_ip | +| ipv4-addr | value | external_ip | +| ipv4-addr | resolves_to_refs | local_ip | +|
| | | +| ipv6-addr | value | local_ip | +| ipv6-addr | value | external_ip | +| ipv6-addr | resolves_to_refs | local_ip | +|
| | | +| mac-addr | value | mac_address | +| process | name | filename | +| process | command_line | cmdline | +| process | command_line | parent_cmdline | +| process | x_process_graph_id | triggering_process_graph_id | +| process | x_process_graph_id | parent_process_graph_id | +| process | parent_ref | parent_cmdline | +| process | parent_ref | parent_process_graph_id | +| process | binary_ref | md5 | +| process | binary_ref | parent_sha256 | +| process | binary_ref | parent_md5 | +| process | binary_ref | filename | +| process | creator_user_ref | user_id | +|
| | | +| file | name | filename | +| file | hashes.MD5 | md5 | +| file | hashes.MD5 | parent_md5 | +| file | hashes.SHA-256 | parent_sha256 | +| file | hashes.SHA-256 | sha256 | +| file | x_extension | alleged_filetype | +| file | x_path | filepath | +| file | parent_directory_ref | filepath | +|
| | | +| directory | path | filepath | +|
| | | +| user-account | user_id | user_id | +| user-account | display_name | user_name | +|
| | | +| software | name | platform_name | +| software | version | os_version | +| software | x_id | platform_id | +| software | x_minor_version | minor_version | +| software | x_major_version | major_version | +|
| | | +| x-crowdstrike-detection-behavior | description | description | +| x-crowdstrike-detection-behavior | objective | objective | +| x-crowdstrike-detection-behavior | behavior_id | behavior_id | +| x-crowdstrike-detection-behavior | display_name | display_name | +| x-crowdstrike-detection-behavior | rule_instance_version | rule_instance_version | +| x-crowdstrike-detection-behavior | pattern_disposition | pattern_disposition | +| x-crowdstrike-detection-behavior | pattern_disposition | pattern_disposition_details | +| x-crowdstrike-detection-behavior | rule_instance_id | rule_instance_id | +| x-crowdstrike-detection-behavior | created_time | timestamp | +| x-crowdstrike-detection-behavior | control_graph_id | control_graph_id | +| x-crowdstrike-detection-behavior | severity | severity | +| x-crowdstrike-detection-behavior | confidence | confidence | +| x-crowdstrike-detection-behavior | template_instance_id | template_instance_id | +| x-crowdstrike-detection-behavior | scenario | scenario | +| x-crowdstrike-detection-behavior | ioc_description | ioc_description | +| x-crowdstrike-detection-behavior | ioc_source | ioc_source | +| x-crowdstrike-detection-behavior | ioc_type | ioc_type | +| x-crowdstrike-detection-behavior | ioc_value | ioc_value | +| x-crowdstrike-detection-behavior | ttp_tagging_ref | tactic | +| x-crowdstrike-detection-behavior | ttp_tagging_ref | tactic_id | +| x-crowdstrike-detection-behavior | process_ref | filename | +| x-crowdstrike-detection-behavior | user_ref | user_id | +|
| | | +| x-ibm-finding | name | detection_id | +| x-ibm-finding | confidence | max_confidence | +| x-ibm-finding | time_observed | created_timestamp | +| x-ibm-finding | severity | max_severity | +| x-ibm-finding | x_severity_name | max_severity_displayname | +| x-ibm-finding | x_status | status | +| x-ibm-finding | x_first_behavior_observed | first_behavior | +| x-ibm-finding | x_last_behavior_observed | last_behavior | +| x-ibm-finding | x_last_updated | date_updated | +| x-ibm-finding | x_behaviors_processed | behaviors_processed | +| x-ibm-finding | src_ip_ref | external_ip | +| x-ibm-finding | src_os_ref | platform_name | +| x-ibm-finding | ttp_tagging_refs | sensor_name | +| x-ibm-finding | x_is_email_sent | email_sent | +| x-ibm-finding | ioc_refs | domain | +| x-ibm-finding | x_seconds_to_resolved | seconds_to_resolved | +| x-ibm-finding | x_seconds_to_triaged | seconds_to_triaged | +| x-ibm-finding | x_logscale_event_id | @id | +| x-ibm-finding | x_logscale_repository | #repo | +|
| | | +| x-ibm-ttp-tagging | name | tactic | +| x-ibm-ttp-tagging | extensions.mitre-attack-ext.tactic_id | tactic_id | +| x-ibm-ttp-tagging | extensions.mitre-attack-ext.technique_name | technique | +| x-ibm-ttp-tagging | extensions.mitre-attack-ext.technique_id | technique_id | +|
| | | +| x-oca-asset | ip_refs | local_ip | +| x-oca-asset | ip_refs | external_ip | +| x-oca-asset | mac_refs | mac_address | +| x-oca-asset | hostname | hostname | +| x-oca-asset | device_id | device_id | +| x-oca-asset | x_cid | cid | +| x-oca-asset | host_type | product_type_desc | +| x-oca-asset | x_instance_id | instance_id | +| x-oca-asset | x_host_type_number | product_type | +| x-oca-asset | x_first_seen | first_seen | +| x-oca-asset | x_last_seen | last_seen | +| x-oca-asset | x_bios_manufacturer | bios_manufacturer | +| x-oca-asset | x_bios_version | bios_version | +| x-oca-asset | x_last_modified | modified_timestamp | +| x-oca-asset | x_service_provider | service_provider | +| x-oca-asset | x_service_account_id | service_provider_account_id | +| x-oca-asset | x_status | status | +| x-oca-asset | x_system_product_name | system_product_name | +| x-oca-asset | x_system_manufacturer | system_manufacturer | +| x-oca-asset | x_agent_ref | agent_local_time | +| x-oca-asset | x_agent_ref | agent_version | +| x-oca-asset | x_device_groups | groups | +| x-oca-asset | os_ref | platform_name | +|
| | | +| x-crowdstrike-edr-agent | load_flags | agent_load_flags | +| x-crowdstrike-edr-agent | local_time | agent_local_time | +| x-crowdstrike-edr-agent | version | agent_version | +| x-crowdstrike-edr-agent | config_id_base | config_id_base | +| x-crowdstrike-edr-agent | config_id_build | config_id_build | +| x-crowdstrike-edr-agent | config_id_platform | config_id_platform | +|
| | | diff --git a/stix_shifter_modules/crowdstrike_logscale/entry_point.py b/stix_shifter_modules/crowdstrike_logscale/entry_point.py new file mode 100644 index 000000000..00b4181aa --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/entry_point.py @@ -0,0 +1,13 @@ +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint + + +class EntryPoint(BaseEntryPoint): + + # python main.py translate crowdstrike_logscale query '{}' "[ipv4-addr:value = '127.0.0.1']" + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + self.set_async(True) + if connection: + self.setup_transmission_simple(connection, configuration) + self.setup_translation_simple(dialect_default='crowdstrikeedr') diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/__init__.py b/stix_shifter_modules/crowdstrike_logscale/stix_translation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/config_map.json b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/config_map.json new file mode 100644 index 000000000..5a9c7026b --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/config_map.json @@ -0,0 +1,13 @@ +{ + "crowdstrikeedr": + { + "integer_fields": [ + "behaviors[*].severity", + "behaviors[*].confidence", + "behaviors[*].pattern_disposition", + "behaviors[*].rule_instance_version" + ] + } +} + + diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/crowdstrikeedr_from_stix_map.json b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/crowdstrikeedr_from_stix_map.json new file mode 100644 index 000000000..4cb963606 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/crowdstrikeedr_from_stix_map.json @@ -0,0 +1,339 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "device.local_ip", + "device.external_ip" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "device.local_ip", + "device.external_ip" + ] + } + }, + "mac-addr": { + "fields": { + "value": [ + "device.mac_address" + ] + } + }, + "process": { + "fields": { + "name": [ + "behaviors[*].filename" + ], + "x_process_graph_id": [ + "behaviors[*].triggering_process_graph_id" + ], + "command_line": [ + "behaviors[*].cmdline" + ], + "parent_ref.command_line": [ + "behaviors[*].parent_details.parent_cmdline" + ], + "parent_ref.x_process_graph_id": [ + "behaviors[*].parent_details.parent_process_graph_id" + ], + "binary_ref.name": [ + "behaviors[*].filename" + ], + "binary_ref.hashes.MD5": [ + "behaviors[*].md5" + ], + "binary_ref.hashes.'SHA-256'": [ + "behaviors[*].sha256" + ], + "creator_user_ref.user_id": [ + "behaviors[*].user_id" + ] + } + }, + "file": { + "fields": { + "name": [ + "behaviors[*].filename" + ], + "hashes.MD5": [ + "behaviors[*].parent_details.parent_md5", + "behaviors[*].md5" + ], + "hashes.'SHA-256'": [ + "behaviors[*].parent_details.parent_sha256", + "behaviors[*].sha256" + ], + "x_extension": [ + "behaviors[*].alleged_filetype" + ], + "x_path": [ + "behaviors[*].filepath" + ], + "parent_directory_ref.path": [ + "behaviors[*].filepath" + ] + } + }, + "directory": { + "fields": { + "path": [ + "behaviors[*].filepath" + ] + } + }, + "user-account": { + "fields": { + "user_id": [ + "behaviors[*].user_id" + ], + "display_name": [ + "behaviors[*].user_name" + ] + } + }, + "software": { + "fields": { + "name": [ + "device.platform_name" + ], + "version": [ + "device.os_version" + ], + "x_id": [ + "device.platform_id" + ], + "x_minor_version": [ + "device.minor_version" + ], + "x_major_version": [ + "device.major_version" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "hostinfo.domain" + ] + } + }, + "x-oca-asset": { + "fields": { + "hostname": [ + "device.hostname" + ], + "device_id": [ + "device.device_id" + ], + "ip_refs[*].value": [ + "device.local_ip", + "device.external_ip" + ], + "mac_refs[*].value": [ + "device.mac_address" + ], + "host_type": [ + "device.product_type_desc" + ], + "x_instance_id": [ + "device.instance_id" + ], + "x_host_type_number": [ + "device.product_type" + ], + "x_first_seen": [ + "device.first_seen" + ], + "x_last_seen": [ + "device.last_seen" + ], + "x_bios_manufacturer": [ + "device.bios_manufacturer" + ], + "x_bios_version": [ + "device.bios_version" + ], + "x_last_modified": [ + "device.modified_timestamp" + ], + "x_service_provider": [ + "device.service_provider" + ], + "x_service_account_id": [ + "device.service_provider_account_id" + ], + "x_status": [ + "device.status" + ], + "x_system_product_name": [ + "device.system_product_name" + ], + "x_system_manufacturer": [ + "device.system_manufacturer" + ], + "x_agent_ref.version": [ + "device.agent_version" + ], + "x_cid": [ + "cid" + ], + "x_device_groups[*]": [ + "device.groups[*]" + ], + "os_ref.name": [ + "device.platform_name" + ] + } + }, + "x-ibm-finding": { + "fields": { + "x_logscale_event_id": [ + "@id" + ], + "confidence": [ + "max_confidence" + ], + "time_observed": [ + "created_timestamp" + ], + "name": [ + "detection_id" + ], + "severity": [ + "max_severity" + ], + "x_severity_name": [ + "max_severity_displayname" + ], + "x_status": [ + "status" + ], + "x_first_behavior_observed": [ + "first_behavior" + ], + "x_last_behavior_observed": [ + "last_behavior" + ], + "x_last_updated": [ + "date_updated" + ], + "x_behaviors_processed[*]": [ + "behaviors_processed[*]" + ], + "x_behavior_refs[*].behavior_id": [ + "behaviors[*].behavior_id" + ], + "src_ip_ref.value": [ + "device.external_ip" + ], + "src_os_ref.name": [ + "device.platform_name" + ], + "ttp_tagging_refs.name": [ + "behaviors[*].tactic" + ] + } + }, + "x-crowdstrike-detection-behavior": { + "fields": { + "description": [ + "behaviors[*].description" + ], + "objective": [ + "behaviors[*].objective" + ], + "behavior_id": [ + "behaviors[*].behavior_id" + ], + "display_name": [ + "behaviors[*].display_name" + ], + "rule_instance_version": [ + "behaviors[*].rule_instance_version" + ], + "pattern_disposition": [ + "behaviors[*].pattern_disposition" + ], + "rule_instance_id": [ + "behaviors[*].rule_instance_id" + ], + "created_time": [ + "behaviors[*].timestamp" + ], + "control_graph_id": [ + "behaviors[*].control_graph_id" + ], + "severity": [ + "behaviors[*].severity" + ], + "confidence": [ + "behaviors[*].confidence" + ], + "template_instance_id": [ + "behaviors[*].template_instance_id" + ], + "scenario": [ + "behaviors[*].scenario" + ], + "ioc_description": [ + "behaviors[*].ioc_description" + ], + "ioc_source": [ + "behaviors[*].ioc_source" + ], + "ioc_type": [ + "behaviors[*].ioc_type" + ], + "ioc_value": [ + "behaviors[*].ioc_value" + ], + "ttp_tagging_ref.name": [ + "behaviors[*].tactic" + ], + "process_ref.name": [ + "behaviors[*].filename" + ], + "user_ref.user_id": [ + "behaviors[*].user_id" + ] + } + }, + "x-crowdstrike-edr-agent": { + "fields": { + "local_time": [ + "device.agent_local_time" + ], + "version": [ + "device.agent_version" + ], + "config_id_platform": [ + "device.config_id_platform" + ], + "config_id_base": [ + "device.config_id_base" + ], + "config_id_build": [ + "device.config_id_build" + ] + } + }, + "x-ibm-ttp-tagging": { + "fields": { + "name": [ + "behaviors[*].tactic" + ], + "extensions.'mitre-attack-ext'.technique_name": [ + "behaviors[*].technique" + ], + "extensions.'mitre-attack-ext'.technique_id": [ + "behaviors[*].technique_id" + ], + "extensions.'mitre-attack-ext'.tactic_id": [ + "behaviors[*].tactic_id" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/crowdstrikeedr_to_stix_map.json b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/crowdstrikeedr_to_stix_map.json new file mode 100644 index 000000000..51152e291 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/crowdstrikeedr_to_stix_map.json @@ -0,0 +1,558 @@ +{ + "behaviors": { + "alleged_filetype": { + "key": "file.x_extension", + "object": "file" + }, + "behavior_id": { + "key": "x-crowdstrike-detection-behavior.behavior_id", + "object": "behavior" + }, + "cmdline": { + "key": "process.command_line", + "object": "process" + }, + "confidence": { + "key": "x-crowdstrike-detection-behavior.confidence", + "object": "behavior", + "transformer": "ToInteger" + }, + "control_graph_id": { + "key": "x-crowdstrike-detection-behavior.control_graph_id", + "object": "behavior" + }, + "description": { + "key": "x-crowdstrike-detection-behavior.description", + "object": "behavior" + }, + "display_name": { + "key": "x-crowdstrike-detection-behavior.display_name", + "object": "behavior" + }, + "filename": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "process.name", + "object": "process" + }, + { + "key": "x-crowdstrike-detection-behavior.process_ref", + "object": "behavior", + "references": "process" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "file" + } + ], + "filepath": [ + { + "key": "file.x_path", + "object": "file" + }, + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "template_instance_id": { + "key": "x-crowdstrike-detection-behavior.template_instance_id", + "object": "behavior" + }, + "ioc_description": { + "key": "x-crowdstrike-detection-behavior.ioc_description", + "object": "behavior" + }, + "ioc_source": { + "key": "x-crowdstrike-detection-behavior.ioc_source", + "object": "behavior" + }, + "ioc_type": { + "key": "x-crowdstrike-detection-behavior.ioc_type", + "object": "behavior" + }, + "ioc_value": { + "key": "x-crowdstrike-detection-behavior.ioc_value", + "object": "behavior" + }, + "md5": [ + { + "key": "file.hashes.MD5", + "object": "file" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "file" + } + ], + "objective": { + "key": "x-crowdstrike-detection-behavior.objective", + "object": "behavior" + }, + "parent_details": { + "parent_cmdline": [ + { + "key": "process.command_line", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "parent_md5": [ + { + "key": "file.hashes.MD5", + "object": "parent_file" + }, + { + "key": "process.binary_ref", + "object": "parent_process", + "references": "parent_file" + } + ], + "parent_process_graph_id": [ + { + "key": "process.x_process_graph_id", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "parent_sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "parent_file" + }, + { + "key": "process.binary_ref", + "object": "parent_process", + "references": "parent_file" + } + ] + }, + "pattern_disposition": { + "key": "x-crowdstrike-detection-behavior.pattern_disposition", + "object": "behavior", + "transformer": "ToInteger" + }, + "pattern_disposition_details": { + "key": "x-crowdstrike-detection-behavior.pattern_disposition_details", + "object": "behavior" + }, + "rule_instance_id": { + "key": "x-crowdstrike-detection-behavior.rule_instance_id", + "object": "behavior", + "transformer": "ToInteger" + }, + "rule_instance_version": { + "key": "x-crowdstrike-detection-behavior.rule_instance_version", + "object": "behavior" + }, + "scenario": { + "key": "x-crowdstrike-detection-behavior.scenario", + "object": "behavior" + }, + "severity": { + "key": "x-crowdstrike-detection-behavior.severity", + "object": "behavior", + "transformer": "ToInteger" + }, + "sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "tactic": [ + { + "key": "x-ibm-ttp-tagging.name", + "object": "ttp" + }, + { + "key": "x-crowdstrike-detection-behavior.ttp_tagging_ref", + "object": "behavior", + "references": "ttp" + } + ], + "tactic_id": [ + { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.tactic_id", + "object": "ttp" + }, + { + "key": "x-crowdstrike-detection-behavior.ttp_tagging_ref", + "object": "behavior", + "references": "ttp" + } + ], + "technique": { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.technique_name", + "object": "ttp" + }, + "technique_id": { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.technique_id", + "object": "ttp" + }, + "timestamp": { + "key": "x-crowdstrike-detection-behavior.created_time", + "object": "behavior" + }, + "triggering_process_graph_id": { + "key": "process.x_process_graph_id", + "object": "process" + }, + "user_id": [ + { + "key": "user-account.user_id", + "object": "user" + }, + { + "key": "x-crowdstrike-detection-behavior.user_ref", + "object": "behavior", + "references": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user" + } + ], + "user_name": { + "key": "user-account.display_name", + "object": "user" + }, + "GroupBehaviorsReferences": { + "key": "x-ibm-finding.x_behavior_refs", + "object": "finding", + "references": [ + "behavior" + ], + "group_ref": true + }, + "GroupTtpReferences": { + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "finding", + "references": [ + "ttp" + ], + "group_ref": "true" + } + }, + "finding_type": { + "key": "x-ibm-finding.finding_type", + "object": "finding" + }, + "behaviors_processed": { + "key": "x-ibm-finding.x_behaviors_processed", + "object": "finding" + }, + "cid": [ + { + "key": "x-oca-asset.x_cid", + "object": "asset" + } + ], + "created_timestamp": [ + { + "key": "x-ibm-finding.time_observed", + "object": "finding", + "transformer": "LogscaleToTimestamp" + }, + { + "key": "first_observed", + "transformer": "LogscaleToTimestamp" + } + ], + "detection_id": { + "key": "x-ibm-finding.name", + "object": "finding" + }, + "date_updated": { + "key": "x-ibm-finding.x_last_updated", + "object": "finding" + }, + "device": { + "agent_load_flags": { + "key": "x-crowdstrike-edr-agent.load_flags", + "object": "edr" + }, + "agent_local_time": [ + { + "key": "x-crowdstrike-edr-agent.local_time", + "object": "edr" + }, + { + "key": "x-oca-asset.x_agent_ref", + "object": "asset", + "references": "edr" + } + ], + "agent_version": [ + { + "key": "x-crowdstrike-edr-agent.version", + "object": "edr" + }, + { + "key": "x-oca-asset.x_agent_ref", + "object": "asset", + "references": "edr" + } + ], + "bios_manufacturer": { + "key": "x-oca-asset.x_bios_manufacturer", + "object": "asset" + }, + "bios_version": { + "key": "x-oca-asset.x_bios_version", + "object": "asset" + }, + "config_id_base": { + "key": "x-crowdstrike-edr-agent.config_id_base", + "object": "edr" + }, + "config_id_build": { + "key": "x-crowdstrike-edr-agent.config_id_build", + "object": "edr" + }, + "config_id_platform": { + "key": "x-crowdstrike-edr-agent.config_id_platform", + "object": "edr" + }, + "device_id": { + "key": "x-oca-asset.device_id", + "object": "asset" + }, + "external_ip": [ + { + "key": "ipv4-addr.value", + "object": "external" + }, + { + "key": "ipv6-addr.value", + "object": "external" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "external" + ], + "group": true + }, + { + "key": "x-ibm-finding.src_ip_ref", + "object": "finding", + "references": "external" + } + ], + "first_seen": { + "key": "x-oca-asset.x_first_seen", + "object": "asset" + }, + "groups": { + "key": "x-oca-asset.x_device_groups", + "object": "asset" + }, + "hostname": { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + "instance_id": { + "key": "x-oca-asset.x_instance_id", + "object": "asset" + }, + "last_seen": { + "key": "x-oca-asset.x_last_seen", + "object": "asset" + }, + "local_ip": [ + { + "key": "ipv4-addr.value", + "object": "internal" + }, + { + "key": "ipv6-addr.value", + "object": "internal" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "internal" + ], + "group": true + } + ], + "mac_address": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacAddress" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": [ + "mac" + ] + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "internal", + "references": [ + "mac" + ] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "internal", + "references": [ + "mac" + ] + } + ], + "major_version": { + "key": "software.x_major_version", + "object": "software" + }, + "minor_version": { + "key": "software.x_minor_version", + "object": "software" + }, + "modified_timestamp": { + "key": "x-oca-asset.x_last_modified", + "object": "asset" + }, + "os_version": { + "key": "software.version", + "object": "software" + }, + "platform_id": { + "key": "software.x_id", + "object": "software" + }, + "platform_name": [ + { + "key": "software.name", + "object": "software" + }, + { + "key": "x-oca-asset.os_ref", + "object": "asset", + "references": "software" + }, + { + "key": "x-ibm-finding.src_os_ref", + "object": "finding", + "references": "software" + } + ], + "product_type": { + "key": "x-oca-asset.x_host_type_number", + "object": "asset" + }, + "product_type_desc": { + "key": "x-oca-asset.host_type", + "object": "asset" + }, + "service_provider": { + "key": "x-oca-asset.x_service_provider", + "object": "asset" + }, + "service_provider_account_id": { + "key": "x-oca-asset.x_service_account_id", + "object": "asset" + }, + "status": { + "key": "x-oca-asset.x_status", + "object": "asset" + }, + "system_manufacturer": { + "key": "x-oca-asset.x_system_manufacturer", + "object": "asset" + }, + "system_product_name": { + "key": "x-oca-asset.x_system_product_name", + "object": "asset" + } + }, + "email_sent": { + "key": "x-ibm-finding.x_is_email_sent", + "object": "finding" + }, + "first_behavior": { + "key": "x-ibm-finding.x_first_behavior_observed", + "object": "finding" + }, + "hostinfo": { + "domain": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-ibm-finding.ioc_refs", + "object": "finding", + "references": [ + "domain" + ] + } + ] + }, + "@id": { + "key": "x-ibm-finding.x_logscale_event_id", + "object": "finding" + }, + "#repo": { + "key": "x-ibm-finding.x_logscale_repository", + "object": "finding" + }, + "@timestamp": { + "key": "last_observed", + "transformer": "EpochToTimestamp" + }, + "last_behavior": { + "key": "x-ibm-finding.x_last_behavior_observed", + "object": "finding" + }, + "max_confidence": { + "key": "x-ibm-finding.confidence", + "object": "finding", + "transformer": "ToInteger" + }, + "max_severity": { + "key": "x-ibm-finding.severity", + "object": "finding", + "transformer": "ToInteger" + }, + "max_severity_displayname": { + "key": "x-ibm-finding.x_severity_name", + "object": "finding" + }, + "seconds_to_resolved": { + "key": "x-ibm-finding.x_seconds_to_resolved", + "object": "finding" + }, + "status": { + "key": "x-ibm-finding.x_status", + "object": "finding" + }, + "seconds_to_triaged": { + "key": "x-ibm-finding.x_seconds_to_triaged", + "object": "finding" + } +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/operators.json b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/operators.json new file mode 100644 index 000000000..cb7a2362e --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/operators.json @@ -0,0 +1,16 @@ +{ + "ComparisonExpressionOperators.And": "and", + "ComparisonExpressionOperators.Or": "or", + "ComparisonComparators.GreaterThan": ">", + "ComparisonComparators.GreaterThanOrEqual": ">=", + "ComparisonComparators.LessThan": "<", + "ComparisonComparators.LessThanOrEqual": "<=", + "ComparisonComparators.Equal": "=", + "ComparisonComparators.NotEqual": "!=", + "ComparisonComparators.Like": "=", + "ComparisonComparators.In": "=", + "ComparisonComparators.Matches": "=", + "ObservationOperators.Or": "or", + "ObservationOperators.And": "or", + "ComparisonComparators.IsSubSet": "=~" +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/stix_2_1/crowdstrikeedr_from_stix_map.json b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/stix_2_1/crowdstrikeedr_from_stix_map.json new file mode 100644 index 000000000..e74162bc8 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/stix_2_1/crowdstrikeedr_from_stix_map.json @@ -0,0 +1,336 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "device.local_ip", + "device.external_ip" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "device.local_ip", + "device.external_ip" + ] + } + }, + "mac-addr": { + "fields": { + "value": [ + "device.mac_address" + ] + } + }, + "process": { + "fields": { + "x_process_graph_id": [ + "behaviors[*].triggering_process_graph_id" + ], + "command_line": [ + "behaviors[*].cmdline" + ], + "parent_ref.command_line": [ + "behaviors[*].parent_details.parent_cmdline" + ], + "parent_ref.x_process_graph_id": [ + "behaviors[*].parent_details.parent_process_graph_id" + ], + "image_ref.name": [ + "behaviors[*].filename" + ], + "image_ref.hashes.MD5": [ + "behaviors[*].md5" + ], + "image_ref.hashes.'SHA-256'": [ + "behaviors[*].sha256" + ], + "creator_user_ref.user_id": [ + "behaviors[*].user_id" + ] + } + }, + "file": { + "fields": { + "name": [ + "behaviors[*].filename" + ], + "hashes.MD5": [ + "behaviors[*].parent_details.parent_md5", + "behaviors[*].md5" + ], + "hashes.'SHA-256'": [ + "behaviors[*].parent_details.parent_sha256", + "behaviors[*].sha256" + ], + "x_extension": [ + "behaviors[*].alleged_filetype" + ], + "x_path": [ + "behaviors[*].filepath" + ], + "parent_directory_ref.path": [ + "behaviors[*].filepath" + ] + } + }, + "directory": { + "fields": { + "path": [ + "behaviors[*].filepath" + ] + } + }, + "user-account": { + "fields": { + "user_id": [ + "behaviors[*].user_id" + ], + "display_name": [ + "behaviors[*].user_name" + ] + } + }, + "software": { + "fields": { + "name": [ + "device.platform_name" + ], + "version": [ + "device.os_version" + ], + "x_id": [ + "device.platform_id" + ], + "x_minor_version": [ + "device.minor_version" + ], + "x_major_version": [ + "device.major_version" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "hostinfo.domain" + ] + } + }, + "x-oca-asset": { + "fields": { + "hostname": [ + "device.hostname" + ], + "device_id": [ + "device.device_id" + ], + "ip_refs[*].value": [ + "device.local_ip", + "device.external_ip" + ], + "mac_refs[*].value": [ + "device.mac_address" + ], + "host_type": [ + "device.product_type_desc" + ], + "x_instance_id": [ + "device.instance_id" + ], + "x_host_type_number": [ + "device.product_type" + ], + "x_first_seen": [ + "device.first_seen" + ], + "x_last_seen": [ + "device.last_seen" + ], + "x_bios_manufacturer": [ + "device.bios_manufacturer" + ], + "x_bios_version": [ + "device.bios_version" + ], + "x_last_modified": [ + "device.modified_timestamp" + ], + "x_service_provider": [ + "device.service_provider" + ], + "x_service_account_id": [ + "device.service_provider_account_id" + ], + "x_status": [ + "device.status" + ], + "x_system_product_name": [ + "device.system_product_name" + ], + "x_system_manufacturer": [ + "device.system_manufacturer" + ], + "x_agent_ref.version": [ + "device.agent_version" + ], + "x_cid": [ + "cid" + ], + "x_device_groups[*]": [ + "device.groups[*]" + ], + "os_ref.name": [ + "device.platform_name" + ] + } + }, + "x-ibm-finding": { + "fields": { + "x_logscale_event_id": [ + "@id" + ], + "confidence": [ + "max_confidence" + ], + "time_observed": [ + "created_timestamp" + ], + "name": [ + "detection_id" + ], + "x_severity": [ + "max_severity" + ], + "x_severity_name": [ + "max_severity_displayname" + ], + "x_status": [ + "status" + ], + "x_first_behavior_observed": [ + "first_behavior" + ], + "x_last_behavior_observed": [ + "last_behavior" + ], + "x_last_updated": [ + "date_updated" + ], + "x_behaviors_processed[*]": [ + "behaviors_processed[*]" + ], + "x_behavior_refs[*].behavior_id": [ + "behaviors[*].behavior_id" + ], + "src_ip_ref.value": [ + "device.external_ip" + ], + "src_os_ref.name": [ + "device.platform_name" + ], + "ttp_tagging_refs.name": [ + "behaviors[*].tactic" + ] + } + }, + "x-crowdstrike-detection-behavior": { + "fields": { + "description": [ + "behaviors[*].description" + ], + "objective": [ + "behaviors[*].objective" + ], + "behavior_id": [ + "behaviors[*].behavior_id" + ], + "display_name": [ + "behaviors[*].display_name" + ], + "rule_instance_version": [ + "behaviors[*].rule_instance_version" + ], + "pattern_disposition": [ + "behaviors[*].pattern_disposition" + ], + "rule_instance_id": [ + "behaviors[*].rule_instance_id" + ], + "created_time": [ + "behaviors[*].timestamp" + ], + "control_graph_id": [ + "behaviors[*].control_graph_id" + ], + "x_severity": [ + "behaviors[*].severity" + ], + "confidence": [ + "behaviors[*].confidence" + ], + "template_instance_id": [ + "behaviors[*].template_instance_id" + ], + "scenario": [ + "behaviors[*].scenario" + ], + "ioc_description": [ + "behaviors[*].ioc_description" + ], + "ioc_source": [ + "behaviors[*].ioc_source" + ], + "ioc_type": [ + "behaviors[*].ioc_type" + ], + "ioc_value": [ + "behaviors[*].ioc_value" + ], + "ttp_tagging_ref.name": [ + "behaviors[*].tactic" + ], + "process_ref.name": [ + "behaviors[*].filename" + ], + "user_ref.user_id": [ + "behaviors[*].user_id" + ] + } + }, + "x-crowdstrike-edr-agent": { + "fields": { + "local_time": [ + "device.agent_local_time" + ], + "version": [ + "device.agent_version" + ], + "config_id_platform": [ + "device.config_id_platform" + ], + "config_id_base": [ + "device.config_id_base" + ], + "config_id_build": [ + "device.config_id_build" + ] + } + }, + "x-ibm-ttp-tagging": { + "fields": { + "name": [ + "behaviors[*].tactic" + ], + "extensions.'mitre-attack-ext'.technique_name": [ + "behaviors[*].technique" + ], + "extensions.'mitre-attack-ext'.technique_id": [ + "behaviors[*].technique_id" + ], + "extensions.'mitre-attack-ext'.tactic_id": [ + "behaviors[*].tactic_id" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/stix_2_1/crowdstrikeedr_to_stix_map.json b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/stix_2_1/crowdstrikeedr_to_stix_map.json new file mode 100644 index 000000000..b2ec9e9c5 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/json/stix_2_1/crowdstrikeedr_to_stix_map.json @@ -0,0 +1,554 @@ +{ + "behaviors": { + "alleged_filetype": { + "key": "file.x_extension", + "object": "file" + }, + "behavior_id": { + "key": "x-crowdstrike-detection-behavior.behavior_id", + "object": "behavior" + }, + "cmdline": { + "key": "process.command_line", + "object": "process" + }, + "confidence": { + "key": "x-crowdstrike-detection-behavior.confidence", + "object": "behavior", + "transformer": "ToInteger" + }, + "control_graph_id": { + "key": "x-crowdstrike-detection-behavior.control_graph_id", + "object": "behavior" + }, + "description": { + "key": "x-crowdstrike-detection-behavior.description", + "object": "behavior" + }, + "display_name": { + "key": "x-crowdstrike-detection-behavior.display_name", + "object": "behavior" + }, + "filename": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "x-crowdstrike-detection-behavior.process_ref", + "object": "behavior", + "references": "process" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file" + } + ], + "filepath": [ + { + "key": "file.x_path", + "object": "file" + }, + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "template_instance_id": { + "key": "x-crowdstrike-detection-behavior.template_instance_id", + "object": "behavior" + }, + "ioc_description": { + "key": "x-crowdstrike-detection-behavior.ioc_description", + "object": "behavior" + }, + "ioc_source": { + "key": "x-crowdstrike-detection-behavior.ioc_source", + "object": "behavior" + }, + "ioc_type": { + "key": "x-crowdstrike-detection-behavior.ioc_type", + "object": "behavior" + }, + "ioc_value": { + "key": "x-crowdstrike-detection-behavior.ioc_value", + "object": "behavior" + }, + "md5": [ + { + "key": "file.hashes.MD5", + "object": "file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file" + } + ], + "objective": { + "key": "x-crowdstrike-detection-behavior.objective", + "object": "behavior" + }, + "parent_details": { + "parent_cmdline": [ + { + "key": "process.command_line", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "parent_md5": [ + { + "key": "file.hashes.MD5", + "object": "parent_file" + }, + { + "key": "process.image_ref", + "object": "parent_process", + "references": "parent_file" + } + ], + "parent_process_graph_id": [ + { + "key": "process.x_process_graph_id", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "parent_sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "parent_file" + }, + { + "key": "process.image_ref", + "object": "parent_process", + "references": "parent_file" + } + ] + }, + "pattern_disposition": { + "key": "x-crowdstrike-detection-behavior.pattern_disposition", + "object": "behavior", + "transformer": "ToInteger" + }, + "pattern_disposition_details": { + "key": "x-crowdstrike-detection-behavior.pattern_disposition_details", + "object": "behavior" + }, + "rule_instance_id": { + "key": "x-crowdstrike-detection-behavior.rule_instance_id", + "object": "behavior", + "transformer": "ToInteger" + }, + "rule_instance_version": { + "key": "x-crowdstrike-detection-behavior.rule_instance_version", + "object": "behavior" + }, + "scenario": { + "key": "x-crowdstrike-detection-behavior.scenario", + "object": "behavior" + }, + "severity": { + "key": "x-crowdstrike-detection-behavior.x_severity", + "object": "behavior", + "transformer": "ToInteger" + }, + "sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "tactic": [ + { + "key": "x-ibm-ttp-tagging.name", + "object": "ttp" + }, + { + "key": "x-crowdstrike-detection-behavior.ttp_tagging_ref", + "object": "behavior", + "references": "ttp" + } + ], + "tactic_id": [ + { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.tactic_id", + "object": "ttp" + }, + { + "key": "x-crowdstrike-detection-behavior.ttp_tagging_ref", + "object": "behavior", + "references": "ttp" + } + ], + "technique": { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.technique_name", + "object": "ttp" + }, + "technique_id": { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.technique_id", + "object": "ttp" + }, + "timestamp": { + "key": "x-crowdstrike-detection-behavior.created_time", + "object": "behavior" + }, + "triggering_process_graph_id": { + "key": "process.x_process_graph_id", + "object": "process" + }, + "user_id": [ + { + "key": "user-account.user_id", + "object": "user" + }, + { + "key": "x-crowdstrike-detection-behavior.user_ref", + "object": "behavior", + "references": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user" + } + ], + "user_name": { + "key": "user-account.display_name", + "object": "user" + }, + "GroupBehaviorsReferences": { + "key": "x-ibm-finding.x_behavior_refs", + "object": "finding", + "references": [ + "behavior" + ], + "group_ref": true + }, + "GroupTtpReferences": { + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "finding", + "references": [ + "ttp" + ], + "group_ref": "true" + } + }, + "finding_type": { + "key": "x-ibm-finding.finding_type", + "object": "finding" + }, + "behaviors_processed": { + "key": "x-ibm-finding.x_behaviors_processed", + "object": "finding" + }, + "cid": [ + { + "key": "x-oca-asset.x_cid", + "object": "asset" + } + ], + "created_timestamp": [ + { + "key": "x-ibm-finding.time_observed", + "object": "finding", + "transformer": "LogscaleToTimestamp" + }, + { + "key": "first_observed", + "transformer": "LogscaleToTimestamp" + } + ], + "detection_id": { + "key": "x-ibm-finding.name", + "object": "finding" + }, + "date_updated": { + "key": "x-ibm-finding.x_last_updated", + "object": "finding" + }, + "device": { + "agent_load_flags": { + "key": "x-crowdstrike-edr-agent.load_flags", + "object": "edr" + }, + "agent_local_time": [ + { + "key": "x-crowdstrike-edr-agent.local_time", + "object": "edr" + }, + { + "key": "x-oca-asset.x_agent_ref", + "object": "asset", + "references": "edr" + } + ], + "agent_version": [ + { + "key": "x-crowdstrike-edr-agent.version", + "object": "edr" + }, + { + "key": "x-oca-asset.x_agent_ref", + "object": "asset", + "references": "edr" + } + ], + "bios_manufacturer": { + "key": "x-oca-asset.x_bios_manufacturer", + "object": "asset" + }, + "bios_version": { + "key": "x-oca-asset.x_bios_version", + "object": "asset" + }, + "config_id_base": { + "key": "x-crowdstrike-edr-agent.config_id_base", + "object": "edr" + }, + "config_id_build": { + "key": "x-crowdstrike-edr-agent.config_id_build", + "object": "edr" + }, + "config_id_platform": { + "key": "x-crowdstrike-edr-agent.config_id_platform", + "object": "edr" + }, + "device_id": { + "key": "x-oca-asset.device_id", + "object": "asset" + }, + "external_ip": [ + { + "key": "ipv4-addr.value", + "object": "external" + }, + { + "key": "ipv6-addr.value", + "object": "external" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "external" + ], + "group": true + }, + { + "key": "x-ibm-finding.src_ip_ref", + "object": "finding", + "references": "external" + } + ], + "first_seen": { + "key": "x-oca-asset.x_first_seen", + "object": "asset" + }, + "groups": { + "key": "x-oca-asset.x_device_groups", + "object": "asset" + }, + "hostname": { + "key": "x-oca-asset.hostname", + "object": "asset" + }, + "instance_id": { + "key": "x-oca-asset.x_instance_id", + "object": "asset" + }, + "last_seen": { + "key": "x-oca-asset.x_last_seen", + "object": "asset" + }, + "local_ip": [ + { + "key": "ipv4-addr.value", + "object": "internal" + }, + { + "key": "ipv6-addr.value", + "object": "internal" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "asset", + "references": [ + "internal" + ], + "group": true + } + ], + "mac_address": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacAddress" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "asset", + "references": [ + "mac" + ] + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "internal", + "references": [ + "mac" + ] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "internal", + "references": [ + "mac" + ] + } + ], + "major_version": { + "key": "software.x_major_version", + "object": "software" + }, + "minor_version": { + "key": "software.x_minor_version", + "object": "software" + }, + "modified_timestamp": { + "key": "x-oca-asset.x_last_modified", + "object": "asset" + }, + "os_version": { + "key": "software.version", + "object": "software" + }, + "platform_id": { + "key": "software.x_id", + "object": "software" + }, + "platform_name": [ + { + "key": "software.name", + "object": "software" + }, + { + "key": "x-oca-asset.os_ref", + "object": "asset", + "references": "software" + }, + { + "key": "x-ibm-finding.src_os_ref", + "object": "finding", + "references": "software" + } + ], + "product_type": { + "key": "x-oca-asset.x_host_type_number", + "object": "asset" + }, + "product_type_desc": { + "key": "x-oca-asset.host_type", + "object": "asset" + }, + "service_provider": { + "key": "x-oca-asset.x_service_provider", + "object": "asset" + }, + "service_provider_account_id": { + "key": "x-oca-asset.x_service_account_id", + "object": "asset" + }, + "status": { + "key": "x-oca-asset.x_status", + "object": "asset" + }, + "system_manufacturer": { + "key": "x-oca-asset.x_system_manufacturer", + "object": "asset" + }, + "system_product_name": { + "key": "x-oca-asset.x_system_product_name", + "object": "asset" + } + }, + "email_sent": { + "key": "x-ibm-finding.x_is_email_sent", + "object": "finding" + }, + "first_behavior": { + "key": "x-ibm-finding.x_first_behavior_observed", + "object": "finding" + }, + "hostinfo": { + "domain": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-ibm-finding.ioc_refs", + "object": "finding", + "references": [ + "domain" + ] + } + ] + }, + "@id": { + "key": "x-ibm-finding.x_logscale_event_id", + "object": "finding" + }, + "#repo": { + "key": "x-ibm-finding.x_logscale_repository", + "object": "finding" + }, + "@timestamp": { + "key": "last_observed", + "transformer": "EpochToTimestamp" + }, + "last_behavior": { + "key": "x-ibm-finding.x_last_behavior_observed", + "object": "finding" + }, + "max_confidence": { + "key": "x-ibm-finding.confidence", + "object": "finding", + "transformer": "ToInteger" + }, + "max_severity": { + "key": "x-ibm-finding.x_severity", + "object": "finding", + "transformer": "ToInteger" + }, + "max_severity_displayname": { + "key": "x-ibm-finding.x_severity_name", + "object": "finding" + }, + "seconds_to_resolved": { + "key": "x-ibm-finding.x_seconds_to_resolved", + "object": "finding" + }, + "status": { + "key": "x-ibm-finding.x_status", + "object": "finding" + }, + "seconds_to_triaged": { + "key": "x-ibm-finding.x_seconds_to_triaged", + "object": "finding" + } +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/query_constructor.py b/stix_shifter_modules/crowdstrike_logscale/stix_translation/query_constructor.py new file mode 100644 index 000000000..9810ecbe3 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/query_constructor.py @@ -0,0 +1,615 @@ +import re +import json +import logging +from os import path +from itertools import product +from datetime import datetime, timedelta +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, \ + ComparisonExpression, ComparisonComparators, Pattern, CombinedComparisonExpression, CombinedObservationExpression + +START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)" +logger = logging.getLogger(__name__) +CONFIG_MAP_PATH = "json/config_map.json" +STOP_TIME = datetime.utcnow() + + +class FileNotFoundException(Exception): + pass + + +class StartStopQualifierValueException(Exception): + pass + + +class QueryStringPatternTranslator: + + def __init__(self, pattern: Pattern, data_model_mapper, options): + logger.info("CrowdStrike LogScale Connector") + self.dmm = data_model_mapper + self.config_map = QueryStringPatternTranslator.load_json(CONFIG_MAP_PATH) + self.options = options + self.all_queries = [] + self.comparator_lookup = self.dmm.map_comparator() + self.current_observation_operator = "" + self.source = self.dmm.dialect + self.parse_expression(pattern) + self.translated_query = self._create_formatted_query() + + @staticmethod + def load_json(rel_path_of_file): + """ + Consumes a json file and returns a dictionary + :param rel_path_of_file: str + :return: dict + """ + _json_path = path.dirname(path.abspath(__file__)) + "/" + rel_path_of_file + try: + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundException + except FileNotFoundException as e: + raise FileNotFoundError(f'{rel_path_of_file} not found') from e + + @staticmethod + def _format_set(values, list_of_dict_field=False): + """ + Formats value in the event of IN operation + :param: values + :return: formatted values,str + """ + gen = values.element_iterator() + if list_of_dict_field: + formatted_values = [QueryStringPatternTranslator._escape_value(value, list_of_dict_field=list_of_dict_field) + for value in gen] + else: + formatted_values = \ + [f'\"{QueryStringPatternTranslator._escape_value(value, list_of_dict_field=list_of_dict_field)}\"' + for value in gen] + return formatted_values + + @staticmethod + def _format_subset(expression, values): + """ + Formats value in the event of subset operation + :param expression, values + :return: formatted values,str + """ + if expression.negated: + return f'!cidr(subnet=\"{values}\")' + return f'cidr(subnet=\"{values}\")' + + @staticmethod + def _format_match(value, array_fields=None): + """ + Formats value in the event of match operation + :param value: str, array_fields: list + :return: formatted string type value, str + """ + value = QueryStringPatternTranslator._escape_value(value, is_regex=True, array_fields=array_fields) + return value + + @staticmethod + def _format_like(value, array_fields=None): + """ + Formatting value in the event of like operation + :param value: str, array_fields: list + :return: formatted string type value, str + """ + value = value.replace('%', '.*').replace('_', '.') + value = QueryStringPatternTranslator._escape_value(value, is_regex=True, array_fields=array_fields) + return value + + @staticmethod + def format_equals(value): + """ + Formats value in the event of equals operation + :param value: str + :return: formatted values,str + """ + + if isinstance(value, str): + value = value.replace('\\', '\\\\').replace('\"', '\\"') + value = f'\"{value}\"' + return value + + @staticmethod + def _escape_value(value, is_regex=False, list_of_dict_field=False, array_fields=None): + """ + Format the value with escape characters + :param value: str or int, is_regex: bool, list_of_dict_field: bool, array_fields: list + :return: value, str or int + """ + value = str(value) + if list_of_dict_field: + value = re.escape(value).replace('/', '\\/').replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"') + + elif is_regex: + value = (value.replace('\\', '\\\\').replace('\"', '\\"').replace('(', '\\(').replace(')', '\\)'). + replace('/', '\\/')) + + value = f'/{value}/i' if not array_fields else value + + return value + + @staticmethod + def _validate_numerical_value(comparator, value): + """ + Validate the input value with numerical operators and raise exception in case of invalid type + :param comparator, value: int + :return: value,int + """ + if isinstance(value,str): + if not value.isdigit(): + raise NotImplementedError(f'{comparator} operator is not supported for string type value: {value}') + return value + + def _create_formatted_query(self): + """ + Creates formatted native data source query + :return: formatted query: list + """ + final_query = [] + merged_queries = [] + self.all_queries = [i for n, i in enumerate(self.all_queries) if i not in self.all_queries[:n]] + + for queries in self.all_queries: + if not merged_queries: + merged_queries.append(queries) + else: + matched = False + for query in merged_queries: + if queries["operator"] != "|" and query["operator"] != "|" and \ + queries["timestamp"] == query["timestamp"]: + if not query["is_ts_matched"]: + query["query"] = f'({query["query"]})' + query["is_ts_matched"] = True + query["query"] += f' {queries["operator"]} ({queries["query"]})' + matched = True + if not matched: + merged_queries.append(queries) + + for queries in merged_queries: + result_query_string = f'{queries["query"]} | tail()' + formatted_query = { + "source": self.source, + "queryString": result_query_string, + "start": queries["timestamp"][0], + "end": queries["timestamp"][1] + } + final_query.append(formatted_query) + return final_query + + def validate_values(self, field_name, list_of_dict_value, array_not_in_list_of_dict): + """ + Validate the input values + :param: field_name: str, list_of_dict_value: list, array_not_in_list_of_dict: bool + :return: formatted query: str or int + """ + if field_name in self.config_map.get(self.source, {}).get('integer_fields', []): + if isinstance(list_of_dict_value, list): + if len(list_of_dict_value) > 1: + return f'({"|".join(list_of_dict_value)})' + return "|".join(list_of_dict_value) + return int(list_of_dict_value) + + if isinstance(list_of_dict_value, list): + if array_not_in_list_of_dict: + return f'{"|".join(list_of_dict_value)}' + if len(list_of_dict_value) > 1: + joined_str = '|'.join([f'\"{val}\"' for val in list_of_dict_value]) + return f'({joined_str})' + return '|'.join([f'\"{val}\"' for val in list_of_dict_value]) + return f'\"{list_of_dict_value}\"' + + @staticmethod + def _attribute_processing(field_name): + """ + Removing the "[*]" from mapped attributes + :param field_name: str + :return: list_attribute: list + """ + list_attribute = [] + split_field = field_name.split(".") + for field in split_field: + if "[*]" in field: + field = field[:-3] + '[]' + list_attribute.append(field) + return list_attribute + + def _create_parsed_query_for_list_of_dict(self, field_name, list_of_dict_value, expression_comparator): + """ + create parsed query for list of dict fields + :param: field_name: str, list_of_dict_value: list, expression_comparator: object + :return: formatted query: list + """ + array_in_list_of_dict = False + array_not_in_list_of_dict = False + + if field_name.endswith("[*]"): + if field_name.count("[*]") > 1: + array_in_list_of_dict = True + else: + array_not_in_list_of_dict = True + list_of_dict_value = self.validate_values(field_name, list_of_dict_value, array_not_in_list_of_dict) + + list_attribute = self._attribute_processing(field_name) + + if array_not_in_list_of_dict: + new_field_name = ".".join(list_attribute) + else: + new_field_name = "" + for index, value in enumerate(list_attribute): + if value.endswith('[]'): + value = f'\"{value[:-2]}\"' + r"\s*:\s*\[.*" + elif index < len(list_attribute) - 1: + value = f'\"{value}\"' + r"\s*:\s*\{.*" + else: + value = f'\"{value}\"' + new_field_name += value + + operator = "!=" if expression_comparator == ComparisonComparators.NotEqual else "=" + + if array_in_list_of_dict: + field_mappings = f'@rawstring {operator} /{new_field_name}' + f'{list_of_dict_value}/' + elif array_not_in_list_of_dict: + if expression_comparator in [ComparisonComparators.Like, ComparisonComparators.Matches]: + field_mappings = f'array:regex(array=\"{new_field_name}\",regex = \"{list_of_dict_value}\", flags=i)' + else: + if operator == "!=": + field_mappings = f'!array:contains(array=\"{new_field_name}\",value = {list_of_dict_value})' + else: + field_mappings = f'array:contains(array=\"{new_field_name}\",value = {list_of_dict_value})' + else: + field_mappings = f'@rawstring {operator} /{new_field_name}' + r'\s*:\s*' + f'{list_of_dict_value}/' + + return field_mappings + + def _create_parsed_query(self, field_name, list_of_dict_value, value, comparator, is_negated, + expression_comparator): + """ + Creates comparison string for each field_name in mapped_field_array + :param field_name: str, + :param list_of_dict_value: list + :param value: str + :param comparator: str + :param is_negated: boolean + :param expression_comparator: str + :return: comparison_string, str + """ + comparison_string = "" + if not isinstance(value, list): + value = [value] + + all_mappings = [] + if "[*]" in field_name: + list_value = value if field_name.endswith("[*]") and field_name.count("[*]") == 1 else list_of_dict_value + + field_mappings = self._create_parsed_query_for_list_of_dict(field_name, list_value, + expression_comparator) + if is_negated: + field_mappings = f'not {field_mappings}' + all_mappings.append(field_mappings) + else: + for values in value: + if expression_comparator == ComparisonComparators.NotEqual and not is_negated: + field_mappings = f'({field_name} {comparator} {values} and {field_name} = "*")' + elif (expression_comparator != ComparisonComparators.IsSubSet) and is_negated: + field_mappings = (f'(not {field_name} {comparator} {values} ' + f'and {field_name} = "*")') + else: + field_mappings = f'{field_name} {comparator} {values}' + all_mappings.append(field_mappings) + + if all_mappings: + if len(all_mappings) == 1: + comparison_string += all_mappings[0] + else: + in_comparison_string = ' and '.join(all_mappings) if is_negated else ' or '.join(all_mappings) + comparison_string += in_comparison_string + + if len(all_mappings) > 1: + comparison_string = "(" + comparison_string + ")" + + return comparison_string + + @staticmethod + def _format_datetime(value): + """ + Converts timestamp to seconds + :param value: str + :return: converted epoch value: int + """ + try: + time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ' + if re.search(r"\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}Z", str(value)): + time_pattern = '%Y-%m-%dT%H:%M:%SZ' + epoch = datetime(1970, 1, 1) + converted_time = int(((datetime.strptime(value, time_pattern) - epoch).total_seconds()) * 1000) + return converted_time + except ValueError: + logger.error("Cannot convert the timestamp %s to seconds", value) + raise NotImplementedError(f'cannot convert the timestamp {value} to seconds') + + @staticmethod + def _parse_time_range(qualifier, time_range): + """ + Converts qualifier to timestamp format + :param qualifier: str + :param time_range: int + return: list of formatted timestamps + """ + try: + compile_timestamp_regex = re.compile(START_STOP_PATTERN) + if qualifier and compile_timestamp_regex.search(qualifier): + time_range_iterator = compile_timestamp_regex.finditer(qualifier) + time_range_list = [each.group() for each in time_range_iterator] + else: + start_time = STOP_TIME - timedelta(minutes=time_range) + converted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + # limit 3 digit value for millisecond + converted_stop_time = STOP_TIME.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + time_range_list = [converted_start_time, converted_stop_time] + return time_range_list + except (KeyError, IndexError, TypeError) as e: + raise e + + @staticmethod + def _check_time_range_values(time_range_list): + """ + checks for valid start and stop time + :param time_range_list: list + :return: converted_timestamp, list + """ + utc_timestamp = STOP_TIME.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + converted_utc_timestamp = QueryStringPatternTranslator._format_datetime(utc_timestamp) + converted_timestamp = [] + for timestamp in time_range_list: + converted_time = QueryStringPatternTranslator._format_datetime(timestamp) + if converted_time > converted_utc_timestamp: + raise StartStopQualifierValueException('Start/Stop time should not be in the future UTC timestamp') + converted_timestamp.append(converted_time) + if converted_timestamp[0] >= converted_timestamp[1]: + raise StartStopQualifierValueException('Start time should be lesser than Stop time') + return converted_timestamp + + def _eval_combined_observation_exp(self, expression, qualifier=None): + """ + Function for parsing combined observation expression + :param expression: object + :param qualifier: str, default in None + """ + combined_obs_operator = self._lookup_comparison_operator(expression.operator) + self._parse_expression(expression.expr1, qualifier, combined_observation=True) + self.current_observation_operator = combined_obs_operator + self._parse_expression(expression.expr2, qualifier, combined_observation=True) + + def _parse_mapped_fields(self, list_of_dict_value, value, expression, mapped_fields_array): + """ + creates comparison expression for the mapped fields + :param list_of_dict_value: list + :param expression: expression + :param mapped_fields_array: list + :return comparison_string,str + """ + subset_expression = [] + is_negated = expression.negated + comparison_string = "" + comparison_string_new_count = 0 + for field_name in mapped_fields_array: + comparator = self._lookup_comparison_operator(expression.comparator) + comparison_string_new = self._create_parsed_query(field_name, list_of_dict_value, + value, comparator, is_negated, + expression.comparator) + + if comparison_string_new: + if expression.comparator in [ComparisonComparators.IsSubSet] or ( + field_name.endswith("[*]") and field_name.count("[*]") == 1): + subset_expression.append(comparison_string_new) + else: + comparison_string_new_count += 1 + if comparison_string: + comparison_string += " or " + comparison_string += comparison_string_new + + if len(mapped_fields_array) > 1 and comparison_string and comparison_string_new_count > 1: + # More than one data source field maps to the STIX attribute, so group comparisons together. + comparison_string = "(" + comparison_string + ")" + + return comparison_string, subset_expression + + def _lookup_comparison_operator(self, expression_operator): + """ + Fetch the comparison operator of the expression + :param expression_operator: + return operator,str + """ + if str(expression_operator) not in self.comparator_lookup: + raise NotImplementedError(f"Comparison operator {expression_operator.name} unsupported for connector") + return self.comparator_lookup[str(expression_operator)] + + def _eval_combined_comparison_exp(self, expression): + """ + Function for parsing combined comparison expression + :param expression: object + :return query_string: str + """ + operator = self._lookup_comparison_operator(expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + + if isinstance(expression.expr1, CombinedComparisonExpression) and expression_01: + expression_01 = f'({expression_01})' + if isinstance(expression.expr2, CombinedComparisonExpression) and expression_02: + expression_02 = f'({expression_02})' + + if not expression_01 or not expression_02: + query_string = f'{expression_01}' if expression_01 else f'{expression_02}' + else: + query_string = f'{expression_01} {operator} {expression_02}' + return query_string + + def _eval_comparison_value(self, expression, list_of_dict_fields, individual_fields, array_fields): + """ + Function for parsing comparison expression value + :param expression: expression object + :param list_of_dict_fields: list + :param individual_fields: list + :param array_fields: list + :return: formatted expression value + """ + list_of_dict_value, value = None, None + if list_of_dict_fields and expression.comparator not in [ComparisonComparators.In, + ComparisonComparators.NotEqual, + ComparisonComparators.Equal]: + raise NotImplementedError(f'{expression.comparator} is not supported for list of dictionary ' + f'fields [{",".join(list_of_dict_fields)}]') + if array_fields and expression.comparator not in (ComparisonComparators.Like, ComparisonComparators.Matches, + ComparisonComparators.NotEqual, + ComparisonComparators.Equal): + raise NotImplementedError(f'{expression.comparator} is not supported for array attribute ' + f' [{",".join(array_fields)}]') + + if expression.comparator == ComparisonComparators.Like and (individual_fields or array_fields): + value = self._format_like(expression.value, array_fields) + elif expression.comparator == ComparisonComparators.Matches and (individual_fields or array_fields): + value = self._format_match(expression.value, array_fields=array_fields) + elif expression.comparator == ComparisonComparators.In: + if list_of_dict_fields: + list_of_dict_value = self._format_set(expression.value, list_of_dict_fields) + if individual_fields: + value = self._format_set(expression.value) + elif expression.comparator == ComparisonComparators.IsSubSet and individual_fields: + value = self._format_subset(expression, expression.value) + elif expression.comparator in [ComparisonComparators.GreaterThan, ComparisonComparators.GreaterThanOrEqual, + ComparisonComparators.LessThan, ComparisonComparators.LessThanOrEqual] and \ + individual_fields: + value = self._validate_numerical_value(expression.comparator, expression.value) + elif expression.comparator in [ComparisonComparators.Equal, ComparisonComparators.NotEqual]: + if list_of_dict_fields: + list_of_dict_value = (QueryStringPatternTranslator._escape_value + (expression.value, list_of_dict_field=list_of_dict_fields)) + if individual_fields or array_fields: + value = self.format_equals(expression.value) + else: + raise NotImplementedError('Unknown comparator expression operator') + + return list_of_dict_value, value + + def _merge_observation_exp(self, expression, query_string, converted_time_range, obs_operator): + """ + Function for merging observation expression + :param expression: expression object + :param query_string: str + :param converted_time_range: str + :param obs_operator: str + """ + query_combinations = list(product(*self.query_list)) + if 'ComparisonExpressionOperators.Or' not in str(expression): + if query_string: + for row in query_combinations: + joined_query = f'{" | ".join(row)} | {query_string}' + self.all_queries.append( + {"query": joined_query, "timestamp": converted_time_range, "operator": "|", + "is_ts_matched": False}) + else: + for row in query_combinations: + self.all_queries.append( + {"query": " | ".join(row), "timestamp": converted_time_range, "operator": "|", + "is_ts_matched": False}) + else: + merged_list = sum(self.query_list, []) + for row in merged_list: + self.all_queries.append({"query": row, "timestamp": converted_time_range, "operator": "|", + "is_ts_matched": False}) + + if 'ComparisonExpressionOperators.Or' in str(expression) and query_string: + self.all_queries.append({"query": query_string, "timestamp": converted_time_range, + "operator": obs_operator, "is_ts_matched": False}) + + def _eval_observation_exp(self, expression, combined_observation, qualifier): + """ + Function for parsing observation expression value + :param expression: expression object + :param combined_observation: bool + :param qualifier: expression object + :return: formatted expression value + """ + self.query_list = [] + query_string = f'{self._parse_expression(expression.comparison_expression)}' + if isinstance(expression.comparison_expression, ComparisonExpression) and query_string and \ + query_string[0] == "(" and query_string[-1] == ")": + query_string = query_string[1:-1] # removes extra brackets () + + time_range_list = QueryStringPatternTranslator._parse_time_range(qualifier, self.options["time_range"]) + converted_time_range = QueryStringPatternTranslator._check_time_range_values(time_range_list) + obs_operator = self.current_observation_operator if self.current_observation_operator else "or" + + if combined_observation: + if self.query_list: + merged_list = sum(self.query_list, []) + for row in merged_list: + self.all_queries.append({"query": row, "timestamp": converted_time_range, "operator": "|", + "is_ts_matched": False}) + if query_string: + self.all_queries.append( + {"query": query_string, "timestamp": converted_time_range, "operator": obs_operator, + "is_ts_matched": False}) + else: + if self.query_list: + self._merge_observation_exp(expression, query_string, converted_time_range, obs_operator) + else: + self.all_queries.append({"query": query_string, "timestamp": converted_time_range, + "operator": obs_operator, "is_ts_matched": False}) + + def _parse_expression(self, expression, qualifier=None, combined_observation=False): + """ + parse ANTLR pattern to CrowdStrike LogScale query format + :param expression: expression object, ANTLR parsed expression object + :param qualifier: str, default in None + """ + if isinstance(expression, ComparisonExpression): + stix_object, stix_field = expression.object_path.split(':') + mapped_fields_array = self.dmm.map_field(stix_object, stix_field) + list_of_dict_fields, individual_fields, array_fields = ([j for j in mapped_fields_array if "[*]" in j + and + not (j.endswith("[*]") and j.count("[*]") == 1)], + [i for i in mapped_fields_array if "[*]" not in i], + [k for k in mapped_fields_array if k.endswith("[*]") + and k.count("[*]") == 1]) + list_of_dict_value, value = self._eval_comparison_value(expression, list_of_dict_fields, + individual_fields, array_fields) + comparison_string, comparison_list = self._parse_mapped_fields(list_of_dict_value, value, expression, + mapped_fields_array) + if comparison_list: + self.query_list.append(comparison_list) + return comparison_string + elif isinstance(expression, CombinedComparisonExpression): + return self._eval_combined_comparison_exp(expression) + elif isinstance(expression, ObservationExpression): + return self._eval_observation_exp(expression, combined_observation, qualifier) + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + self._parse_expression(expression.observation_expression, expression.qualifier, combined_observation) + elif isinstance(expression, CombinedObservationExpression): + self._eval_combined_observation_exp(expression, qualifier) + elif isinstance(expression, Pattern): + self._parse_expression(expression.expression) + else: + raise RuntimeError(f'Unknown Recursion Case for expression={expression},' + f' type(expression)={type(expression)}') + return None + + def parse_expression(self, pattern: Pattern): + return self._parse_expression(pattern) + + +def translate_pattern(pattern: Pattern, data_model_mapping, options): + """ + Conversion of ANTLR pattern to Crowdstrike LogScale query + :param pattern: expression object, ANTLR parsed expression object + :param data_model_mapping: DataMapper object, mapping object obtained by parsing json + :param options: dict + :return: list + """ + translator = QueryStringPatternTranslator(pattern, data_model_mapping, options) + return translator.translated_query diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/query_translator.py b/stix_shifter_modules/crowdstrike_logscale/stix_translation/query_translator.py new file mode 100644 index 000000000..01b910a39 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/query_translator.py @@ -0,0 +1,23 @@ +import logging +from . import query_constructor +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file + :param antlr_parsing_object: Antlr parsing objects for the STIX pattern + :type antlr_parsing_object: object + :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into + another format. This should default to something if one isn't passed in + :type mapping: str (filepath) + :return: transformed query string + :rtype: str + """ + logger.info("Converting STIX2 Pattern to data source query") + query_string = query_constructor.translate_pattern(antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_translation/transformers.py b/stix_shifter_modules/crowdstrike_logscale/stix_translation/transformers.py new file mode 100644 index 000000000..ba9999555 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_translation/transformers.py @@ -0,0 +1,39 @@ +from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer +from stix_shifter_utils.utils import logger +import re + +LOGGER = logger.set_logger(__name__) +connector = __name__.split('.')[1] + +class FormatMacAddress(ValueTransformer): + @staticmethod + def transform(mac_value): + """correcting mac address presentation. The mac address should be separated + by only colon (:) not by any other special character. + Example: + Input: 10-10-10-10-10-10 Output: 10:10:10:10:10:10 + """ + try: + colon_converted = re.sub("[^A-Fa-f0-9]", ":", mac_value) + return colon_converted.lower() + + except Exception: + LOGGER.error(f'{connector} connector error -> cannot convert {mac_value} into valid the MAC address') + raise + + +class LogscaleToTimestamp(ValueTransformer): + """A value transformer to truncate milliseconds + Example: + Input : 2024-01-23T12:33:15.170758259Z Output: 2024-01-23T12:33:15.170Z + """ + + @staticmethod + def transform(value): + try: + time_array = value.split('.') + converted_time = time_array[0] + '.' + time_array[1][:3] + 'Z' if len(time_array) > 1 else time_array[0] + 'Z' + return converted_time + except Exception: + LOGGER.error(f'{connector} connector error -> cannot convert {value} into valid timestamp') + raise diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_transmission/__init__.py b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_transmission/api_client.py b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/api_client.py new file mode 100644 index 000000000..f9ffe30bb --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/api_client.py @@ -0,0 +1,55 @@ +from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync +import json + +PING_ENDPOINT = 'api/v1/status' +JOB_ENDPOINT = 'api/v1/repositories' + +class APIClient: + + def __init__(self, connection, configuration): + auth = configuration.get('auth') + self.headers = {"Authorization": f"Bearer {auth.get('api_token')}", "Content-Type": "application/json", + "Accept": "application/json"} + self.client = RestApiClientAsync(connection.get('host'), headers=self.headers) + self.timeout = connection['options'].get('timeout') + self.result_limit = connection['options'].get('result_limit') + self.api_page_size = connection['options'].get('api_page_size') + self.api_endpoint = f"{JOB_ENDPOINT}/{connection.get('repository')}/queryjobs" + + async def ping_data_source(self): + """ + Pings the data source + :return: response object + """ + return await self.client.call_api(PING_ENDPOINT, 'GET', headers=self.headers, data={}) + + async def create_search(self, query_expression): + """ + Create an Enterprise search for the input query + :param query_expression: dict + :return: response object + """ + return await self.client.call_api(self.api_endpoint, 'POST', headers=self.headers, + data=json.dumps(query_expression), + timeout=self.timeout) + + async def poll_query_job(self, search_id): + """ + Fetch the status and results of the query job + :param search_id: str + :return: response object + """ + poll_endpoint = f'{self.api_endpoint}/{search_id}' + return await self.client.call_api(poll_endpoint, 'GET', headers=self.headers, + timeout=self.timeout) + + async def delete_search(self, search_id): + """ + Delete the corresponding search + :param search_id: str + :return: response object + """ + poll_endpoint = f'{self.api_endpoint}/{search_id}' + return await self.client.call_api(poll_endpoint, 'DELETE', headers=self.headers, + timeout=self.timeout) + diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_transmission/delete_connector.py b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/delete_connector.py new file mode 100644 index 000000000..ad5fffb1d --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/delete_connector.py @@ -0,0 +1,49 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + +class DeleteConnector(BaseDeleteConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def delete_query_connection(self, search_id): + """ + Detete the search id + :param search_id: str + :return: return_obj, dict + """ + return_obj = {} + try: + search_id = search_id.split(":")[0] + response = await self.api_client.delete_search(search_id) + response_code = response.code + response_content = response.read().decode('utf-8') + if response_code == 204: + return_obj['success'] = True + else: + return_obj = self.handle_api_exception(response_code, response_content,response.response.reason) + + except Exception as ex: + self.logger.error('error while Deleting Query Job Id in Crowdstrike Falcon Logscale: %s', ex) + code = 408 if "timeout_error" in str(ex) else None + return_obj = self.handle_api_exception(code, str(ex)) + + return return_obj + + def handle_api_exception(self, code, response_txt, client_response=None): + """ + create the exception response + :param code, int + :param response_txt, dict + :param client_response, str + :return: return_obj, dict + """ + return_obj = {} + if code == 404 and str(response_txt) == "": + response_txt = "404: Resource not found" if client_response == "Not Found" else f'404: {client_response}' + + response_dict = {'code': code, 'message': str(response_txt)} if code else {'message': str(response_txt)} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_transmission/error_mapper.py b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/error_mapper.py new file mode 100644 index 000000000..d884abbee --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/error_mapper.py @@ -0,0 +1,40 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + 100: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + 101: ErrorCode.TRANSMISSION_INVALID_PARAMETER, + 408: ErrorCode.TRANSMISSION_CONNECT, + 400: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + 401: ErrorCode.TRANSMISSION_AUTH_CREDENTIALS, + 403: ErrorCode.TRANSMISSION_FORBIDDEN, + 404: ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS, + 500: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + 406: ErrorCode.TRANSMISSION_CONNECT, + 503: ErrorCode.TRANSMISSION_CONNECT +} + + +class ErrorMapper: + + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + logger = logger.set_logger(__name__) + + @staticmethod + def set_error_code(json_data, return_obj, connector=None): + code = None + try: + code = int(json_data['code']) + except Exception: + pass + + error_code = ErrorMapper.DEFAULT_ERROR + + if code in error_mapping: + error_code = error_mapping.get(code) + + if error_code == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: %s", str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_code, connector=connector) \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_transmission/ping_connector.py b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/ping_connector.py new file mode 100644 index 000000000..68440b4e3 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/ping_connector.py @@ -0,0 +1,44 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + + +class PingConnector(BasePingConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def ping_connection(self): + """ + Pings the data source to check the status of the server + :return: return_obj, dict + """ + return_obj = {} + try: + response = await self.api_client.ping_data_source() + response_code = response.code + response_str = response.read().decode('utf-8') + + if response_code == 200: + return_obj['success'] = True + else: + return_obj = self.handle_api_exception(response_code, response_str) + except Exception as ex: + self.logger.error('error while pinging in Crowdstrike Falcon Logscale: %s', ex) + code = 408 if "timeout_error" in str(ex) else None + return_obj = self.handle_api_exception(code, str(ex)) + + return return_obj + + def handle_api_exception(self, code, response_txt): + """ + create the exception response + :param code, int + :param response_txt, dict + :return: return_obj, dict + """ + return_obj = {} + response_dict = {'code': code, 'message': str(response_txt)} if code else {'message': str(response_txt)} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_transmission/query_connector.py b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/query_connector.py new file mode 100644 index 000000000..4cf85a6e4 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/query_connector.py @@ -0,0 +1,50 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_query_connector import BaseQueryConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + +class QueryConnector(BaseQueryConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def create_query_connection(self, query): + """ + Create the query Job for the input query + :param query: dict + :return: return_obj, dict + """ + return_obj = {} + try: + if isinstance(query, str): + query = json.loads(query) + source = query.pop('source') + response = await self.api_client.create_search(query) + response_content = response.read().decode('utf-8') + response_code = response.code + if response_code == 200: + return_obj['success'] = True + response_content = json.loads(response_content) + return_obj['search_id'] = f"{response_content['id']}:{source}" + else: + return_obj = self.handle_api_exception(response_code, response_content) + + except Exception as ex: + self.logger.error('error while creating query in Crowdstrike Falcon Logscale: %s', ex) + code = 408 if "timeout_error" in str(ex) else None + return_obj = self.handle_api_exception(code, str(ex)) + + return return_obj + + def handle_api_exception(self, code, response_txt): + """ + create the exception response + :param code, int + :param response_txt, dict + :return: return_obj, dict + """ + return_obj = {} + response_dict = {'code': code, 'message': str(response_txt)} if code else {'message': str(response_txt)} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_transmission/results_connector.py b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/results_connector.py new file mode 100644 index 000000000..dadf9c2d4 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/results_connector.py @@ -0,0 +1,298 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +from .status_connector import StatusConnector +from .delete_connector import DeleteConnector +import regex +import json +from flatten_json import unflatten + +class InvalidSearchIdException(Exception): + pass +class InvalidMetadataException(Exception): + pass + + +# LogScale event fields starts with @timestamp and @error_msg will be overwritten while un-flattening. +# These fields are excluded while un flattening +DS_FLATTEN_KEY_EXCLUDE = ["@timestamp", "@error_msg"] + + +class ResultsConnector(BaseJsonResultsConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def create_results_connection(self, search_id, offset, length, metadata=None): + """ + Fetch the results of the search by creating and polling the query job + :param search_id: str + :param offset: str + :param length: str + :param metadata: dict + :return: return_obj, dict + """ + response_dict = {} + return_obj = {} + response_query_status_details = {} + try: + offset = int(offset) + length = int(length) + data = [] + total_records = self.fetch_total_records(metadata, offset, length) + first_iteration = False + if not metadata: + first_iteration = True # setting the variable to apply offset and length on data when metadata is not passed as input + return_obj, data, metadata, response_query_status_details = await self.process_first_query_job(search_id, data, total_records) + if metadata: + while len(data) < total_records: + job_id = await self.fetch_query_job_id(metadata, self.api_client.api_page_size) + if isinstance(job_id,dict): + return job_id + return_obj, response_query_status_details = await self.fetch_status_and_response(job_id, length) + if not return_obj.get('success'): + break + if return_obj.get('data'): + data.extend(return_obj['data']) + #prepare metadata for next iteration + metadata = ResultsConnector.prepare_metadata(return_obj['data'], response_query_status_details) + else: + break + # deleting the internal query jobs that has been created for pagination + delete_obj = DeleteConnector(self.api_client) + await delete_obj.delete_query_connection(job_id) + + if data: + return_obj = await self.format_results(data, offset,total_records, first_iteration) + if (offset + len(return_obj['data'])) < self.api_client.result_limit: + return_obj['metadata'] = ResultsConnector.prepare_metadata(return_obj['data'], response_query_status_details) + + except InvalidSearchIdException: + return_obj = self.handle_api_exception(404,"Invalid Search Id") + + except InvalidMetadataException: + return_obj = self.handle_api_exception(101, "Invalid Metadata") + + except Exception as ex: + if "timeout_error" in str(ex): + response_dict['code'] = 408 + response_dict['message'] = str(ex) + self.logger.error('error while getting results in Crowdstrike Falcon Logscale: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + + return return_obj + + def handle_api_exception(self, code, response_txt): + """ + Create the exception response + :param code, int + :param response_txt, dict + :return: return_obj, dict + """ + return_obj = {} + response_dict = {'code': code, 'message': str(response_txt)} if code else {'message': str(response_txt)} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + async def process_first_query_job(self, search_id, data, total_records): + """ + Fetch the response of the First Query Job Id + :param search_id: str + :param data: list + :param total_records: int + :return: dict + """ + metadata = {} + if ":" not in search_id: + raise InvalidSearchIdException + # split the input search id which is in the format job_id:source into individual fields for fetching results + job_id = search_id.split(":")[0] + return_obj, response_query_status_details = await self.fetch_query_job_response(job_id) + if return_obj['success']: + if return_obj.get('data'): + data.extend(return_obj['data']) + if len(return_obj['data']) < total_records: + metadata = ResultsConnector.prepare_metadata(return_obj['data'], response_query_status_details) + else: + return return_obj, data, metadata, response_query_status_details + return {}, data, metadata, response_query_status_details + async def format_results(self, data, offset, total_records, first_iteration): + """ + Slice the records and format the results + :param data: dict + :param offset: int + :param total_records: int + :param first_iteration: bool + :return: dict + """ + if first_iteration: + formatted_data = [self.unflatten_json(event) for event in data[offset: total_records]] + else: + formatted_data = [self.unflatten_json(event) for event in data[:total_records]] + + return_obj = {'success': True, 'data': formatted_data} + return return_obj + + def fetch_total_records(self,metadata, offset, length): + """ + Calculate the value of total records to be fetched based on metadata + :param metadata: dict + :param offset: int + :param length: int + :return: int + """ + if not metadata: + total_records = offset + length + if self.api_client.result_limit < total_records: + total_records = self.api_client.result_limit + else: + records_fetched = offset + total_records = length + if abs(self.api_client.result_limit - records_fetched) < total_records: + total_records = abs(self.api_client.result_limit - records_fetched) + return total_records + async def fetch_query_job_id(self,metadata, length): + """ + Create a new job id using metadata + :param metadata: dict + :param length: int + :return: dict/str + """ + # Fetch the input details from metadata and create a new job id to fetch the next set of results through pagination + if isinstance(metadata, str): + metadata = json.loads(metadata) + if (metadata.get('last_event_id') and metadata.get('last_event_timestamp') and + metadata.get('input_query_string') and metadata.get('start')): + search_query = { + 'queryString': metadata['input_query_string'], + 'around': {"eventId" : metadata['last_event_id'], + "numberOfEventsAfter" : 0, + "numberOfEventsBefore" : length, + "timestamp" : metadata['last_event_timestamp']}, + 'start': metadata['start'], + 'end': metadata['last_event_timestamp'] + } + query_response = await self.api_client.create_search(search_query) + query_response_content = query_response.read().decode('utf-8') + if query_response.code == 200: + query_response_text = json.loads(query_response_content) + job_id = query_response_text['id'] + else: + return self.handle_api_exception(query_response.code, query_response_content) + else: + raise InvalidMetadataException + return job_id + + async def fetch_status_and_response(self, search_id, length): + """ + Fetch the status and results of the intermediate query job + :param search_id: str + :param length: int + :return: dict,dict + """ + return_obj = {} + status_obj = StatusConnector(self.api_client) + status = await status_obj.create_status_connection(search_id) + if status['success']: + # check if the intermediate queryjob's status is still running + while status.get('progress') < 100 and status.get('status') == 'RUNNING': + status = await status_obj.create_status_connection(search_id,{'length':length}) + if status['success'] is False: + return status, {} + if status.get('status') == 'CANCELED': + return_obj['success'] = True + return_obj['data'] = [] + return return_obj, {} + # Fetch the response if the intermediate queryjob's status is completed + return await self.fetch_query_job_response(search_id) + return status, {} + + async def fetch_query_job_response(self, search_id, metadata=None): + """ + Fetch the results for the Query Job ID + :param search_id: str + :param metadata: dict + :return: dict, dict + """ + return_obj = {} + response_query_status_details = {} + result_response = await self.api_client.poll_query_job(search_id) + result_response_text = result_response.read().decode('utf-8') + if result_response.code == 200: + return_obj['success'] = True + result_response_json = json.loads(result_response_text) + if not metadata or (metadata and result_response_json.get('done')): + return_obj['data'] = result_response_json['events'] + # store the filter query details in order to prepare the metadata + response_query_status_details = {'filterQuery': result_response_json['metaData']['filterQuery'], + 'done': result_response_json['done'] + } + else: + return_obj = self.handle_api_exception(result_response.code, result_response_text) + + return return_obj, response_query_status_details + @staticmethod + def prepare_metadata(data, response_query_status_details): + """ + Create the metadata + :param data: list + :param response_query_status_details: dict + :return: return_obj, dict + """ + metadata = {} + + querystring = response_query_status_details.get('filterQuery').get('queryString') + tail_index = querystring.find("| tail") + formatted_query = querystring[:tail_index] if tail_index != -1 else querystring + metadata['last_event_id'] = data[-1]['@id'] # -1 index represents the last record in the list + metadata['last_event_timestamp'] = data[-1]['@timestamp'] + metadata['input_query_string'] = formatted_query + metadata['start'] = response_query_status_details['filterQuery']['start'] + return metadata + + def unflatten_json(self, flatten_json): + """ + Unflatten the flattened JSON event + :param flatten_json: dict + :return res,dict + """ + res = {} + # Excluded flatten keys + excluded_keys = [key for key in flatten_json if key.startswith(tuple(DS_FLATTEN_KEY_EXCLUDE))] + # copy excluded items to res and remove from flatten_json + for key in excluded_keys: + res[key] = flatten_json.pop(key) + + unflatten_json = unflatten(flatten_json, separator='.') + # Format array indexed keys to proper json format + unflatten_json = self.format_indexed_keys(unflatten_json) + res.update(unflatten_json) + res['finding_type'] = 'alert' + return res + + def format_indexed_keys(self, event): + """ + Format array indexed string keys to single key + :param event: dict + :return res,dict + """ + res = {} + for key, value in event.items(): + match = regex.search(r'(.*)\[(\d+)\]$', key) + if match: + indexed_key, _ = match.groups() + val = res.get(indexed_key, {}) or [] + if isinstance(value, dict): + response = self.format_indexed_keys(value) + val.append(response) + else: + if value != {} and value != '' and value != [] and value is not None: + val.append(value) + res[indexed_key] = val + else: + if isinstance(value, dict): + value = self.format_indexed_keys(value) + if value != {} and value != '' and value != [] and value is not None: + res[key] = value + return res diff --git a/stix_shifter_modules/crowdstrike_logscale/stix_transmission/status_connector.py b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/status_connector.py new file mode 100644 index 000000000..237a46c5e --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/stix_transmission/status_connector.py @@ -0,0 +1,84 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import BaseStatusConnector +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from enum import Enum +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json +class LogScaleStatus(Enum): + EXECUTE = 'EXECUTE' + COMPLETED = 'COMPLETED' + CANCELED = 'CANCELED' + +TAIL_MAXIMUM_LIMIT = 10000 # maximum tail limit of logscale + + +class StatusConnector(BaseStatusConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + @staticmethod + def get_status( status): + switcher = { + LogScaleStatus.EXECUTE.value: Status.RUNNING, + LogScaleStatus.COMPLETED.value: Status.COMPLETED, + LogScaleStatus.CANCELED.value: Status.CANCELED + } + return switcher.get(status).value + + async def create_status_connection(self, search_id, metadata=None): + """ + Poll the QueryJob to find the status of the query job + :param search_id: str + :param metadata: dict + :return: return_obj, dict + """ + return_obj = {} + try: + search_id = search_id.split(":")[0] + + response = await self.api_client.poll_query_job(search_id) + response_code = response.code + response_content = response.read().decode('utf-8') + + if response_code == 200: + response_content = json.loads(response_content) + return_obj['success'] = True + + if response_content.get('cancelled'): + return_obj['status'] = StatusConnector.get_status('CANCELED') + return_obj['progress'] = 0 + elif response_content.get('done'): + return_obj['status'] = StatusConnector.get_status('COMPLETED') + return_obj['progress'] = 100 + elif not response_content.get('done'): + + return_obj['status'] = StatusConnector.get_status('EXECUTE') + event_count = response_content['metaData']['eventCount'] + if metadata and metadata.get("length"): + progress_percent = int(event_count / metadata['length'] * 100) + else: + progress_percent = int(event_count/TAIL_MAXIMUM_LIMIT * 100) + return_obj['progress'] = progress_percent + else: + return_obj = self.handle_api_exception(response_code, response_content) + + except Exception as ex: + self.logger.error('Error while fetching status from CrowdStrike Falcon Logscale API: %s', ex) + code = 408 if "timeout_error" in str(ex) else None + return_obj = self.handle_api_exception(code, str(ex)) + + return return_obj + + def handle_api_exception(self, code, response_txt): + """ + create the exception response + :param code, int + :param response_txt, dict + :return: return_obj, dict + """ + return_obj = {} + response_dict = {'code': code, 'message': str(response_txt)} if code else {'message': str(response_txt)} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj diff --git a/stix_shifter_modules/crowdstrike_logscale/test/stix_translation/test_crowdstrike_logscale_json_to_stix.py b/stix_shifter_modules/crowdstrike_logscale/test/stix_translation/test_crowdstrike_logscale_json_to_stix.py new file mode 100644 index 000000000..4d1fa6c29 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/test/stix_translation/test_crowdstrike_logscale_json_to_stix.py @@ -0,0 +1,429 @@ +""" test script to perform unit test case for CrowdStrike Falcon LogScale translate results """ +import unittest +from stix_shifter_modules.crowdstrike_logscale.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers + +MODULE = "crowdstrike_logscale" +entry_point = EntryPoint() +map_data = entry_point.get_results_translator().map_data +data_source = { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "crowdstrike_logscale", + "identity_class": "events" +} +options = {} + +logscale_sample_response = { + '@timestamp': 1700059736377, + '@timestamp.nanos': '312000', + '#repo': 'TestRepository', + '#type': 'CrowdStrike_Spotlight', + '@id': '3sAoxxxxWl4Y_3_190_17xxxx36', + '@ingesttimestamp': '1700059737501', + '@rawstring': '{"cid": "ef21xxxxxxxxxxxxxxxxxxxxxxxxxxx440d",' + '"created_timestamp": "2023-09-12T11:46:19.787962809Z",' + ' "detection_id": "ldt:7adbxxxxxxxx0d49:103079284165",' + ' "device": {"device_id": "7adbxxxxxxxx0d49",' + ' "cid": "ef21xxxxxxxxxxxxxxxxxxxxxxxxxxx440d", "agent_load_flags": "1",' + ' "agent_local_time": "2023-09-12T08:30:15.487Z", "agent_version": "6.58.17212.0",' + ' "bios_manufacturer": "Xen", "bios_version": "4.11.hp",' + ' "config_id_base": "65994763", "config_id_build": "17212", "config_id_platform": "3",' + ' "external_ip": "1.1.2.2", "hostname": "CROWDST",' + ' "first_seen": "2023-05-16T05:10:55Z", "last_seen": "2023-09-12T11:24:19Z",' + ' "local_ip": "172.0.0.1", "mac_address": "12-11-11-11-11-11",' + ' "major_version": "10", "minor_version": "0", "os_version": "Windows Server 2022",' + ' "platform_id": "0", "platform_name": "Windows", "product_type": "3",' + ' "product_type_desc": "Server", "status": "normal", "system_manufacturer": "Xen",' + ' "system_product_name": "HVM domU",' + ' "groups": ["97350feebe4541e8a615c0d3f18acdf3","bb1exxxxxxxxxxxxxxxxxxxxxxxxxxb23d"],' + ' "modified_timestamp": "2023-09-12T11:45:57Z", "instance_id": "065fxxxxxxxxxx27ce",' + ' "service_provider": "WINDOWS_V2", "service_provider_account_id": "1xxxxxxxxxxxx2"},' + ' "behaviors": [{"device_id": "7adbxxxxxxxx0d49",' + ' "timestamp": "2023-09-12T11:46:12Z", "template_instance_id": "3",' + ' "behavior_id": "41002", "filename": "calc.exe",' + ' "filepath": "\\\\Device\\\\HarddiskVolume1\\\\Windows\\\\System32\\\\calc.exe",' + ' "alleged_filetype": "exe", "cmdline": "calc", "scenario": "suspicious_activity",' + ' "objective": "Falcon Detection Method", "tactic": "Custom Intelligence",' + ' "tactic_id": "CSTA0005", "technique": "Indicator of Attack", "technique_id": "CST0004",' + ' "display_name": "CustomIOAWinMedium",' + ' "description": "A process triggered a medium severity custom rule.",' + ' "severity": 50, "confidence": 100, "ioc_type": "hash_sha256",' + ' "ioc_value": "4208xxxx2554da89f45xxxx3bbd",' + ' "ioc_source": "library_load",' + ' "ioc_description": "\\\\Device\\\\HarddiskVolume1\\\\Windows\\\\System32\\\\calc.exe",' + ' "user_name": "testuser", "user_id": "S-1-5-21-xxxx-16xxxx15-312xxxx6-500",' + ' "control_graph_id": "ctg:7adbxxxx0d49:1030xxxx4165",' + ' "triggering_process_graph_id": "pid:7adbxxxxxxxx0d49:103512883608",' + ' "sha256": "4208xxxx2554da89f45xxxx3bbd",' + ' "md5": "1fd4xxxxxxxxxxxxxxxxxxxxxxxxxx86231e",' + ' "parent_details": ' + '{"parent_sha256":"eb71xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc208",' + ' "parent_md5": "e7a6xxxxxxxxxxxxxxxxxxxxxxxxxxxx90f4",' + ' "parent_cmdline": "\\"C:\\\\Windows\\\\system32\\\\cmd.exe\\" ",' + ' "parent_process_graph_id": "pid:7adbxxxxxxxx0d49:103472906247"},' + ' "pattern_disposition": 2048,' + ' "pattern_disposition_details": {"indicator": false, "detect": false, "inddet_mask": false,' + ' "sensor_only": false, "rooting": false, "kill_process": false, "kill_subprocess": false,' + ' "quarantine_machine": false, "quarantine_file": false, "policy_disabled": false,' + ' "kill_parent": false, "operation_blocked": false, "process_blocked": true,' + ' "registry_operation_blocked": false, "critical_process_disabled": false,' + ' "bootup_safeguard_enabled": false, "fs_operation_blocked": false,' + ' "handle_operation_downgraded": false, "kill_action_failed": false,' + ' "blocking_unsupported_or_disabled": false, "suspend_process": false,' + ' "suspend_parent": false}, "rule_instance_id": "3", "rule_instance_version": 2}],' + ' "email_sent": true, "first_behavior": "2023-09-12T11:46:12Z",' + ' "last_behavior": "2023-09-12T11:46:12Z", "max_confidence": 100, "max_severity": 50,' + ' "max_severity_displayname": "Medium", "show_in_ui": true, "status": "new",' + ' "hostinfo": {"domain": ""}, "seconds_to_triaged": 0, "seconds_to_resolved": 0,' + ' "behaviors_processed": ["pid:7adbxxxxxxxx0d49:103512883608:41002"],' + ' "date_updated": "2023-11-15T14:48:56.377312Z"}', '@timezone': 'Z', + 'behaviors': [ + { + 'alleged_filetype': 'exe', + 'behavior_id': '41002', + 'cmdline': 'calc', + 'confidence': '100', + 'control_graph_id': 'ctg:7adbxxxx0d49:1030xxxx4165', + 'description': 'A process triggered a medium severity custom rule.', + 'device_id': '7adbxxxxxxxx0d49', + 'display_name': 'CustomIOAWinMedium', + 'filename': 'calc.exe', + 'filepath': '\\Device\\HarddiskVolume1\\Windows\\System32\\calc.exe', + 'ioc_description': '\\Device\\HarddiskVolume1\\Windows\\System32\\calc.exe', + 'ioc_source': 'library_load', + 'ioc_type': 'hash_sha256', + 'ioc_value': '4208xxxx2554da89f45xxxx3bbd', + 'md5': '1fd4xxxxxxxxxxxxxxxxxxxxxxxxxx86231e', + 'objective': 'Falcon Detection Method', + 'parent_details': + { + 'parent_cmdline': '"C:\\Windows\\system32\\cmd.exe" ', + 'parent_md5': 'e7a6xxxxxxxxxxxxxxxxxxxxxxxxxxxx90f4', + 'parent_process_graph_id': 'pid:7adbxxxxxxxx0d49:103472906247', + 'parent_sha256': 'eb71xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc208' + }, + 'pattern_disposition': '2048', + 'pattern_disposition_details': + { + 'blocking_unsupported_or_disabled': 'false', + 'bootup_safeguard_enabled': 'false', + 'critical_process_disabled': 'false', + 'detect': 'false', + 'fs_operation_blocked': 'false', + 'handle_operation_downgraded': 'false', + 'inddet_mask': 'false', + 'indicator': 'false', + 'kill_action_failed': 'false', + 'kill_parent': 'false', + 'kill_process': 'false', + 'kill_subprocess': 'false', + 'operation_blocked': 'false', + 'policy_disabled': 'false', + 'process_blocked': 'true', + 'quarantine_file': 'false', + 'quarantine_machine': 'false', + 'registry_operation_blocked': 'false', + 'rooting': 'false', + 'sensor_only': 'false', + 'suspend_parent': 'false', + 'suspend_process': 'false' + }, + 'rule_instance_id': '3', + 'rule_instance_version': '2', + 'scenario': 'suspicious_activity', + 'severity': '50', + 'sha256': '4208xxxx2554da89f45xxxx3bbd', + 'tactic': 'Custom Intelligence', + 'tactic_id': 'CSTA0005', + 'technique': 'Indicator of Attack', + 'technique_id': 'CST0004', + 'template_instance_id': '3', + 'timestamp': '2023-09-12T11:46:12Z', + 'triggering_process_graph_id': 'pid:7adbxxxxxxxx0d49:103512883608', + 'user_id': 'S-1-5-21-xxxx-16xxxx15-312xxxx6-500', + 'user_name': 'testuser' + } + ], + 'behaviors_processed': 'pid:7adbxxxxxxxx0d49:103512883608:41002', + 'cid': 'ef21xxxxxxxxxxxxxxxxxxxxxxxxxxx440d', + 'created_timestamp': '2023-09-12T11:46:19.787962809Z', + 'date_updated': '2023-11-15T14:48:56.377312Z', + 'detection_id': 'ldt:7adbxxxxxxxx0d49:103079284165', + 'device': { + 'agent_load_flags': '1', + 'agent_local_time': '2023-09-12T08:30:15.487Z', + 'agent_version': '6.58.17212.0', + 'bios_manufacturer': 'Xen', + 'bios_version': '4.11.hp', + 'cid': 'ef21xxxxxxxxxxxxxxxxxxxxxxxxxxx440d', + 'config_id_base': '65994763', + 'config_id_build': '17212', + 'config_id_platform': '3', + 'device_id': '7adbxxxxxxxx0d49', + 'external_ip': '1.1.2.2', + 'first_seen': '2023-05-16T05:10:55Z', + 'groups': 'bb1exxxxxxxxxxxxxxxxxxxxxxxxxxb23d', + 'hostname': 'CROWDST', + 'instance_id': '065fxxxxxxxxxx27ce', + 'last_seen': '2023-09-12T11:24:19Z', + 'local_ip': '172.0.0.1', + 'mac_address': '12-11-11-11-11-11', + 'major_version': '10', + 'minor_version': '0', + 'modified_timestamp': '2023-09-12T11:45:57Z', + 'os_version': 'Windows Server 2022', + 'platform_id': '0', + 'platform_name': 'Windows', + 'product_type': '3', + 'product_type_desc': 'Server', + 'service_provider': 'WINDOWS_V2', + 'service_provider_account_id': '1xxxxxxxxxxxx2', + 'status': 'normal', + 'system_manufacturer': 'Xen', + 'system_product_name': 'HVM domU' + }, + 'email_sent': 'true', + 'first_behavior': '2023-09-12T11:46:12Z', + 'hostinfo': { + 'domain': '' + }, + 'last_behavior': '2023-09-12T11:46:12Z', + 'max_confidence': '100', + 'max_severity': '50', + 'max_severity_displayname': 'Medium', + 'seconds_to_resolved': '0', + 'seconds_to_triaged': '0', + 'show_in_ui': 'true', + 'status': 'new' +} + + +class TestLogScaleResultsToStix(unittest.TestCase): + """ + class to perform unit test case for CrowdStrike LogScale translate results + """ + + @staticmethod + def get_first(itr, constraint): + """ + return the obj in the itr if constraint is true + """ + return next((obj for obj in itr if constraint(obj)), None) + + @staticmethod + def get_first_of_type(itr, typ): + """ + to check whether the object belongs to respective stix object + """ + return TestLogScaleResultsToStix.get_first(itr, lambda o: isinstance(o, dict) and o.get('type') == typ) + + @staticmethod + def get_observed_data_objects(data): + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + return observed_data['objects'] + + def test_ipv4_addr_json_to_stix(self): + """ + to test ipv4-addr stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + ipv4_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'ipv4-addr') + ipv6_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'ipv6-addr') + assert (ipv4_obj is not None), 'ipv4 object type not found' + assert ipv4_obj['type'] == 'ipv4-addr' + assert ipv4_obj['value'] == '1.1.2.2' + assert ipv6_obj is None + + + def test_file_json_to_stix(self): + """ + to test file stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + file_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'file') + assert (file_obj.keys() == {'type', 'name', 'x_path', 'x_extension', 'parent_directory_ref', 'hashes'}) + assert (file_obj is not None), 'file object type not found' + assert file_obj['type'] == 'file' + assert file_obj['name'] == 'calc.exe' + assert file_obj['x_extension'] == 'exe' + assert file_obj['hashes']['MD5'] == '1fd4xxxxxxxxxxxxxxxxxxxxxxxxxx86231e' + assert file_obj['hashes']['SHA-256'] == '4208xxxx2554da89f45xxxx3bbd' + + def test_process_json_to_stix(self): + """ + to test process stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + process_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'process') + assert (process_obj.keys() == {'type', 'name', 'binary_ref', 'parent_ref', 'x_process_graph_id', + 'command_line', 'creator_user_ref'}) + assert (process_obj is not None), 'process object type not found' + assert process_obj['type'] == 'process' + assert process_obj['name'] == 'calc.exe' + assert process_obj['command_line'] == 'calc' + assert process_obj['x_process_graph_id'] == 'pid:7adbxxxxxxxx0d49:103512883608' + assert (all(index in objects for ref, index in process_obj.items() if '_ref' in ref and + not isinstance(index, list))), "one of the references in process object is not found" + + def test_x_crowdstrike_behavior_json_to_stix(self): + """ + to test behavior stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + behavior_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'x-crowdstrike-detection-behavior') + assert (behavior_obj.keys() == {'type', 'display_name', 'behavior_id', 'confidence', 'control_graph_id', + 'description', 'objective', 'pattern_disposition', + 'pattern_disposition_details', + 'rule_instance_id', 'rule_instance_version', 'scenario', 'severity', + 'created_time', 'user_ref', 'process_ref', 'ttp_tagging_ref', 'ioc_description', + 'ioc_source', 'ioc_type', 'ioc_value', 'template_instance_id'}) + assert (behavior_obj is not None), 'behavior object type not found' + assert behavior_obj['type'] == 'x-crowdstrike-detection-behavior' + assert behavior_obj['display_name'] == 'CustomIOAWinMedium' + assert behavior_obj['behavior_id'] == '41002' + assert behavior_obj['confidence'] == 100 + assert behavior_obj['control_graph_id'] == 'ctg:7adbxxxx0d49:1030xxxx4165' + assert behavior_obj['objective'] == 'Falcon Detection Method' + assert (all(index in objects for ref, index in behavior_obj.items() if '_ref' in ref and + not isinstance(index, list))), "one of the references in behavior object is not found" + + def test_directory_json_to_stix(self): + """ + to test directory stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + directory_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'directory') + assert (directory_obj.keys() == {'type', 'path'}) + assert (directory_obj is not None), 'directory object type not found' + assert directory_obj['type'] == 'directory' + assert directory_obj['path'] == '\\Device\\HarddiskVolume1\\Windows\\System32' + + def test_x_ibm_ttp_tagging_json_to_stix(self): + """ + to test x-ibm-ttp-tagging stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + ttp_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'x-ibm-ttp-tagging') + assert (ttp_obj.keys() == {'type', 'name', 'extensions'}) + assert (ttp_obj is not None), 'x-ibm-ttp-tagging object type not found' + assert ttp_obj['type'] == 'x-ibm-ttp-tagging' + assert ttp_obj['name'] == 'Custom Intelligence' + assert ttp_obj['extensions']['mitre-attack-ext']['tactic_id'] == 'CSTA0005' + assert ttp_obj['extensions']['mitre-attack-ext']['technique_name'] == 'Indicator of Attack' + assert ttp_obj['extensions']['mitre-attack-ext']['technique_id'] == 'CST0004' + + def test_user_account_json_to_stix(self): + """ + to test user-account stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + user_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'user-account') + assert (user_obj.keys() == {'type', 'user_id', 'display_name'}) + assert (user_obj is not None), 'user-account object type not found' + assert user_obj['type'] == 'user-account' + assert user_obj['user_id'] == 'S-1-5-21-xxxx-16xxxx15-312xxxx6-500' + assert user_obj['display_name'] == 'testuser' + + def test_x_ibm_finding_json_to_stix(self): + """ + to test x-ibm-finding stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + finding_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'x-ibm-finding') + assert (finding_obj.keys() == {'type', 'name', 'x_behavior_refs', 'ttp_tagging_refs', 'x_behaviors_processed', + 'time_observed', 'x_last_updated', 'src_os_ref', + 'src_ip_ref', 'x_is_email_sent', 'x_first_behavior_observed', + 'x_last_behavior_observed', 'confidence', 'severity', + 'x_severity_name', 'x_seconds_to_resolved', 'x_seconds_to_triaged', 'x_status', + 'x_logscale_repository','x_logscale_event_id'}) + assert (finding_obj is not None), 'x-ibm-finding object type not found' + assert finding_obj['type'] == 'x-ibm-finding' + assert finding_obj['name'] == 'ldt:7adbxxxxxxxx0d49:103079284165' + assert finding_obj['x_behaviors_processed'] == 'pid:7adbxxxxxxxx0d49:103512883608:41002' + assert finding_obj['time_observed'] == '2023-09-12T11:46:19.787Z' + assert finding_obj['x_last_updated'] == '2023-11-15T14:48:56.377312Z' + assert finding_obj['x_is_email_sent'] == 'true' + assert finding_obj['confidence'] == 100 + assert finding_obj['severity'] == 50 + assert finding_obj['x_status'] == 'new' + assert (all(index in objects for ref, index in finding_obj.items() if '_ref' in ref and + not isinstance(index, list))), "one of the references in finding object is not found" + assert (all(x_behavior_ref in objects for x_behavior_ref in finding_obj['x_behavior_refs'])),\ + "behavior object is not found" + assert (all(ttp_tagging_ref in objects for ttp_tagging_ref in finding_obj['ttp_tagging_refs'])),\ + "ttp_tagging object is not found" + + def test_x_crowdstrike_edr_agent_json_to_stix(self): + """ + to test x-crowdstrike-edr-agent stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + edr_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'x-crowdstrike-edr-agent') + assert (edr_obj.keys() == {'type', 'load_flags', 'local_time', 'version', + 'config_id_base', 'config_id_build', 'config_id_platform'}) + assert (edr_obj is not None), 'x-crowdstrike-edr object type not found' + assert edr_obj['type'] == 'x-crowdstrike-edr-agent' + assert edr_obj['load_flags'] == '1' + assert edr_obj['local_time'] == '2023-09-12T08:30:15.487Z' + assert edr_obj['version'] == '6.58.17212.0' + assert edr_obj['config_id_base'] == '65994763' + + def test_x_ocs_asset_json_to_stix(self): + """ + to test x-oca-asset stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + asset_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'x-oca-asset') + assert (asset_obj.keys() == {'type', 'x_cid', 'x_device_groups', 'x_agent_ref', 'x_bios_manufacturer', + 'x_bios_version', 'device_id', 'ip_refs', + 'x_first_seen', 'hostname', 'x_instance_id', 'x_last_seen', 'mac_refs', + 'x_last_modified', 'os_ref', 'x_host_type_number', 'host_type', + 'x_service_provider', 'x_service_account_id', 'x_status', 'x_system_manufacturer', + 'x_system_product_name'}) + assert (asset_obj is not None), 'x-oca-asset object type not found' + assert asset_obj['type'] == 'x-oca-asset' + assert asset_obj['device_id'] == '7adbxxxxxxxx0d49' + assert asset_obj['x_bios_version'] == '4.11.hp' + assert asset_obj['hostname'] == 'CROWDST' + assert asset_obj['x_instance_id'] == '065fxxxxxxxxxx27ce' + assert asset_obj['x_last_seen'] == '2023-09-12T11:24:19Z' + assert asset_obj['host_type'] == 'Server' + assert asset_obj['x_system_product_name'] == 'HVM domU' + assert (all(index in objects for ref, index in asset_obj.items() if '_ref' in ref and + not isinstance(index, list))), "one of the references in oca asset object is not found" + assert (all(ip_ref in objects for ip_ref in asset_obj['ip_refs'])), "ip object is not found" + assert (all(mac_ref in objects for mac_ref in asset_obj['mac_refs'])), "mac object is not found" + + def test_mac_addr_json_to_stix(self): + """ + to test mac-addr stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + mac_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'mac-addr') + assert (mac_obj.keys() == {'type', 'value'}) + assert (mac_obj is not None), 'mac-addr object type not found' + assert mac_obj['type'] == 'mac-addr' + assert mac_obj['value'] == '12:11:11:11:11:11' + + def test_software_json_to_stix(self): + """ + to test software stix object properties + """ + objects = TestLogScaleResultsToStix.get_observed_data_objects(logscale_sample_response) + software_obj = TestLogScaleResultsToStix.get_first_of_type(objects.values(), 'software') + assert (software_obj.keys() == {'type', 'name', 'version', 'x_minor_version', 'x_major_version', 'x_id'}) + assert (software_obj is not None), 'software object type not found' + assert software_obj['type'] == 'software' + assert software_obj['name'] == 'Windows' + assert software_obj['version'] == 'Windows Server 2022' + assert software_obj['x_major_version'] == '10' diff --git a/stix_shifter_modules/crowdstrike_logscale/test/stix_translation/test_crowdstrike_logscale_stix_to_query.py b/stix_shifter_modules/crowdstrike_logscale/test/stix_translation/test_crowdstrike_logscale_stix_to_query.py new file mode 100644 index 000000000..066f749b5 --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/test/stix_translation/test_crowdstrike_logscale_stix_to_query.py @@ -0,0 +1,423 @@ +from stix_shifter.stix_translation import stix_translation +import unittest + +translation = stix_translation.StixTranslation() + + +def _remove_timestamp_from_query(queries): + """ + remove the timestamp in the query + : param: queries: list + : return: queries: list + """ + if isinstance(queries, list): + for query in queries: + del query['start'] + del query['end'] + return queries + + +class TestQueryTranslator(unittest.TestCase): + """ + class to perform unittest case CrowdStrike LogScale translate query + """ + if __name__ == "__main__": + unittest.main() + + def _test_query_assertions(self, query, queries): + """ + to assert the each query in the list against expected result + """ + self.assertIsInstance(queries, list) + self.assertIsInstance(query, dict) + self.assertIsInstance(query['queries'], list) + for index, each_query in enumerate(query.get('queries'), start=0): + self.assertEqual(each_query, queries[index]) + + def test_ipv4_query(self): + stix_pattern = "[ipv4-addr:value = '111.111.11.111']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': 'device.local_ip = \"111.111.11.111\" or ' + 'device.external_ip = \"111.111.11.111\" | tail()', + 'start': 1700213537397, 'end': 1700213837397}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_equals_operator_for_list_of_dict(self): + stix_pattern = "[file:name = 'cmd.exe']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': '@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"filename\"' + '\\s*:\\s*\"cmd\\.exe\"/ | tail()', + 'start': 1700213537397, 'end': 1700213837397}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_equals_operator_for_list_of_dict_attribute(self): + stix_pattern = "[file:name != 'cmd.exe']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': '@rawstring != /\"behaviors\"\\s*:\\s*\\[.*\"filename\"' + '\\s*:\\s*\"cmd\\.exe\"/ | tail()', + 'start': 1700213537397, 'end': 1700213837397}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_equals_operator(self): + stix_pattern = "[x-oca-asset:hostname != 'EC2AMAZ']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': 'device.hostname != \"EC2AMAZ\" ' + 'and device.hostname = \"*\" | tail()', + 'start': 1700213537397, 'end': 1700213837397}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_gt_operator(self): + stix_pattern = "[x-ibm-finding:severity > 50]" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': 'max_severity > 50 | tail()', + 'start': 1700214399519, 'end': 1700214699519}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_less_than_operator(self): + stix_pattern = "[x-ibm-finding:severity NOT < 50]" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': 'not max_severity < 50 and max_severity = \"*\"' + ' | tail()', + 'start': 1700214399519, 'end': 1700214699519}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_in_operator_for_list_of_dict_field(self): + stix_pattern = "[process:name IN ('mstsc.exe', 'test.exe')] START t'2023-11-04T16:43:26.000Z' " \ + "STOP t'2023-11-12T00:43:26.003Z'" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': '@rawstring = /"behaviors"\\s*:\\s*\\[' + '.*"filename"\\s*:\\s*("mstsc\\.exe"|"test\\.exe")/ | ' + 'tail()', 'start': 1699116206000, + 'end': 1699749806003}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_in_operator(self): + stix_pattern = "[x-ibm-finding:name IN ('123','456')]" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': 'detection_id = \"123\" or detection_id = ' + '\"456\" | tail()', 'start': 1699116206000, + 'end': 1699749806003}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_in_operator_for_list_of_dict_field(self): + stix_pattern = "[process:name NOT IN ('mstsc.exe', 'test.exe')] START t'2023-11-04T16:43:26.000Z' " \ + "STOP t'2023-11-12T00:43:26.003Z'" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', 'queryString': 'not @rawstring = /\"behaviors\"\\s*:\\s*\\' + '[.*\"filename\"\\s*:\\s*(\"mstsc\\.exe\"|\"test\\.exe' + '\")/ | tail()', 'start': 1699116206000, + 'end': 1699749806003}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_in_operator(self): + stix_pattern = "[ipv4-addr:value NOT IN ('1.1.1.1','2.2.2.2')]" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': '((not device.local_ip = \"1.1.1.1\" and device.local_ip = \"*\") and ' + '(not device.local_ip = \"2.2.2.2\" and device.local_ip = \"*\")) or ' + '((not device.external_ip = \"1.1.1.1\" and device.external_ip = \"*\") ' + 'and (not device.external_ip = \"2.2.2.2\" and device.external_ip = \"*\")) ' + '| tail()', 'start': 1699116206000, + 'end': 1699749806003}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_like_operator(self): + stix_pattern = "[mac-addr:value LIKE '11-22-28-67%']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': 'device.mac_address = /11-22-28-67.*/i | tail()', + 'start': 1700215236531, 'end': 1700215536531}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_matches_operator(self): + stix_pattern = "[x-oca-asset:device_id NOT MATCHES '^7adb1f5eb5164fde90279ab0a1600d49']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': 'not device.device_id = /^7adb1f5eb5164fde90279ab0a1600d49/i and ' + 'device.device_id = \"*\" | tail()', + 'start': 1700215742090, 'end': 1700216042090}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_subset_operator(self): + stix_pattern = "[ipv6-addr:value NOT ISSUBSET '1.2.3.4/30']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': 'device.local_ip =~ !cidr(subnet="1.2.3.4/30") | tail()', + 'start': 1700216196932, 'end': 1700216496932}, + {'source': 'crowdstrikeedr', + 'queryString': 'device.external_ip =~ !cidr(subnet="1.2.3.4/30") | tail()', + 'start': 1700216196932, 'end': 1700216496932}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_equals_for_array_attribute(self): + stix_pattern = "[x-ibm-finding:x_behaviors_processed[*] = '123']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': 'array:contains(array=\"behaviors_processed[]\",value = ' + '\"123\") | tail()', + 'start': 1700216196932, 'end': 1700216496932}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_matches_operator_for_array_attribute(self): + stix_pattern = "[ x-oca-asset:x_device_groups[*] MATCHES '97350feebe4541e8a615c0d3f18acdf3']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': 'array:regex(array=\"device.groups[]\",regex = \"97350feebe4541e8a615c0d3f18acdf3\"' + ', flags=i) | tail()', + 'start': 1700216196932, 'end': 1700216496932}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_equals_operator_for_array_attribute(self): + stix_pattern = "[ x-oca-asset:x_device_groups[*] != '97350feebe4541e8a615c0d3f18acdf3']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': '!array:contains(array=\"device.groups[]\",value = ' + '\"97350feebe4541e8a615c0d3f18acdf3\") | tail()', + 'start': 1700216196932, 'end': 1700216496932}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_like_operator_for_array_attribute(self): + stix_pattern = "[ x-oca-asset:x_device_groups[*] NOT LIKE '97350feebe4541e8a615c0d3f18acdf3']" + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': 'not array:regex(array=\"device.groups[]\",regex = ' + '\"97350feebe4541e8a615c0d3f18acdf3\"' + ', flags=i) | tail()', + 'start': 1700216196932, 'end': 1700216496932}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combined_comparison_without_timestamp(self): + stix_pattern = ("[(x-ibm-ttp-tagging:extensions.'mitre-attack-ext'.technique_name != 'test' OR " + "ipv4-addr:value = '11.111.111.111') AND (mac-addr:value = '11-11-11-11-1a-1b' OR " + "file:hashes.MD5 = '11111111114a00996a9f5aaf9c0db84b')]") + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': '((@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"parent_details\"\\s*:\\s*\\' + '{.*\"parent_md5\"\\s*:\\s*\"11111111114a00996a9f5aaf9c0db84b\"/ or @rawstring ' + '= /\"behaviors\"\\s*:\\s*\\[.*\"md5\"\\s*:\\s*' + '\"11111111114a00996a9f5aaf9c0db84b\"/) or device.mac_address = ' + '\"11-11-11-11-1a-1b\") and ((device.local_ip = \"11.111.111.111\" or ' + 'device.external_ip = \"11.111.111.111\") or @rawstring != /\"behaviors\"\\s*:' + '\\s*\\[.*\"technique\"\\s*:\\s*\"test\"/) | tail()', + 'start': 1700559682224, 'end': 1700559982224}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_common_timestamp(self): + stix_pattern = ("([x-crowdstrike-detection-behavior:severity IN (50,30)] OR [directory:path != " + "'\\\\Device\\\\HarddiskVolume1\\\\Windows\\\\System32\\\\cmd.exe']) START " + "t'2023-11-15T01:43:26Z' STOP t'2023-11-20T00:43:26Z'") + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{'source': 'crowdstrikeedr', + 'queryString': '(@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"severity\"\\s*:\\s*(50|30)/) ' + 'or (@rawstring != /\"behaviors\"\\s*:\\s*\\[.*\"filepath\"\\s*:\\s*\"\\\\\\\\' + 'Device\\\\\\\\HarddiskVolume1\\\\\\\\Windows\\\\\\\\System32\\\\\\\\cmd\\.exe\"/) ' + '| tail()', + 'start': 1700012606000, 'end': 1700441006000}] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_different_timestamp(self): + stix_pattern = ("[software:name = 'Windows'] START t'2023-11-15T01:43:26Z' STOP t'2023-11-20T00:43:26Z' " + "AND [software:x_minor_version = '0'] START t'2023-12-10T01:43:26Z' STOP " + "t'2023-12-17T00:43:26Z'") + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{ + 'source': 'crowdstrikeedr', + 'queryString': 'device.platform_name = \"Windows\" | tail()', + "start": 1700012606000, + "end": 1700441006000 + }, { + 'source': 'crowdstrikeedr', + 'queryString': 'device.minor_version = \"0\" | tail()', + "start": 1702172606000, + "end": 1702172606000 + }] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_expression_with_filter_function_joined_by_and_operator(self): + stix_pattern = ("[user-account:user_id = 'S-1-5-18' AND ipv4-addr:value ISSUBSET '1.2.3.4/32' AND " + "process:parent_ref.command_line = '\"C:\\\\Windows\\\\system32\\\\cmd.exe\" /d " + "/c C:\\\\Windows\\\\system32\\\\silcollector.cmd configure']START t'2023-12-10T01:43:26Z' " + "STOP t'2023-12-16T00:43:26Z'") + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{ + 'source': 'crowdstrikeedr', + 'queryString': 'device.local_ip =~ cidr(subnet=\"1.2.3.4/32\") | @rawstring = ' + '/\"behaviors\"\\s*:\\s*\\[.*\"parent_details\"\\s*:\\s*\\{.*\"parent_cmdline\"' + '\\s*:\\s*\"\\\\\"C:\\\\\\\\Windows\\\\\\\\system32\\\\\\\\cmd\\.exe\\\\\"\\ \\/d\\ ' + '\\/c\\ C:\\\\\\\\Windows\\\\\\\\system32\\\\\\\\silcollector\\.cmd\\ configure\"/ ' + 'and (@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"user_id\"\\s*:\\s*\"S\\-1\\-5\\-18\"/) ' + '| tail()', + 'start': 1702172606000, + 'end': 1702687406000 + }, { + 'source': 'crowdstrikeedr', + 'queryString': 'device.external_ip =~ cidr(subnet=\"1.2.3.4/32\") | @rawstring = ' + '/\"behaviors\"\\s*:\\s*\\[.*\"parent_details\"\\s*:\\s*\\{.*\"parent_cmdline\"' + '\\s*:\\s*\"\\\\\"C:\\\\\\\\Windows\\\\\\\\system32\\\\\\\\cmd\\.exe\\\\\"\\ \\/d\\ ' + '\\/c\\ C:\\\\\\\\Windows\\\\\\\\system32\\\\\\\\silcollector\\.cmd\\ configure\"/ and ' + '(@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"user_id\"\\s*:\\s*\"S\\-1\\-5\\-18\"/) ' + '| tail()', + 'start': 1702172606000, + 'end': 1702687406000 + }] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_expression_with_filter_function_joined_by_combination_of_or_and_operator(self): + stix_pattern = ("[user-account:user_id = 'S-1-5-18' AND ipv4-addr:value ISSUBSET '1.2.3.4/32' AND " + "process:parent_ref.command_line = '\"C:\\\\Windows\\\\system32\\\\cmd.exe\" /d " + "/c C:\\\\Windows\\\\system32\\\\silcollector.cmd configure' OR x-ibm-finding:" + "x_behaviors_processed[*] NOT LIKE '123']START t'2023-12-10T01:43:26Z' " + "STOP t'2023-12-16T00:43:26Z'") + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{ + 'source': 'crowdstrikeedr', + 'queryString': 'not array:regex(array=\"behaviors_processed[]\",regex = \"123\", flags=i) | tail()', + 'start': 1702172606000, + 'end': 1702687406000 + }, + { + 'source': 'crowdstrikeedr', + 'queryString': 'device.local_ip =~ cidr(subnet=\"1.2.3.4/32\") | tail()', + 'start': 1702172606000, + 'end': 1702687406000 + }, + { + 'source': 'crowdstrikeedr', + 'queryString': 'device.external_ip =~ cidr(subnet=\"1.2.3.4/32\") | tail()', + 'start': 1702172606000, + 'end': 1702687406000 + }, + { + 'source': 'crowdstrikeedr', + 'queryString': '(@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"parent_details\"\\s*:\\s*\\' + '{.*\"parent_cmdline\"\\s*:\\s*\"\\\\\"C:\\\\\\\\Windows\\\\\\\\system32\\\\\\\\cmd\\.' + 'exe\\\\\"\\ \\/d\\ \\/c\\ C:\\\\\\\\Windows\\\\\\\\system32\\\\\\\\silcollector\\.' + 'cmd\\ configure\"/ and (@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"user_id\"\\s*:\\' + 's*\"S\\-1\\-5\\-18\"/)) | tail()', + 'start': 1702172606000, + 'end': 1702687406000 + }] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_filter_functions_in_multiple_observation(self): + + stix_pattern = ("[user-account:user_id = 'S-1-5-18' AND ipv4-addr:value ISSUBSET '1.2.3.4/32' AND " + "process:parent_ref.command_line = '\"C:\\\\Windows\\\\system32\\\\cmd.exe\" /d /c " + "C:\\\\Windows\\\\system32\\\\silcollector.cmd configure'] START t'2023-12-10T01:43:26Z' " + "STOP t'2023-12-16T00:43:26Z' OR [x-ibm-finding:x_behaviors_processed LIKE " + "'pid:84f9f480747a43469228f876063b0ece:38864991970:41002' AND " + "x-oca-asset:hostname = 'EC2AMAZ']") + query = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [{ + 'source': 'crowdstrikeedr', + 'queryString': 'device.local_ip =~ cidr(subnet=\"1.2.3.4/32\") | @rawstring = ' + '/\"behaviors\"\\s*:\\s*\\[.*\"parent_details\"\\s*:\\s*\\{.*\"parent_cmdline\"\\s*:' + '\\s*\"\\\\\"C:\\\\\\\\Windows\\\\\\\\system32\\\\\\\\cmd\\.exe\\\\\"\\ \\/d\\ \\/c\\ ' + 'C:\\\\\\\\Windows\\\\\\\\system32\\\\\\\\silcollector\\.cmd\\ configure\"/ and ' + '(@rawstring = /\"behaviors\"\\s*:\\s*\\[.*\"user_id\"\\s*:\\s*\"S\\-1\\-5\\-18\"/) |' + ' tail()', + 'start': 1702172606000, + 'end': 1702687406000 + }, + { + 'source': 'crowdstrikeedr', + 'queryString': 'device.external_ip =~ cidr(subnet=\"1.2.3.4/32\") | @rawstring = ' + '/\"behaviors\"\\s*:' + '\\s*\\[.*\"parent_details\"\\s*:\\s*\\{.*\"parent_cmdline\"\\s*:\\s*\"\\\\\"C:\\\\\\\\' + 'Windows\\\\\\\\system32\\\\\\\\cmd\\.exe\\\\\"\\ \\/d\\ \\/c\\ C:\\\\\\\\Windows' + '\\\\\\\\' + 'system32\\\\\\\\silcollector\\.cmd\\ configure\"/ and (@rawstring = /\"behaviors\"\\s*:' + '\\s*\\[.*\"user_id\"\\s*:\\s*\"S\\-1\\-5\\-18\"/) | tail()', + 'start': 1702172606000, + 'end': 1702687406000 + }, + { + 'source': 'crowdstrikeedr', + 'queryString': 'array:regex(array=\"behaviors_processed[]\",regex = ' + '\"pid:84f9f480747a43469228f876063b0ece' + ':38864991970:41002\", flags=i) | tail()', + 'start': 1702830883582, + 'end': 1702831183582 + }, + { + 'source': 'crowdstrikeedr', + 'queryString': 'device.hostname = \"EC2AMAZ\" | tail()', + 'start': 1702830883582, + 'end': 1702831183582 + }] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_invalid_int_value(self): + stix_pattern = "[software:version < 'fifty']" + result = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + assert result['success'] is False + assert "not_implemented" == result['code'] + assert 'ComparisonComparators.LessThan operator is not supported for string type value: fifty' in result['error'] + + def test_unsupported_like_field(self): + stix_pattern = "[process:name LIKE 'test']" + result = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + assert result['success'] is False + assert "not_implemented" == result['code'] + assert 'ComparisonComparators.Like is not supported for list of dictionary fields' \ + in result['error'] + + def test_invalid_timestamp_range(self): + stix_pattern = "[file:name = 'mstsc.exe'] START t'2023-11-20T01:43:26.000Z' STOP t'2023-11-19T00:43:26.003Z'" + result = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + assert result['success'] is False + assert "translation_error" == result['code'] + assert 'Start time should be lesser than Stop time' in result['error'] + + def test_invalid_operator_for_array_attribute(self): + stix_pattern = "[x-ibm-finding:x_behaviors_processed[*] IN ('123')]" + result = translation.translate('crowdstrike_logscale', 'query', '{}', stix_pattern) + assert result['success'] is False + assert "not_implemented" == result['code'] + assert 'ComparisonComparators.In is not supported for array attribute' in result['error'] diff --git a/stix_shifter_modules/crowdstrike_logscale/test/stix_transmission/test_crowdstrike_logscale.py b/stix_shifter_modules/crowdstrike_logscale/test/stix_transmission/test_crowdstrike_logscale.py new file mode 100644 index 000000000..aace27e3e --- /dev/null +++ b/stix_shifter_modules/crowdstrike_logscale/test/stix_transmission/test_crowdstrike_logscale.py @@ -0,0 +1,647 @@ +from stix_shifter_modules.crowdstrike_logscale.entry_point import EntryPoint +from stix_shifter.stix_transmission.stix_transmission import run_in_thread +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from tests.utils.async_utils import get_mock_response +from unittest.mock import patch +import unittest +import json + + +class TestCrowdstrikeLogscaleConnection(unittest.TestCase, object): + + def connection(self): + return { + "host": "hostbla", + "repository": "testrepo", + } + def connection_with_result_limit(self): + return { + "host": "hostbla", + "repository": "testrepo", + "options": {"result_limit": 2} + } + + def configuration(self): + return { + "auth": { + "api_token": "123" + } + } + + mocked_ping_response = {"status": "OK", + "version": "1.x.0--build-4xxx0--sha-123"} + + mocked_result_response = { + "cancelled": False, + "done": True, + "events": [ + { + "behaviors[0].technique": "Indicator of Attack", + "seconds_to_resolved": "0", + "device.local_ip": "1.1.1.1", + "device.product_type_desc": "Server", + "device.os_version": "Windows Server 2022", + "behaviors[0].tactic": "Custom Intelligence", + "behaviors[0].ioc_description": "", + "max_severity": "50", + "@timestamp": 1711549347952, + "@ingesttimestamp": "1711549348586", + "hostinfo.domain": "", + "#type": "CrowdStrike_Spotlight", + "device.product_type": "3", + "behaviors[0].description": "A process triggered a medium severity custom rule.", + "device.agent_local_time": "2024-03-13T07:33:59.693Z", + "device.config_id_platform": "3", + "behaviors[0].behavior_id": "41002", + "behaviors[0].ioc_source": "", + "device.platform_name": "Windows", + "behaviors[0].user_name": "user1", + "device.first_seen": "2023-05-16T05:10:55Z", + "detection_id": "ldt:123:123", + "device.external_ip": "2.2.2.2", + "device.bios_manufacturer": "Xen", + "first_behavior": "2024-03-13T07:34:08Z", + "behaviors[0].technique_id": "CST0004", + "behaviors[0].parent_details.parent_md5": "", + "device.platform_id": "0", + "#repo": "TestRepository", + "device.modified_timestamp": "2024-03-13T07:34:11Z", + "device.minor_version": "0", + "device.bios_version": "4.2.amazon", + "behaviors[0].display_name": "CustomIOAWinMedium", + "device.hostname": "host", + "behaviors[0].parent_details.parent_cmdline": "", + "device.service_provider": "aws", + "behaviors[0].filepath": "\\Device\\conhost.exe", + "date_updated": "2024-03-27T14:22:27.952818Z", + "behaviors[0].md5": "1234xxxxxxxxxxxx", + "max_severity_displayname": "Medium", + "behaviors[0].pattern_disposition": "10240", + "email_sent": "false", + "behaviors[0].triggering_process_graph_id": "pid:12:12", + "device.last_login_user": "Administrator", + "device.status": "normal", + "behaviors[0].rule_instance_id": "3", + "behaviors[0].objective": "Falcon Detection Method", + "behaviors[0].timestamp": "2024-03-13T07:34:08Z", + "device.instance_id": "i-123ab", + "behaviors[0].parent_details.parent_sha256": "", + "device.config_id_base": "65994763", + "behaviors[0].user_id": "S-1-5-18", + "behaviors_processed[0]": "pid:12:33221:42", + "seconds_to_triaged": "0", + "status": "new", + "device.major_version": "10", + "max_confidence": "100", + "device.agent_version": "7.06.1xxx.0", + "@timezone": "Z", + "behaviors[0].filename": "conhost.exe", + "behaviors[0].parent_details.parent_process_graph_id": "pid:12:33", + "last_behavior": "2024-03-13T07:34:08Z", + "device.last_seen": "2024-03-13T07:34:10Z", + "behaviors[0].control_graph_id": "ctg:xx:yy", + "device.system_product_name": "HVM domU", + "behaviors[0].sha256": "1234567000", + "created_timestamp": "2024-03-13T07:34:15.869091531Z", + "device.config_id_build": "17807", + "behaviors[0].cmdline": "C:\\conhost.exe 0xffffffff -ForceV1", + "behaviors[0].ioc_type": "", + "behaviors[0].tactic_id": "CSTA0005", + "device.last_login_timestamp": "2024-03-12T16:50:17Z", + "behaviors[0].severity": "50", + "device.system_manufacturer": "Xen", + "device.agent_load_flags": "1", + "behaviors[0].confidence": "100", + "@timestamp.nanos": "818000", + "behaviors[0].ioc_value": "", + "behaviors[0].template_instance_id": "3", + "device.mac_address": "01-01-01-01-01-01", + "behaviors[0].alleged_filetype": "exe", + "behaviors[0].device_id": "device", + "behaviors[0].scenario": "suspicious_activity", + "behaviors[0].rule_instance_version": "3", + "device.service_provider_account_id": "1234", + "@id": "ATzrtyg4xCKOqQnD9NodpvsY_363_23_1711549347" + }], + "metaData": + {"eventCount": 0, + "filterQuery": + {"end": 1711549347952, + "queryString": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\"", + "start": 1710921332365}}, + "warnings": []} + + mocked_result_response_2 = { + "cancelled": False, + "done": True, + "events": [{ + "behaviors[0].technique": "Indicator of Attack", + "seconds_to_resolved": "0", + "device.local_ip": "2.2.2.2", + "device.product_type_desc": "Server", + "device.os_version": "Windows Server 2022", + "behaviors[0].tactic": "Custom Intelligence", + "behaviors[0].ioc_description": "\\Device\\cmd.exe", + "max_severity": "50", + "@timestamp": 1711549342830, + "@ingesttimestamp": "1711549348586", + "device.groups[0]": "1234", + "hostinfo.domain": "", + "#type": "CrowdStrike_Spotlight", + "device.product_type": "3", + "behaviors[0].description": "A process triggered a medium severity custom rule.", + "device.agent_local_time": "2024-03-14T14:40:04.490Z", + "device.config_id_platform": "3", + "behaviors[0].behavior_id": "41002", + "behaviors[0].ioc_source": "library_load", + "device.platform_name": "Windows", + "behaviors[0].user_name": "user2", + "device.first_seen": "2023-05-16T05:10:55Z", + "detection_id": "ldt:7adxxxx00d49:33", + "device.external_ip": "4.5.6.7", + "device.bios_manufacturer": "Xen", + "first_behavior": "2024-03-14T14:40:26Z", + "behaviors[0].technique_id": "CST0004", + "behaviors[0].parent_details.parent_md5": "", + "device.platform_id": "0", + "#repo": "TestRepository", + "device.modified_timestamp": "2024-03-14T14:40:19Z", + "device.minor_version": "0", + "device.bios_version": "4.11.amazon", + "behaviors[0].display_name": "CustomIOAWinMedium", + "device.hostname": "host", + "behaviors[0].parent_details.parent_cmdline": "", + "device.service_provider": "aws", + "behaviors[0].filepath": "\\Device\\cmd.exe", + "date_updated": "2024-03-27T14:22:27.952818Z", + "behaviors[0].md5": "axxtyya", + "max_severity_displayname": "Medium", + "behaviors[0].pattern_disposition": "2048", + "email_sent": "false", + "behaviors[0].triggering_process_graph_id": "pid:11:00", + "device.last_login_user": "Administrator", + "device.status": "normal", + "behaviors[0].rule_instance_id": "3", + "behaviors[0].objective": "Falcon Detection Method", + "behaviors[0].timestamp": "2024-03-14T14:40:26Z", + "device.instance_id": "i-123", + "behaviors[0].parent_details.parent_sha256": "", + "device.config_id_base": "65994763", + "behaviors[0].user_id": "user_id_1", + "behaviors_processed[0]": "pid:22:33:41002", + "seconds_to_triaged": "0", + "status": "new", + "device.major_version": "10", + "max_confidence": "100", + "device.agent_version": "7.11.xx.0", + "@timezone": "Z", + "behaviors[0].filename": "cmd.exe", + "behaviors[0].parent_details.parent_process_graph_id": "pid:22:33", + "last_behavior": "2024-03-14T14:40:26Z", + "device.last_seen": "2024-03-14T14:40:18Z", + "behaviors[0].control_graph_id": "ctg:xx:33", + "device.system_product_name": "HVM domU", + "behaviors[0].sha256": "exxxx123", + "created_timestamp": "2024-03-14T14:41:22.092316953Z", + "device.config_id_build": "18110", + "behaviors[0].cmdline": "C:\\cmd.exe /c reg query \"HKLM\\Software\\Npcap\" /ve 2>nul | find \"REG_SZ\"", + "behaviors[0].ioc_type": "hash_sha256", + "behaviors[0].tactic_id": "CSTA0005", + "device.last_login_timestamp": "2024-03-14T13:14:27Z", + "behaviors[0].severity": "50", + "device.system_manufacturer": "Xen", + "device.agent_load_flags": "1", + "behaviors[0].confidence": "100", + "@timestamp.nanos": "818000", + "behaviors[0].ioc_value": "xxyy", + "behaviors[0].template_instance_id": "3", + "device.groups[1]": "bb1", + "device.mac_address": "01-01-01-01-01-01", + "behaviors[0].alleged_filetype": "exe", + "behaviors[0].scenario": "suspicious_activity", + "behaviors[0].rule_instance_version": "3", + "device.service_provider_account_id": "123", + "@id": "ATzrtyg4xCKOqQnD9NodpvsY_363_21_1711549347" + }], + "metaData": + {"eventCount": 0, + "filterQuery": + {"end": 1711549347952, + "queryString": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\"", + "start": 1710921332365}}, + "warnings": []} + + + def test_is_async(self): + """ test async""" + entry_point = EntryPoint(self.connection(), self.configuration()) + check_async = entry_point.is_async() + assert check_async + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_ping_success(self, mock_ping_response): + """ test success response for ping""" + mock_ping_response.return_value = \ + get_mock_response(200, json.dumps(TestCrowdstrikeLogscaleConnection.mocked_ping_response), 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_result = run_in_thread(entry_point.ping_connection) + assert ping_result["success"] is True + + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_query_success(self, mock_query_response): + """ test success response for query""" + query = {"source": "edr", "queryString": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\" | tail(10000)", "start": 1710921332365, "end": 1711549347952} + query_response = '{"hashedQueryOnView": "123ac", "id": "P3-ohFBXrfxvfy3U1xk28PLFJKL"}' + mock_query_response.return_value = \ + get_mock_response(200, query_response, 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + query_response = run_in_thread(entry_point.create_query_connection, json.dumps(query)) + assert query_response['success'] is True + assert query_response['search_id'] == "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_status_completed(self, mock_status_response): + """" test success response for status with completed status""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + status_response = ('{"cancelled": false,"done": true,"events": [], "metaData": {"eventCount": 1,' + '"filterQuery": {"end": 1711549347952,"queryString": "behaviors[0].filename=\\\"cmd.exe\\\" OR ' + 'behaviors[0].filename=\\\"conhost.exe\\\"","start": 1710921332365}},"warnings": []}') + mock_status_response.return_value = \ + get_mock_response(200, status_response, 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, search_id) + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.COMPLETED.value + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_status_running(self, mock_status_response): + """ test success response for status with running status""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + status_response = ('{"cancelled": false,"done": false,"events": [], "metaData": {"eventCount": 3000,' + '"filterQuery": {"end": 1711549347952,"queryString": "behaviors[0].filename=\\\"cmd.exe\\\" OR ' + 'behaviors[0].filename=\\\"conhost.exe\\\"","start": 1710921332365}},"warnings": []}') + mock_status_response.return_value = \ + get_mock_response(200, status_response, 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, search_id) + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.RUNNING.value + assert status_response['progress'] == 30 + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_status_running_using_metadata(self, mock_status_response): + """ test success response for status when called from results connector with metadata""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + status_response = ('{"cancelled": false,"done": false,"events": [], "metaData": {"eventCount": 1000,' + '"filterQuery": {"end": 1711549347952,"queryString": "behaviors[0].filename=\\\"cmd.exe\\\" OR ' + 'behaviors[0].filename=\\\"conhost.exe\\\"","start": 1710921332365}},"warnings": []}') + metadata = {'length': 2000} + mock_status_response.return_value = \ + get_mock_response(200, status_response, 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, search_id, metadata) + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.RUNNING.value + assert status_response['progress'] == 50 + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_status_canceled(self, mock_status_response): + """ test success response with status as cancelled """ + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + status_response = ('{"cancelled": true,"done": false,"events": [], "metaData": {"eventCount": 0,' + '"filterQuery": {"end": 1711549347952,"queryString": "behaviors[0].filename=\\\"cmd.exe\\\" OR ' + 'behaviors[0].filename=\\\"conhost.exe\\\"","start": 1710921332365}},"warnings": []}') + mock_status_response.return_value = \ + get_mock_response(200, status_response, 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, search_id) + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.CANCELED.value + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_success_response(self, mock_results_response): + """ test results with success response""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + mock_delete_job_id_response = get_mock_response(204, "", 'byte') + query_response = get_mock_response(200, '{"hashedQueryOnView": "123ac", "id": "P3-ohFBXrfxvfy3U1xk28PLFJKL"}', 'byte') + first_response = get_mock_response(200, json.dumps(TestCrowdstrikeLogscaleConnection.mocked_result_response), 'byte') + second_response = get_mock_response(200, json.dumps(TestCrowdstrikeLogscaleConnection.mocked_result_response_2), 'byte') + mock_results_response.side_effect = [first_response, query_response, second_response, second_response, mock_delete_job_id_response] + + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 2) + success = results_response["success"] + assert success + data = results_response["data"] + assert data + assert (results_response['metadata'] == + {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_21_1711549347', + 'last_event_timestamp': 1711549342830, + 'start': 1710921332365}) + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_success_response_with_result_limit_less_than_length(self, mock_results_response): + """ test results with success response""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + mock_delete_job_id_response = get_mock_response(204, "", 'byte') + query_response = get_mock_response(200, '{"hashedQueryOnView": "123ac", "id": "P3-ohFBXrfxvfy3U1xk28PLFJKL"}', 'byte') + first_response = get_mock_response(200, json.dumps(TestCrowdstrikeLogscaleConnection.mocked_result_response), 'byte') + second_response = get_mock_response(200, json.dumps(TestCrowdstrikeLogscaleConnection.mocked_result_response_2), 'byte') + mock_results_response.side_effect = [first_response, query_response, second_response, second_response, mock_delete_job_id_response] + + entry_point = EntryPoint(self.connection_with_result_limit(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 5) + success = results_response["success"] + assert success + data = results_response["data"] + assert data + assert len(data) == 2 + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_success_response_using_metadata_for_next_iteration(self, mock_results_response): + """ test success response for results with metadata as input to be used for next iteration""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + metadata = {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'start': 1710921332365, + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_23_1711549347', + 'last_event_timestamp': 1711549347952} + mock_delete_job_id_response = get_mock_response(204, "", 'byte') + mock_query_job_id_response = get_mock_response(200, json.dumps({"hashedQueryOnView": "456ac", "id": "P6-o1FbXrfxvfy3U1xk28PLFijk"}), 'byte') + mock_query_poll_response = get_mock_response(200, json.dumps(TestCrowdstrikeLogscaleConnection.mocked_result_response_2), 'byte') + mock_results_response.side_effect = [mock_query_job_id_response,mock_query_poll_response, mock_query_poll_response,mock_delete_job_id_response] + + entry_point = EntryPoint(self.connection_with_result_limit(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 1, metadata) + success = results_response["success"] + assert success + data = results_response["data"] + assert data + assert (results_response['metadata'] == + {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'start': 1710921332365, + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_21_1711549347', + 'last_event_timestamp': 1711549342830}) + + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_using_metadata_for_next_iteration_and_call_status(self, mock_results_response): + """ test success response for results with metadata as input and call status connector to check status for + intermediate query job id""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + mocked_running_result_response = { + "cancelled": False, + "done": False, + "events": [], + "metaData": + {"eventCount": 0, + "filterQuery": + {"end": 1711549347952, + "queryString": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\"", + "start": 1710921332365}}, + "warnings": []} + + metadata = {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'start': 1710921332365, + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_23_1711549347', + 'last_event_timestamp': 1711549347952} + + mock_delete_job_id_response = get_mock_response(204, "", 'byte') + mock_query_job_id_response = get_mock_response(200, json.dumps({"hashedQueryOnView": "456ac", "id": "P6-o1FbXrfxvfy3U1xk28PLFijk"}), 'byte') + mock_query_poll_response = get_mock_response(200, json.dumps(TestCrowdstrikeLogscaleConnection.mocked_result_response_2), 'byte') + mock_query_poll_running_response = get_mock_response(200, json.dumps(mocked_running_result_response), 'byte') + + mock_results_response.side_effect = [mock_query_job_id_response,mock_query_poll_running_response,mock_query_poll_running_response,mock_query_poll_response,mock_query_poll_response,mock_delete_job_id_response] + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 1, metadata) + success = results_response["success"] + assert success + data = results_response["data"] + assert data + assert (results_response['metadata'] == + {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'start': 1710921332365, + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_21_1711549347', + 'last_event_timestamp': 1711549342830}) + + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_using_metadata_for_next_iteration_with_timeout_exception(self, mock_results_response): + """ test failure response for results with timeout exception when status connector + is called to check status of intermediate query job id from results""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + mocked_running_result_response = { + "cancelled": False, + "done": False, + "events": [], + "metaData": + {"eventCount": 0, + "filterQuery": + {"end": 1711549347952, + "queryString": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\"", + "start": 1710921332365}}, + "warnings": []} + + metadata = {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'start': 1710921332365, + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_23_1711549347', + 'last_event_timestamp': 1711549347952} + + mock_delete_job_id_response = get_mock_response(204, "", 'byte') + mock_query_job_id_response = get_mock_response(200, json.dumps({"hashedQueryOnView": "456ac", "id": "P6-o1FbXrfxvfy3U1xk28PLFijk"}), 'byte') + mock_query_poll_running_response = get_mock_response(200, json.dumps(mocked_running_result_response), 'byte') + mock_results_response.side_effect = [mock_query_job_id_response,mock_query_poll_running_response,mock_query_poll_running_response,Exception("timeout_error"),mock_delete_job_id_response] + entry_point = EntryPoint(self.connection_with_result_limit(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 2000, 1, metadata) + assert results_response["success"] is False + assert 'timeout_error' in results_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_using_metadata_for_next_iteration_with_status_cancelled(self, mock_results_response): + """ test response for results with status connector being called to check status + for cancelled intermediate query job id """ + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + mocked_running_result_response = { + "cancelled": False, + "done": False, + "events": [], + "metaData": + {"eventCount": 0, + "filterQuery": + {"end": 1711549347952, + "queryString": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\"", + "start": 1710921332365}}, + "warnings": []} + mocked_cancelled_result_response = { + "cancelled": True, + "done": False, + "events": [], + "metaData": + {"eventCount": 0, + "filterQuery": + {"end": 1711549347952, + "queryString": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\"", + "start": 1710921332365}}, + "warnings": []} + + metadata = {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'start': 1710921332365, + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_23_1711549347', + 'last_event_timestamp': 1711549347952} + + mock_delete_job_id_response = get_mock_response(204, "", 'byte') + mock_query_job_id_response = get_mock_response(200, json.dumps({"hashedQueryOnView": "456ac", "id": "P6-o1FbXrfxvfy3U1xk28PLFijk"}), 'byte') + mock_query_poll_running_response = get_mock_response(200, json.dumps(mocked_running_result_response), 'byte') + mock_status_cancelled_response = get_mock_response(200, json.dumps(mocked_cancelled_result_response), 'byte') + + mock_results_response.side_effect = [mock_query_job_id_response,mock_query_poll_running_response,mock_query_poll_running_response,mock_status_cancelled_response,mock_delete_job_id_response] + entry_point = EntryPoint(self.connection_with_result_limit(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 2000, 1, metadata) + assert results_response["success"] is True + assert results_response["data"] == [] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_with_failure_response_during_job_id_creation_with_metadata(self, mock_results_response): + """ test failure response for results when query job id creation failed when metadata is passed""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + mock_results_response.return_value = \ + get_mock_response(401, "The supplied authentication is invalid", 'byte') + + metadata = {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'start': 1710921332365, + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_23_1711549347', + 'last_event_timestamp': 1711549347952} + entry_point = EntryPoint(self.connection_with_result_limit(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 2000, 1, metadata) + assert results_response["success"] is False + assert "The supplied authentication is invalid" in results_response["error"] + assert results_response["code"] == "authentication_fail" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_with_failure_response_during_status_check_with_metadata(self, mock_results_response): + """ test failure response for results during status check with metadata""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + invalid_response = \ + get_mock_response(401, "The supplied authentication is invalid", 'byte') + mock_query_job_id_response = get_mock_response(200, json.dumps( + {"hashedQueryOnView": "456ac", "id": "P6-o1FbXrfxvfy3U1xk28PLFijk"}), 'byte') + mock_results_response.side_effect = [mock_query_job_id_response, invalid_response] + + metadata = {'input_query_string': 'behaviors[0].filename="cmd.exe" OR ' + 'behaviors[0].filename="conhost.exe"', + 'start': 1710921332365, + 'last_event_id': 'ATzrtyg4xCKOqQnD9NodpvsY_363_23_1711549347', + 'last_event_timestamp': 1711549347952} + entry_point = EntryPoint(self.connection_with_result_limit(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 2000, 1, metadata) + assert results_response["success"] is False + assert "The supplied authentication is invalid" in results_response["error"] + assert results_response["code"] == "authentication_fail" + + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_authentication_failure_in_transmit_query(self, mock_query_response): + """ test authentication fail""" + query = {"source": "edr", + "queryString": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\" | tail(10000)", + "start": 1710921332365, "end": 1711549347952} + mock_query_response.return_value = \ + get_mock_response(401, "The supplied authentication is invalid", 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + query_result = run_in_thread(entry_point.create_query_connection, json.dumps(query)) + assert query_result["success"] is False + assert "The supplied authentication is invalid" in query_result["error"] + assert query_result["code"] == "authentication_fail" + + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_query_in_transmit(self, mock_query_response): + query = {"source": "edr", + "querystring": "behaviors[0].filename=\"cmd.exe\" OR behaviors[0].filename=\"conhost.exe\" | tail(10000)", + "start": 1710921332365, "end": 1711549347952} + mock_query_response.return_value = \ + get_mock_response(400, "The request content was malformed:\nObject is missing required member 'queryString'", 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + query_result = run_in_thread(entry_point.create_query_connection, json.dumps(query)) + assert query_result["success"] is False + assert "The request content was malformed:\nObject is missing required member 'queryString'" in query_result["error"] + assert query_result["code"] == "invalid_query" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_search_id_in_status(self, mock_status_response): + """ test invalid search id in status""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + mock_status_response.return_value = \ + get_mock_response(404, "No query with id=P3-ohFBXrfxvfy3U1xk28PLFJKL", 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + status_response = run_in_thread(entry_point.create_status_connection, search_id) + assert status_response["success"] is False + assert "No query with id=P3-ohFBXrfxvfy3U1xk28PLFJKL" in status_response[ + "error"] + assert status_response["code"] == "no_results" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_search_id_in_results(self, mock_results_response): + """ test invalid search id in results""" + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + mock_results_response.return_value = \ + get_mock_response(404, "No query with id=P3-ohFBXrfxvfy3U1xk28PLFJKL", 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + results_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 10000) + assert results_response["success"] is False + assert "No query with id=P3-ohFBXrfxvfy3U1xk28PLFJKL" in results_response[ + "error"] + assert results_response["code"] == "no_results" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_time_out_exception_for_results(self, mock_result_response): + """ test time out exception in results""" + mock_result_response.side_effect = Exception("timeout_error") + search_id = "P3-ohFBXrfxvfy3U1xk28PLFJKL:edr" + entry_point = EntryPoint(self.connection(), self.configuration()) + result_response = run_in_thread(entry_point.create_results_connection, search_id, 0, 10000) + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert 'timeout_error' in result_response['error'] + assert result_response['code'] == 'service_unavailable' + + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_time_out_exception_for_ping(self, mock_ping_response): + """ test timeout exception for ping""" + mock_ping_response.side_effect = Exception("timeout_error") + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_result = run_in_thread(entry_point.ping_connection) + assert ping_result is not None + assert ping_result['success'] is False + assert 'error' in ping_result + assert 'timeout_error' in ping_result['error'] + assert ping_result['code'] == 'service_unavailable' + + + + + + + + +