From 7041efa10e6d981c34aa1214812ccbe278b9c1bf Mon Sep 17 00:00:00 2001 From: tkishel Date: Mon, 28 Nov 2022 13:52:27 -0800 Subject: [PATCH 1/4] correct date format and math and make all reads optional --- scripts/pcs_compute_forward_to_siem.py | 61 +++++++++++++------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/scripts/pcs_compute_forward_to_siem.py b/scripts/pcs_compute_forward_to_siem.py index 63a00a1..1bd4866 100644 --- a/scripts/pcs_compute_forward_to_siem.py +++ b/scripts/pcs_compute_forward_to_siem.py @@ -5,18 +5,19 @@ # It depends upon the SIEM to deduplicate data, and requires you to modify the `send_data_to_siem()` function for your SIEM API. import concurrent.futures -import datetime import json import inspect import time +import sys from pathlib import Path from typing import Union import requests +from datetime import datetime, timedelta, timezone from dateutil import parser, tz - + # pylint: disable=import-error from prismacloud.api import pc_api, pc_utility @@ -43,9 +44,9 @@ default=DEFAULT_MINUTES_OVERLAP, help=f'(Optional) - Minutes of overlap for time period to collect. (Default: {DEFAULT_MINUTES_OVERLAP})') this_parser.add_argument( - '--no_audit_events', + '--audit_events', action='store_true', - help='(Optional) - Do not collect Audit Events. (Default: disabled)') + help='(Optional) - Collect Audit Events. (Default: disabled)') this_parser.add_argument( '--host_forensic_activities', action='store_true', @@ -59,10 +60,10 @@ action='store_true', help='(Optional) - Collect Console Logs. (Default: disabled)') this_parser.add_argument( - '--console_log_limit', + '--console_logs_limit', type=int, default=DEFAULT_CONSOLE_LOG_LIMIT, - help=f'(Optional) - Number of console logs to collect, requires --console_logs. (Default: {DEFAULT_CONSOLE_LOG_LIMIT})') + help=f'(Optional) - Number of messages to collect, requires --console_logs. (Default: {DEFAULT_CONSOLE_LOG_LIMIT})') args = this_parser.parse_args() # -- User Defined Functions-- # @@ -176,31 +177,29 @@ def profile_log(detail: str, state: str, initialize=False): print('Collect Compute Audits, History, and Logs') print() -# Date Ranges +# Dates -date_time_1 = datetime.datetime.now().replace(microsecond=0) -date_time_0 = date_time_1 - datetime.timedelta(hours=args.hours, minutes=args.minutes_overlap) -zone_time_1 = date_time_1.astimezone(tz.tzlocal()) -zone_time_0 = zone_time_1 - datetime.timedelta(hours=args.hours, minutes=args.minutes_overlap) +date_time_1 = datetime.now(timezone.utc).replace(microsecond=0) +date_time_0 = date_time_1 - timedelta(hours=args.hours, minutes=args.minutes_overlap) -audit_query_params = { - 'from': f"{date_time_0.isoformat(sep='T')}Z", - 'to': f"{date_time_1.isoformat(sep='T')}Z", +datetime_range = { + 'from': f"{date_time_0.isoformat()}Z".replace('+00:00Z', '.000Z'), + 'to': f"{date_time_1.isoformat()}Z".replace('+00:00Z', '.000Z'), 'sort': 'time' } -console_log_query_params = { - 'lines': args.console_log_limit +console_logs_datetime_range = { + 'from': date_time_0, + 'to': date_time_1, } -console_log_time_range = { - 'from': zone_time_0, - 'to': zone_time_1, +console_logs_query_params = { + 'lines': args.console_logs_limit } print('Query Period:') -print(f' From: {date_time_0}') -print(f' To: {date_time_1}') +print(' From: %s' % datetime_range['from']) +print(' To: %s' % datetime_range['to']) print() # Calculon Compute! @@ -208,13 +207,13 @@ def profile_log(detail: str, state: str, initialize=False): outer_futures = [] with concurrent.futures.ThreadPoolExecutor(OUTER_CONCURRENY) as executor: - if not args.no_audit_events: + if args.audit_events: print('Collecting Audits') print() for this_audit_type in pc_api.compute_audit_types(): outer_futures.append(executor.submit( - #process_audit_events(this_audit_type, audit_query_params) - process_audit_events, this_audit_type, audit_query_params + # aka: process_audit_events(this_audit_type, datetime_range) + process_audit_events, this_audit_type, datetime_range ) ) concurrent.futures.wait(outer_futures) @@ -224,8 +223,8 @@ def profile_log(detail: str, state: str, initialize=False): print('Collecting Host Forensic Activity Audits (high-volume/time-intensive, please wait)') print() outer_futures.append(executor.submit( - #process_host_forensic_activities(audit_query_params) - process_host_forensic_activities, audit_query_params + # aka: process_host_forensic_activities(datetime_range) + process_host_forensic_activities, datetime_range ) ) print() @@ -234,18 +233,18 @@ def profile_log(detail: str, state: str, initialize=False): print('Collecting Console History') print() outer_futures.append(executor.submit( - #process_console_history(audit_query_params) - process_console_history, audit_query_params + # aka: process_console_history(datetime_range) + process_console_history, datetime_range ) ) print() if args.console_logs: - print(f'Collecting Console History (Log Limit: {args.console_log_limit})') + print(f'Collecting Console Logs (Limit: {args.console_logs_limit})') print() outer_futures.append(executor.submit( - #process_console_logs(console_log_query_params, console_log_time_range) - process_console_logs, console_log_query_params, console_log_time_range + # aka: process_console_logs(console_logs_query_params, console_logs_datetime_range) + process_console_logs, console_logs_query_params, console_logs_datetime_range ) ) print() From e55c713db7622bdc71fb377ab0f3ec103f2e1e14 Mon Sep 17 00:00:00 2001 From: tkishel Date: Mon, 28 Nov 2022 13:53:11 -0800 Subject: [PATCH 2/4] a none respose is not an error --- prismacloud/api/code_security/code_security.py | 13 ++++++++----- prismacloud/api/compute/_audits.py | 10 ++++++---- prismacloud/api/compute/compute.py | 13 ++++++++----- prismacloud/api/posture/posture.py | 13 ++++++++----- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/prismacloud/api/code_security/code_security.py b/prismacloud/api/code_security/code_security.py index b28cfd0..22d5b5b 100644 --- a/prismacloud/api/code_security/code_security.py +++ b/prismacloud/api/code_security/code_security.py @@ -34,6 +34,9 @@ def execute_code_security(self, action, endpoint, query_params=None, body_params url = 'https://%s/%s' % (self.api, endpoint) if self.token: request_headers['authorization'] = self.token + self.debug_print('API URL: %s' % url) + self.debug_print('API Query Params: %s' % query_params) + self.debug_print('API Body Params: %s' % body_params_json) api_response = requests.request(action, url, headers=request_headers, params=query_params, data=body_params_json, verify=self.verify, timeout=self.timeout) self.debug_print('API Response Status Code: %s' % api_response.status_code) if api_response.status_code in self.retry_status_codes: @@ -49,11 +52,11 @@ def execute_code_security(self, action, endpoint, query_params=None, body_params return api_response.content.decode('utf-8') try: result = json.loads(api_response.content) - if result is None: - self.logger.error('JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) - if force: - return results # or continue - self.error_and_exit(api_response.status_code, 'JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) + #if result is None: + # self.logger.error('JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) + # if force: + # return results # or continue + # self.error_and_exit(api_response.status_code, 'JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) except ValueError: self.logger.error('JSON raised ValueError, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) if force: diff --git a/prismacloud/api/compute/_audits.py b/prismacloud/api/compute/_audits.py index 2834927..9b78bc1 100644 --- a/prismacloud/api/compute/_audits.py +++ b/prismacloud/api/compute/_audits.py @@ -10,7 +10,7 @@ class AuditsPrismaCloudAPIComputeMixin: # Reference: https://prisma.pan.dev/api/cloud/cwpp/audits def audits_list_read(self, audit_type='incidents', query_params=None): - audits = self.execute_compute('GET', 'api/v1/audits/%s?' % audit_type, query_params=query_params, paginated=True) + audits = self.execute_compute('GET', 'api/v1/audits/%s' % audit_type, query_params=query_params, paginated=True) return audits def audits_ack_incident(self, incident_id, ack_status = True): @@ -29,13 +29,15 @@ def compute_audit_types(): 'access', 'admission', 'firewall/app/app-embedded', - 'firewall/app/container', + 'firewall/app/app-embedded', + 'firewall/network/container', 'kubernetes', 'runtime/app-embedded', 'runtime/container', 'trust', # Hosts 'firewall/app/host', + 'firewall/network/host', 'runtime/file-integrity', 'runtime/host', 'runtime/log-inspection', @@ -49,11 +51,11 @@ def compute_audit_types(): # Hosts > Host Activities def host_forensic_activities_list_read(self, query_params=None): - audits = self.execute_compute('GET', 'api/v1/forensic/activities?', query_params=query_params, paginated=True) + audits = self.execute_compute('GET', 'api/v1/forensic/activities', query_params=query_params, paginated=True) return audits # Compute > Manage > History def console_history_list_read(self, query_params=None): - logs = self.execute_compute('GET', 'api/v1/audits/mgmt?', query_params=query_params) + logs = self.execute_compute('GET', 'api/v1/audits/mgmt', query_params=query_params) return logs diff --git a/prismacloud/api/compute/compute.py b/prismacloud/api/compute/compute.py index 3a80da0..a63d961 100644 --- a/prismacloud/api/compute/compute.py +++ b/prismacloud/api/compute/compute.py @@ -53,6 +53,9 @@ def execute_compute(self, action, endpoint, query_params=None, body_params=None, else: # Authenticate via CWP request_headers['Authorization'] = "Bearer %s" % self.token + self.debug_print('API URL: %s' % url) + self.debug_print('API Query Params: %s' % query_params) + self.debug_print('API Body Params: %s' % body_params_json) api_response = requests.request(action, url, headers=request_headers, params=query_params, data=body_params_json, verify=self.verify, timeout=self.timeout) self.debug_print('API Response Status Code: (%s)' % api_response.status_code) if api_response.status_code in self.retry_status_codes: @@ -68,11 +71,11 @@ def execute_compute(self, action, endpoint, query_params=None, body_params=None, return api_response.content.decode('utf-8') try: result = json.loads(api_response.content) - if result is None: - self.logger.error('JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) - if force: - return results # or continue - self.error_and_exit(api_response.status_code, 'JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) + #if result is None: + # self.logger.error('JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) + # if force: + # return results # or continue + # self.error_and_exit(api_response.status_code, 'JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) except ValueError: self.logger.error('JSON raised ValueError, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) if force: diff --git a/prismacloud/api/posture/posture.py b/prismacloud/api/posture/posture.py index e56ed1f..4a531ae 100644 --- a/prismacloud/api/posture/posture.py +++ b/prismacloud/api/posture/posture.py @@ -85,6 +85,9 @@ def execute(self, action, endpoint, query_params=None, body_params=None, request if self.token: request_headers['x-redlock-auth'] = self.token body_params_json = json.dumps(body_params) + self.debug_print('API URL: %s' % url) + self.debug_print('API Query Params: %s' % query_params) + self.debug_print('API Body Params: %s' % body_params_json) api_response = requests.request(action, url, headers=request_headers, params=query_params, data=body_params_json, verify=self.verify, timeout=self.timeout) if self.debug: print('API Response Status Code: %s' % api_response.status_code) @@ -101,11 +104,11 @@ def execute(self, action, endpoint, query_params=None, body_params=None, request return api_response.content.decode('utf-8') try: result = json.loads(api_response.content) - if result is None: - self.logger.error('JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) - if force: - return results # or continue - self.error_and_exit(api_response.status_code, 'JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) + #if result is None: + # self.logger.error('JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) + # if force: + # return results # or continue + # self.error_and_exit(api_response.status_code, 'JSON returned None, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) except ValueError: self.logger.error('JSON raised ValueError, API: (%s) with query params: (%s) and body params: (%s) parsing response: (%s)' % (url, query_params, body_params, api_response.content)) if force: From 4ec3cc174f78edb6fd27f230dc21dcdc425f7bdc Mon Sep 17 00:00:00 2001 From: tkishel Date: Mon, 28 Nov 2022 13:53:45 -0800 Subject: [PATCH 3/4] Version 5.0.8 --- prismacloud/api/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prismacloud/api/version.py b/prismacloud/api/version.py index d5cac17..e7bfaba 100644 --- a/prismacloud/api/version.py +++ b/prismacloud/api/version.py @@ -1,3 +1,3 @@ """ version file """ -version = '5.0.7' +version = '5.0.8' From 3e4b0b30d3de1bf5b7887e56af047b114bef6ca9 Mon Sep 17 00:00:00 2001 From: tkishel Date: Mon, 28 Nov 2022 13:55:21 -0800 Subject: [PATCH 4/4] pylint --- scripts/pcs_compute_forward_to_siem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/pcs_compute_forward_to_siem.py b/scripts/pcs_compute_forward_to_siem.py index 1bd4866..42adf49 100644 --- a/scripts/pcs_compute_forward_to_siem.py +++ b/scripts/pcs_compute_forward_to_siem.py @@ -8,16 +8,16 @@ import json import inspect import time -import sys from pathlib import Path from typing import Union -import requests from datetime import datetime, timedelta, timezone from dateutil import parser, tz - + +import requests + # pylint: disable=import-error from prismacloud.api import pc_api, pc_utility