[salesforce] Implement apex data stream for Salesforce

packages/salesforce/changelog.yml

@@ -1,5 +1,10 @@
 # newer versions go on top
 
+- version: 0.3.0
+  changes:
+    - description: Salesforce integration package with "apex" data stream.
+      link: https://github.com/elastic/integrations/pull/4410
+      type: enhancement
 - version: 0.2.0
   changes:
     - description: Salesforce integration package with "logout_rest" data stream.

packages/salesforce/data_stream/apex/_dev/test/pipeline/test-apex.log

@@ -0,0 +1,2 @@
+{"CALLOUT_TIME":"0","CLIENT_IP":"81.2.69.142","CPU_TIME":"174","DB_TOTAL_TIME":"12","ENTRY_POINT":"Salesforce Migration","EVENT_TYPE":"ApexExecution","EXEC_TIME":"25","IS_LONG_RUNNING_REQUEST":"0","LOGIN_KEY":"QfNecrLXSII6fsBq","NUMBER_SOQL_QUERIES":"1","ORGANIZATION_ID":"00D5j000000VI3n","QUIDDITY":"QTXF","REQUEST_ID":"4lc-XEdmLH121-l1cJ0001","RUN_TIME":"291","SESSION_KEY":"d7DEqANa7nNZZVD","TIMESTAMP":"20221001170000.885","TIMESTAMP_DERIVED":"2022-10-01T17:00:00.885Z","URI":"/home/home.jsp","URI_ID_DERIVED":"AQB00530000009M943","USER_ID":"0055j000000u123","USER_ID_DERIVED":"0055j000000utlP001"}
+{"CLIENT_IP":"81.2.69.142","CPU_TIME":"10","EVENT_TYPE":"ApexCallout","LOGIN_KEY":"","METHOD":"GET","ORGANIZATION_ID":"00D5j000000XY0P","REQUEST_ID":"00D5j00XYZBDS","REQUEST_SIZE":"10","RESPONSE_SIZE":"61","RUN_TIME":"237","SESSION_KEY":"","SUCCESS":"1","TIME":"227","TIMESTAMP":"20220930055317.461","TIMESTAMP_DERIVED":"2022-09-30T05:53:17.461Z","TYPE":"OData","URI":"CALLOUT-LOG","URI_ID_DERIVED":"","URL":"https://temp.website.here.sh/odata/Random","USER_ID":"0055j000000PQ01","USER_ID_DERIVED":"0055j000000utll12"}

packages/salesforce/data_stream/apex/_dev/test/pipeline/test-apex.log-expected.json

@@ -0,0 +1,138 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-10-01T17:00:00.885Z",
+            "ecs": {
+                "version": "8.5.0"
+            },
+            "event": {
+                "action": "apex-execution",
+                "dataset": "salesforce.apex",
+                "duration": 25,
+                "kind": "event",
+                "module": "salesforce",
+                "original": "{\"CALLOUT_TIME\":\"0\",\"CLIENT_IP\":\"81.2.69.142\",\"CPU_TIME\":\"174\",\"DB_TOTAL_TIME\":\"12\",\"ENTRY_POINT\":\"Salesforce Migration\",\"EVENT_TYPE\":\"ApexExecution\",\"EXEC_TIME\":\"25\",\"IS_LONG_RUNNING_REQUEST\":\"0\",\"LOGIN_KEY\":\"QfNecrLXSII6fsBq\",\"NUMBER_SOQL_QUERIES\":\"1\",\"ORGANIZATION_ID\":\"00D5j000000VI3n\",\"QUIDDITY\":\"QTXF\",\"REQUEST_ID\":\"4lc-XEdmLH121-l1cJ0001\",\"RUN_TIME\":\"291\",\"SESSION_KEY\":\"d7DEqANa7nNZZVD\",\"TIMESTAMP\":\"20221001170000.885\",\"TIMESTAMP_DERIVED\":\"2022-10-01T17:00:00.885Z\",\"URI\":\"/home/home.jsp\",\"URI_ID_DERIVED\":\"AQB00530000009M943\",\"USER_ID\":\"0055j000000u123\",\"USER_ID_DERIVED\":\"0055j000000utlP001\"}",
+                "url": "/home/home.jsp"
+            },
+            "related": {
+                "ip": [
+                    "81.2.69.142"
+                ]
+            },
+            "salesforce": {
+                "apex": {
+                    "access_mode": "REST",
+                    "callout_time": 0,
+                    "cpu_time": 174,
+                    "db_time": {
+                        "total": 12
+                    },
+                    "entry_point": "Salesforce Migration",
+                    "event_type": "ApexExecution",
+                    "is_long_running_request": "0",
+                    "login_key": "QfNecrLXSII6fsBq",
+                    "organization_id": "00D5j000000VI3n",
+                    "quiddity": "QTXF",
+                    "request_id": "4lc-XEdmLH121-l1cJ0001",
+                    "run_time": 291,
+                    "soql_queries": {
+                        "count": "1"
+                    },
+                    "uri_derived_id": "AQB00530000009M943",
+                    "user_id_derived": "0055j000000utlP001"
+                }
+            },
+            "source": {
+                "geo": {
+                    "city_name": "London",
+                    "continent_name": "Europe",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lat": 51.5142,
+                        "lon": -0.0931
+                    },
+                    "region_iso_code": "GB-ENG",
+                    "region_name": "England"
+                },
+                "ip": "81.2.69.142"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "id": "0055j000000u123"
+            }
+        },
+        {
+            "@timestamp": "2022-09-30T05:53:17.461Z",
+            "ecs": {
+                "version": "8.5.0"
+            },
+            "event": {
+                "action": "apex-callout",
+                "category": [
+                    "network"
+                ],
+                "dataset": "salesforce.apex",
+                "duration": 227,
+                "kind": "event",
+                "module": "salesforce",
+                "original": "{\"CLIENT_IP\":\"81.2.69.142\",\"CPU_TIME\":\"10\",\"EVENT_TYPE\":\"ApexCallout\",\"LOGIN_KEY\":\"\",\"METHOD\":\"GET\",\"ORGANIZATION_ID\":\"00D5j000000XY0P\",\"REQUEST_ID\":\"00D5j00XYZBDS\",\"REQUEST_SIZE\":\"10\",\"RESPONSE_SIZE\":\"61\",\"RUN_TIME\":\"237\",\"SESSION_KEY\":\"\",\"SUCCESS\":\"1\",\"TIME\":\"227\",\"TIMESTAMP\":\"20220930055317.461\",\"TIMESTAMP_DERIVED\":\"2022-09-30T05:53:17.461Z\",\"TYPE\":\"OData\",\"URI\":\"CALLOUT-LOG\",\"URI_ID_DERIVED\":\"\",\"URL\":\"https://temp.website.here.sh/odata/Random\",\"USER_ID\":\"0055j000000PQ01\",\"USER_ID_DERIVED\":\"0055j000000utll12\"}",
+                "outcome": "success",
+                "type": [
+                    "connection"
+                ],
+                "url": "https://temp.website.here.sh/odata/Random"
+            },
+            "http": {
+                "request": {
+                    "bytes": 10,
+                    "method": "GET"
+                },
+                "response": {
+                    "bytes": 61
+                }
+            },
+            "related": {
+                "ip": [
+                    "81.2.69.142"
+                ]
+            },
+            "salesforce": {
+                "apex": {
+                    "access_mode": "REST",
+                    "cpu_time": 10,
+                    "event_type": "ApexCallout",
+                    "organization_id": "00D5j000000XY0P",
+                    "request_id": "00D5j00XYZBDS",
+                    "run_time": 237,
+                    "type": "OData",
+                    "uri": "CALLOUT-LOG",
+                    "user_id_derived": "0055j000000utll12"
+                }
+            },
+            "source": {
+                "geo": {
+                    "city_name": "London",
+                    "continent_name": "Europe",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lat": 51.5142,
+                        "lon": -0.0931
+                    },
+                    "region_iso_code": "GB-ENG",
+                    "region_name": "England"
+                },
+                "ip": "81.2.69.142"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "id": "0055j000000PQ01"
+            }
+        }
+    ]
+}

packages/salesforce/data_stream/apex/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,5 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event

packages/salesforce/data_stream/apex/_dev/test/system/test-default-config.yml

@@ -0,0 +1,12 @@
+input: httpjson
+service: salesforce
+vars:
+  instance_url: http://{{Hostname}}:{{Port}}
+  client_id: temp_client_id
+  client_secret: forty_characters_long_secret_key
+  username: temp_user
+  password: temp_password
+  token_url: http://{{Hostname}}:{{Port}}/services/oauth2/token
+data_stream:
+  vars:
+    preserve_original_event: true

packages/salesforce/data_stream/apex/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,47 @@
+config_version: 2
+interval: {{period}}
+request.method: GET
+auth.oauth2:
+  enabled: true
+  client.id: {{client_id}}
+  client.secret: {{client_secret}}
+  token_url: {{token_url}}
+  user: {{username}}
+  password: {{password}}
+request.url: {{instance_url}}/services/data/v54.0/query?q=apex
+request.transforms:
+  - set:
+      target: url.params.q
+      value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND LogDate > [[.cursor.last_published_apex]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST"
+      default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST"
+response.pagination:
+- set:
+    target: url.value
+    value: '[[if (ne .last_response.body.done true)]]{{instance_url}}[[.last_response.body.nextRecordsUrl]][[end]]'
+    fail_on_template_error: true
+chain:
+  - step:
+      request.url: {{instance_url}}/services/data/v54.0/sobjects/EventLogFile/$.records[:].Id/LogFile
+      request.method: GET
+      replace: $.records[:].Id
+cursor:
+  last_published_apex:
+    value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]'
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+- add_fields:
+    target: salesforce
+    fields:
+      instance_url: {{instance_url}}
+{{#if processors}}
+{{processors}}
+{{/if}}