cel: new generic integration

.github/CODEOWNERS

@@ -36,6 +36,7 @@
 /packages/carbonblack_edr @elastic/security-external-integrations
 /packages/cassandra @elastic/obs-service-integrations
 /packages/cef @elastic/security-external-integrations
+/packages/cel @elastic/security-external-integrations
 /packages/ceph @elastic/obs-service-integrations
 /packages/checkpoint @elastic/security-external-integrations
 /packages/cisco_aironet @elastic/security-external-integrations

packages/cel/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "2.3"
+services:
+  cel:
+    image: docker.elastic.co/observability/stream:v0.7.0
+    ports:
+      - 8080
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8080
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml

packages/cel/_dev/deploy/docker/files/config.yml

@@ -0,0 +1,68 @@
+rules:
+  - path: /test/api
+    methods: [GET]
+    responses:
+      - status_code: 200
+        body: |
+          {"message": "success"}
+  #basic authentication is not supported until 8.7.1:
+  # - path: /testbasicauth/api
+  #   methods: [GET]
+  #   request_headers:
+  #     Authorization: "Basic dGVzdDp0ZXN0"
+  #   responses:
+  #     - status_code: 200
+  #       body: |
+  #         {"message": "success"}
+  #
+  #Associated test: add as data_stream/generic/_dev/test/system:
+  # input: cel
+  # service: cel
+  # data_stream:
+  #   vars:
+  #     data_stream.dataset: cel.generic
+  #     username: test
+  #     password: test
+  #     resource_url: http://{{Hostname}}:{{Port}}/testbasicauth/api
+  #     program: |
+  #       bytes(get(state.url).Body).as(body, {"events": [body.decode_json()]})
+
+
+  - path: /testoauth/token
+    methods: [POST]
+    query_params:
+      grant_type: client_credentials
+    request_headers:
+      Content-Type:
+        - application/x-www-form-urlencoded
+      Authorization:
+        - "Basic dGVzdDp0ZXN0"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {"access_token": "testaccess","token_type": "Bearer","expires_in": 172799,"refresh_token": "testrefresh"}
+  - path: /testoauth/scopedtoken
+    methods: [POST]
+    query_params:
+      scope: token_scope
+    request_headers:
+      Content-Type:
+        - application/x-www-form-urlencoded
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {"access_token": "testaccess","token_type": "Bearer","expires_in": 172799,"refresh_token": "testrefresh"}
+  - path: /testoauth/api
+    methods: [GET]
+    request_headers:
+      Authorization: "Bearer testaccess"
+    responses:
+      - status_code: 200
+        body: |-
+          {"message": "success"}

packages/cel/_dev/test/system/test-get-config.yml

@@ -0,0 +1,8 @@
+vars:
+  resource_url: http://{{Hostname}}:{{Port}}/test/api
+  program: |
+    bytes(
+      get(state.url).Body).as(body, {
+        "events": [body.decode_json()]
+      }
+    )

packages/cel/_dev/test/system/test-oauth-config.yml

@@ -0,0 +1,11 @@
+vars:
+  oauth_id: test
+  oauth_secret: test
+  oauth_token_url: http://{{Hostname}}:{{Port}}/testoauth/token
+  resource_url: http://{{Hostname}}:{{Port}}/testoauth/api
+  program: |
+    bytes(
+      get(state.url).Body).as(body, {
+        "events": [body.decode_json()]
+      }
+    )

packages/cel/_dev/test/system/test-oauth-scope-config.yml

@@ -0,0 +1,12 @@
+vars:
+  oauth_id: test
+  oauth_secret: test
+  oauth_scopes: ["token_scope"]
+  oauth_token_url: http://{{Hostname}}:{{Port}}/testoauth/scopedtoken
+  resource_url: http://{{Hostname}}:{{Port}}/testoauth/api
+  program: |
+    bytes(
+      get(state.url).Body).as(body, {
+        "events": [body.decode_json()]
+      }
+    )

packages/cel/agent/input/input.yml.hbs

@@ -0,0 +1,128 @@
+data_stream:
+  dataset: {{data_stream.dataset}}
+interval: {{resource_interval}}
+
+program: {{escape_string program}}
+
+{{#if state}}
+state:
+  {{state}}
+{{/if}}
+
+{{#if regexp}}
+regexp:
+  {{regexp}}
+{{/if}}
+
+{{#if username}}
+auth.basic.user: {{username}}
+{{/if}}
+{{#if password}}
+auth.basic.password: {{password}}
+{{/if}}
+
+{{#if pipeline}}
+pipeline: {{pipeline}}
+{{/if}}
+
+{{#unless username}}
+{{#unless password}}
+{{#if oauth_id}}
+auth.oauth2.client.id: {{oauth_id}}
+{{/if}}
+{{#if oauth_secret}}
+auth.oauth2.client.secret: {{oauth_secret}}
+{{/if}}
+{{#if oauth_token_url}}
+auth.oauth2.token_url: {{oauth_token_url}}
+{{/if}}
+{{#if oauth_provider}}
+auth.oauth2.provider: {{oauth_provider}}
+{{/if}}
+{{#if oauth_scopes}}
+auth.oauth2.scopes:
+{{#each oauth_scopes as |scope|}}
+  - {{scope}}
+{{/each}}
+{{/if}}
+{{#if oauth_google_credentials_file}}
+auth.oauth2.google.credentials_file: {{oauth_google_credentials_file}}
+{{/if}}
+{{#if oauth_google_credentials_json}}
+auth.oauth2.google.credentials_json: '{{oauth_google_credentials_json}}'
+{{/if}}
+{{#if oauth_google_jwt_file}}
+auth.oauth2.google.jwt_file: {{oauth_google_jwt_file}}
+{{/if}}
+{{#if oauth_google_jwt_json}}
+auth.oauth2.google.jwt_json: {{oauth_google_jwt_json}}
+{{/if}}
+{{#if oauth_google_delegated_account}}
+auth.oauth2.google.delegated_account: {{oauth_google_delegated_account}}
+{{/if}}
+{{#if oauth_azure_tenant_id}}
+auth.oauth2.azure.tenant_id: {{oauth_azure_tenant_id}}
+{{/if}}
+{{#if oauth_azure_resource}}
+auth.oauth2.azure.resource: {{oauth_azure_resource}}
+{{/if}}
+{{#if oauth_endpoint_params}}
+auth.oauth2.endpoint_params: 
+  {{oauth_endpoint_params}}
+{{/if}}
+{{/unless}}
+{{/unless}}
+
+resource.url: {{resource_url}}
+{{#if resource_ssl}}
+resource.ssl: 
+  {{resource_ssl}}
+{{/if}}
+{{#if resource_proxy_url}}
+resource.proxy_url: {{resource_proxy_url}}
+{{/if}}
+{{#if resource_retry_max_attempts}}
+resource.retry.max_attempts: {{resource_retry_max_attempts}}
+{{/if}}
+{{#if resource_retry_wait_min}}
+resource.retry.wait_min: {{resource_retry_wait_min}}
+{{/if}}
+{{#if resource_retry_wait_max}}
+resource.retry.wait_max: {{resource_retry_wait_max}}
+{{/if}}
+{{#if resource_redirect_forward_headers}}
+resource.redirect.forward_headers: {{resource_redirect_forward_headers}}
+{{/if}}
+{{#if resource_redirect_headers_ban_list}}
+resource.redirect.headers_ban_list:
+{{#each resource_redirect_headers_ban_list as |item|}}
+  - {{item}}
+{{/each}}
+{{/if}}
+{{#if resource_redirect_max_redirects}}
+resource.redirect.max_redirects: {{resource_redirect_max_redirects}}
+{{/if}}
+{{#if resource_rate_limit_limit}}
+resource.rate_limit.limit: {{resource_rate_limit_limit}}
+{{/if}}
+{{#if resource_rate_limit_burst}}
+resource.rate_limit.burst: {{resource_rate_limit_burst}}
+{{/if}}
+
+{{#if enable_request_tracer}}
+resource.tracer.filename: http-request-trace-cel.ndjson
+{{/if}}
+
+{{#if tags}}
+tags:
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{/if}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/cel/changelog.yml

@@ -0,0 +1,5 @@
+- version: "0.1.0"
+  changes:
+    - description: Initial Implementation
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5539

packages/cel/docs/README.md

@@ -0,0 +1,19 @@
+# CEL Custom API input integration
+
+The CEL custom API input integration is used to ingest data from custom HTTP and local file-system APIs that do not currently have an existing integration.
+
+The input itself supports making both HTTP requests and file-system read operations, and keeping running and persistent state on information from the last collected events. The input performs [Common Expression Language](https://opensource.google.com/projects/cel) with a [set of standard extensions](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_cel_extension_libraries) to both obtain input data from the API and then process the data into events that are published to Elasticsearch.
+
+## Configuration
+
+The full documentation for the input are currently available [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html).
+
+The most commonly used configuration options are available on the main integration page, while more advanced and customizable options currently resides under the "Advanced options" part of the integration settings page.
+
+Configuration is split into two main parts: Program and State, with an associated named Resource.
+
+The program is a CEL program that will obtain data from the API either by an HTTP request or a local file-system read operation, and then transform the data into a set of events and cursor states. The CEL environment that the program is run in is provided with an HTTP client that has been set-up with the user-specified authentication, proxy, rate limit and other options, and with a set of user-defined regular expressions that are available to use during execution.
+
+The state is passed into the program on start of execution and may contain configuration details not included in the standard set of options. The CEL program return value will be the state used during the next cycle of CEL execution, and will include the events to be published to Elasticsearch and then removed from the state. State values in general will not persist over restarts, but it may contain a cursor that is persisted. The cursor part of state is used when there is a need to keep long-lived state that will persist over restarts.
+
+The named resource is a string value that the CEL program can use to identify the location of the API and will usually either be a URL or a file path. It is included in the state as the `state.url`. CEL programs are not required to make use of this configuration, but its presence is required in the configuration.

packages/cel/fields/input.yml

@@ -0,0 +1,28 @@
+- name: "@timestamp"
+  external: ecs
+- name: ecs.version
+  external: ecs
+- name: message
+  external: ecs
+- name: input.name
+  type: constant_keyword
+- name: input.type
+  type: keyword
+- name: data_stream.type
+  external: ecs
+  type: constant_keyword
+- name: data_stream.dataset
+  external: ecs
+  type: constant_keyword
+- name: data_stream.namespace
+  external: ecs
+  type: constant_keyword
+- name: event.module
+  external: ecs
+  type: constant_keyword
+  value: cel
+- name: event.dataset
+  external: ecs
+  type: constant_keyword
+- name: tags
+  external: ecs