From 61255b76531dfc95b6fbb8a9f4e9d434846d051f Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 15 Apr 2024 14:57:28 -0700 Subject: [PATCH] HAProxy Flint Integration (#1692) * Add draft queries for haproxy Signed-off-by: Simeon Widdis * OTEL-ify more fields Signed-off-by: Simeon Widdis * Fix event domain and add config updates Signed-off-by: Simeon Widdis * Rename remaining fields Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../haproxy/assets/create_mv-1.0.0.sql | 150 ++++++++++++++++++ .../haproxy/assets/create_table-1.0.0.sql | 7 + .../repository/haproxy/haproxy-1.0.0.json | 30 +++- 3 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql new file mode 100644 index 000000000..5e2ce8047 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql @@ -0,0 +1,150 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS +SELECT + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 1 + ) AS `connection.source.ip`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 2 + ) AS int) AS `connection.source.port`, + to_timestamp(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 3 + ), "dd/MMM/yyyy:HH:mm:ss.SSS") AS `@timestamp`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 4 + ) AS `haproxy.service_name`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 5 + ) AS `haproxy.backend_name`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 6 + ) AS `haproxy.proxy_name`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 7 + ) AS int) AS `haproxy.time_total_ms`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 8 + ) AS int) AS `haproxy.time_queue_ms`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 9 + ) AS int) AS `haproxy.time_connection_ms`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 10 + ) AS int) AS `haproxy.time_server_ms`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 11 + ) AS int) AS `haproxy.time_active_ms`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 12 + ) AS int) AS `http.response.status_code`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 13 + ) AS int) AS `http.response.bytes`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 14 + ) AS `http.request.captured_cookie`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 15 + ) AS `http.response.captured_cookie`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 16 + ) AS termination_state, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 17 + ) AS int) AS `haproxy.active_connections`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 18 + ) AS int) AS `haproxy.frontend_connections`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 19 + ) AS int) AS `haproxy.backend_connections`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 20 + ) AS int) AS `haproxy.server_connections`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 21 + ) AS int) AS `haproxy.retries`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 22 + ) AS int) AS `haproxy.server_queue`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 23 + ) AS int) AS `haproxy.backend_queue`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 24 + ) AS `http.request.captured_headers`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 25 + ) AS `http.response.captured_headers`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 26 + ) AS `http.request.method`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 27 + ) AS `http.url`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 28 + ) AS `http.flavor`, + 'haproxy.access' AS `event.domain` +FROM {table_name} +WITH ( + auto_refresh = true, + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +); diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql new file mode 100644 index 000000000..5e749e099 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql @@ -0,0 +1,7 @@ +CREATE EXTERNAL TABLE {table_name} ( + record STRING +) USING csv +LOCATION '{s3_bucket_location}' +OPTIONS ( + sep='\0x1E' +); diff --git a/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json b/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json index c88e42c31..131a3f696 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json @@ -8,6 +8,20 @@ "labels": ["Observability", "Logs"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/haproxy/info", + "workflows": [ + { + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true + }, + { + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false + } + ], "statics": { "logo": { "annotation": "HAProxy Logo", @@ -43,7 +57,21 @@ "name": "haproxy", "version": "1.0.0", "extension": "ndjson", - "type": "savedObjectBundle" + "type": "savedObjectBundle", + "workflows": ["dashboards"] + }, + { + "name": "create_table", + "version": "1.0.0", + "extension": "sql", + "type": "query" + }, + { + "name": "create_mv", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["dashboards"] } ], "sampleData": {