diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql new file mode 100644 index 000000000..2612638e7 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql @@ -0,0 +1,24 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS + SELECT + CAST(FROM_UNIXTIME(start) AS TIMESTAMP) as `@timestamp`, + version as `aws.vpc.version`, + account_id as `aws.vpc.account-id`, + interface_id as `aws.vpc.interface-id`, + srcaddr as `aws.vpc.srcaddr`, + dstaddr as `aws.vpc.dstaddr`, + CAST(srcport AS LONG) as `aws.vpc.srcport`, + CAST(dstport AS LONG) as `aws.vpc.dstport`, + protocol as `aws.vpc.protocol`, + CAST(packets AS LONG) as `aws.vpc.packets`, + CAST(bytes AS LONG) as `aws.vpc.bytes`, + CAST(FROM_UNIXTIME(start) AS TIMESTAMP) as `aws.vpc.start`, + CAST(FROM_UNIXTIME(end) AS TIMESTAMP) as `aws.vpc.end`, + action as `aws.vpc.action`, + log_status as `aws.vpc.log-status`, + CASE + WHEN regexp(dstaddr, '(10\\..*)|(192\\.168\\..*)|(172\\.1[6-9]\\..*)|(172\\.2[0-9]\\..*)|(172\\.3[0-1]\\.*)') + THEN 'ingress' + ELSE 'egress' + END AS `aws.vpc.flow-direction` +FROM + {table_name}; diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql new file mode 100644 index 000000000..bb73fa934 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql @@ -0,0 +1,20 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( + version INT, + account_id STRING, + interface_id STRING, + srcaddr STRING, + dstaddr STRING, + srcport STRING, + dstport STRING, + protocol STRING, + packets STRING, + bytes STRING, + start BIGINT, + end BIGINT, + action STRING, + log_status STRING +) USING csv +LOCATION '{s3_bucket_location}' +OPTIONS ( + sep=' ' +); diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql new file mode 100644 index 000000000..458202eae --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql @@ -0,0 +1 @@ +REFRESH MATERIALIZED VIEW {table_name}_mview diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index 1d92e2665..2a2fbf3f5 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS VPC Flow log collector", "license": "Apache-2.0", "type": "logs_vpc", - "labels": ["Observability", "Logs", "AWS", "Cloud"], + "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "Haidong Wang", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info", "statics": { @@ -46,7 +46,24 @@ "savedObjects": { "name": "aws_vpc_flow", "version": "1.0.0" - } + }, + "queries": [ + { + "name": "create_table_vpc", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "create_mv_vpc", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "refresh_mv_vpc", + "version": "1.0.0", + "language": "sql" + } + ] }, "sampleData": { "path": "sample.json" diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql new file mode 100644 index 000000000..8b29fdf0b --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql @@ -0,0 +1,10 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS +SELECT + to_timestamp(trim(BOTH '[]' FROM concat(time_local_1, ' ', time_local_2)), 'dd/MMM/yyyy:HH:mm:ss Z') AS `@timestamp`, + split_part (request, ' ', 1) as `http.request.method`, + split_part (request, ' ', 2) as `http.url`, + split_part (request, ' ', 3) as `http.flavor`, + status AS `http.response.status_code`, + body_bytes_sent AS `http.response.bytes`, + 'nginx.access' AS `event.domain` +FROM {table_name} diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql new file mode 100644 index 000000000..38f719421 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql @@ -0,0 +1,18 @@ +CREATE EXTERNAL TABLE {table_name} ( + remote_addr STRING, + empty_col STRING, + remote_user STRING, + time_local_1 STRING, + time_local_2 STRING, + request STRING, + status INT, + body_bytes_sent INT, + http_referer STRING, + http_user_agent STRING, + gzip_ratio STRING +) USING csv +LOCATION '{s3_bucket_location}' +OPTIONS ( + sep=' ', + nullValue='-' +) diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql new file mode 100644 index 000000000..458202eae --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql @@ -0,0 +1 @@ +REFRESH MATERIALIZED VIEW {table_name}_mview diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index 975430d0d..aadcae2f0 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -5,7 +5,7 @@ "description": "Nginx HTTP server collector", "license": "Apache-2.0", "type": "logs", - "labels": ["Observability", "Logs"], + "labels": ["Observability", "Logs", "Flint S3"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info", "statics": { @@ -38,7 +38,24 @@ "savedObjects": { "name": "nginx", "version": "1.0.0" - } + }, + "queries": [ + { + "name": "create_table", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "create_mv", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "refresh_mv", + "version": "1.0.0", + "language": "sql" + } + ] }, "sampleData": { "path": "sample.json"