Skip to content

Commit

Permalink
Add VPC S3 integration
Browse files Browse the repository at this point in the history
Signed-off-by: Simeon Widdis <sawiddis@amazon.com>
  • Loading branch information
Swiddis committed Oct 31, 2023
1 parent fb011cf commit 882b441
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 2 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
CREATE MATERIALIZED VIEW {table_name}_mview AS
SELECT
CAST(FROM_UNIXTIME(start) AS TIMESTAMP) as `@timestamp`,
version as `aws.vpc.version`,
account_id as `aws.vpc.account-id`,
interface_id as `aws.vpc.interface-id`,
srcaddr as `aws.vpc.srcaddr`,
dstaddr as `aws.vpc.dstaddr`,
CAST(srcport AS LONG) as `aws.vpc.srcport`,
CAST(dstport AS LONG) as `aws.vpc.dstport`,
protocol as `aws.vpc.protocol`,
CAST(packets AS LONG) as `aws.vpc.packets`,
CAST(bytes AS LONG) as `aws.vpc.bytes`,
CAST(FROM_UNIXTIME(start) AS TIMESTAMP) as `aws.vpc.start`,
CAST(FROM_UNIXTIME(end) AS TIMESTAMP) as `aws.vpc.end`,
action as `aws.vpc.action`,
log_status as `aws.vpc.log-status`,
CASE
WHEN regexp(dstaddr, '(10\\..*)|(192\\.168\\..*)|(172\\.1[6-9]\\..*)|(172\\.2[0-9]\\..*)|(172\\.3[0-1]\\.*)')
THEN 'ingress'
ELSE 'egress'
END AS `aws.vpc.flow-direction`
FROM
{table_name};
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
version INT,
account_id STRING,
interface_id STRING,
srcaddr STRING,
dstaddr STRING,
srcport STRING,
dstport STRING,
protocol STRING,
packets STRING,
bytes STRING,
start BIGINT,
end BIGINT,
action STRING,
log_status STRING
) USING csv
LOCATION '{s3_bucket_location}'
OPTIONS (
sep=' '
);
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
REFRESH MATERIALIZED VIEW {table_name}_mview
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"description": "AWS VPC Flow log collector",
"license": "Apache-2.0",
"type": "logs_vpc",
"labels": ["Observability", "Logs", "AWS", "Cloud"],
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
"author": "Haidong Wang",
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
"statics": {
Expand Down Expand Up @@ -46,7 +46,24 @@
"savedObjects": {
"name": "aws_vpc_flow",
"version": "1.0.0"
}
},
"queries": [
{
"name": "create_table_vpc",
"version": "1.0.0",
"language": "sql"
},
{
"name": "create_mv_vpc",
"version": "1.0.0",
"language": "sql"
},
{
"name": "refresh_mv_vpc",
"version": "1.0.0",
"language": "sql"
}
]
},
"sampleData": {
"path": "sample.json"
Expand Down

0 comments on commit 882b441

Please sign in to comment.