feat(ec2): change log format in Vpc flow logs (#22430)

refer to #16279 and #16279 (comment).  The difference is below

- I don't think you need custom for `LogFormatField`. "custom" in the document below does not mean that the user can specify any Key. It means that you can specify any Key of "Available fields" with a space-separated string.
https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html#flow-logs-fields


fixes #19316

----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/aws-ec2/README.md

@@ -1402,6 +1402,38 @@ vpc.addFlowLog('FlowLogCloudWatch', {
 });
 ```
 
+### Custom Formatting
+
+You can also custom format flow logs.
+
+```ts
+const vpc = new ec2.Vpc(this, 'Vpc');
+
+vpc.addFlowLog('FlowLog', {
+  logFormat: [
+    ec2.LogFormat.DST_PORT,
+    ec2.LogFormat.SRC_PORT,
+  ],
+});
+
+// If you just want to add a field to the default field
+vpc.addFlowLog('FlowLog', {
+  logFormat: [
+    ec2.LogFormat.VERSION,
+    ec2.LogFormat.ALL_DEFAULT_FIELDS,
+  ],
+});
+
+// If AWS CDK does not support the new fields
+vpc.addFlowLog('FlowLog', {
+  logFormat: [
+    ec2.LogFormat.SRC_PORT,
+    ec2.LogFormat.custom('${new-field}'),
+  ],
+});
+```
+
+
 By default, the CDK will create the necessary resources for the destination. For the CloudWatch Logs destination
 it will create a CloudWatch Logs Log Group as well as the IAM role with the necessary permissions to publish to
 the log group. In the case of an S3 destination, it will create the S3 bucket.

packages/@aws-cdk/aws-ec2/lib/vpc-flow-logs.ts

@@ -401,6 +401,141 @@ export enum FlowLogMaxAggregationInterval {
 
 }
 
+/**
+ * The following table describes all of the available fields for a flow log record.
+ */
+export class LogFormat {
+  /**
+   * The VPC Flow Logs version.
+   */
+  public static readonly VERSION = new LogFormat('${version}');
+
+  /**
+   * The AWS account ID of the owner of the source network interface for which traffic is recorded.
+   */
+  public static readonly ACCOUNT_ID = new LogFormat('${account-id}');
+
+  /**
+   * The ID of the network interface for which the traffic is recorded.
+   */
+  public static readonly INTERFACE_ID = new LogFormat('${interface-id');
+
+  /**
+   * The source address for incoming traffic, or the IPv4 or IPv6 address of the network interface
+   * for outgoing traffic on the network interface.
+   */
+  public static readonly SRC_ADDR = new LogFormat('${srcaddr}');
+
+  /**
+   * The destination address for outgoing traffic, or the IPv4 or IPv6 address of the network interface
+   * for incoming traffic on the network interface.
+   */
+  public static readonly DST_ADDR = new LogFormat('${dstaddr}');
+
+  /**
+   * The source port of the traffic.
+   */
+  public static readonly SRC_PORT = new LogFormat('${srcport}');
+
+  /**
+   * The destination port of the traffic.
+   */
+  public static readonly DST_PORT = new LogFormat('${dstport}');
+
+  /**
+   * The IANA protocol number of the traffic.
+   */
+  public static readonly PROTOCOL = new LogFormat('${protocol}');
+
+  /**
+   * The number of packets transferred during the flow.
+   */
+  public static readonly PACKETS = new LogFormat('${packets}');
+
+  /**
+   * The number of bytes transferred during the flow.
+   */
+  public static readonly BYTES = new LogFormat('${bytes}');
+
+  /**
+   * The packet-level (original) source IP address of the traffic.
+   */
+  public static readonly PKT_SRC_ADDR = new LogFormat('${pkt-srcaddr}');
+
+  /**
+   * The packet-level (original) destination IP address for the traffic.
+   */
+  public static readonly PKT_DST_ADDR = new LogFormat('${pkt-dstaddr}');
+
+  /**
+   * The Region that contains the network interface for which traffic is recorded.
+   */
+  public static readonly REGION = new LogFormat('${region}');
+
+  /**
+   * The ID of the Availability Zone that contains the network interface for which traffic is recorded.
+   */
+  public static readonly AZ_ID = new LogFormat('${az-id}');
+
+  /**
+   * The type of sublocation that's returned in the sublocation-id field.
+   */
+  public static readonly SUBLOCATION_TYPE = new LogFormat('${sublocation-type}');
+
+  /**
+   * The ID of the sublocation that contains the network interface for which traffic is recorded.
+   */
+  public static readonly SUBLOCATION_ID = new LogFormat('${sublocation-id}');
+
+  /**
+   * The name of the subset of IP address ranges for the pkt-srcaddr field,
+   * if the source IP address is for an AWS service.
+   */
+  public static readonly PKT_SRC_AWS_SERVICE = new LogFormat('${pkt-src-aws-service}');
+
+  /**
+   * The name of the subset of IP address ranges for the pkt-dstaddr field,
+   * if the destination IP address is for an AWS service.
+   */
+  public static readonly PKT_DST_AWS_SERVICE = new LogFormat('${pkt-dst-aws-service}');
+
+  /**
+   * The direction of the flow with respect to the interface where traffic is captured.
+   */
+  public static readonly FLOW_DIRECTION = new LogFormat('${flow-direction}');
+
+  /**
+   * The path that egress traffic takes to the destination.
+   */
+  public static readonly TRAFFIC_PATH = new LogFormat('${traffic-path}');
+
+  /**
+   * The default format.
+   */
+  public static readonly ALL_DEFAULT_FIELDS = new LogFormat('${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}');
+
+  /**
+   * A custom format string.
+   *
+   * Gives full control over the format string fragment.
+   */
+  public static custom(formatString: string): LogFormat {
+    return new LogFormat(formatString);
+  }
+
+  /**
+   * A custom field name.
+   *
+   * If there is no ready-made constant for a new field yet, you can use this.
+   * The field name will automatically be wrapped in `${ ... }`.
+   */
+  public static field(field: string): LogFormat {
+    return new LogFormat(`\${${field}}`);
+  }
+
+  protected constructor(public readonly value: string) {}
+}
+
 /**
  * Options to add a flow log to a VPC
  */
@@ -420,6 +555,18 @@ export interface FlowLogOptions {
    */
   readonly destination?: FlowLogDestination;
 
+  /**
+   * The fields to include in the flow log record, in the order in which they should appear.
+   *
+   * If multiple fields are specified, they will be separated by spaces. For full control over the literal log format
+   * string, pass a single field constructed with `LogFormat.custom()`.
+   *
+   * See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html#flow-log-records
+   *
+   * @default - default log format is used.
+   */
+  readonly logFormat?: LogFormat[];
+
   /**
    * The maximum interval of time during which a flow of packets is captured
    * and aggregated into a flow log record.
@@ -521,6 +668,12 @@ export class FlowLog extends FlowLogBase {
     if (this.bucket) {
       logDestination = this.keyPrefix ? this.bucket.arnForObjects(this.keyPrefix) : this.bucket.bucketArn;
     }
+    let customLogFormat: string | undefined = undefined;
+    if (props.logFormat) {
+      customLogFormat = props.logFormat.map(elm => {
+        return elm.value;
+      }).join(' ');
+    }
 
     const flowLog = new CfnFlowLog(this, 'FlowLog', {
       destinationOptions: destinationConfig.destinationOptions,
@@ -533,6 +686,7 @@ export class FlowLog extends FlowLogBase {
       trafficType: props.trafficType
         ? props.trafficType
         : FlowLogTrafficType.ALL,
+      logFormat: customLogFormat,
       logDestination,
     });
 

packages/@aws-cdk/aws-ec2/package.json

@@ -364,6 +364,7 @@
       "docs-public-apis:@aws-cdk/aws-ec2.InterfaceVpcEndpointAwsService.SECURITYHUB",
       "docs-public-apis:@aws-cdk/aws-ec2.InterfaceVpcEndpointAwsService.EMAIL_SMTP",
       "docs-public-apis:@aws-cdk/aws-ec2.InterfaceVpcEndpointAwsService.WORKSPACES",
+      "docs-public-apis:@aws-cdk/aws-ec2.LogFormat.value",
       "docs-public-apis:@aws-cdk/aws-ec2.Port.toString",
       "docs-public-apis:@aws-cdk/aws-ec2.PrivateSubnet.fromPrivateSubnetAttributes",
       "docs-public-apis:@aws-cdk/aws-ec2.PublicSubnet.fromPublicSubnetAttributes",
@@ -749,7 +750,8 @@
       "props-physical-name:@aws-cdk/aws-ec2.ClientVpnRouteProps",
       "duration-prop-type:@aws-cdk/aws-ec2.ClientVpnEndpointOptions.sessionTimeout",
       "duration-prop-type:@aws-cdk/aws-ec2.ClientVpnEndpointProps.sessionTimeout",
-      "resource-attribute:@aws-cdk/aws-ec2.VpnGateway.vpnGatewayId"
+      "resource-attribute:@aws-cdk/aws-ec2.VpnGateway.vpnGatewayId",
+      "docs-public-apis:@aws-cdk/aws-ec2.LogFormatField.value"
     ]
   },
   "stability": "stable",

packages/@aws-cdk/aws-ec2/test/integ.vpc-flow-logs-customformat.js.snapshot/FlowLogsDefaultTestDeployAssert6AFD1854.assets.json

@@ -0,0 +1,19 @@
+{
+  "version": "21.0.0",
+  "files": {
+    "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
+      "source": {
+        "path": "FlowLogsDefaultTestDeployAssert6AFD1854.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    }
+  },
+  "dockerImages": {}
+}

packages/@aws-cdk/aws-ec2/test/integ.vpc-flow-logs-customformat.js.snapshot/FlowLogsDefaultTestDeployAssert6AFD1854.template.json

@@ -0,0 +1,36 @@
+{
+ "Parameters": {
+  "BootstrapVersion": {
+   "Type": "AWS::SSM::Parameter::Value<String>",
+   "Default": "/cdk-bootstrap/hnb659fds/version",
+   "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
+  }
+ },
+ "Rules": {
+  "CheckBootstrapVersion": {
+   "Assertions": [
+    {
+     "Assert": {
+      "Fn::Not": [
+       {
+        "Fn::Contains": [
+         [
+          "1",
+          "2",
+          "3",
+          "4",
+          "5"
+         ],
+         {
+          "Ref": "BootstrapVersion"
+         }
+        ]
+       }
+      ]
+     },
+     "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
+    }
+   ]
+  }
+ }
+}

packages/@aws-cdk/aws-ec2/test/integ.vpc-flow-logs-customformat.js.snapshot/FlowLogsTestStack.assets.json

@@ -0,0 +1,32 @@
+{
+  "version": "21.0.0",
+  "files": {
+    "33e2651435a0d472a75c1e033c9832b21321d9e56711926b04c5705e5f63874c": {
+      "source": {
+        "path": "asset.33e2651435a0d472a75c1e033c9832b21321d9e56711926b04c5705e5f63874c",
+        "packaging": "zip"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "33e2651435a0d472a75c1e033c9832b21321d9e56711926b04c5705e5f63874c.zip",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    },
+    "9dcef326beebc49accefb3f0f234ec72b4de2a2aa5f1dc4ed26408fcc22c1dd7": {
+      "source": {
+        "path": "FlowLogsTestStack.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "9dcef326beebc49accefb3f0f234ec72b4de2a2aa5f1dc4ed26408fcc22c1dd7.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    }
+  },
+  "dockerImages": {}
+}