Fix integ-dev make target and update README (#866)

- fix integ-dev to be able to run integ tests locally.
- update README with all necessary pre-reqs.
- change default DOCKER_BUILD_OPTS when running local dev builds so that
  the docker build cache is used, not using the cache means that
  re-running integ-dev takes ~30 minutes every run.
- Minor fixes for re-running and using AWS_REGION in integ test
  cloudformation.

.gitignore

@@ -1,3 +1,3 @@
 bin
-integ/out
+integ/out/
 .venv

Makefile

@@ -13,8 +13,12 @@
 
 all: release
 
-# Improve build speeds during development by removing the --no-cache flag
-export DOCKER_BUILD_FLAGS=--no-cache
+# Execute set-cache to turn docker cache back on for faster development.
+DOCKER_BUILD_FLAGS := "--no-cache"
+
+.PHONY: dev
+dev: DOCKER_BUILD_FLAGS =
+dev: release
 
 .PHONY: release
 release: build build-init linux-plugins
@@ -150,40 +154,43 @@ kinesis-dev:
 	$(DOCKER_BUILD_FLAGS) -t aws-fluent-bit-plugins:latest -f ./scripts/dockerfiles/Dockerfile.plugins .
 	docker build -t amazon/aws-for-fluent-bit:latest -f ./scripts/dockerfiles/Dockerfile .
 
+integ/out:
+	mkdir -p integ/out
+
 .PHONY: integ-cloudwatch
-integ-cloudwatch: release
+integ-cloudwatch: integ/out release
 	./integ/integ.sh cloudwatch
 
 .PHONY: integ-cloudwatch-dev
-integ-cloudwatch-dev: cloudwatch-dev
+integ-cloudwatch-dev: integ/out cloudwatch-dev
 	./integ/integ.sh cloudwatch
 
 .PHONY: integ-clean-cloudwatch
-integ-clean-cloudwatch:
+integ-clean-cloudwatch: integ/out
 	./integ/integ.sh clean-cloudwatch
 
 .PHONY: integ-kinesis
-integ-kinesis: release
+integ-kinesis: integ/out release
 	./integ/integ.sh kinesis
 
 .PHONY: integ-kinesis-dev
-integ-kinesis-dev: kinesis-dev
+integ-kinesis-dev: integ/out kinesis-dev
 	./integ/integ.sh kinesis
 
 .PHONY: integ-firehose
-integ-firehose: release
+integ-firehose: integ/out release
 	./integ/integ.sh firehose
 
 .PHONY: integ-firehose-dev
-integ-firehose-dev: firehose-dev
+integ-firehose-dev: integ/out firehose-dev
 	./integ/integ.sh firehose
 
 .PHONY: integ-clean-s3
-integ-clean-s3:
+integ-clean-s3: integ/out
 	./integ/integ.sh clean-s3
 
 .PHONY: integ-dev
-integ-dev: release
+integ-dev: integ/out dev
 	./integ/integ.sh kinesis
 	./integ/integ.sh kinesis_streams
 	./integ/integ.sh firehose
@@ -192,7 +199,7 @@ integ-dev: release
 	./integ/integ.sh cloudwatch_logs
 
 .PHONY: integ
-integ:
+integ: integ/out
 	./integ/integ.sh cicd
 
 .PHONY: delete-resources
@@ -202,6 +209,7 @@ delete-resources:
 .PHONY: clean
 clean:
 	rm -rf ./build
+	rm -rf ./integ/out
 	docker image remove -f aws-fluent-bit-plugins:latest
 
 	docker image remove -f amazon/aws-for-fluent-bit:latest

README.md

@@ -354,16 +354,32 @@ For more details about running Fluent Bit Windows containers in Amazon ECS, plea
 
 ### Development
 
-#### Local testing
+#### Local integ testing
 
-Use `make release` to build the image.
+Use `make dev` to build the image.
 
-To run the integration tests, run `make integ-dev`. The `make integ-dev` command will run the integration tests for all of our plugins-
+The `make integ-dev` command will run the integration tests for all of our plugins-
 kinesis streams, kinesis firehose, and cloudwatch.
 
-The integ tests require the following env vars to be set:
-* `CW_INTEG_VALIDATOR_IMAGE`: Build the [integ/validate_cloudwatch/](integ/validate_cloudwatch/) folder with `docker build` and set the resulting image as the value of this env var.
-* `S3_INTEG_VALIDATOR_IMAGE`: Build the [integ/s3/](integ/s3/) folder with `docker build` and set the resulting image as the value of this env var.
+Note that these steps rely on creating Cfn stacks in an AWS account in region us-west-2,
+so AWS credentials must be setup before they are run.
+
+Instructions:
+1. Setup AWS access via EC2 instance role or AWS_* env vars
+2. Install dependent packages: `docker awscli`
+3. Install docker-compose:
+```
+sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+```
+4. Build validator images:
+```
+pushd integ/validate_cloudwatch && docker build -t flbcwinteg . && popd
+pushd integ/s3 && docker build -t flbs3integ . && popd
+export CW_INTEG_VALIDATOR_IMAGE="flbcwinteg"
+export S3_INTEG_VALIDATOR_IMAGE="flbs3integ"
+```
+5. Run `make integ-dev`
 
 To run integration tests separately, execute `make integ-cloudwatch` or `make integ-kinesis` or `make integ-firehose`.
 

integ/integ.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+
+set -ex
+
 export AWS_REGION="us-west-2"
 export PROJECT_ROOT="$(pwd)"
 export VOLUME_MOUNT_CONTAINER="/out"
@@ -9,8 +12,14 @@ if [ "$ARCHITECTURE" = "aarch64" ]; then
     ARCHITECTURE="arm64"
 fi
 
-export CW_INTEG_VALIDATOR_IMAGE="${CW_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
-export S3_INTEG_VALIDATOR_IMAGE="${S3_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
+# If we're testing locally, then these are set to local images rather than pulling
+# from ECR. See https://github.com/aws/aws-for-fluent-bit?tab=readme-ov-file#local-testing
+if [ -z "$CW_INTEG_VALIDATOR_IMAGE" ]; then
+	export CW_INTEG_VALIDATOR_IMAGE="${CW_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
+fi
+if [ -z "$S3_INTEG_VALIDATOR_IMAGE" ]; then
+	export S3_INTEG_VALIDATOR_IMAGE="${S3_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
+fi
 
 test_cloudwatch() {
 	export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}"
@@ -287,7 +296,7 @@ if [ "${1}" = "cicd" ]; then
 	export TEST_FILE="kinesis-test"
 	export EXPECTED_EVENTS_LEN="1000"
 	clean_s3 && test_kinesis
-	
+
 	# golang firehose plugin
 	export S3_PREFIX="firehose-test"
 	export TEST_FILE="firehose-test"
@@ -315,4 +324,4 @@ fi
 
 if [ "${1}" = "delete" ]; then
 	source ./integ/resources/delete_test_resources.sh
-fi
+fi

integ/resources/create_test_resources.sh

@@ -8,4 +8,4 @@ ARCHITECTURE=$(uname -m | tr '_' '-')
 if [ "$ARCHITECTURE" = "aarch64" ]; then
     ARCHITECTURE="arm64"
 fi
-aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region us-west-2 --capabilities CAPABILITY_NAMED_IAM
+aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region "$AWS_REGION" --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset

integ/resources/setup_test_environment.sh

@@ -7,7 +7,7 @@ ARCHITECTURE=$(uname -m | tr '_' '-')
 if [ "$ARCHITECTURE" = "aarch64" ]; then
     ARCHITECTURE="arm64"
 fi
-stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue')
+stackOutputs=$(aws cloudformation describe-stacks --region "$AWS_REGION" --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue')
 read -r -a outputArray <<< "$stackOutputs"
 export FIREHOSE_STREAM="${outputArray[0]}"
 export KINESIS_STREAM="${outputArray[1]}"

scripts/build_plugins.sh

@@ -35,7 +35,7 @@ PLUGIN_BUILD_ARGS=""
 # set by env var in Makefile right now
 # setting this by env var ensures it works even on platforms where getopt and longoptions does not work
 OS_TYPE="${OS_TYPE}"
-DOCKER_BUILD_FLAGS="${DOCKER_BUILD_FLAGS}"
+DOCKER_BUILD_FLAGS="${DOCKER_BUILD_FLAGS:-}"
 
 # Go plugin versions can either be set by args to the script, or they will be sourced
 # from the windows.versions or linux.version file