From fad65b7448d6be2162329fc4c212b6c42f418a22 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 11 Oct 2024 15:16:49 -0700 Subject: [PATCH] Fix integ-dev make target and update README (#866) - fix integ-dev to be able to run integ tests locally. - update README with all necessary pre-reqs. - change default DOCKER_BUILD_OPTS when running local dev builds so that the docker build cache is used, not using the cache means that re-running integ-dev takes ~30 minutes every run. - Minor fixes for re-running and using AWS_REGION in integ test cloudformation. --- .gitignore | 2 +- Makefile | 32 ++++++++++++++--------- README.md | 28 +++++++++++++++----- integ/integ.sh | 17 +++++++++--- integ/out/expected-metric-name | 1 - integ/out/s3-test | 0 integ/resources/create_test_resources.sh | 2 +- integ/resources/setup_test_environment.sh | 2 +- scripts/build_plugins.sh | 2 +- 9 files changed, 59 insertions(+), 27 deletions(-) delete mode 100644 integ/out/expected-metric-name delete mode 100644 integ/out/s3-test diff --git a/.gitignore b/.gitignore index a7347b7c4..9e8a609c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ bin -integ/out +integ/out/ .venv diff --git a/Makefile b/Makefile index 95fab8ad8..2c56a29a1 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,12 @@ all: release -# Improve build speeds during development by removing the --no-cache flag -export DOCKER_BUILD_FLAGS=--no-cache +# Execute set-cache to turn docker cache back on for faster development. +DOCKER_BUILD_FLAGS := "--no-cache" + +.PHONY: dev +dev: DOCKER_BUILD_FLAGS = +dev: release .PHONY: release release: build build-init linux-plugins @@ -150,40 +154,43 @@ kinesis-dev: $(DOCKER_BUILD_FLAGS) -t aws-fluent-bit-plugins:latest -f ./scripts/dockerfiles/Dockerfile.plugins . docker build -t amazon/aws-for-fluent-bit:latest -f ./scripts/dockerfiles/Dockerfile . +integ/out: + mkdir -p integ/out + .PHONY: integ-cloudwatch -integ-cloudwatch: release +integ-cloudwatch: integ/out release ./integ/integ.sh cloudwatch .PHONY: integ-cloudwatch-dev -integ-cloudwatch-dev: cloudwatch-dev +integ-cloudwatch-dev: integ/out cloudwatch-dev ./integ/integ.sh cloudwatch .PHONY: integ-clean-cloudwatch -integ-clean-cloudwatch: +integ-clean-cloudwatch: integ/out ./integ/integ.sh clean-cloudwatch .PHONY: integ-kinesis -integ-kinesis: release +integ-kinesis: integ/out release ./integ/integ.sh kinesis .PHONY: integ-kinesis-dev -integ-kinesis-dev: kinesis-dev +integ-kinesis-dev: integ/out kinesis-dev ./integ/integ.sh kinesis .PHONY: integ-firehose -integ-firehose: release +integ-firehose: integ/out release ./integ/integ.sh firehose .PHONY: integ-firehose-dev -integ-firehose-dev: firehose-dev +integ-firehose-dev: integ/out firehose-dev ./integ/integ.sh firehose .PHONY: integ-clean-s3 -integ-clean-s3: +integ-clean-s3: integ/out ./integ/integ.sh clean-s3 .PHONY: integ-dev -integ-dev: release +integ-dev: integ/out dev ./integ/integ.sh kinesis ./integ/integ.sh kinesis_streams ./integ/integ.sh firehose @@ -192,7 +199,7 @@ integ-dev: release ./integ/integ.sh cloudwatch_logs .PHONY: integ -integ: +integ: integ/out ./integ/integ.sh cicd .PHONY: delete-resources @@ -202,6 +209,7 @@ delete-resources: .PHONY: clean clean: rm -rf ./build + rm -rf ./integ/out docker image remove -f aws-fluent-bit-plugins:latest docker image remove -f amazon/aws-for-fluent-bit:latest diff --git a/README.md b/README.md index 65aa670dd..96eafdd17 100644 --- a/README.md +++ b/README.md @@ -354,16 +354,32 @@ For more details about running Fluent Bit Windows containers in Amazon ECS, plea ### Development -#### Local testing +#### Local integ testing -Use `make release` to build the image. +Use `make dev` to build the image. -To run the integration tests, run `make integ-dev`. The `make integ-dev` command will run the integration tests for all of our plugins- +The `make integ-dev` command will run the integration tests for all of our plugins- kinesis streams, kinesis firehose, and cloudwatch. -The integ tests require the following env vars to be set: -* `CW_INTEG_VALIDATOR_IMAGE`: Build the [integ/validate_cloudwatch/](integ/validate_cloudwatch/) folder with `docker build` and set the resulting image as the value of this env var. -* `S3_INTEG_VALIDATOR_IMAGE`: Build the [integ/s3/](integ/s3/) folder with `docker build` and set the resulting image as the value of this env var. +Note that these steps rely on creating Cfn stacks in an AWS account in region us-west-2, +so AWS credentials must be setup before they are run. + +Instructions: +1. Setup AWS access via EC2 instance role or AWS_* env vars +2. Install dependent packages: `docker awscli` +3. Install docker-compose: +``` +sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +``` +4. Build validator images: +``` +pushd integ/validate_cloudwatch && docker build -t flbcwinteg . && popd +pushd integ/s3 && docker build -t flbs3integ . && popd +export CW_INTEG_VALIDATOR_IMAGE="flbcwinteg" +export S3_INTEG_VALIDATOR_IMAGE="flbs3integ" +``` +5. Run `make integ-dev` To run integration tests separately, execute `make integ-cloudwatch` or `make integ-kinesis` or `make integ-firehose`. diff --git a/integ/integ.sh b/integ/integ.sh index 8aee9c10c..0ee5f2590 100755 --- a/integ/integ.sh +++ b/integ/integ.sh @@ -1,4 +1,7 @@ #!/bin/bash + +set -ex + export AWS_REGION="us-west-2" export PROJECT_ROOT="$(pwd)" export VOLUME_MOUNT_CONTAINER="/out" @@ -9,8 +12,14 @@ if [ "$ARCHITECTURE" = "aarch64" ]; then ARCHITECTURE="arm64" fi -export CW_INTEG_VALIDATOR_IMAGE="${CW_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}" -export S3_INTEG_VALIDATOR_IMAGE="${S3_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}" +# If we're testing locally, then these are set to local images rather than pulling +# from ECR. See https://github.com/aws/aws-for-fluent-bit?tab=readme-ov-file#local-testing +if [ -z "$CW_INTEG_VALIDATOR_IMAGE" ]; then + export CW_INTEG_VALIDATOR_IMAGE="${CW_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}" +fi +if [ -z "$S3_INTEG_VALIDATOR_IMAGE" ]; then + export S3_INTEG_VALIDATOR_IMAGE="${S3_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}" +fi test_cloudwatch() { export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}" @@ -287,7 +296,7 @@ if [ "${1}" = "cicd" ]; then export TEST_FILE="kinesis-test" export EXPECTED_EVENTS_LEN="1000" clean_s3 && test_kinesis - + # golang firehose plugin export S3_PREFIX="firehose-test" export TEST_FILE="firehose-test" @@ -315,4 +324,4 @@ fi if [ "${1}" = "delete" ]; then source ./integ/resources/delete_test_resources.sh -fi \ No newline at end of file +fi diff --git a/integ/out/expected-metric-name b/integ/out/expected-metric-name deleted file mode 100644 index 736dc8745..000000000 --- a/integ/out/expected-metric-name +++ /dev/null @@ -1 +0,0 @@ -fluent-bit-integ-test-1264 diff --git a/integ/out/s3-test b/integ/out/s3-test deleted file mode 100644 index e69de29bb..000000000 diff --git a/integ/resources/create_test_resources.sh b/integ/resources/create_test_resources.sh index b61ae42b8..15457dbd4 100755 --- a/integ/resources/create_test_resources.sh +++ b/integ/resources/create_test_resources.sh @@ -8,4 +8,4 @@ ARCHITECTURE=$(uname -m | tr '_' '-') if [ "$ARCHITECTURE" = "aarch64" ]; then ARCHITECTURE="arm64" fi -aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region us-west-2 --capabilities CAPABILITY_NAMED_IAM \ No newline at end of file +aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region "$AWS_REGION" --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset diff --git a/integ/resources/setup_test_environment.sh b/integ/resources/setup_test_environment.sh index d25a90ced..a58bdfe41 100644 --- a/integ/resources/setup_test_environment.sh +++ b/integ/resources/setup_test_environment.sh @@ -7,7 +7,7 @@ ARCHITECTURE=$(uname -m | tr '_' '-') if [ "$ARCHITECTURE" = "aarch64" ]; then ARCHITECTURE="arm64" fi -stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue') +stackOutputs=$(aws cloudformation describe-stacks --region "$AWS_REGION" --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue') read -r -a outputArray <<< "$stackOutputs" export FIREHOSE_STREAM="${outputArray[0]}" export KINESIS_STREAM="${outputArray[1]}" diff --git a/scripts/build_plugins.sh b/scripts/build_plugins.sh index f5bdcd379..5fc1a70a4 100755 --- a/scripts/build_plugins.sh +++ b/scripts/build_plugins.sh @@ -35,7 +35,7 @@ PLUGIN_BUILD_ARGS="" # set by env var in Makefile right now # setting this by env var ensures it works even on platforms where getopt and longoptions does not work OS_TYPE="${OS_TYPE}" -DOCKER_BUILD_FLAGS="${DOCKER_BUILD_FLAGS}" +DOCKER_BUILD_FLAGS="${DOCKER_BUILD_FLAGS:-}" # Go plugin versions can either be set by args to the script, or they will be sourced # from the windows.versions or linux.version file