Add bb_asg_lifecycle_hook

In the following commit, we removed logic from bb_scheduler to talk to AWS ASG lifecycle hooks: buildbarn/bb-remote-execution@78e027f There is no need to let this be part of bb_scheduler itself, as we can easily move that into a separate binary. This change adds such a binary to the bb-autoscaler repository. Even if people don't need this binary in literal form, it still acts as useful documentation for how to add graceful worker termination to your cluster.
buildbarn · Jul 3, 2023 · 19862fe · 19862fe
1 parent 1e92910
commit 19862fe
Show file tree

Hide file tree

Showing 19 changed files with 1,362 additions and 9 deletions.
diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml
@@ -55,6 +55,18 @@
                "name": "linux_amd64: build and test",
                "run": "bazel test --test_output=errors --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."
             },
+            {
+               "name": "linux_amd64: copy bb_asg_lifecycle_hook",
+               "run": "rm -f bb_asg_lifecycle_hook && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //cmd/bb_asg_lifecycle_hook $(pwd)/bb_asg_lifecycle_hook"
+            },
+            {
+               "name": "linux_amd64: upload bb_asg_lifecycle_hook",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_asg_lifecycle_hook.linux_amd64",
+                  "path": "bb_asg_lifecycle_hook"
+               }
+            },
             {
                "name": "linux_amd64: copy bb_autoscaler",
                "run": "rm -f bb_autoscaler && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //cmd/bb_autoscaler $(pwd)/bb_autoscaler"
@@ -71,6 +83,18 @@
                "name": "linux_386: build and test",
                "run": "bazel test --test_output=errors --platforms=@io_bazel_rules_go//go/toolchain:linux_386 //..."
             },
+            {
+               "name": "linux_386: copy bb_asg_lifecycle_hook",
+               "run": "rm -f bb_asg_lifecycle_hook && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_386 //cmd/bb_asg_lifecycle_hook $(pwd)/bb_asg_lifecycle_hook"
+            },
+            {
+               "name": "linux_386: upload bb_asg_lifecycle_hook",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_asg_lifecycle_hook.linux_386",
+                  "path": "bb_asg_lifecycle_hook"
+               }
+            },
             {
                "name": "linux_386: copy bb_autoscaler",
                "run": "rm -f bb_autoscaler && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_386 //cmd/bb_autoscaler $(pwd)/bb_autoscaler"
@@ -87,6 +111,18 @@
                "name": "linux_arm: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_arm //..."
             },
+            {
+               "name": "linux_arm: copy bb_asg_lifecycle_hook",
+               "run": "rm -f bb_asg_lifecycle_hook && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_arm //cmd/bb_asg_lifecycle_hook $(pwd)/bb_asg_lifecycle_hook"
+            },
+            {
+               "name": "linux_arm: upload bb_asg_lifecycle_hook",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_asg_lifecycle_hook.linux_arm",
+                  "path": "bb_asg_lifecycle_hook"
+               }
+            },
             {
                "name": "linux_arm: copy bb_autoscaler",
                "run": "rm -f bb_autoscaler && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_arm //cmd/bb_autoscaler $(pwd)/bb_autoscaler"
@@ -103,6 +139,18 @@
                "name": "linux_arm64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64 //..."
             },
+            {
+               "name": "linux_arm64: copy bb_asg_lifecycle_hook",
+               "run": "rm -f bb_asg_lifecycle_hook && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64 //cmd/bb_asg_lifecycle_hook $(pwd)/bb_asg_lifecycle_hook"
+            },
+            {
+               "name": "linux_arm64: upload bb_asg_lifecycle_hook",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_asg_lifecycle_hook.linux_arm64",
+                  "path": "bb_asg_lifecycle_hook"
+               }
+            },
             {
                "name": "linux_arm64: copy bb_autoscaler",
                "run": "rm -f bb_autoscaler && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64 //cmd/bb_autoscaler $(pwd)/bb_autoscaler"
@@ -119,6 +167,18 @@
                "name": "darwin_amd64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64 //..."
             },
+            {
+               "name": "darwin_amd64: copy bb_asg_lifecycle_hook",
+               "run": "rm -f bb_asg_lifecycle_hook && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64 //cmd/bb_asg_lifecycle_hook $(pwd)/bb_asg_lifecycle_hook"
+            },
+            {
+               "name": "darwin_amd64: upload bb_asg_lifecycle_hook",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_asg_lifecycle_hook.darwin_amd64",
+                  "path": "bb_asg_lifecycle_hook"
+               }
+            },
             {
                "name": "darwin_amd64: copy bb_autoscaler",
                "run": "rm -f bb_autoscaler && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64 //cmd/bb_autoscaler $(pwd)/bb_autoscaler"
@@ -135,6 +195,18 @@
                "name": "darwin_arm64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:darwin_arm64 //..."
             },
+            {
+               "name": "darwin_arm64: copy bb_asg_lifecycle_hook",
+               "run": "rm -f bb_asg_lifecycle_hook && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:darwin_arm64 //cmd/bb_asg_lifecycle_hook $(pwd)/bb_asg_lifecycle_hook"
+            },
+            {
+               "name": "darwin_arm64: upload bb_asg_lifecycle_hook",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_asg_lifecycle_hook.darwin_arm64",
+                  "path": "bb_asg_lifecycle_hook"
+               }
+            },
             {
                "name": "darwin_arm64: copy bb_autoscaler",
                "run": "rm -f bb_autoscaler && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:darwin_arm64 //cmd/bb_autoscaler $(pwd)/bb_autoscaler"
@@ -149,7 +221,19 @@
             },
             {
                "name": "freebsd_amd64: build and test",
-               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_autoscaler"
+               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_asg_lifecycle_hook //cmd/bb_autoscaler"
+            },
+            {
+               "name": "freebsd_amd64: copy bb_asg_lifecycle_hook",
+               "run": "rm -f bb_asg_lifecycle_hook && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_asg_lifecycle_hook $(pwd)/bb_asg_lifecycle_hook"
+            },
+            {
+               "name": "freebsd_amd64: upload bb_asg_lifecycle_hook",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_asg_lifecycle_hook.freebsd_amd64",
+                  "path": "bb_asg_lifecycle_hook"
+               }
             },
             {
                "name": "freebsd_amd64: copy bb_autoscaler",
@@ -167,6 +251,18 @@
                "name": "windows_amd64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64 //..."
             },
+            {
+               "name": "windows_amd64: copy bb_asg_lifecycle_hook",
+               "run": "rm -f bb_asg_lifecycle_hook.exe && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64 //cmd/bb_asg_lifecycle_hook $(pwd)/bb_asg_lifecycle_hook.exe"
+            },
+            {
+               "name": "windows_amd64: upload bb_asg_lifecycle_hook",
+               "uses": "actions/upload-artifact@v2-preview",
+               "with": {
+                  "name": "bb_asg_lifecycle_hook.windows_amd64",
+                  "path": "bb_asg_lifecycle_hook.exe"
+               }
+            },
             {
                "name": "windows_amd64: copy bb_autoscaler",
                "run": "rm -f bb_autoscaler.exe && bazel run --run_under cp --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64 //cmd/bb_autoscaler $(pwd)/bb_autoscaler.exe"
@@ -186,6 +282,10 @@
                "name": "Install Docker credentials",
                "run": "echo \"${GITHUB_TOKEN}\" | docker login ghcr.io -u $ --password-stdin"
             },
+            {
+               "name": "Push container bb_asg_lifecycle_hook:bb_asg_lifecycle_hook",
+               "run": "bazel run --stamp //cmd/bb_asg_lifecycle_hook:bb_asg_lifecycle_hook_container_push"
+            },
             {
                "name": "Push container bb_autoscaler:bb_autoscaler",
                "run": "bazel run --stamp //cmd/bb_autoscaler:bb_autoscaler_container_push"

diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml
@@ -77,7 +77,7 @@
             },
             {
                "name": "freebsd_amd64: build and test",
-               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_autoscaler"
+               "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:freebsd_amd64 //cmd/bb_asg_lifecycle_hook //cmd/bb_autoscaler"
             },
             {
                "name": "windows_amd64: build and test",

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -5,6 +5,10 @@ load("@bazel_gazelle//:def.bzl", "gazelle")
 # gazelle:resolve proto go build/bazel/remote/execution/v2/remote_execution.proto @com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution
 # gazelle:resolve proto pkg/proto/configuration/cloud/aws/aws.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/cloud/aws:aws_proto
 # gazelle:resolve proto go pkg/proto/configuration/cloud/aws/aws.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/cloud/aws
+# gazelle:resolve proto pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto
+# gazelle:resolve proto go pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global
+# gazelle:resolve proto pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc:grpc_proto
+# gazelle:resolve proto go pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc
 # gazelle:resolve proto pkg/proto/configuration/http/http.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/http:http_proto
 # gazelle:resolve proto go pkg/proto/configuration/http/http.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/http
 gazelle(

diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 # Buildbarn Autoscaler [![Build status](https://github.com/buildbarn/bb-autoscaler/workflows/master/badge.svg)](https://github.com/buildbarn/bb-autoscaler/actions) [![PkgGoDev](https://pkg.go.dev/badge/github.com/buildbarn/bb-autoscaler)](https://pkg.go.dev/github.com/buildbarn/bb-autoscaler) [![Go Report Card](https://goreportcard.com/badge/github.com/buildbarn/bb-autoscaler)](https://goreportcard.com/report/github.com/buildbarn/bb-autoscaler)
 
-This repository provides a utility that can be used in combination with
-[Buildbarn Remote Execution](https://github.com/buildbarn/bb-remote-execution)
+This repository provides a utility named `bb_autoscaler` that can be used in
+combination with [Buildbarn Remote Execution](https://github.com/buildbarn/bb-remote-execution)
 to automatically adjust the size of Amazon EC2
 [Auto Scaling Groups (ASGs)](https://docs.aws.amazon.com/autoscaling/ec2/userguide/AutoScalingGroup.html),
 EKS [Managed Node Groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html),
@@ -12,9 +12,14 @@ querying Prometheus, which in its turn extracts metrics from
 into a desired number of workers (e.g., by using
 [`quantile_over_time()`](https://prometheus.io/docs/prometheus/latest/querying/functions/#aggregation_over_time)).
 
-Note that it may not always be necessary to use a utility like this.
+Furthermore, this repository provides a tool named
+`bb_asg_lifecycle_hook`, which may be used to gracefully downscale
+workers running on plain EC2 instances, using
+[EC2 ASG lifecycle hooks](https://docs.aws.amazon.com/autoscaling/ec2/userguide/lifecycle-hooks.html).
+
+Note that it may not always be necessary to use utilities like these.
 When using Kubernetes, it may be sufficient to create a
 [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)
 that uses the [Custom Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md).
-Using this tool may still be preferable if it is undesirable if your
-cluster is not configured to use the Custom Metrics API.
+Using these tools may still be preferable if it is undesirable to
+reconfigure your cluster to use the Custom Metrics API.
diff --git a/WORKSPACE b/WORKSPACE
@@ -26,6 +26,20 @@ http_archive(
     ],
 )
 
+load("@bazel_gazelle//:deps.bzl", "go_repository")
+
+# Override the version of gomock to one that includes support for
+# generating mocks for function types. We can't do this through go.mod,
+# as it causes almost all of our package dependencies to be downgraded.
+go_repository(
+    name = "com_github_golang_mock",
+    importpath = "github.com/golang/mock",
+    patches = ["@com_github_buildbarn_bb_storage//:patches/com_github_golang_mock/mocks-for-funcs.diff"],
+    replace = "github.com/golang/mock",
+    sum = "h1:DxRM2MRFDKF8JGaT1ZSsCZ9KxoOki+rrOoB011jIEDc=",
+    version = "v1.6.1-0.20220512030613-73266f9366fc",
+)
+
 # gazelle:repository_macro go_dependencies.bzl%go_dependencies
 load(":go_dependencies.bzl", "go_dependencies")
 
@@ -104,3 +118,54 @@ jsonnet_go_repositories()
 load("@google_jsonnet_go//bazel:deps.bzl", "jsonnet_go_dependencies")
 
 jsonnet_go_dependencies()
+
+http_archive(
+    name = "com_github_twbs_bootstrap",
+    build_file_content = """exports_files(["css/bootstrap.min.css", "js/bootstrap.min.js"])""",
+    sha256 = "395342b2974e3350560e65752d36aab6573652b11cc6cb5ef79a2e5e83ad64b1",
+    strip_prefix = "bootstrap-5.1.0-dist",
+    urls = ["https://github.com/twbs/bootstrap/releases/download/v5.1.0/bootstrap-5.1.0-dist.zip"],
+)
+
+http_archive(
+    name = "aspect_rules_js",
+    sha256 = "00e7b97b696af63812df0ca9e9dbd18579f3edd3ab9a56f227238b8405e4051c",
+    strip_prefix = "rules_js-1.23.0",
+    url = "https://github.com/aspect-build/rules_js/releases/download/v1.23.0/rules_js-v1.23.0.tar.gz",
+)
+
+load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies")
+
+rules_js_dependencies()
+
+load("@rules_nodejs//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", "nodejs_register_toolchains")
+
+nodejs_register_toolchains(
+    name = "nodejs",
+    node_version = DEFAULT_NODE_VERSION,
+)
+
+load("@aspect_rules_js//npm:npm_import.bzl", "npm_translate_lock")
+
+npm_translate_lock(
+    name = "npm",
+    pnpm_lock = "@com_github_buildbarn_bb_storage//:pnpm-lock.yaml",
+)
+
+load("@npm//:repositories.bzl", "npm_repositories")
+
+npm_repositories()
+
+http_archive(
+    name = "io_opentelemetry_proto",
+    build_file_content = """
+proto_library(
+    name = "common_proto",
+    srcs = ["opentelemetry/proto/common/v1/common.proto"],
+    visibility = ["//visibility:public"],
+)
+""",
+    sha256 = "464bc2b348e674a1a03142e403cbccb01be8655b6de0f8bfe733ea31fcd421be",
+    strip_prefix = "opentelemetry-proto-0.19.0",
+    urls = ["https://github.com/open-telemetry/opentelemetry-proto/archive/refs/tags/v0.19.0.tar.gz"],
+)
diff --git a/cmd/bb_asg_lifecycle_hook/BUILD.bazel b/cmd/bb_asg_lifecycle_hook/BUILD.bazel
@@ -0,0 +1,42 @@
+load("@com_github_buildbarn_bb_storage//tools:container.bzl", "container_push_official")
+load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "bb_asg_lifecycle_hook_lib",
+    srcs = ["main.go"],
+    importpath = "github.com/buildbarn/bb-autoscaler/cmd/bb_asg_lifecycle_hook",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//pkg/cloud/aws",
+        "//pkg/proto/configuration/bb_asg_lifecycle_hook",
+        "@com_github_aws_aws_sdk_go_v2_service_autoscaling//:autoscaling",
+        "@com_github_aws_aws_sdk_go_v2_service_sqs//:sqs",
+        "@com_github_buildbarn_bb_remote_execution//pkg/proto/buildqueuestate",
+        "@com_github_buildbarn_bb_storage//pkg/cloud/aws",
+        "@com_github_buildbarn_bb_storage//pkg/global",
+        "@com_github_buildbarn_bb_storage//pkg/program",
+        "@com_github_buildbarn_bb_storage//pkg/util",
+        "@org_golang_google_grpc//codes",
+        "@org_golang_google_grpc//status",
+    ],
+)
+
+go_binary(
+    name = "bb_asg_lifecycle_hook",
+    embed = [":bb_asg_lifecycle_hook_lib"],
+    visibility = ["//visibility:public"],
+)
+
+go_image(
+    name = "bb_asg_lifecycle_hook_container",
+    embed = [":bb_asg_lifecycle_hook_lib"],
+    pure = "on",
+    visibility = ["//visibility:public"],
+)
+
+container_push_official(
+    name = "bb_asg_lifecycle_hook_container_push",
+    component = "bb-asg-lifecycle-hook",
+    image = ":bb_asg_lifecycle_hook_container",
+)