From e8f766c2cd8824d1a5b7d6e2f4e8845c4beb73e2 Mon Sep 17 00:00:00 2001 From: William Findlay Date: Wed, 21 Feb 2024 13:04:31 -0500 Subject: [PATCH 1/2] makefile: containerize crd generation [upstream commit: 5df7458b9afa99b056e612f035915e45a2e53987] We encountered some version-specific differences related to golang tooling which impact the resulting CRD generation from make generate. To solve this problem, let's containerize the CRD generation workflow so that we can get consistent output through consistent tooling. while we're at it, rename the generate target to crds to disambiguate it from protobuf code generation. We keep the old generate target around as an alias for backward compatibility. Signed-off-by: William Findlay --- Makefile | 7 ++++--- .../docs/contribution-guide/development-setup.md | 2 +- pkg/k8s/Makefile | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 01890622f27..3cbf8bc3f87 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ help: @echo ' tarball-release - build Tetragon release tarball' @echo ' Generated files:' @echo ' codegen - generate code based on .proto files' - @echo ' generate - generate kubebuilder files' + @echo ' crds - generate kubebuilder files' @echo ' generate-flags - generate Tetragon daemon flags for documentation' @echo ' Linting and chores:' @echo ' vendor - tidy and vendor Go modules' @@ -334,8 +334,9 @@ tarball-clean: fetch-testdata: wget -nc -P testdata/btf 'https://github.com/cilium/tetragon-testdata/raw/main/btf/vmlinux-5.4.104+' -.PHONY: generate codegen protoc-gen-go-tetragon -generate: +.PHONY: crdgen generate codegen protoc-gen-go-tetragon +generate: | crds +crds: # Need to call vendor twice here, once before and once after generate, the reason # being we need to grab changes first plus pull in whatever gets generated here. $(MAKE) vendor diff --git a/docs/content/en/docs/contribution-guide/development-setup.md b/docs/content/en/docs/contribution-guide/development-setup.md index a1d1d07838a..285957eeebc 100644 --- a/docs/content/en/docs/contribution-guide/development-setup.md +++ b/docs/content/en/docs/contribution-guide/development-setup.md @@ -70,7 +70,7 @@ make codegen And to run k8s CRD generation, run the following command from the root of the repository: ```shell -make generate +make crds ``` Finally, should you wish to modify any of the resulting codegen files (ending diff --git a/pkg/k8s/Makefile b/pkg/k8s/Makefile index afa195c4151..d0e52759a32 100644 --- a/pkg/k8s/Makefile +++ b/pkg/k8s/Makefile @@ -1,15 +1,25 @@ +# renovate: datasource=docker +BUILDER_IMAGE=docker.io/library/golang:1.22.0@sha256:7b297d9abee021bab9046e492506b3c2da8a3722cbf301653186545ecc1e00bb GO ?= go all: generate -.PHONY: generate - .PHONY: help help: @echo 'Generated files:' - @echo ' generate - genereate kubebuilder files' + @echo ' generate - generate kubebuilder files' +.PHONY: generate generate: + docker container run --rm \ + --volume $(CURDIR)/../..:/src \ + --user "$(shell id -u):$(shell id -g)" \ + --env GOCACHE=/go/.cache \ + $(BUILDER_IMAGE) \ + make -C /src/pkg/k8s __do_generate + +.PHONY: __do_generate +__do_generate: ./tools/controller-gen crd paths=./apis/... output:dir=apis/cilium.io/client/crds/v1alpha1 chmod +x vendor/k8s.io/code-generator/generate-internal-groups.sh # Do two invocations of generate-groups.sh, one with --plural-exceptions, one without From 99f23a7f4dc5773a1b0f08aa9befd55ffb4002e8 Mon Sep 17 00:00:00 2001 From: William Findlay Date: Wed, 21 Feb 2024 16:19:22 -0500 Subject: [PATCH 2/2] makefile: rename codegen to protogen [upstream commit: 6df735d7a815b88d77bf6ebc83b26174679577f7] Now that we have renamed the generate target to crds, it makes sense to also rename to codegen target to make it more clear that it specifically refers to protobuf codegen. Just as with make generate, we keep the old codegen target around to use as an alias for backward compatbility. We also remove the original protogen dockerfile as it is no longer needed. Instead, we can simply use the base cilium-builder image. Signed-off-by: William Findlay --- Dockerfile.codegen | 4 ---- Makefile | 16 ++++++---------- api/Makefile | 7 ++++--- api/v1/Makefile.protoc | 2 +- .../contribution-guide/development-setup.md | 19 ++++++++++++++++--- .../protoc-gen-go-tetragon/common/common.go | 0 .../eventchecker/enum.go | 2 +- .../eventchecker/eventchecker.go | 2 +- .../eventchecker/field.go | 4 ++-- .../eventchecker/message.go | 2 +- .../eventchecker/multichecker.go | 2 +- .../eventchecker/yaml.go | 2 +- .../generate/generate.go | 6 +++--- .../protoc-gen-go-tetragon/helpers/helpers.go | 2 +- .../protoc-gen-go-tetragon/imports/imports.go | 0 {cmd => tools}/protoc-gen-go-tetragon/main.go | 2 +- .../protoc-gen-go-tetragon/types/types.go | 2 +- 17 files changed, 40 insertions(+), 34 deletions(-) delete mode 100644 Dockerfile.codegen rename {cmd => tools}/protoc-gen-go-tetragon/common/common.go (100%) rename {cmd => tools}/protoc-gen-go-tetragon/eventchecker/enum.go (98%) rename {cmd => tools}/protoc-gen-go-tetragon/eventchecker/eventchecker.go (99%) rename {cmd => tools}/protoc-gen-go-tetragon/eventchecker/field.go (99%) rename {cmd => tools}/protoc-gen-go-tetragon/eventchecker/message.go (98%) rename {cmd => tools}/protoc-gen-go-tetragon/eventchecker/multichecker.go (99%) rename {cmd => tools}/protoc-gen-go-tetragon/eventchecker/yaml.go (99%) rename {cmd => tools}/protoc-gen-go-tetragon/generate/generate.go (74%) rename {cmd => tools}/protoc-gen-go-tetragon/helpers/helpers.go (98%) rename {cmd => tools}/protoc-gen-go-tetragon/imports/imports.go (100%) rename {cmd => tools}/protoc-gen-go-tetragon/main.go (66%) rename {cmd => tools}/protoc-gen-go-tetragon/types/types.go (98%) diff --git a/Dockerfile.codegen b/Dockerfile.codegen deleted file mode 100644 index a98daa86220..00000000000 --- a/Dockerfile.codegen +++ /dev/null @@ -1,4 +0,0 @@ -FROM quay.io/cilium/cilium-builder:3f94a291166286657c38a108d6eb092c5a603ab7@sha256:af052fdec4787a587aedbd79086952925e00ceff47dc6d25d62da09b6aacdbb9 -RUN --mount=type=bind,readwrite,target=/go/src/github.com/cilium/tetragon cd /go/src/github.com/cilium/tetragon && go install ./cmd/protoc-gen-go-tetragon - -#- vi:ft=dockerfile -# diff --git a/Makefile b/Makefile index 3cbf8bc3f87..335bed02c42 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,7 @@ help: @echo ' tarball - build Tetragon compressed tarball' @echo ' tarball-release - build Tetragon release tarball' @echo ' Generated files:' - @echo ' codegen - generate code based on .proto files' + @echo ' protogen - generate code based on .proto files' @echo ' crds - generate kubebuilder files' @echo ' generate-flags - generate Tetragon daemon flags for documentation' @echo ' Linting and chores:' @@ -282,7 +282,7 @@ update-copyright: lint: golint -set_exit_status $$(go list ./...) -.PHONY: image image-operator image-test image-codegen +.PHONY: image image-operator image-test image: $(CONTAINER_ENGINE) build -t "cilium/tetragon:${DOCKER_IMAGE_TAG}" --target release --build-arg TETRAGON_VERSION=$(VERSION) . $(QUIET)@echo "Push like this when ready:" @@ -298,11 +298,6 @@ image-test: image-clang $(QUIET)@echo "Push like this when ready:" $(QUIET)@echo "${CONTAINER_ENGINE} push cilium/tetragon-test:$(DOCKER_IMAGE_TAG)" -image-codegen: - $(CONTAINER_ENGINE) build -f Dockerfile.codegen -t "cilium/tetragon-codegen:${DOCKER_IMAGE_TAG}" . - $(QUIET)@echo "Push like this when ready:" - $(QUIET)@echo "${CONTAINER_ENGINE} push cilium/tetragon-codegen:$(DOCKER_IMAGE_TAG)" - .PHONY: image-clang image-clang: $(CONTAINER_ENGINE) build -f Dockerfile.clang --build-arg VERSION=1:15.0.7-0ubuntu0.22.04.2 -t "cilium/clang:${DOCKER_IMAGE_TAG}" . @@ -334,7 +329,7 @@ tarball-clean: fetch-testdata: wget -nc -P testdata/btf 'https://github.com/cilium/tetragon-testdata/raw/main/btf/vmlinux-5.4.104+' -.PHONY: crdgen generate codegen protoc-gen-go-tetragon +.PHONY: crdgen generate protogen codegen protoc-gen-go-tetragon generate: | crds crds: # Need to call vendor twice here, once before and once after generate, the reason @@ -343,7 +338,8 @@ crds: $(MAKE) -C pkg/k8s/ $(MAKE) vendor -codegen: image-codegen +codegen: | protogen +protogen: protoc-gen-go-tetragon # Need to call vendor twice here, once before and once after codegen the reason # being we need to grab changes first plus pull in whatever gets generated here. $(MAKE) vendor @@ -351,7 +347,7 @@ codegen: image-codegen $(MAKE) vendor protoc-gen-go-tetragon: - $(GO_BUILD) -o bin/$@ ./cmd/protoc-gen-go-tetragon/ + $(GO_BUILD) -o bin/$@ ./tools/protoc-gen-go-tetragon/ .PHONY: check ifneq (,$(findstring $(GOLANGCILINT_WANT_VERSION),$(GOLANGCILINT_VERSION))) diff --git a/api/Makefile b/api/Makefile index abf3d846f42..2461d8d801a 100644 --- a/api/Makefile +++ b/api/Makefile @@ -1,8 +1,9 @@ # Copyright 2017-2020 Authors of Cilium # SPDX-License-Identifier: Apache-2.0 +# renovate: datasource=docker +BUILDER_IMAGE=quay.io/cilium/cilium-builder@sha256:a2036a3f6420647e6702695dabd2ffa4d2832db45157042d0255bdc707b8e1f2 GO ?= go -BUILDER_IMAGE=cilium/tetragon-codegen:latest .PHONY: all all: proto @@ -13,10 +14,10 @@ proto: v1 .PHONY: v1 v1: docker container run --rm \ - --volume $(CURDIR)/v1:/src \ + --volume $(CURDIR)/..:/src \ --user "$(shell id -u):$(shell id -g)" \ $(BUILDER_IMAGE) \ - make -C /src -f Makefile.protoc + make -C /src/api/v1 -f Makefile.protoc ./export-doc.sh ../docs/content/en/docs/reference/grpc-api.md .PHONY: vendor diff --git a/api/v1/Makefile.protoc b/api/v1/Makefile.protoc index 71be6ce7589..930131373ad 100644 --- a/api/v1/Makefile.protoc +++ b/api/v1/Makefile.protoc @@ -18,7 +18,7 @@ TETRAGON_PROTOC_PLUGINS := \ --plugin=$(GOPATH)/bin/protoc-gen-go \ --plugin=$(GOPATH)/bin/protoc-gen-go-grpc \ --plugin=$(GOPATH)/bin/protoc-gen-go-json \ - --plugin=$(GOPATH)/bin/protoc-gen-go-tetragon \ + --plugin=../../bin/protoc-gen-go-tetragon \ .PHONY: all all: protobuf codegen diff --git a/docs/content/en/docs/contribution-guide/development-setup.md b/docs/content/en/docs/contribution-guide/development-setup.md index 285957eeebc..a26752bf2b0 100644 --- a/docs/content/en/docs/contribution-guide/development-setup.md +++ b/docs/content/en/docs/contribution-guide/development-setup.md @@ -65,7 +65,7 @@ you will be required to re-run code generation before your PR can be accepted. To run codegen from protoc, run the following command from the root of the repository: ```shell -make codegen +make protogen ``` And to run k8s CRD generation, run the following command from the root of the repository: @@ -75,9 +75,22 @@ make crds Finally, should you wish to modify any of the resulting codegen files (ending in` .pb.go`), do not modify them directly. Instead, you can edit the files in -`cmd/protoc-gen-go-tetragon` and then re-run `make codegen`. +`cmd/protoc-gen-go-tetragon` and then re-run `make protogen`. -### Building and running a Docker image +## Running vendor + +Tetragon uses multiple modules to separate the main module, from `api` from +`pkg/k8s`. Depending on your changes you might need to vendor those changes, +you can use: + +```shell +make vendor +``` + +Note that the `make protogen` and `make crds` commands already vendor +changes automatically. + +## Building and running a Docker image The base kernel should support [BTF](https://github.com/cilium/tetragon#btf-requirement) or a BTF file should be bind mounted on top of `/var/lib/tetragon/btf` inside diff --git a/cmd/protoc-gen-go-tetragon/common/common.go b/tools/protoc-gen-go-tetragon/common/common.go similarity index 100% rename from cmd/protoc-gen-go-tetragon/common/common.go rename to tools/protoc-gen-go-tetragon/common/common.go diff --git a/cmd/protoc-gen-go-tetragon/eventchecker/enum.go b/tools/protoc-gen-go-tetragon/eventchecker/enum.go similarity index 98% rename from cmd/protoc-gen-go-tetragon/eventchecker/enum.go rename to tools/protoc-gen-go-tetragon/eventchecker/enum.go index 7f9ba87079a..d5c60ed557f 100644 --- a/cmd/protoc-gen-go-tetragon/eventchecker/enum.go +++ b/tools/protoc-gen-go-tetragon/eventchecker/enum.go @@ -6,7 +6,7 @@ package eventchecker import ( "fmt" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/common" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/common" "github.com/jpillora/longestcommon" "google.golang.org/protobuf/compiler/protogen" ) diff --git a/cmd/protoc-gen-go-tetragon/eventchecker/eventchecker.go b/tools/protoc-gen-go-tetragon/eventchecker/eventchecker.go similarity index 99% rename from cmd/protoc-gen-go-tetragon/eventchecker/eventchecker.go rename to tools/protoc-gen-go-tetragon/eventchecker/eventchecker.go index f2d201a5f6e..6348a0e96fc 100644 --- a/cmd/protoc-gen-go-tetragon/eventchecker/eventchecker.go +++ b/tools/protoc-gen-go-tetragon/eventchecker/eventchecker.go @@ -6,7 +6,7 @@ package eventchecker import ( "fmt" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/common" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/common" "google.golang.org/protobuf/compiler/protogen" ) diff --git a/cmd/protoc-gen-go-tetragon/eventchecker/field.go b/tools/protoc-gen-go-tetragon/eventchecker/field.go similarity index 99% rename from cmd/protoc-gen-go-tetragon/eventchecker/field.go rename to tools/protoc-gen-go-tetragon/eventchecker/field.go index b1b36d0cc11..f328233d7fd 100644 --- a/cmd/protoc-gen-go-tetragon/eventchecker/field.go +++ b/tools/protoc-gen-go-tetragon/eventchecker/field.go @@ -8,8 +8,8 @@ import ( "path/filepath" "strings" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/common" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/imports" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/common" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/imports" "github.com/iancoleman/strcase" "google.golang.org/protobuf/compiler/protogen" "google.golang.org/protobuf/reflect/protoreflect" diff --git a/cmd/protoc-gen-go-tetragon/eventchecker/message.go b/tools/protoc-gen-go-tetragon/eventchecker/message.go similarity index 98% rename from cmd/protoc-gen-go-tetragon/eventchecker/message.go rename to tools/protoc-gen-go-tetragon/eventchecker/message.go index edc39005320..afb4d73f466 100644 --- a/cmd/protoc-gen-go-tetragon/eventchecker/message.go +++ b/tools/protoc-gen-go-tetragon/eventchecker/message.go @@ -8,7 +8,7 @@ import ( "path/filepath" "strings" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/common" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/common" "google.golang.org/protobuf/compiler/protogen" ) diff --git a/cmd/protoc-gen-go-tetragon/eventchecker/multichecker.go b/tools/protoc-gen-go-tetragon/eventchecker/multichecker.go similarity index 99% rename from cmd/protoc-gen-go-tetragon/eventchecker/multichecker.go rename to tools/protoc-gen-go-tetragon/eventchecker/multichecker.go index 640b8ed175b..7653853da7f 100644 --- a/cmd/protoc-gen-go-tetragon/eventchecker/multichecker.go +++ b/tools/protoc-gen-go-tetragon/eventchecker/multichecker.go @@ -4,7 +4,7 @@ package eventchecker import ( - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/common" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/common" "google.golang.org/protobuf/compiler/protogen" ) diff --git a/cmd/protoc-gen-go-tetragon/eventchecker/yaml.go b/tools/protoc-gen-go-tetragon/eventchecker/yaml.go similarity index 99% rename from cmd/protoc-gen-go-tetragon/eventchecker/yaml.go rename to tools/protoc-gen-go-tetragon/eventchecker/yaml.go index b8957b449db..b6b07ac59a2 100644 --- a/cmd/protoc-gen-go-tetragon/eventchecker/yaml.go +++ b/tools/protoc-gen-go-tetragon/eventchecker/yaml.go @@ -7,7 +7,7 @@ import ( "fmt" "strings" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/common" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/common" "github.com/iancoleman/strcase" "google.golang.org/protobuf/compiler/protogen" ) diff --git a/cmd/protoc-gen-go-tetragon/generate/generate.go b/tools/protoc-gen-go-tetragon/generate/generate.go similarity index 74% rename from cmd/protoc-gen-go-tetragon/generate/generate.go rename to tools/protoc-gen-go-tetragon/generate/generate.go index a0e63633ab6..07407fdf6d0 100644 --- a/cmd/protoc-gen-go-tetragon/generate/generate.go +++ b/tools/protoc-gen-go-tetragon/generate/generate.go @@ -6,9 +6,9 @@ package generate import ( "fmt" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/eventchecker" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/helpers" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/types" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/eventchecker" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/helpers" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/types" "google.golang.org/protobuf/compiler/protogen" ) diff --git a/cmd/protoc-gen-go-tetragon/helpers/helpers.go b/tools/protoc-gen-go-tetragon/helpers/helpers.go similarity index 98% rename from cmd/protoc-gen-go-tetragon/helpers/helpers.go rename to tools/protoc-gen-go-tetragon/helpers/helpers.go index 638263dad99..4d6dcb5ea50 100644 --- a/cmd/protoc-gen-go-tetragon/helpers/helpers.go +++ b/tools/protoc-gen-go-tetragon/helpers/helpers.go @@ -7,7 +7,7 @@ import ( "fmt" "strings" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/common" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/common" "google.golang.org/protobuf/compiler/protogen" ) diff --git a/cmd/protoc-gen-go-tetragon/imports/imports.go b/tools/protoc-gen-go-tetragon/imports/imports.go similarity index 100% rename from cmd/protoc-gen-go-tetragon/imports/imports.go rename to tools/protoc-gen-go-tetragon/imports/imports.go diff --git a/cmd/protoc-gen-go-tetragon/main.go b/tools/protoc-gen-go-tetragon/main.go similarity index 66% rename from cmd/protoc-gen-go-tetragon/main.go rename to tools/protoc-gen-go-tetragon/main.go index e48dde5a6c6..1ea68c84a8f 100644 --- a/cmd/protoc-gen-go-tetragon/main.go +++ b/tools/protoc-gen-go-tetragon/main.go @@ -4,7 +4,7 @@ package main import ( - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/generate" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/generate" ) func main() { diff --git a/cmd/protoc-gen-go-tetragon/types/types.go b/tools/protoc-gen-go-tetragon/types/types.go similarity index 98% rename from cmd/protoc-gen-go-tetragon/types/types.go rename to tools/protoc-gen-go-tetragon/types/types.go index e99d5032928..fdaa4b505ed 100644 --- a/cmd/protoc-gen-go-tetragon/types/types.go +++ b/tools/protoc-gen-go-tetragon/types/types.go @@ -6,7 +6,7 @@ package types import ( "path/filepath" - "github.com/cilium/tetragon/cmd/protoc-gen-go-tetragon/common" + "github.com/cilium/tetragon/tools/protoc-gen-go-tetragon/common" "google.golang.org/protobuf/compiler/protogen" )