From 120d86fda9680d220d1a2463731559070ee52d0a Mon Sep 17 00:00:00 2001 From: kpango Date: Tue, 30 Jul 2024 00:27:40 +0900 Subject: [PATCH 01/35] update deps Signed-off-by: kpango --- .gitfiles | 2 + .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .../ISSUE_TEMPLATE/security_issue_report.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/actions/setup-e2e/action.yaml | 2 +- .github/actions/setup-k3d/action.yaml | 24 ++- .github/workflows/e2e-chaos.yaml | 18 ++- .github/workflows/e2e-max-dim.yml | 6 +- .github/workflows/e2e-profiling.yml | 6 +- .github/workflows/e2e.yml | 34 +++- .github/workflows/helm.yml | 3 + Makefile | 47 ++++-- Makefile.d/dependencies.mk | 15 +- Makefile.d/docker.mk | 6 - Makefile.d/k3d.mk | 8 +- Makefile.d/tools.mk | 9 +- dockers/agent/core/agent/Dockerfile | 34 +++- dockers/agent/core/faiss/Dockerfile | 34 +++- dockers/agent/core/ngt/Dockerfile | 34 +++- dockers/agent/sidecar/Dockerfile | 31 +++- dockers/ci/base/Dockerfile | 68 ++++++-- dockers/dev/Dockerfile | 63 ++++++-- dockers/discoverer/k8s/Dockerfile | 31 +++- dockers/gateway/filter/Dockerfile | 31 +++- dockers/gateway/lb/Dockerfile | 31 +++- dockers/gateway/mirror/Dockerfile | 31 +++- dockers/index/job/correction/Dockerfile | 31 +++- dockers/index/job/creation/Dockerfile | 31 +++- .../index/job/readreplica/rotate/Dockerfile | 31 +++- dockers/index/job/save/Dockerfile | 31 +++- dockers/index/operator/Dockerfile | 31 +++- dockers/manager/index/Dockerfile | 31 +++- dockers/operator/helm/Dockerfile | 33 +++- dockers/tools/benchmark/job/Dockerfile | 34 +++- dockers/tools/benchmark/operator/Dockerfile | 31 +++- dockers/tools/cli/loadtest/Dockerfile | 34 +++- go.mod | 50 +++--- go.sum | 52 +++--- hack/docker/gen/main.go | 148 +++++++++++++----- hack/go.mod.default | 16 +- hack/license/gen/main.go | 5 +- pkg/agent/core/ngt/handler/grpc/insert.go | 2 +- .../core/ngt/handler/grpc/insert_test.go | 6 +- .../core/ngt/handler/grpc/search_test.go | 3 +- rust/Cargo.lock | 8 +- rust/rust-toolchain | 1 + rust/rust-toolchain.toml | 2 +- versions/CMAKE_VERSION | 1 + versions/OPERATOR_SDK_VERSION | 2 +- versions/PROMETHEUS_STACK_VERSION | 2 +- versions/RUST_VERSION | 2 +- versions/actions/DOCKER_SETUP_BUILDX_ACTION | 2 +- 52 files changed, 889 insertions(+), 305 deletions(-) create mode 100644 rust/rust-toolchain create mode 100644 versions/CMAKE_VERSION diff --git a/.gitfiles b/.gitfiles index 13f61ddc34..3019a82c21 100644 --- a/.gitfiles +++ b/.gitfiles @@ -1971,6 +1971,7 @@ rust/libs/proto/src/sidecar.v1.rs rust/libs/proto/src/sidecar.v1.tonic.rs rust/libs/proto/src/vald.v1.rs rust/libs/proto/src/vald.v1.tonic.rs +rust/rust-toolchain rust/rust-toolchain.toml tests/chaos/chart/.helmignore tests/chaos/chart/Chart.yaml @@ -2000,6 +2001,7 @@ tests/e2e/sidecar/sidecar_test.go tests/performance/max_vector_dim_test.go versions/BUF_VERSION versions/CHAOS_MESH_VERSION +versions/CMAKE_VERSION versions/DOCKER_VERSION versions/FAISS_VERSION versions/GOLANGCILINT_VERSION diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8b4d110a67..c5db91ca9c 100755 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,7 +24,7 @@ assignees: "" - Vald Version: v1.7.12 - Go Version: v1.22.5 -- Rust Version: v1.77.2 +- Rust Version: v1.80.0 - Docker Version: v27.1.1 - Kubernetes Version: v1.30.3 - Helm Version: v3.15.3 diff --git a/.github/ISSUE_TEMPLATE/security_issue_report.md b/.github/ISSUE_TEMPLATE/security_issue_report.md index 0d78df8f92..3474581d6c 100644 --- a/.github/ISSUE_TEMPLATE/security_issue_report.md +++ b/.github/ISSUE_TEMPLATE/security_issue_report.md @@ -18,7 +18,7 @@ assignees: "" - Vald Version: v1.7.12 - Go Version: v1.22.5 -- Rust Version: v1.77.2 +- Rust Version: v1.80.0 - Docker Version: v27.1.1 - Kubernetes Version: v1.30.3 - Helm Version: v3.15.3 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 831b078b81..8f43359802 100755 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -17,7 +17,7 @@ - Vald Version: v1.7.12 - Go Version: v1.22.5 -- Rust Version: v1.77.2 +- Rust Version: v1.80.0 - Docker Version: v27.1.1 - Kubernetes Version: v1.30.3 - Helm Version: v3.15.3 diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index f2c11c0aec..e3a18d35de 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -19,7 +19,7 @@ inputs: require_libhdf5: description: "If libhdf5 is not required, set this to false" required: false - default: "true" + default: "false" require_go: description: "If go is not required, set this to false" required: false diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 48f7556818..8f39cdc8de 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -93,9 +93,31 @@ runs: shell: bash id: start_k3d run: | - k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} + k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} \ + --image docker.io/rancher/k3s:latest \ + --k3s-arg "--disable=traefik@server:*" \ + --api-port 0.0.0.0:6443 \ + -v "/lib/modules:/lib/modules" + k3d cluster start ${{ inputs.name }} + k3d kubeconfig merge ${{ inputs.name }} --kubeconfig-switch-context - name: Ready k8s cluster shell: bash run: | k3d cluster list + server="k3d-${{ inputs.name }}-server-0" + docker logs ${server} + docker inspect ${server} + - name: Set k3d cluster config to KUBECONFIG + shell: bash + run: | + echo "KUBECONFIG=$(k3d kubeconfig write ${{ inputs.name }})" >> $GITHUB_ENV + - name: Check KUBECONFIG environment variable + shell: bash + run: | + echo $KUBECONFIG + cat $KUBECONFIG + cat /etc/hosts + - name: Show Kubernetes Cluster Info + shell: bash + run: | kubectl cluster-info dump diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index d571227e1f..ed93101f73 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -35,11 +35,15 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml agent-failure: name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -75,9 +79,11 @@ jobs: e2e/insert/search random-pod-failure: name: "E2E chaos test (random Pod failure: to test redundancy)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -113,9 +119,11 @@ jobs: e2e/insert/search agent-network-partition: name: "E2E chaos test (agent network partition: to test retries)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -151,9 +159,11 @@ jobs: e2e/insert/search clusterwide-network-bandwidth: name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index ef2b137547..00bf9e5360 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -32,11 +32,15 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml e2e-max-dimension-insert: name: "E2E test (Max Dimension Insert: skip strict exist check)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index d54c79722d..38b0d9332a 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -33,11 +33,15 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml e2e-profiling: name: "E2E profiling" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 780ddd8b66..104878d243 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -33,11 +33,15 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml e2e-stream-crud: name: "E2E test (Stream CRUD)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -73,9 +77,11 @@ jobs: POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-for-operator: name: "E2E test (Stream CRUD) for operator" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -133,9 +139,11 @@ jobs: POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-under-index-management-jobs: name: "E2E test (Stream CRUD) under index management jobs" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -175,9 +183,11 @@ jobs: POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-skip-exist-check: name: "E2E test (Stream CRUD: skip strict exist check)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -213,9 +223,11 @@ jobs: POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-multiapis-crud: name: "E2E test (Multi-APIs CRUD)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -247,9 +259,11 @@ jobs: POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-jobs: name: "E2E test (Jobs)" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -278,9 +292,11 @@ jobs: POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-with-readreplica: name: "E2E test (Stream CRUD) with read replica" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config @@ -326,9 +342,11 @@ jobs: POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }} e2e-stream-crud-with-mirror: name: "E2E test (Stream CRUD) with mirror" - needs: [dump-contexts-to-log] + needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index f72f943530..a360808379 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -75,6 +75,9 @@ jobs: name: CRUD test on remote Helm chart runs-on: ubuntu-latest timeout-minutes: 60 + needs: [detect-ci-container] + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} steps: - uses: actions/checkout@v4 - name: Set Git config diff --git a/Makefile b/Makefile index 9986a575ac..ff723f0bf7 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,8 @@ CARGO_HOME ?= $(RUST_HOME)/cargo RUST_VERSION := $(eval RUST_VERSION := $(shell cat versions/RUST_VERSION))$(RUST_VERSION) BUF_VERSION := $(eval BUF_VERSION := $(shell cat versions/BUF_VERSION))$(BUF_VERSION) -DOCKER_VERSION := $(eval DOCKER_VERSION := $(shell cat versions/DOCKER_VERSION))$(DOCKER_VERSION) +CMAKE_VERSION := $(eval CMAKE_VERSION := $(shell cat versions/CMAKE_VERSION))$(CMAKE_VERSION) +DOCKER_VERSION := $(eval DOCKER_VERSION := $(shell cat versions/DOCKER_VERSION))$(DOCKER_VERSION) FAISS_VERSION := $(eval FAISS_VERSION := $(shell cat versions/FAISS_VERSION))$(FAISS_VERSION) GOLANGCILINT_VERSION := $(eval GOLANGCILINT_VERSION := $(shell cat versions/GOLANGCILINT_VERSION))$(GOLANGCILINT_VERSION) HDF5_VERSION := $(eval HDF5_VERSION := $(shell cat versions/HDF5_VERSION))$(HDF5_VERSION) @@ -139,7 +140,7 @@ PBGOS = $(PROTOS:apis/proto/%.proto=apis/grpc/%.pb.go) SWAGGERS = $(PROTOS:apis/proto/%.proto=apis/swagger/%.swagger.json) PBDOCS = apis/docs/v1/docs.md -LDFLAGS = -static -fPIC -pthread -std=gnu++20 -lstdc++ -lm -z relro -z now -flto=auto -march=native -mtune=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast -fomit-frame-pointer -fmerge-all-constants -funroll-loops -falign-functions=32 -ffunction-sections -fdata-sections +LDFLAGS = -static -fPIC -pthread -std=gnu++23 -lstdc++ -lm -z relro -z now -flto=auto -march=native -mtune=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast -fomit-frame-pointer -fmerge-all-constants -funroll-loops -falign-functions=32 -ffunction-sections -fdata-sections NGT_LDFLAGS = -fopenmp -lopenblas -llapack FAISS_LDFLAGS = $(NGT_LDFLAGS) -lgfortran @@ -150,17 +151,14 @@ ifeq ($(GOARCH),amd64) CFLAGS ?= -mno-avx512f -mno-avx512dq -mno-avx512cd -mno-avx512bw -mno-avx512vl CXXFLAGS ?= $(CFLAGS) EXTLDFLAGS ?= -m64 -NGT_EXTRA_FLAGS ?= else ifeq ($(GOARCH),arm64) CFLAGS ?= CXXFLAGS ?= $(CFLAGS) EXTLDFLAGS ?= -march=armv8-a -NGT_EXTRA_FLAGS ?= else CFLAGS ?= CXXFLAGS ?= $(CFLAGS) EXTLDFLAGS ?= -NGT_EXTRA_FLAGS ?= endif BENCH_DATASET_MD5S := $(eval BENCH_DATASET_MD5S := $(shell find $(BENCH_DATASET_MD5_DIR) -type f -regex ".*\.md5"))$(BENCH_DATASET_MD5S) @@ -622,9 +620,9 @@ $(USR_LOCAL)/include/NGT/Capi.h: -DCMAKE_C_FLAGS="$(CFLAGS)" \ -DCMAKE_CXX_FLAGS="$(CXXFLAGS)" \ -DCMAKE_INSTALL_PREFIX=$(USR_LOCAL) \ - "$(NGT_EXTRA_FLAGS)" . - make -j$(CORES) -C $(TEMP_DIR)/NGT-$(NGT_VERSION) - make install -C $(TEMP_DIR)/NGT-$(NGT_VERSION) + -B $(TEMP_DIR)/NGT-$(NGT_VERSION)/build $(TEMP_DIR)/NGT-$(NGT_VERSION) + make -C $(TEMP_DIR)/NGT-$(NGT_VERSION)/build -j$(CORES) ngt + make -C $(TEMP_DIR)/NGT-$(NGT_VERSION)/build install cd $(ROOTDIR) rm -rf $(TEMP_DIR)/NGT-$(NGT_VERSION) ldconfig @@ -640,18 +638,37 @@ $(LIB_PATH)/libfaiss.a: -DBUILD_SHARED_LIBS=OFF \ -DBUILD_STATIC_EXECS=ON \ -DBUILD_TESTING=OFF \ - -DCMAKE_C_FLAGS="$(LDFLAGS)" \ - -DCMAKE_INSTALL_PREFIX=$(USR_LOCAL) \ -DFAISS_ENABLE_PYTHON=OFF \ -DFAISS_ENABLE_GPU=OFF \ -DBLA_VENDOR=OpenBLAS \ + -DCMAKE_C_FLAGS="$(LDFLAGS)" \ -DCMAKE_EXE_LINKER_FLAGS="$(FAISS_LDFLAGS)" \ - -B build . && \ - make -C build -j$(CORES) faiss && \ - make -C build install - rm -rf v$(FAISS_VERSION).tar.gz - rm -rf $(TEMP_DIR)/faiss-$(FAISS_VERSION) + -DCMAKE_INSTALL_PREFIX=$(USR_LOCAL) \ + -B $(TEMP_DIR)/faiss-$(FAISS_VERSION)/build $(TEMP_DIR)/faiss-$(FAISS_VERSION) + make -C $(TEMP_DIR)/faiss-$(FAISS_VERSION)/build -j$(CORES) faiss + make -C $(TEMP_DIR)/faiss-$(FAISS_VERSION)/build install + cd $(ROOTDIR) + rm -rf $(TEMP_DIR)/v$(FAISS_VERSION).tar.gz $(TEMP_DIR)/faiss-$(FAISS_VERSION) + ldconfig + +.PHONY: cmake/install +## install CMAKE +cmake/install: + git clone --depth 1 --branch v$(CMAKE_VERSION) https://github.com/Kitware/CMake.git $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION) + cd $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION) && \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DCMAKE_C_FLAGS="$(CFLAGS)" \ + -DCMAKE_CXX_FLAGS="$(CXXFLAGS)" \ + -DCMAKE_INSTALL_PREFIX=$(USR_LOCAL) \ + -B $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION)/build $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION) + make -C $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION)/build -j$(CORES) cmake + make -C $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION)/build install + cd $(ROOTDIR) + rm -rf $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION) ldconfig + # -DCMAKE_USE_OPENSSL=OFF .PHONY: lint ## run lints diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk index 8e01f2fb07..c764da0309 100644 --- a/Makefile.d/dependencies.mk +++ b/Makefile.d/dependencies.mk @@ -18,9 +18,12 @@ ## update vald libraries including tools update/libs: \ update/chaos-mesh \ + update/cmake \ + update/docker \ update/faiss \ update/go \ update/golangci-lint \ + update/hdf5 \ update/helm \ update/helm-docs \ update/helm-operator \ @@ -33,12 +36,12 @@ update/libs: \ update/prometheus-stack \ update/protobuf \ update/reviewdog \ + update/rust \ update/telepresence \ update/vald \ update/valdcli \ update/yq \ - update/zlib \ - update/hdf5 + update/zlib .PHONY: go/download ## download Go package dependencies @@ -87,6 +90,8 @@ go/example/deps: rust/deps: \ rust/install sed -i "17s/channel = \"[0-9]\+\.[0-9]\+\(\.[0-9]\+\)\?\"/channel = \"$(RUST_VERSION)\"/g" $(ROOTDIR)/rust/rust-toolchain.toml + rustup toolchain install $(RUST_VERSION) + rustup default $(RUST_VERSION) cd $(ROOTDIR)/rust && $(CARGO_HOME)/bin/cargo update && cd - .PHONY: update/chaos-mesh @@ -120,6 +125,7 @@ update/golangci-lint: ## update rust version update/rust: curl -fsSL https://releases.rs | grep -Po 'Stable: \K[\d.]+\s' | head -n 1 > $(ROOTDIR)/versions/RUST_VERSION + cp -f $(ROOTDIR)/versions/RUST_VERSION $(ROOTDIR)/rust/rust-toolchain .PHONY: update/docker ## update docker version @@ -186,6 +192,11 @@ update/ngt: update/faiss: curl -fsSL https://api.github.com/repos/facebookresearch/faiss/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' > $(ROOTDIR)/versions/FAISS_VERSION +.PHONY: update/cmake +## update CMAKE version +update/cmake: + curl -fsSL https://api.github.com/repos/Kitware/CMAKE/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' > $(ROOTDIR)/versions/CMAKE_VERSION + .PHONY: update/reviewdog ## update reviewdog version update/reviewdog: diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 540eca3eb4..1e62d969a6 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -53,8 +53,6 @@ ifeq ($(REMOTE),true) --build-arg BUILDKIT_INLINE_CACHE=$(BUILDKIT_INLINE_CACHE) \ --build-arg GO_VERSION=$(GO_VERSION) \ --build-arg RUST_VERSION=$(RUST_VERSION) \ - --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ - --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ --build-arg MAINTAINER=$(MAINTAINER) \ --sbom=true \ --provenance=mode=max \ @@ -70,8 +68,6 @@ else --build-arg BUILDKIT_INLINE_CACHE=$(BUILDKIT_INLINE_CACHE) \ --build-arg GO_VERSION=$(GO_VERSION) \ --build-arg RUST_VERSION=$(RUST_VERSION) \ - --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ - --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ --build-arg MAINTAINER=$(MAINTAINER) \ $(EXTRA_ARGS) \ -t $(CRORG)/$(IMAGE):$(TAG) \ @@ -120,8 +116,6 @@ docker/name/agent: docker/build/agent: @make DOCKERFILE="$(ROOTDIR)/dockers/agent/core/agent/Dockerfile" \ IMAGE=$(AGENT_IMAGE) \ - DISTROLESS_IMAGE=gcr.io/distroless/cc-debian12 \ - EXTRA_ARGS="--build-arg RUST_VERSION=$(RUST_VERSION)" \ docker/build/image .PHONY: docker/name/discoverer-k8s diff --git a/Makefile.d/k3d.mk b/Makefile.d/k3d.mk index 7449df203c..c446cd6d9a 100644 --- a/Makefile.d/k3d.mk +++ b/Makefile.d/k3d.mk @@ -36,10 +36,12 @@ k3d/start: --host-pid-mode=true \ --port 8081:80@loadbalancer \ --k3s-arg "--disable=traefik@server:*" \ + --api-port 0.0.0.0:6443 \ -v "/lib/modules:/lib/modules" - # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) --agents $(K3D_NODES) -v "/lib/modules:/lib/modules" - # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) -p "8081:80@loadbalancer" --agents $(K3D_NODES) --k3s-arg '--disable=traefik@all' - export KUBECONFIG="$(shell sudo $(K3D_COMMAND) kubeconfig merge -o $(TEMP_DIR)/k3d_$(K3D_CLUSTER_NAME)_kubeconfig.yaml $(K3D_CLUSTER_NAME))" + export KUBECONFIG="$(shell $(K3D_COMMAND) kubeconfig merge -o $(TEMP_DIR)/k3d_$(K3D_CLUSTER_NAME)_kubeconfig.yaml $(K3D_CLUSTER_NAME)) --kubeconfig-switch-context" + docker logs k3d-$(K3D_CLUSTER_NAME)-server-0 + docker inspect k3d-$(K3D_CLUSTER_NAME)-server-0 + kubectl cluster-info dump .PHONY: k3d/restart ## restart k3d (kubernetes in docker) cluster diff --git a/Makefile.d/tools.mk b/Makefile.d/tools.mk index 59b06c60d7..242b5b3adb 100644 --- a/Makefile.d/tools.mk +++ b/Makefile.d/tools.mk @@ -53,9 +53,10 @@ $(GOBIN)/crlfmt: $(call go-install, github.com/cockroachdb/crlfmt) .PHONY: prettier/install -prettier/install: $(BINDIR)/prettier -$(BINDIR)/prettier: - npm config set registry http://registry.npmjs.org/ +prettier/install: $(NPM_GLOBAL_PREFIX)/bin/prettier +$(NPM_GLOBAL_PREFIX)/bin/prettier: + npm config -g set registry http://registry.npmjs.org/ + npm cache clean --force type prettier || npm install -g prettier .PHONY: reviewdog/install @@ -170,6 +171,8 @@ rust/install: $(CARGO_HOME)/bin/cargo $(CARGO_HOME)/bin/cargo: curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | CARGO_HOME=${CARGO_HOME} RUSTUP_HOME=${RUSTUP_HOME} sh -s -- --default-toolchain $(RUST_VERSION) -y + rustup toolchain install $(RUST_VERSION) + rustup default $(RUST_VERSION) source "${CARGO_HOME}/env" .PHONY: zlib/install diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile index c13f258679..e3be959db4 100644 --- a/dockers/agent/core/agent/Dockerfile +++ b/dockers/agent/core/agent/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -32,17 +35,24 @@ ENV DEBIAN_FRONTEND=noninteractive ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=agent/core/agent ENV REPO=vald ENV RUST_HOME=/usr/loacl/lib/rust +ENV TZ=Etc/UTC +ENV USER=root ENV RUSTUP_HOME=${RUST_HOME}/rustup ENV CARGO_HOME=${RUST_HOME}/cargo ENV PATH=${CARGO_HOME}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,21 +62,30 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ cmake \ - g++ \ gcc \ + g++ \ unzip \ + libssl-dev \ liblapack-dev \ libomp-dev \ libopenblas-dev \ gfortran \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -79,14 +98,14 @@ WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO}/rust COPY rust . WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN make RUST_VERSION="${RUST_VERSION}" rust/install \ && make ngt/install \ && make faiss/install \ && make rust/target/release/${APP_NAME} \ && mv "rust/target/release/${APP_NAME}" "/usr/bin/${APP_NAME}" \ && rm -rf rust/target -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/cc-debian12:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -94,5 +113,6 @@ LABEL maintainer="${MAINTAINER}" ENV APP_NAME=agent COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/agent"] diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile index d9755d36c1..492d165add 100644 --- a/dockers/agent/core/faiss/Dockerfile +++ b/dockers/agent/core/faiss/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=agent/core/faiss ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,21 +62,30 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ cmake \ - g++ \ gcc \ + g++ \ unzip \ + libssl-dev \ liblapack-dev \ libomp-dev \ libopenblas-dev \ gfortran \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -90,7 +109,7 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ @@ -98,7 +117,7 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ && make faiss/install \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -107,5 +126,6 @@ ENV APP_NAME=faiss COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/faiss"] diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index f1ef1aa20e..b96bcf6298 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=agent/core/ngt ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,20 +62,29 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ cmake \ - g++ \ gcc \ + g++ \ unzip \ + libssl-dev \ liblapack-dev \ libomp-dev \ libopenblas-dev \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -89,7 +108,7 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ @@ -97,7 +116,7 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ && make ngt/install \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -106,5 +125,6 @@ ENV APP_NAME=ngt COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/ngt"] diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile index 21cdfea4d0..696ca75b06 100644 --- a/dockers/agent/sidecar/Dockerfile +++ b/dockers/agent/sidecar/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=agent/sidecar ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -81,14 +99,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ LABEL maintainer="${MAINTAINER}" ENV APP_NAME=sidecar COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/sidecar"] diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index f6e4aae0c6..039797428e 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -18,16 +18,21 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 -FROM ghcr.io/vdaas/vald/vald-buildbase:nightly +# skipcq: DOK-DL3026,DOK-DL3007 +FROM ghcr.io/actions-runner-controller/actions-runner-controller/actions-runner-dind:ubuntu-22.04 ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION ARG RUST_VERSION ENV APP_NAME=ci-container +ENV CC=gcc +ENV CXX=g++ ENV DEBIAN_FRONTEND=noninteractive ENV GO111MODULE=on ENV GOPATH=/go @@ -35,53 +40,71 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=ci/base ENV REPO=vald ENV RUST_HOME=/usr/loacl/lib/rust -ENV RUSTUP_HOME=${RUST_HOME}/rustup +ENV TZ=Etc/UTC +ENV USER=root ENV CARGO_HOME=${RUST_HOME}/cargo +ENV RUSTUP_HOME=${RUST_HOME}/rustup ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ && apt-get update -y \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends --fix-missing \ + curl \ + gnupg \ + software-properties-common \ + && add-apt-repository ppa:ubuntu-toolchain-r/test -y \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ ca-certificates \ - curl \ + tzdata \ + locales \ git \ cmake \ - g++ \ gcc \ + g++ \ unzip \ + libssl-dev \ liblapack-dev \ libomp-dev \ libopenblas-dev \ gfortran \ - curl \ gawk \ - git \ gnupg2 \ graphviz \ jq \ libhdf5-dev \ libaec-dev \ - nodejs \ - npm \ sed \ zip \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -90,19 +113,27 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY . . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make RUST_VERSION="${RUST_VERSION}" rust/install \ - && sysctl -w net.ipv6.conf.all.disable_ipv6=1 \ - && sysctl -w net.ipv6.conf.default.disable_ipv6=1 \ - && sysctl -w net.ipv6.conf.lo.disable_ipv6=1 \ - && sysctl -p \ + && curl -fsSL https://deb.nodesource.com/setup_current.x | bash - \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/cache/* \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing nodejs \ + && npm install -g npm@latest \ + && apt-get clean \ + && apt-get autoclean -y \ + && rm -rf /var/lib/apt/lists/* /var/cache/* \ + && apt-get autoremove -y \ && make GOARCH=${TARGETARCH} GOOS=${TARGETOS} deps GO_CLEAN_DEPS=false \ && make GOARCH=${TARGETARCH} GOOS=${TARGETOS} golangci-lint/install \ && make GOARCH=${TARGETARCH} GOOS=${TARGETOS} gotestfmt/install \ + && make cmake/install \ && make buf/install \ && make hdf5/install \ && make helm-docs/install \ @@ -122,3 +153,6 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ && make ngt/install \ && make faiss/install \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* +# skipcq: DOK-DL3002 +USER root:root +ENTRYPOINT ["/bin/bash", "-c", "/usr/bin/entrypoint-dind.sh"] diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index d7409e637e..3acbe914a9 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -18,16 +18,21 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 -FROM mcr.microsoft.com/vscode/devcontainers/base:debian +# skipcq: DOK-DL3026,DOK-DL3007 +FROM mcr.microsoft.com/devcontainers/base:ubuntu22.04 ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION ARG RUST_VERSION ENV APP_NAME=dev-container +ENV CC=gcc +ENV CXX=g++ ENV DEBIAN_FRONTEND=noninteractive ENV GO111MODULE=on ENV GOPATH=/go @@ -35,53 +40,71 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=dev ENV REPO=vald ENV RUST_HOME=/usr/loacl/lib/rust -ENV RUSTUP_HOME=${RUST_HOME}/rustup +ENV TZ=Etc/UTC +ENV USER=root ENV CARGO_HOME=${RUST_HOME}/cargo +ENV RUSTUP_HOME=${RUST_HOME}/rustup ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ && apt-get update -y \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends --fix-missing \ + curl \ + gnupg \ + software-properties-common \ + && add-apt-repository ppa:ubuntu-toolchain-r/test -y \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ ca-certificates \ - curl \ + tzdata \ + locales \ git \ cmake \ - g++ \ gcc \ + g++ \ unzip \ + libssl-dev \ liblapack-dev \ libomp-dev \ libopenblas-dev \ gfortran \ - curl \ gawk \ - git \ gnupg2 \ graphviz \ jq \ libhdf5-dev \ libaec-dev \ - nodejs \ - npm \ sed \ zip \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -90,15 +113,27 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY . . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make RUST_VERSION="${RUST_VERSION}" rust/install \ + && curl -fsSL https://deb.nodesource.com/setup_current.x | bash - \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/cache/* \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing nodejs \ + && npm install -g npm@latest \ + && apt-get clean \ + && apt-get autoclean -y \ + && rm -rf /var/lib/apt/lists/* /var/cache/* \ + && apt-get autoremove -y \ && make GOARCH=${TARGETARCH} GOOS=${TARGETOS} deps GO_CLEAN_DEPS=false \ && make GOARCH=${TARGETARCH} GOOS=${TARGETOS} golangci-lint/install \ && make GOARCH=${TARGETARCH} GOOS=${TARGETOS} gotestfmt/install \ + && make cmake/install \ && make buf/install \ && make hdf5/install \ && make helm-docs/install \ @@ -125,3 +160,5 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ && make ngt/install \ && make faiss/install \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* +# skipcq: DOK-DL3002 +USER root:root diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile index 28fd4770c7..0f41ed0390 100644 --- a/dockers/discoverer/k8s/Dockerfile +++ b/dockers/discoverer/k8s/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=discoverer/k8s ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=discoverer COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/discoverer"] diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile index b2f5478a5c..c7f4622cf7 100644 --- a/dockers/gateway/filter/Dockerfile +++ b/dockers/gateway/filter/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=gateway/filter ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=filter COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/filter"] diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile index bf78cc7ad8..4d84be8249 100644 --- a/dockers/gateway/lb/Dockerfile +++ b/dockers/gateway/lb/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=gateway/lb ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=lb COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/lb"] diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile index c52c633a04..9a2011a1ab 100644 --- a/dockers/gateway/mirror/Dockerfile +++ b/dockers/gateway/mirror/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=gateway/mirror ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=mirror COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/mirror"] diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile index 269648c226..a4a58e071a 100644 --- a/dockers/index/job/correction/Dockerfile +++ b/dockers/index/job/correction/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=index/job/correction ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=index-correction COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/index-correction"] diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile index 6af94e2436..fd1f004b8b 100644 --- a/dockers/index/job/creation/Dockerfile +++ b/dockers/index/job/creation/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=index/job/creation ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=index-creation COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/index-creation"] diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile index 4fac5a23a2..cce459c2fb 100644 --- a/dockers/index/job/readreplica/rotate/Dockerfile +++ b/dockers/index/job/readreplica/rotate/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=index/job/readreplica/rotate ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=readreplica-rotate COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/readreplica-rotate"] diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile index 2af2997b22..98bc29725d 100644 --- a/dockers/index/job/save/Dockerfile +++ b/dockers/index/job/save/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=index/job/save ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=index-save COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/index-save"] diff --git a/dockers/index/operator/Dockerfile b/dockers/index/operator/Dockerfile index ab6931a089..e06ff0b151 100644 --- a/dockers/index/operator/Dockerfile +++ b/dockers/index/operator/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=index/operator ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=index-operator COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/index-operator"] diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile index 0475251f4c..dd62d8aa59 100644 --- a/dockers/manager/index/Dockerfile +++ b/dockers/manager/index/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=manager/index ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=index COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/index"] diff --git a/dockers/operator/helm/Dockerfile b/dockers/operator/helm/Dockerfile index 1e7f1a4bf5..f53bd6ee68 100644 --- a/dockers/operator/helm/Dockerfile +++ b/dockers/operator/helm/Dockerfile @@ -19,13 +19,16 @@ ARG UPX_OPTIONS=-9 ARG OPERATOR_SDK_VERSION=latest -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM quay.io/operator-framework/helm-operator:${OPERATOR_SDK_VERSION} AS operator -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -38,14 +41,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=operator/helm ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -55,14 +65,22 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ upx \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -82,7 +100,7 @@ COPY hack . COPY --from=operator /usr/local/bin/${APP_NAME} /usr/bin/${APP_NAME} WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ @@ -103,7 +121,7 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ && make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald-helm-operator \ && cp -r charts/* /opt/helm/charts/ \ && upx "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -114,5 +132,6 @@ COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /opt/helm/watches.yaml /opt/helm/watches.yaml COPY --from=builder /opt/helm/charts/vald /opt/helm/charts/vald COPY --from=builder /opt/helm/charts/vald-helm-operator /opt/helm/charts/vald-helm-operator +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"] diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile index a75d7645d7..5d9f6e672c 100644 --- a/dockers/tools/benchmark/job/Dockerfile +++ b/dockers/tools/benchmark/job/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=tools/benchmark/job ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,19 +62,28 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ cmake \ - g++ \ gcc \ + g++ \ unzip \ + libssl-dev \ libhdf5-dev \ libaec-dev \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -86,7 +105,7 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ @@ -94,7 +113,7 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ && make hdf5/install \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -103,5 +122,6 @@ ENV APP_NAME=job COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/job"] diff --git a/dockers/tools/benchmark/operator/Dockerfile b/dockers/tools/benchmark/operator/Dockerfile index 56a011fcab..1e67eb393d 100644 --- a/dockers/tools/benchmark/operator/Dockerfile +++ b/dockers/tools/benchmark/operator/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=tools/benchmark/operator ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,13 +62,21 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -80,14 +98,14 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ && make go/download \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -96,5 +114,6 @@ ENV APP_NAME=operator COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/operator"] diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile index f92a30cbe1..8b3d06adf3 100644 --- a/dockers/tools/cli/loadtest/Dockerfile +++ b/dockers/tools/cli/loadtest/Dockerfile @@ -18,11 +18,14 @@ # DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER root:root + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -35,14 +38,21 @@ ENV GOROOT=/opt/go ENV HOME=/root ENV INITRD=No ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=tools/cli/loadtest ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ @@ -52,19 +62,28 @@ RUN apt-get clean \ build-essential \ ca-certificates \ curl \ + tzdata \ + locales \ git \ cmake \ - g++ \ gcc \ + g++ \ unzip \ + libssl-dev \ libhdf5-dev \ libaec-dev \ && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . @@ -88,7 +107,7 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make GO_VERSION="${GO_VERSION}" go/install \ @@ -96,7 +115,7 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ && make hdf5/install \ && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM gcr.io/distroless/static:nonroot ARG MAINTAINER="vdaas.org vald team " LABEL maintainer="${MAINTAINER}" @@ -105,5 +124,6 @@ ENV APP_NAME=loadtest COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +# skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/loadtest"] diff --git a/go.mod b/go.mod index 9d60dce779..f6bae1e793 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ replace ( cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.13.5 cloud.google.com/go/storage => cloud.google.com/go/storage v1.43.0 cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.11 - code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20240725181214-870a2a4a34a6 + code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20240727181252-5b6127bbde80 contrib.go.opencensus.io/exporter/aws => contrib.go.opencensus.io/exporter/aws v0.0.0-20230502192102-15967c811cec contrib.go.opencensus.io/exporter/prometheus => contrib.go.opencensus.io/exporter/prometheus v0.4.2 contrib.go.opencensus.io/integrations/ocsql => contrib.go.opencensus.io/integrations/ocsql v0.1.7 @@ -44,7 +44,7 @@ replace ( github.com/akrylysov/pogreb => github.com/akrylysov/pogreb v0.10.2 github.com/antihax/optional => github.com/antihax/optional v1.0.0 github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.55.3 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.55.4 github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.30.3 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.27.27 @@ -154,7 +154,7 @@ replace ( github.com/google/gofuzz => github.com/google/gofuzz v1.2.0 github.com/google/martian => github.com/google/martian v2.1.0+incompatible github.com/google/martian/v3 => github.com/google/martian/v3 v3.3.3 - github.com/google/pprof => github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 + github.com/google/pprof => github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 github.com/google/shlex => github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/subcommands => github.com/google/subcommands v1.2.0 github.com/google/uuid => github.com/google/uuid v1.6.0 @@ -192,7 +192,7 @@ replace ( github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report v1.0.0 github.com/kisielk/errcheck => github.com/kisielk/errcheck v1.7.0 github.com/kisielk/gotool => github.com/kisielk/gotool v1.0.0 - github.com/klauspost/compress => github.com/klauspost/compress v1.17.10-0.20240719110028-cfab8bd586ec + github.com/klauspost/compress => github.com/klauspost/compress v1.17.10-0.20240726110848-d76f801616d1 github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 v2.2.8 github.com/kpango/fastime => github.com/kpango/fastime v1.1.9 github.com/kpango/fuid => github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 @@ -225,8 +225,8 @@ replace ( github.com/niemeyer/pretty => github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e github.com/nxadm/tail => github.com/nxadm/tail v1.4.11 github.com/onsi/ginkgo => github.com/onsi/ginkgo v1.16.5 - github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 v2.19.0 - github.com/onsi/gomega => github.com/onsi/gomega v1.34.0 + github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 v2.19.1 + github.com/onsi/gomega => github.com/onsi/gomega v1.34.1 github.com/peterbourgon/diskv => github.com/peterbourgon/diskv v2.0.1+incompatible github.com/phpdave11/gofpdf => github.com/phpdave11/gofpdf v1.4.2 github.com/phpdave11/gofpdi => github.com/phpdave11/gofpdi v1.0.13 @@ -320,7 +320,7 @@ replace ( google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f google.golang.org/grpc => google.golang.org/grpc v1.65.0 - google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.4.0 + google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 google.golang.org/protobuf => google.golang.org/protobuf v1.34.2 gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/inconshreveable/log15.v2 => gopkg.in/inconshreveable/log15.v2 v2.16.0 @@ -329,16 +329,16 @@ replace ( gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1 honnef.co/go/tools => honnef.co/go/tools v0.4.7 - k8s.io/api => k8s.io/api v0.30.2 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.2 - k8s.io/apimachinery => k8s.io/apimachinery v0.30.2 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.30.2 - k8s.io/client-go => k8s.io/client-go v0.30.2 - k8s.io/component-base => k8s.io/component-base v0.30.2 + k8s.io/api => k8s.io/api v0.30.3 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.3 + k8s.io/apimachinery => k8s.io/apimachinery v0.30.3 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.30.3 + k8s.io/client-go => k8s.io/client-go v0.30.3 + k8s.io/component-base => k8s.io/component-base v0.30.3 k8s.io/klog/v2 => k8s.io/klog/v2 v2.130.1 - k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f - k8s.io/kubernetes => k8s.io/kubernetes v0.30.2 - k8s.io/metrics => k8s.io/metrics v0.30.2 + k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20240726031636-6f6746feab9c + k8s.io/kubernetes => k8s.io/kubernetes v0.30.3 + k8s.io/metrics => k8s.io/metrics v0.30.3 nhooyr.io/websocket => nhooyr.io/websocket v1.8.11 rsc.io/pdf => rsc.io/pdf v0.1.1 sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.18.4 @@ -410,14 +410,14 @@ require ( google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.30.2 - k8s.io/apimachinery v0.30.2 + k8s.io/api v0.30.3 + k8s.io/apimachinery v0.30.3 k8s.io/cli-runtime v0.0.0-00010101000000-000000000000 - k8s.io/client-go v0.30.2 + k8s.io/client-go v0.30.3 k8s.io/metrics v0.0.0-00010101000000-000000000000 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 - sigs.k8s.io/yaml v1.3.0 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -437,7 +437,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -447,7 +447,7 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-pdf/fpdf v0.9.0 // indirect github.com/go-toolsmith/astcopy v1.0.2 // indirect @@ -461,7 +461,7 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 // indirect + github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.6.0 // indirect @@ -506,7 +506,7 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/exp/typeparams v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/image v0.18.0 // indirect golang.org/x/mod v0.19.0 // indirect @@ -518,7 +518,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect - k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect diff --git a/go.sum b/go.sum index 5e3d7727bb..fcb5488444 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ cloud.google.com/go/vpcaccess v1.7.10/go.mod h1:69kdbMh8wvGcM3agEHP1YnHPyxIBSRcZ cloud.google.com/go/webrisk v1.9.10/go.mod h1:wDxtALjJMXlGR2c3qtZaVI5jRKcneIMTYqV1IA1jPmo= cloud.google.com/go/websecurityscanner v1.6.10/go.mod h1:ndil05bWkG/KDgWAXwFFAuvOYcOKu+mk/wC/nIfLQwE= cloud.google.com/go/workflows v1.12.9/go.mod h1:g9S8NdA20MnQTReKVrXCDsnPrOsNgwonY7xZn+vr3SY= -code.cloudfoundry.org/bytefmt v0.0.0-20240725181214-870a2a4a34a6 h1:5wA7H4/Uyij80SkebgeIbSSUw14EzaaWw8/VSt/9ke8= -code.cloudfoundry.org/bytefmt v0.0.0-20240725181214-870a2a4a34a6/go.mod h1:0WabI+CfmksKbGuDTQSdOtS0iIfrWLzjLVbgviM82Hg= +code.cloudfoundry.org/bytefmt v0.0.0-20240727181252-5b6127bbde80 h1:32ycowbmEGb+g1raMsf6VnEycfByxPtgnGY0UOeYWYI= +code.cloudfoundry.org/bytefmt v0.0.0-20240727181252-5b6127bbde80/go.mod h1:gsyTe3p9/Wh4+2zPVyMQwBN9ltHPAHvpixE0q2RDsE4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20221208032759-85de2813cf6b/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= eliasnaur.com/font v0.0.0-20230308162249-dd43949cb42d/go.mod h1:OYVuxibdk9OSLX8vAqydtRPP87PyTFcT9uH3MlEGBQA= @@ -210,8 +210,8 @@ github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+ye github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E= -github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.55.4 h1:u7sFWQQs5ivGuYvCxi7gJI8nN/P9Dq04huLaw39a4lg= +github.com/aws/aws-sdk-go v1.55.4/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= @@ -407,8 +407,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 h1:ssNFCCVmib/GQSzx3uCWyfMgOamLGWuGqlMS77Y1m3Y= -github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= @@ -480,8 +480,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/FqKluHJQ= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.17.10-0.20240719110028-cfab8bd586ec h1:S0mztzoHWmIpI4VuqO7MPIf3DQQxWwW5e3Uurb+HMHA= -github.com/klauspost/compress v1.17.10-0.20240719110028-cfab8bd586ec/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.10-0.20240726110848-d76f801616d1 h1:j6tQDRDYFVnxj0AHKoolYgdATzzNUR8aySZLFF3cuag= +github.com/klauspost/compress v1.17.10-0.20240726110848-d76f801616d1/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84= @@ -547,10 +547,10 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= -github.com/onsi/gomega v1.34.0 h1:eSSPsPNp6ZpsG8X1OVmOTxig+CblTc4AxpPBykhe2Os= -github.com/onsi/gomega v1.34.0/go.mod h1:MIKI8c+f+QLWk+hxbePD4i0LMJSExPaZOVfkoex4cAo= +github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0= +github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= @@ -748,22 +748,22 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/js/dom v0.0.0-20210725211120-f030747120f2/go.mod h1:sUMDUKNB2ZcVjt92UnLy3cdGs+wDAcrPdV3JP6sVgA4= honnef.co/go/tools v0.4.7/go.mod h1:+rnGS1THNh8zMwnd2oVOTL9QF6vmfyG6ZXBULae2uc0= -k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= -k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= -k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE= -k8s.io/apiextensions-apiserver v0.30.2/go.mod h1:lsJFLYyK40iguuinsb3nt+Sj6CmodSI4ACDLep1rgjw= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/cli-runtime v0.30.2 h1:ooM40eEJusbgHNEqnHziN9ZpLN5U4WcQGsdLKVxpkKE= -k8s.io/cli-runtime v0.30.2/go.mod h1:Y4g/2XezFyTATQUbvV5WaChoUGhojv/jZAtdp5Zkm0A= -k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= -k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= +k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= +k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= +k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= +k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= +k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= +k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/cli-runtime v0.30.3 h1:aG69oRzJuP2Q4o8dm+f5WJIX4ZBEwrvdID0+MXyUY6k= +k8s.io/cli-runtime v0.30.3/go.mod h1:hwrrRdd9P84CXSKzhHxrOivAR9BRnkMt0OeP5mj7X30= +k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= +k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f h1:2sXuKesAYbRHxL3aE2PN6zX/gcJr22cjrsej+W784Tc= -k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= -k8s.io/metrics v0.30.2 h1:zj4kIPTCfEbY0RHEogpA7QtlItU7xaO11+Gz1zVDxlc= -k8s.io/metrics v0.30.2/go.mod h1:GpoO5XTy/g8CclVLtgA5WTrr2Cy5vCsqr5Xa/0ETWIk= +k8s.io/kube-openapi v0.0.0-20240726031636-6f6746feab9c h1:CHL3IcTrTI3csK36iwYJy36uQRic+IpSoRMNH+0I8SE= +k8s.io/kube-openapi v0.0.0-20240726031636-6f6746feab9c/go.mod h1:0CVn9SVo8PeW5/JgsBZZIFmmTk5noOM8WXf2e1tCihE= +k8s.io/metrics v0.30.3 h1:gKCpte5zykrOmQhZ8qmsxyJslMdiLN+sqbBfIWNpbGM= +k8s.io/metrics v0.30.3/go.mod h1:W06L2nXRhOwPkFYDJYWdEIS3u6JcJy3ebIPYbndRs6A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 34c9425864..a065d16b1c 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -61,14 +61,17 @@ ARG UPX_OPTIONS=-9 ARG {{$key}}={{$value}} {{- end}} {{- range $image := .ExtraImages }} -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM {{$image}} {{- end}} -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM {{.BuilderImage}}:{{.BuilderTag}}{{- if not .DevContainer}} AS builder {{- end}} ARG MAINTAINER="{{.Maintainer}}" LABEL maintainer="${MAINTAINER}" +# skipcq: DOK-DL3002 +USER {{.BuildUser}} + ARG TARGETARCH ARG TARGETOS ARG GO_VERSION @@ -79,27 +82,49 @@ ENV {{$keyValue}} {{- end}} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# skipcq: DOK-DL3008 -RUN apt-get clean \ +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN rm -f /etc/apt/apt.conf.d/docker-clean \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ && apt-get update -y \ && apt-get upgrade -y \ +{{- if .DevContainer}} + && apt-get install -y --no-install-recommends --fix-missing \ + curl \ + gnupg \ + software-properties-common \ + && add-apt-repository ppa:ubuntu-toolchain-r/test -y \ + && apt-get update -y \ + && apt-get upgrade -y \ +{{- end}} && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ ca-certificates \ +{{- if not .DevContainer}} curl \ +{{- end}} + tzdata \ + locales \ git \ {{- range $epkg := .ExtraPackages }} {{$epkg}} \ {{- end}} && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean \ + && apt-get autoclean -y \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/* \ - && apt-get autoremove + && apt-get autoremove -y WORKDIR {{.RootDir}}/${ORG}/${REPO} COPY Makefile . @@ -120,23 +145,22 @@ COPY cmd/${PKG}/sample.yaml {{$.TmpConfigPath}} WORKDIR {{$.RootDir}}/${ORG}/${REPO}/{{$files}} COPY {{$files}} . {{- end}} -{{- range $files := .ExtraCopies }} -COPY {{$files}} -{{- end}} - {{- else}} WORKDIR {{.RootDir}}/${ORG}/${REPO} COPY . . {{- end}} +{{- range $files := .ExtraCopies }} +COPY {{$files}} +{{- end}} WORKDIR {{.RootDir}}/${ORG}/${REPO} {{- if .RunCommands}} -#skipcq: DOK-W1001, DOK-SC2086 +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 RUN {{RunCommands .RunCommands}} {{- end}} {{- if not .DevContainer}} -# skipcq: DOK-DL3026 +# skipcq: DOK-DL3026,DOK-DL3007 FROM {{.RuntimeImage}}:{{.RuntimeTag}} ARG MAINTAINER="{{.Maintainer}}" LABEL maintainer="${MAINTAINER}" @@ -150,14 +174,16 @@ COPY --from=builder {{$.TmpConfigPath}} /etc/server/config.yaml {{- range $from, $file := .StageFiles }} COPY --from=builder {{$file}} {{$file}} {{- end}} -USER nonroot:nonroot +{{- end}} +# skipcq: DOK-DL3002 +USER {{.RuntimeUser}} {{- if .Entrypoints}} ENTRYPOINT [{{Entrypoint .Entrypoints}}] -{{- else}} +{{- else if not .DevContainer}} ENTRYPOINT ["{{.BinDir}}/{{.AppName}}"] {{- end}} -{{- end}}` +` var docker = template.Must(template.New("Dockerfile").Funcs(template.FuncMap{ "RunCommands": func(commands []string) string { @@ -188,6 +214,7 @@ type Data struct { ContainerType ContainerType AppName string BinDir string + BuildUser string BuilderImage string BuilderTag string Maintainer string @@ -195,6 +222,7 @@ type Data struct { RootDir string RuntimeImage string RuntimeTag string + RuntimeUser string TmpConfigPath string Arguments map[string]string Environments map[string]string @@ -214,15 +242,19 @@ type ContainerType int const ( organization = "vdaas" repository = "vald" - minimumArgumentLength = 2 + defaultBinaryDir = "/usr/bin" + defaultBuilderImage = "ghcr.io/vdaas/vald/vald-buildbase" + defaultBuilderTag = "nightly" + defaultLanguage = "en_US.UTF-8" defaultMaintainer = organization + ".org " + repository + " team <" + repository + "@" + organization + ".org>" - maintainerKey = "MAINTAINER" defaultRuntimeImage = "gcr.io/distroless/static" defaultRuntimeTag = "nonroot" - defaultBuilderImage = "ghcr.io/vdaas/vald/vald-buildbase" - defaultBuilderTag = "nightly" - defaultBinaryDir = "/usr/bin" defaultTmpConfigPath = "/tmp/config.yaml" + defaultRuntimeUser = "nonroot:nonroot" + defaultBuildUser = "root:root" + maintainerKey = "MAINTAINER" + minimumArgumentLength = 2 + ubuntuVersion = "22.04" goWorkdir = "${GOPATH}/src/github.com" rustWorkdir = "${HOME}/rust/src/github.com" @@ -249,9 +281,13 @@ var ( defaultEnvironments = map[string]string{ "DEBIAN_FRONTEND": "noninteractive", "HOME": "/root", + "USER": "root", "INITRD": "No", - "LANG": "en_US.UTF-8", + "LANG": defaultLanguage, + "LANGUAGE": defaultLanguage, + "LC_ALL": defaultLanguage, "ORG": organization, + "TZ": "Etc/UTC", "PATH": "${PATH}:/usr/local/bin", "REPO": repository, } @@ -267,6 +303,10 @@ var ( "CARGO_HOME": "${RUST_HOME}/cargo", "PATH": "${PATH}:${RUSTUP_HOME}/bin:${CARGO_HOME}/bin:/usr/local/bin", } + clangDefaultEnvironments = map[string]string{ + "CC": "gcc", + "CXX": "g++", + } defaultCopyDirectories = []string{ "Makefile.d", @@ -303,9 +343,10 @@ var ( clangBuildDeps = []string{ "cmake", - "g++", "gcc", + "g++", "unzip", + "libssl-dev", } ngtBuildDeps = []string{ "liblapack-dev", @@ -316,24 +357,32 @@ var ( "gfortran", } devContainerDeps = []string{ - "curl", "gawk", - "git", "gnupg2", "graphviz", "jq", "libhdf5-dev", "libaec-dev", - "nodejs", - "npm", "sed", "zip", } ciContainerPreprocess = []string{ + "curl -fsSL https://deb.nodesource.com/setup_current.x | bash -", + "apt-get clean", + "rm -rf /var/lib/apt/lists/* /var/cache/*", + "apt-get update -y", + "apt-get upgrade -y", + "apt-get install -y --no-install-recommends --fix-missing nodejs", + "npm install -g npm@latest", + "apt-get clean", + "apt-get autoclean -y", + "rm -rf /var/lib/apt/lists/* /var/cache/*", + "apt-get autoremove -y", "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} deps GO_CLEAN_DEPS=false", "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} golangci-lint/install", "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} gotestfmt/install", + "make cmake/install", "make buf/install", "make hdf5/install", "make helm-docs/install", @@ -385,6 +434,9 @@ func appendM[K comparable](maps ...map[K]string) map[K]string { if strings.Contains(v, "${PATH}:") { v = strings.TrimPrefix(strings.ReplaceAll(strings.ReplaceAll(v, "${PATH}", ""), "::", ":")+":${PATH}", ":") } + if strings.Contains(v, ":unix") { + v = "unix:" + strings.TrimSuffix(v, ":unix") + } result[k] = v } return result @@ -461,7 +513,7 @@ func main() { defer cancel() maintainer := os.Getenv(maintainerKey) - if len(maintainer) == 0 { + if maintainer == "" { maintainer = defaultMaintainer } year := time.Now().Year() @@ -612,22 +664,23 @@ func main() { AppName: "ci-container", DevContainer: true, ContainerType: DevContainer, + BuilderImage: "ghcr.io/actions-runner-controller/actions-runner-controller/actions-runner-dind", + BuilderTag: "ubuntu-" + ubuntuVersion, PackageDir: "ci/base", + RuntimeUser: defaultBuildUser, ExtraPackages: append(clangBuildDeps, append(ngtBuildDeps, append(faissBuildDeps, devContainerDeps...)...)...), - Preprocess: append([]string{ - "sysctl -w net.ipv6.conf.all.disable_ipv6=1", - "sysctl -w net.ipv6.conf.default.disable_ipv6=1", - "sysctl -w net.ipv6.conf.lo.disable_ipv6=1", - "sysctl -p", - }, append(ciContainerPreprocess, ngtPreprocess, faissPreprocess)...), + Preprocess: append(ciContainerPreprocess, ngtPreprocess, faissPreprocess), + Entrypoints: []string{"/bin/bash", "-c", "/usr/bin/entrypoint-dind.sh"}, }, "vald-dev-container": { AppName: "dev-container", - BuilderImage: "mcr.microsoft.com/vscode/devcontainers/base", - BuilderTag: "debian", + BuilderImage: "mcr.microsoft.com/devcontainers/base", + BuilderTag: "ubuntu" + ubuntuVersion, + BuildUser: defaultBuildUser, + RuntimeUser: defaultBuildUser, DevContainer: true, ContainerType: DevContainer, PackageDir: "dev", @@ -665,6 +718,13 @@ func main() { if data.BuilderTag == "" { data.BuilderTag = defaultBuilderTag } + if data.RuntimeUser == "" { + data.RuntimeUser = defaultRuntimeUser + } + + if data.BuildUser == "" { + data.BuildUser = defaultBuildUser + } if data.CopyDirectories != nil { data.CopyDirectories = append(defaultCopyDirectories, data.CopyDirectories...) @@ -701,12 +761,13 @@ func main() { if data.Preprocess != nil { commands = append(commands, data.Preprocess...) } - data.RunCommands = append(commands, rustBuildCommands...) + commands = append(commands, rustBuildCommands...) + data.RunCommands = commands case DevContainer: data.CopyDirectories = append(data.CopyDirectories, append(goDefaultCopyDirectories, rustDefaultCopyDirectories...)...) - data.Environments = appendM(data.Environments, goDefaultEnvironments, rustDefaultEnvironments) + data.Environments = appendM(data.Environments, goDefaultEnvironments, rustDefaultEnvironments, clangDefaultEnvironments) data.RootDir = goWorkdir - commands := make([]string, 0, len(goInstallCommands)+len(rustInstallCommands)+len(data.Preprocess)) + commands := make([]string, 0, len(goInstallCommands)+len(rustInstallCommands)+len(data.Preprocess)+1) commands = append(commands, append(goInstallCommands, rustInstallCommands...)...) if data.Preprocess != nil { commands = append(commands, data.Preprocess...) @@ -726,6 +787,17 @@ func main() { data.RootDir = "${HOME}" data.Environments["ROOTDIR"] = os.Args[1] } + if strings.Contains(data.BuildUser, "root") { + data.Environments["HOME"] = "/root" + data.Environments["USER"] = "root" + } else { + user := data.BuildUser + if strings.Contains(user, ":") { + user = strings.SplitN(user, ":", 2)[0] + } + data.Environments["HOME"] = "/home/" + user + data.Environments["USER"] = user + } data.Environments["APP_NAME"] = data.AppName data.Environments["PKG"] = data.PackageDir @@ -733,7 +805,7 @@ func main() { data.ConfigExists = file.Exists(file.Join(os.Args[1], "cmd", data.PackageDir, "sample.yaml")) buf := bytes.NewBuffer(make([]byte, 0, len(tmpl))) - log.Infof("generating %s's docker file", name) + log.Infof("Generating %s's Dockerfile", name) docker.Execute(buf, data) tpl := buf.String() buf.Reset() diff --git a/hack/go.mod.default b/hack/go.mod.default index c33a973129..c55303d4ce 100644 --- a/hack/go.mod.default +++ b/hack/go.mod.default @@ -329,16 +329,16 @@ replace ( gopkg.in/yaml.v2 => gopkg.in/yaml.v2 upgrade gopkg.in/yaml.v3 => gopkg.in/yaml.v3 upgrade honnef.co/go/tools => honnef.co/go/tools upgrade - k8s.io/api => k8s.io/api v0.30.2 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.2 - k8s.io/apimachinery => k8s.io/apimachinery v0.30.2 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.30.2 - k8s.io/client-go => k8s.io/client-go v0.30.2 - k8s.io/component-base => k8s.io/component-base v0.30.2 + k8s.io/api => k8s.io/api v0.30.3 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.3 + k8s.io/apimachinery => k8s.io/apimachinery v0.30.3 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.30.3 + k8s.io/client-go => k8s.io/client-go v0.30.3 + k8s.io/component-base => k8s.io/component-base v0.30.3 k8s.io/klog/v2 => k8s.io/klog/v2 upgrade k8s.io/kube-openapi => k8s.io/kube-openapi master - k8s.io/kubernetes => k8s.io/kubernetes v0.30.2 - k8s.io/metrics => k8s.io/metrics v0.30.2 + k8s.io/kubernetes => k8s.io/kubernetes v0.30.3 + k8s.io/metrics => k8s.io/metrics v0.30.3 nhooyr.io/websocket => nhooyr.io/websocket upgrade rsc.io/pdf => rsc.io/pdf upgrade sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.18.4 diff --git a/hack/license/gen/main.go b/hack/license/gen/main.go index a13db09c43..27544ca6dc 100644 --- a/hack/license/gen/main.go +++ b/hack/license/gen/main.go @@ -207,14 +207,15 @@ func dirwalk(dir string) []string { case "AUTHORS", "CONTRIBUTORS", + "FAISS_VERSION", "GO_VERSION", "NGT_VERSION", - "FAISS_VERSION", "Pipefile", "VALD_VERSION", "grp", "obj", "prf", + "rust-toolchain", "src", "tre": default: @@ -260,7 +261,7 @@ func readAndRewrite(path string) error { } buf := bytes.NewBuffer(make([]byte, 0, fi.Size())) maintainer := os.Getenv(maintainerKey) - if len(maintainer) == 0 { + if maintainer == "" { maintainer = defaultMaintainer } d := Data{ diff --git a/pkg/agent/core/ngt/handler/grpc/insert.go b/pkg/agent/core/ngt/handler/grpc/insert.go index ed46791b4e..3e982dbaf1 100644 --- a/pkg/agent/core/ngt/handler/grpc/insert.go +++ b/pkg/agent/core/ngt/handler/grpc/insert.go @@ -99,7 +99,7 @@ func (s *server) Insert( log.Warn(err) attrs = trace.StatusCodeAlreadyExists(err.Error()) } else if errors.Is(err, errors.ErrUUIDNotFound(0)) { - err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API empty uuid \"%s\" was given", vec.GetId()), err, + err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API invalid id: \"%s\" or vector: %v was given", vec.GetId(), vec.GetVector()), err, &errdetails.RequestInfo{ RequestId: req.GetVector().GetId(), ServingData: errdetails.Serialize(req), diff --git a/pkg/agent/core/ngt/handler/grpc/insert_test.go b/pkg/agent/core/ngt/handler/grpc/insert_test.go index fa223ca69d..f03e754476 100644 --- a/pkg/agent/core/ngt/handler/grpc/insert_test.go +++ b/pkg/agent/core/ngt/handler/grpc/insert_test.go @@ -734,7 +734,7 @@ func Test_server_Insert(t *testing.T) { want: want{ err: func() error { err := errors.ErrUUIDNotFound(0) - err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API empty uuid \"%s\" was given", req.GetVector().GetId()), err, + err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API invalid id: \"%s\" or vector: %v was given", req.GetVector().GetId(), req.GetVector().GetVector()), err, &errdetails.RequestInfo{ RequestId: req.GetVector().GetId(), ServingData: errdetails.Serialize(req), @@ -786,7 +786,7 @@ func Test_server_Insert(t *testing.T) { want: want{ err: func() error { err := errors.ErrUUIDNotFound(0) - err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API empty uuid \"%s\" was given", req.GetVector().GetId()), err, + err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API invalid id: \"%s\" or vector: %v was given", req.GetVector().GetId(), req.GetVector().GetVector()), err, &errdetails.RequestInfo{ RequestId: req.GetVector().GetId(), ServingData: errdetails.Serialize(req), @@ -838,7 +838,7 @@ func Test_server_Insert(t *testing.T) { want: want{ err: func() error { err := errors.ErrUUIDNotFound(0) - err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API empty uuid \"%s\" was given", req.GetVector().GetId()), err, + err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API invalid id: \"%s\" or vector: %v was given", req.GetVector().GetId(), req.GetVector().GetVector()), err, &errdetails.RequestInfo{ RequestId: req.GetVector().GetId(), ServingData: errdetails.Serialize(req), diff --git a/pkg/agent/core/ngt/handler/grpc/search_test.go b/pkg/agent/core/ngt/handler/grpc/search_test.go index 936998bdce..1ef340e2c4 100644 --- a/pkg/agent/core/ngt/handler/grpc/search_test.go +++ b/pkg/agent/core/ngt/handler/grpc/search_test.go @@ -92,7 +92,8 @@ func Test_server_Search(t *testing.T) { } } if gotSize := len(gotRes.GetResults()); gotSize != w.resultSize { - return errors.Errorf("got size: \"%#v\",\n\t\t\t\twant size: \"%#v\"", gotSize, w.resultSize) + res, _ := gotRes.MarshalJSON() + return errors.Errorf("got size: \"%#v\",\n\t\t\t\twant size: \"%#v\"\ngotResults: \"%s\"", gotSize, w.resultSize, string(res)) } return nil } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 787aebcaa7..ada07263a6 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -169,9 +169,9 @@ checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "cc" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" [[package]] name = "cfg-if" @@ -958,9 +958,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.39.1" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", diff --git a/rust/rust-toolchain b/rust/rust-toolchain new file mode 100644 index 0000000000..8481a9dfc9 --- /dev/null +++ b/rust/rust-toolchain @@ -0,0 +1 @@ +1.80.0 diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml index 7e37cd2750..f495d3cd86 100644 --- a/rust/rust-toolchain.toml +++ b/rust/rust-toolchain.toml @@ -14,4 +14,4 @@ # limitations under the License. # [toolchain] -channel = "1.77.2" +channel = "1.80.0 " diff --git a/versions/CMAKE_VERSION b/versions/CMAKE_VERSION new file mode 100644 index 0000000000..72bde0ab2a --- /dev/null +++ b/versions/CMAKE_VERSION @@ -0,0 +1 @@ +3.30.1 diff --git a/versions/OPERATOR_SDK_VERSION b/versions/OPERATOR_SDK_VERSION index 251643f6ba..2f2ce0df61 100644 --- a/versions/OPERATOR_SDK_VERSION +++ b/versions/OPERATOR_SDK_VERSION @@ -1 +1 @@ -v1.35 +v1.33 diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION index d16771deca..9c0be4b799 100644 --- a/versions/PROMETHEUS_STACK_VERSION +++ b/versions/PROMETHEUS_STACK_VERSION @@ -1 +1 @@ -61.3.2 +61.6.0 diff --git a/versions/RUST_VERSION b/versions/RUST_VERSION index 6cdeba3855..8481a9dfc9 100644 --- a/versions/RUST_VERSION +++ b/versions/RUST_VERSION @@ -1 +1 @@ -1.77.2 \ No newline at end of file +1.80.0 diff --git a/versions/actions/DOCKER_SETUP_BUILDX_ACTION b/versions/actions/DOCKER_SETUP_BUILDX_ACTION index 1545d96657..9575d51bad 100644 --- a/versions/actions/DOCKER_SETUP_BUILDX_ACTION +++ b/versions/actions/DOCKER_SETUP_BUILDX_ACTION @@ -1 +1 @@ -3.5.0 +3.6.1 From eeb71db8a524b5f2951655434e86cbf9e8e61b02 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 13:49:06 +0900 Subject: [PATCH 02/35] feat: fix refactor Signed-off-by: hlts2 --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index ff723f0bf7..9b85b025dd 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - ORG ?= vdaas NAME = vald GOPKG = github.com/$(ORG)/$(NAME) From f9a0346e1eabf4bd483980e9877af8b62b557167 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 13:55:45 +0900 Subject: [PATCH 03/35] fix: update image tag Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 750 +++++++++++++++++++------------------- 1 file changed, 377 insertions(+), 373 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 104878d243..1be3fa6270 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -22,8 +22,7 @@ on: - "*.*.*-*" - "v*.*.*-*" pull_request: - types: - - "labeled" + env: DATASET: fashion-mnist-784-euclidean.hdf5 jobs: @@ -33,385 +32,390 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context - detect-ci-container: - uses: ./.github/workflows/_detect-ci-container.yml + # detect-ci-container: + # uses: ./.github/workflows/_detect-ci-container.yml e2e-stream-crud: name: "E2E test (Stream CRUD)" - needs: [detect-ci-container] - runs-on: ubuntu-latest - timeout-minutes: 60 - container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - steps: - - uses: actions/checkout@v4 - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml - wait_for_selector: app=vald-lb-gateway - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10000\ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - e2e-stream-crud-for-operator: - name: "E2E test (Stream CRUD) for operator" - needs: [detect-ci-container] - runs-on: ubuntu-latest - timeout-minutes: 60 - container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - steps: - - uses: actions/checkout@v4 - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - name: Create Helm options for vald-helm-operator - uses: ./.github/actions/setup-e2e - id: vald_helm_operator - with: - target_images: "vdaas/vald-helm-operator" - require_libhdf5: false - require_go: false - require_helm: false - require_k3d: false - - name: Merge Docker image tag - run: | - IMAGE_TAGS=(${{ steps.setup_e2e.outputs.IMAGE_TAGS }}) - - for IMAGE_TAG in "${IMAGE_TAGS[@]}" - do - echo "IMAGE_TAG:${IMAGE_TAG}" - - ARR=(${IMAGE_TAG//=/ }) - FIELD=${ARR[0]} - TAG="\"${ARR[1]}\"" - yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml - done - - name: Deploy Vald using Vald Helm Operator - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald-helm-operator - with: - helm_extra_options: ${{ steps.vald_helm_operator.outputs.HELM_EXTRA_OPTIONS }} - valdrelease: ./.github/valdrelease/valdrelease.yaml - wait_for_selector: app=vald-lb-gateway - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - e2e-stream-crud-under-index-management-jobs: - name: "E2E test (Stream CRUD) under index management jobs" - needs: [detect-ci-container] - runs-on: ubuntu-latest - timeout-minutes: 60 - container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - steps: - - uses: actions/checkout@v4 - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - with: - target_images: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-index-creation vdaas/vald-index-save" - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - env: - HELM_EXTRA_OPTIONS: "--values .github/helm/values/values-index-management-jobs.yaml" - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} ${{ env.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml - wait_for_selector: app=vald-lb-gateway - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10000\ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - e2e-stream-crud-skip-exist-check: - name: "E2E test (Stream CRUD: skip strict exist check)" - needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + image: ghcr.io/vdaas/vald/vald-ci-container:pr-2549 steps: - uses: actions/checkout@v4 - name: Set Git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml - wait_for_selector: app=vald-lb-gateway - - name: Run E2E CRUD + - name: Setup K3d environment + uses: ./.github/actions/setup-k3d + - name: Chec run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10 \ - E2E_SEARCH_COUNT=10 \ - E2E_SEARCH_BY_ID_COUNT=10 \ - E2E_GET_OBJECT_COUNT=10 \ - E2E_UPDATE_COUNT=10 \ - E2E_UPSERT_COUNT=10 \ - E2E_REMOVE_COUNT=10 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/skip - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - e2e-multiapis-crud: - name: "E2E test (Multi-APIs CRUD)" - needs: [detect-ci-container] - runs-on: ubuntu-latest - timeout-minutes: 60 - container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - steps: - - uses: actions/checkout@v4 - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml - wait_for_selector: app=vald-lb-gateway - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=100 \ - E2E_SEARCH_COUNT=10 \ - E2E_SEARCH_BY_ID_COUNT=10 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/multi - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - e2e-jobs: - name: "E2E test (Jobs)" - needs: [detect-ci-container] - runs-on: ubuntu-latest - timeout-minutes: 60 - container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - steps: - - uses: actions/checkout@v4 - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml - wait_for_selector: app=vald-lb-gateway - - name: Run E2E Jobs - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_INSERT_COUNT=10000\ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/index/job/correction - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - e2e-stream-crud-with-readreplica: - name: "E2E test (Stream CRUD) with read replica" - needs: [detect-ci-container] - runs-on: ubuntu-latest - timeout-minutes: 60 - container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - steps: - - uses: actions/checkout@v4 - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - with: - require_k3d: "false" - require_minikube: "true" - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-readreplica.yaml - wait_for_selector: app=vald-agent - - name: Deploy Vald Read Replica - id: deploy_vald_readreplica - uses: ./.github/actions/e2e-deploy-vald-readreplica - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-readreplica.yaml - wait_for_selector: app=vald-lb-gateway - - name: Run E2E CRUD with read replica rotation - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=1000 \ - E2E_SEARCH_COUNT=1000 \ - E2E_SEARCH_BY_ID_COUNT=1000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/readreplica - env: - POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }} - e2e-stream-crud-with-mirror: - name: "E2E test (Stream CRUD) with mirror" - needs: [detect-ci-container] - runs-on: ubuntu-latest - timeout-minutes: 60 - container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - steps: - - uses: actions/checkout@v4 - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - with: - target_images: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-manager-index vdaas/vald-mirror-gateway" - - name: Deploy Vald-01 - id: deploy_vald_01 - uses: ./.github/actions/e2e-deploy-vald - with: - namespace: vald-01 - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-mirror-01.yaml - wait_for_selector: app=vald-mirror-gateway - - name: Deploy Vald-02 - id: deploy_vald_02 - uses: ./.github/actions/e2e-deploy-vald - with: - namespace: vald-02 - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-mirror-02.yaml - wait_for_selector: app=vald-mirror-gateway - - name: Deploy Mirror Target - run: | - kubectl apply -f .github/helm/values/vald-mirror-target.yaml -n vald-01 - sleep 5s - kubectl get pods -A && kubectl get vmt -o wide -A - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10000\ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=vald-01 \ - e2e - env: - POD_NAME: ${{ steps.deploy_vald_01.outputs.POD_NAME }} - slack-notification: - name: "Slack notification" - if: startsWith( github.ref, 'refs/tags/') - needs: - - e2e-jobs - - e2e-multiapis-crud - - e2e-stream-crud - - e2e-stream-crud-for-operator - - e2e-stream-crud-skip-exist-check - - e2e-stream-crud-under-index-management-jobs - - e2e-stream-crud-with-mirror - - e2e-stream-crud-with-readreplica - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/notify-slack - with: - author_name: "E2E deploy test" - slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + docker ps -a + kubectl get pods -A + # - name: Setup E2E environment + # id: setup_e2e + # uses: ./.github/actions/setup-e2e + # - name: Deploy Vald + # id: deploy_vald + # uses: ./.github/actions/e2e-deploy-vald + # with: + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-lb.yaml + # wait_for_selector: app=vald-lb-gateway + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${{ env.DATASET }} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${{ env.DATASET }} \ + # E2E_INSERT_COUNT=10000\ + # E2E_SEARCH_COUNT=10000 \ + # E2E_SEARCH_BY_ID_COUNT=10000 \ + # E2E_GET_OBJECT_COUNT=100 \ + # E2E_UPDATE_COUNT=100 \ + # E2E_UPSERT_COUNT=100 \ + # E2E_REMOVE_COUNT=100 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e + # env: + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + # e2e-stream-crud-for-operator: + # name: "E2E test (Stream CRUD) for operator" + # needs: [detect-ci-container] + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # container: + # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + # steps: + # - uses: actions/checkout@v4 + # - name: Set Git config + # run: | + # git config --global --add safe.directory ${GITHUB_WORKSPACE} + # - name: Setup E2E environment + # id: setup_e2e + # uses: ./.github/actions/setup-e2e + # - name: Create Helm options for vald-helm-operator + # uses: ./.github/actions/setup-e2e + # id: vald_helm_operator + # with: + # target_images: "vdaas/vald-helm-operator" + # require_libhdf5: false + # require_go: false + # require_helm: false + # require_k3d: false + # - name: Merge Docker image tag + # run: | + # IMAGE_TAGS=(${{ steps.setup_e2e.outputs.IMAGE_TAGS }}) + # + # for IMAGE_TAG in "${IMAGE_TAGS[@]}" + # do + # echo "IMAGE_TAG:${IMAGE_TAG}" + # + # ARR=(${IMAGE_TAG//=/ }) + # FIELD=${ARR[0]} + # TAG="\"${ARR[1]}\"" + # yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml + # done + # - name: Deploy Vald using Vald Helm Operator + # id: deploy_vald + # uses: ./.github/actions/e2e-deploy-vald-helm-operator + # with: + # helm_extra_options: ${{ steps.vald_helm_operator.outputs.HELM_EXTRA_OPTIONS }} + # valdrelease: ./.github/valdrelease/valdrelease.yaml + # wait_for_selector: app=vald-lb-gateway + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${{ env.DATASET }} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${{ env.DATASET }} \ + # E2E_INSERT_COUNT=10000 \ + # E2E_SEARCH_COUNT=10000 \ + # E2E_SEARCH_BY_ID_COUNT=10000 \ + # E2E_GET_OBJECT_COUNT=100 \ + # E2E_UPDATE_COUNT=100 \ + # E2E_UPSERT_COUNT=100 \ + # E2E_REMOVE_COUNT=100 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e + # env: + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + # e2e-stream-crud-under-index-management-jobs: + # name: "E2E test (Stream CRUD) under index management jobs" + # needs: [detect-ci-container] + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # container: + # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + # steps: + # - uses: actions/checkout@v4 + # - name: Set Git config + # run: | + # git config --global --add safe.directory ${GITHUB_WORKSPACE} + # - name: Setup E2E environment + # id: setup_e2e + # uses: ./.github/actions/setup-e2e + # with: + # target_images: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-index-creation vdaas/vald-index-save" + # - name: Deploy Vald + # id: deploy_vald + # uses: ./.github/actions/e2e-deploy-vald + # env: + # HELM_EXTRA_OPTIONS: "--values .github/helm/values/values-index-management-jobs.yaml" + # with: + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} ${{ env.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-lb.yaml + # wait_for_selector: app=vald-lb-gateway + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${{ env.DATASET }} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${{ env.DATASET }} \ + # E2E_INSERT_COUNT=10000\ + # E2E_SEARCH_COUNT=10000 \ + # E2E_SEARCH_BY_ID_COUNT=10000 \ + # E2E_GET_OBJECT_COUNT=100 \ + # E2E_UPDATE_COUNT=100 \ + # E2E_UPSERT_COUNT=100 \ + # E2E_REMOVE_COUNT=100 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e + # env: + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + # e2e-stream-crud-skip-exist-check: + # name: "E2E test (Stream CRUD: skip strict exist check)" + # needs: [detect-ci-container] + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # container: + # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + # steps: + # - uses: actions/checkout@v4 + # - name: Set Git config + # run: | + # git config --global --add safe.directory ${GITHUB_WORKSPACE} + # - name: Setup E2E environment + # id: setup_e2e + # uses: ./.github/actions/setup-e2e + # - name: Deploy Vald + # id: deploy_vald + # uses: ./.github/actions/e2e-deploy-vald + # with: + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-lb.yaml + # wait_for_selector: app=vald-lb-gateway + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${{ env.DATASET }} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${{ env.DATASET }} \ + # E2E_INSERT_COUNT=10 \ + # E2E_SEARCH_COUNT=10 \ + # E2E_SEARCH_BY_ID_COUNT=10 \ + # E2E_GET_OBJECT_COUNT=10 \ + # E2E_UPDATE_COUNT=10 \ + # E2E_UPSERT_COUNT=10 \ + # E2E_REMOVE_COUNT=10 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/skip + # env: + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + # e2e-multiapis-crud: + # name: "E2E test (Multi-APIs CRUD)" + # needs: [detect-ci-container] + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # container: + # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + # steps: + # - uses: actions/checkout@v4 + # - name: Set Git config + # run: | + # git config --global --add safe.directory ${GITHUB_WORKSPACE} + # - name: Setup E2E environment + # id: setup_e2e + # uses: ./.github/actions/setup-e2e + # - name: Deploy Vald + # id: deploy_vald + # uses: ./.github/actions/e2e-deploy-vald + # with: + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-lb.yaml + # wait_for_selector: app=vald-lb-gateway + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${{ env.DATASET }} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${{ env.DATASET }} \ + # E2E_INSERT_COUNT=100 \ + # E2E_SEARCH_COUNT=10 \ + # E2E_SEARCH_BY_ID_COUNT=10 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/multi + # env: + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + # e2e-jobs: + # name: "E2E test (Jobs)" + # needs: [detect-ci-container] + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # container: + # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + # steps: + # - uses: actions/checkout@v4 + # - name: Set Git config + # run: | + # git config --global --add safe.directory ${GITHUB_WORKSPACE} + # - name: Setup E2E environment + # id: setup_e2e + # uses: ./.github/actions/setup-e2e + # - name: Deploy Vald + # id: deploy_vald + # uses: ./.github/actions/e2e-deploy-vald + # with: + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-lb.yaml + # wait_for_selector: app=vald-lb-gateway + # - name: Run E2E Jobs + # run: | + # make hack/benchmark/assets/dataset/${{ env.DATASET }} + # make E2E_BIND_PORT=8081 \ + # E2E_INSERT_COUNT=10000\ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/index/job/correction + # env: + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + # e2e-stream-crud-with-readreplica: + # name: "E2E test (Stream CRUD) with read replica" + # needs: [detect-ci-container] + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # container: + # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + # steps: + # - uses: actions/checkout@v4 + # - name: Set Git config + # run: | + # git config --global --add safe.directory ${GITHUB_WORKSPACE} + # - name: Setup E2E environment + # id: setup_e2e + # uses: ./.github/actions/setup-e2e + # with: + # require_k3d: "false" + # require_minikube: "true" + # - name: Deploy Vald + # id: deploy_vald + # uses: ./.github/actions/e2e-deploy-vald + # with: + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-readreplica.yaml + # wait_for_selector: app=vald-agent + # - name: Deploy Vald Read Replica + # id: deploy_vald_readreplica + # uses: ./.github/actions/e2e-deploy-vald-readreplica + # with: + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-readreplica.yaml + # wait_for_selector: app=vald-lb-gateway + # - name: Run E2E CRUD with read replica rotation + # run: | + # make hack/benchmark/assets/dataset/${{ env.DATASET }} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${{ env.DATASET }} \ + # E2E_INSERT_COUNT=1000 \ + # E2E_SEARCH_COUNT=1000 \ + # E2E_SEARCH_BY_ID_COUNT=1000 \ + # E2E_GET_OBJECT_COUNT=100 \ + # E2E_UPDATE_COUNT=100 \ + # E2E_UPSERT_COUNT=100 \ + # E2E_REMOVE_COUNT=100 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/readreplica + # env: + # POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }} + # e2e-stream-crud-with-mirror: + # name: "E2E test (Stream CRUD) with mirror" + # needs: [detect-ci-container] + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # container: + # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + # steps: + # - uses: actions/checkout@v4 + # - name: Set Git config + # run: | + # git config --global --add safe.directory ${GITHUB_WORKSPACE} + # - name: Setup E2E environment + # id: setup_e2e + # uses: ./.github/actions/setup-e2e + # with: + # target_images: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-manager-index vdaas/vald-mirror-gateway" + # - name: Deploy Vald-01 + # id: deploy_vald_01 + # uses: ./.github/actions/e2e-deploy-vald + # with: + # namespace: vald-01 + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-mirror-01.yaml + # wait_for_selector: app=vald-mirror-gateway + # - name: Deploy Vald-02 + # id: deploy_vald_02 + # uses: ./.github/actions/e2e-deploy-vald + # with: + # namespace: vald-02 + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # values: .github/helm/values/values-mirror-02.yaml + # wait_for_selector: app=vald-mirror-gateway + # - name: Deploy Mirror Target + # run: | + # kubectl apply -f .github/helm/values/vald-mirror-target.yaml -n vald-01 + # sleep 5s + # kubectl get pods -A && kubectl get vmt -o wide -A + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${{ env.DATASET }} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${{ env.DATASET }} \ + # E2E_INSERT_COUNT=10000\ + # E2E_SEARCH_COUNT=10000 \ + # E2E_SEARCH_BY_ID_COUNT=10000 \ + # E2E_GET_OBJECT_COUNT=100 \ + # E2E_UPDATE_COUNT=100 \ + # E2E_UPSERT_COUNT=100 \ + # E2E_REMOVE_COUNT=100 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=vald-01 \ + # e2e + # env: + # POD_NAME: ${{ steps.deploy_vald_01.outputs.POD_NAME }} + # slack-notification: + # name: "Slack notification" + # if: startsWith( github.ref, 'refs/tags/') + # needs: + # - e2e-jobs + # - e2e-multiapis-crud + # - e2e-stream-crud + # - e2e-stream-crud-for-operator + # - e2e-stream-crud-skip-exist-check + # - e2e-stream-crud-under-index-management-jobs + # - e2e-stream-crud-with-mirror + # - e2e-stream-crud-with-readreplica + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/notify-slack + # with: + # author_name: "E2E deploy test" + # slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} From fa9461a02ebd9d7bb4841ce3cf1e764288f6c777 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 14:06:23 +0900 Subject: [PATCH 04/35] fix: use bind Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1be3fa6270..a76d2711b6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -26,20 +26,24 @@ on: env: DATASET: fashion-mnist-784-euclidean.hdf5 jobs: - dump-contexts-to-log: - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/dump-context + # dump-contexts-to-log: + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/dump-context # detect-ci-container: # uses: ./.github/workflows/_detect-ci-container.yml e2e-stream-crud: name: "E2E test (Stream CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 + services: + docker: + image: docker:20.10-dind + options: --privileged container: - image: ghcr.io/vdaas/vald/vald-ci-container:pr-2549 + image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v4 - name: Set Git config From e3b28b3ce4fb2247a94bf10144ea9ae855b97076 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 14:15:32 +0900 Subject: [PATCH 05/35] fix: add --privileged option Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a76d2711b6..ad137e28ec 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -44,6 +44,7 @@ jobs: options: --privileged container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: --privileged steps: - uses: actions/checkout@v4 - name: Set Git config From c87c1b0db4bc9c8f1bc447403690a17590fb9928 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 14:19:42 +0900 Subject: [PATCH 06/35] fix: deleted unnecessary option Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ad137e28ec..b3867e4c14 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -54,7 +54,6 @@ jobs: uses: ./.github/actions/setup-k3d - name: Chec run: | - docker ps -a kubectl get pods -A # - name: Setup E2E environment # id: setup_e2e From 24e183708542afd75137d558849d8516b5f0013c Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 14:20:45 +0900 Subject: [PATCH 07/35] fix: revert k3d changes Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 8f39cdc8de..48f7556818 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -93,31 +93,9 @@ runs: shell: bash id: start_k3d run: | - k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} \ - --image docker.io/rancher/k3s:latest \ - --k3s-arg "--disable=traefik@server:*" \ - --api-port 0.0.0.0:6443 \ - -v "/lib/modules:/lib/modules" - k3d cluster start ${{ inputs.name }} - k3d kubeconfig merge ${{ inputs.name }} --kubeconfig-switch-context + k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} - name: Ready k8s cluster shell: bash run: | k3d cluster list - server="k3d-${{ inputs.name }}-server-0" - docker logs ${server} - docker inspect ${server} - - name: Set k3d cluster config to KUBECONFIG - shell: bash - run: | - echo "KUBECONFIG=$(k3d kubeconfig write ${{ inputs.name }})" >> $GITHUB_ENV - - name: Check KUBECONFIG environment variable - shell: bash - run: | - echo $KUBECONFIG - cat $KUBECONFIG - cat /etc/hosts - - name: Show Kubernetes Cluster Info - shell: bash - run: | kubectl cluster-info dump From b2caaa4e00e54391fdfbec9efeaee829e307a3f6 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 14:34:23 +0900 Subject: [PATCH 08/35] fix: add port settings Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b3867e4c14..fe3d3b021b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -42,6 +42,8 @@ jobs: docker: image: docker:20.10-dind options: --privileged + ports: + - 2375:2375 container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged @@ -52,9 +54,13 @@ jobs: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup K3d environment uses: ./.github/actions/setup-k3d + env: + DOCKER_HOST: tcp://localhost:2375 - name: Chec run: | kubectl get pods -A + env: + DOCKER_HOST: tcp://localhost:2375 # - name: Setup E2E environment # id: setup_e2e # uses: ./.github/actions/setup-e2e From c6f90321d9cf6a3bb5a68e431e17bbc7861322df Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 14:46:15 +0900 Subject: [PATCH 09/35] fix: docker host address Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fe3d3b021b..4a485e54b7 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -40,7 +40,7 @@ jobs: timeout-minutes: 60 services: docker: - image: docker:20.10-dind + image: docker:23.0.1-dind options: --privileged ports: - 2375:2375 @@ -55,12 +55,12 @@ jobs: - name: Setup K3d environment uses: ./.github/actions/setup-k3d env: - DOCKER_HOST: tcp://localhost:2375 + DOCKER_HOST: tcp://0.0.0.0:2375 - name: Chec run: | kubectl get pods -A env: - DOCKER_HOST: tcp://localhost:2375 + DOCKER_HOST: tcp://0.0.0.0:2375 # - name: Setup E2E environment # id: setup_e2e # uses: ./.github/actions/setup-e2e From 7f9d2e5e32845836d59aa216fb46b3a94d5808a3 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 14:55:40 +0900 Subject: [PATCH 10/35] fix: deleted unnecessary env Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4a485e54b7..91bfe15229 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -54,13 +54,13 @@ jobs: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup K3d environment uses: ./.github/actions/setup-k3d - env: - DOCKER_HOST: tcp://0.0.0.0:2375 + # env: + # DOCKER_HOST: tcp://0.0.0.0:2375 - name: Chec run: | kubectl get pods -A - env: - DOCKER_HOST: tcp://0.0.0.0:2375 + # env: + # DOCKER_HOST: tcp://0.0.0.0:2375 # - name: Setup E2E environment # id: setup_e2e # uses: ./.github/actions/setup-e2e From a8b33b96980d24a468d006eb9c56f8621160a8ab Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 14:58:09 +0900 Subject: [PATCH 11/35] fix: add docker socket settings Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 91bfe15229..fa998a5aab 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -42,8 +42,8 @@ jobs: docker: image: docker:23.0.1-dind options: --privileged - ports: - - 2375:2375 + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged From 8c2349f6ccdd19f46b32f96d588861aeacb1a502 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 15:11:28 +0900 Subject: [PATCH 12/35] fix: add port Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fa998a5aab..e4c330c9bd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -42,6 +42,8 @@ jobs: docker: image: docker:23.0.1-dind options: --privileged + ports: + - 2375:2375 volumes: - "/var/run/docker.sock:/var/run/docker.sock:ro" container: From 8eaae950db30b66ac5ac9a9501d134bc447b1093 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 15:17:58 +0900 Subject: [PATCH 13/35] fix: add env Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 376 +------------------------------------- 1 file changed, 4 insertions(+), 372 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e4c330c9bd..285a6d00b5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -56,378 +56,10 @@ jobs: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup K3d environment uses: ./.github/actions/setup-k3d - # env: - # DOCKER_HOST: tcp://0.0.0.0:2375 + env: + DOCKER_HOST: tcp://0.0.0.0:2375 - name: Chec run: | kubectl get pods -A - # env: - # DOCKER_HOST: tcp://0.0.0.0:2375 - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # - name: Deploy Vald - # id: deploy_vald - # uses: ./.github/actions/e2e-deploy-vald - # with: - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-lb.yaml - # wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${{ env.DATASET }} \ - # E2E_INSERT_COUNT=10000\ - # E2E_SEARCH_COUNT=10000 \ - # E2E_SEARCH_BY_ID_COUNT=10000 \ - # E2E_GET_OBJECT_COUNT=100 \ - # E2E_UPDATE_COUNT=100 \ - # E2E_UPSERT_COUNT=100 \ - # E2E_REMOVE_COUNT=100 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e - # env: - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - # e2e-stream-crud-for-operator: - # name: "E2E test (Stream CRUD) for operator" - # needs: [detect-ci-container] - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # container: - # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - # steps: - # - uses: actions/checkout@v4 - # - name: Set Git config - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # - name: Create Helm options for vald-helm-operator - # uses: ./.github/actions/setup-e2e - # id: vald_helm_operator - # with: - # target_images: "vdaas/vald-helm-operator" - # require_libhdf5: false - # require_go: false - # require_helm: false - # require_k3d: false - # - name: Merge Docker image tag - # run: | - # IMAGE_TAGS=(${{ steps.setup_e2e.outputs.IMAGE_TAGS }}) - # - # for IMAGE_TAG in "${IMAGE_TAGS[@]}" - # do - # echo "IMAGE_TAG:${IMAGE_TAG}" - # - # ARR=(${IMAGE_TAG//=/ }) - # FIELD=${ARR[0]} - # TAG="\"${ARR[1]}\"" - # yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml - # done - # - name: Deploy Vald using Vald Helm Operator - # id: deploy_vald - # uses: ./.github/actions/e2e-deploy-vald-helm-operator - # with: - # helm_extra_options: ${{ steps.vald_helm_operator.outputs.HELM_EXTRA_OPTIONS }} - # valdrelease: ./.github/valdrelease/valdrelease.yaml - # wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${{ env.DATASET }} \ - # E2E_INSERT_COUNT=10000 \ - # E2E_SEARCH_COUNT=10000 \ - # E2E_SEARCH_BY_ID_COUNT=10000 \ - # E2E_GET_OBJECT_COUNT=100 \ - # E2E_UPDATE_COUNT=100 \ - # E2E_UPSERT_COUNT=100 \ - # E2E_REMOVE_COUNT=100 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e - # env: - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - # e2e-stream-crud-under-index-management-jobs: - # name: "E2E test (Stream CRUD) under index management jobs" - # needs: [detect-ci-container] - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # container: - # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - # steps: - # - uses: actions/checkout@v4 - # - name: Set Git config - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # with: - # target_images: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-index-creation vdaas/vald-index-save" - # - name: Deploy Vald - # id: deploy_vald - # uses: ./.github/actions/e2e-deploy-vald - # env: - # HELM_EXTRA_OPTIONS: "--values .github/helm/values/values-index-management-jobs.yaml" - # with: - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} ${{ env.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-lb.yaml - # wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${{ env.DATASET }} \ - # E2E_INSERT_COUNT=10000\ - # E2E_SEARCH_COUNT=10000 \ - # E2E_SEARCH_BY_ID_COUNT=10000 \ - # E2E_GET_OBJECT_COUNT=100 \ - # E2E_UPDATE_COUNT=100 \ - # E2E_UPSERT_COUNT=100 \ - # E2E_REMOVE_COUNT=100 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e - # env: - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - # e2e-stream-crud-skip-exist-check: - # name: "E2E test (Stream CRUD: skip strict exist check)" - # needs: [detect-ci-container] - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # container: - # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - # steps: - # - uses: actions/checkout@v4 - # - name: Set Git config - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # - name: Deploy Vald - # id: deploy_vald - # uses: ./.github/actions/e2e-deploy-vald - # with: - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-lb.yaml - # wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${{ env.DATASET }} \ - # E2E_INSERT_COUNT=10 \ - # E2E_SEARCH_COUNT=10 \ - # E2E_SEARCH_BY_ID_COUNT=10 \ - # E2E_GET_OBJECT_COUNT=10 \ - # E2E_UPDATE_COUNT=10 \ - # E2E_UPSERT_COUNT=10 \ - # E2E_REMOVE_COUNT=10 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/skip - # env: - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - # e2e-multiapis-crud: - # name: "E2E test (Multi-APIs CRUD)" - # needs: [detect-ci-container] - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # container: - # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - # steps: - # - uses: actions/checkout@v4 - # - name: Set Git config - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # - name: Deploy Vald - # id: deploy_vald - # uses: ./.github/actions/e2e-deploy-vald - # with: - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-lb.yaml - # wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${{ env.DATASET }} \ - # E2E_INSERT_COUNT=100 \ - # E2E_SEARCH_COUNT=10 \ - # E2E_SEARCH_BY_ID_COUNT=10 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/multi - # env: - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - # e2e-jobs: - # name: "E2E test (Jobs)" - # needs: [detect-ci-container] - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # container: - # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - # steps: - # - uses: actions/checkout@v4 - # - name: Set Git config - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # - name: Deploy Vald - # id: deploy_vald - # uses: ./.github/actions/e2e-deploy-vald - # with: - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-lb.yaml - # wait_for_selector: app=vald-lb-gateway - # - name: Run E2E Jobs - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_INSERT_COUNT=10000\ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/index/job/correction - # env: - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - # e2e-stream-crud-with-readreplica: - # name: "E2E test (Stream CRUD) with read replica" - # needs: [detect-ci-container] - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # container: - # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - # steps: - # - uses: actions/checkout@v4 - # - name: Set Git config - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # with: - # require_k3d: "false" - # require_minikube: "true" - # - name: Deploy Vald - # id: deploy_vald - # uses: ./.github/actions/e2e-deploy-vald - # with: - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-readreplica.yaml - # wait_for_selector: app=vald-agent - # - name: Deploy Vald Read Replica - # id: deploy_vald_readreplica - # uses: ./.github/actions/e2e-deploy-vald-readreplica - # with: - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-readreplica.yaml - # wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD with read replica rotation - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${{ env.DATASET }} \ - # E2E_INSERT_COUNT=1000 \ - # E2E_SEARCH_COUNT=1000 \ - # E2E_SEARCH_BY_ID_COUNT=1000 \ - # E2E_GET_OBJECT_COUNT=100 \ - # E2E_UPDATE_COUNT=100 \ - # E2E_UPSERT_COUNT=100 \ - # E2E_REMOVE_COUNT=100 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/readreplica - # env: - # POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }} - # e2e-stream-crud-with-mirror: - # name: "E2E test (Stream CRUD) with mirror" - # needs: [detect-ci-container] - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # container: - # image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - # steps: - # - uses: actions/checkout@v4 - # - name: Set Git config - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # with: - # target_images: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-manager-index vdaas/vald-mirror-gateway" - # - name: Deploy Vald-01 - # id: deploy_vald_01 - # uses: ./.github/actions/e2e-deploy-vald - # with: - # namespace: vald-01 - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-mirror-01.yaml - # wait_for_selector: app=vald-mirror-gateway - # - name: Deploy Vald-02 - # id: deploy_vald_02 - # uses: ./.github/actions/e2e-deploy-vald - # with: - # namespace: vald-02 - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-mirror-02.yaml - # wait_for_selector: app=vald-mirror-gateway - # - name: Deploy Mirror Target - # run: | - # kubectl apply -f .github/helm/values/vald-mirror-target.yaml -n vald-01 - # sleep 5s - # kubectl get pods -A && kubectl get vmt -o wide -A - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${{ env.DATASET }} \ - # E2E_INSERT_COUNT=10000\ - # E2E_SEARCH_COUNT=10000 \ - # E2E_SEARCH_BY_ID_COUNT=10000 \ - # E2E_GET_OBJECT_COUNT=100 \ - # E2E_UPDATE_COUNT=100 \ - # E2E_UPSERT_COUNT=100 \ - # E2E_REMOVE_COUNT=100 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=vald-01 \ - # e2e - # env: - # POD_NAME: ${{ steps.deploy_vald_01.outputs.POD_NAME }} - # slack-notification: - # name: "Slack notification" - # if: startsWith( github.ref, 'refs/tags/') - # needs: - # - e2e-jobs - # - e2e-multiapis-crud - # - e2e-stream-crud - # - e2e-stream-crud-for-operator - # - e2e-stream-crud-skip-exist-check - # - e2e-stream-crud-under-index-management-jobs - # - e2e-stream-crud-with-mirror - # - e2e-stream-crud-with-readreplica - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - uses: ./.github/actions/notify-slack - # with: - # author_name: "E2E deploy test" - # slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + env: + DOCKER_HOST: tcp://0.0.0.0:2375 From 6370c7d9d07fc2c7ff6cba2dc899b98fc4e2ad98 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 15:20:39 +0900 Subject: [PATCH 14/35] fix: docker socket Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 285a6d00b5..f480227c44 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,7 +45,7 @@ jobs: ports: - 2375:2375 volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "/var/run/docker.sock:/var/run/docker.sock:rw" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged From b71e57f5cdb9b3840fe9641d7775eb76cff6337e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 15:22:37 +0900 Subject: [PATCH 15/35] fix: debug Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 48f7556818..4fa920abd4 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -94,6 +94,10 @@ runs: id: start_k3d run: | k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} + kubectl config view --minify + kubectl cluster-info + k3d kubeconfig get vald > kubeconfig + export KUBECONFIG=$(pwd)/kubeconfig - name: Ready k8s cluster shell: bash run: | From 2deb3f3c607b69e2889c3b336d98a36fb4033564 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 15:25:23 +0900 Subject: [PATCH 16/35] fix: delete code Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f480227c44..ae256ab73c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -56,10 +56,10 @@ jobs: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup K3d environment uses: ./.github/actions/setup-k3d - env: - DOCKER_HOST: tcp://0.0.0.0:2375 + # env: + # DOCKER_HOST: tcp://0.0.0.0:2375 - name: Chec run: | kubectl get pods -A - env: - DOCKER_HOST: tcp://0.0.0.0:2375 + # env: + # DOCKER_HOST: tcp://0.0.0.0:2375 From d625b28884c9b55368ece78969e63b26654726f0 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 16:09:41 +0900 Subject: [PATCH 17/35] fix Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 3 ++- .github/workflows/e2e.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 4fa920abd4..1fcc0dcc46 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -93,7 +93,8 @@ runs: shell: bash id: start_k3d run: | - k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} + # k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} + k3d cluster create vald --api-port 127.0.0.1:6443 --port '8080:80@loadbalancer' kubectl config view --minify kubectl cluster-info k3d kubeconfig get vald > kubeconfig diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ae256ab73c..7d4d69d17b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,7 +45,7 @@ jobs: ports: - 2375:2375 volumes: - - "/var/run/docker.sock:/var/run/docker.sock:rw" + - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged From 88b8b60ad55d636ae86bb2193a6049b41fbc2d1d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 16:16:14 +0900 Subject: [PATCH 18/35] fix: add port settings Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7d4d69d17b..58914f9e94 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -49,6 +49,8 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged + ports: + - 6443:6443 steps: - uses: actions/checkout@v4 - name: Set Git config From f260169b131ba2ca38c66384bf5a7641646025fb Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 16:26:42 +0900 Subject: [PATCH 19/35] fix: add docker settings Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 58914f9e94..f72908ddf4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -43,6 +43,7 @@ jobs: image: docker:23.0.1-dind options: --privileged ports: + - 6443:6443 - 2375:2375 volumes: - "/var/run/docker.sock:/var/run/docker.sock:ro" @@ -51,6 +52,9 @@ jobs: options: --privileged ports: - 6443:6443 + - 2375:2375 + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" steps: - uses: actions/checkout@v4 - name: Set Git config From a6148e45c9769cde8f94a5cdb77ec0044f193f06 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 16:32:22 +0900 Subject: [PATCH 20/35] fix: debug Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f72908ddf4..7d4d69d17b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -43,18 +43,12 @@ jobs: image: docker:23.0.1-dind options: --privileged ports: - - 6443:6443 - 2375:2375 volumes: - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged - ports: - - 6443:6443 - - 2375:2375 - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" steps: - uses: actions/checkout@v4 - name: Set Git config From 144317e94ecf85fa4592dcb3db88453f3a7ec5d3 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 16:39:20 +0900 Subject: [PATCH 21/35] fix: update port Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7d4d69d17b..b8cd63eb1f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -44,11 +44,15 @@ jobs: options: --privileged ports: - 2375:2375 + - 6443:6443 volumes: - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged + ports: + - 2375:2375 + - 6443:6443 steps: - uses: actions/checkout@v4 - name: Set Git config From 020a8d05be369699903f0e8bc278d589587f3c38 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 17:15:16 +0900 Subject: [PATCH 22/35] fix Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b8cd63eb1f..c6200f5c7b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -38,21 +38,18 @@ jobs: name: "E2E test (Stream CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 - services: - docker: - image: docker:23.0.1-dind - options: --privileged - ports: - - 2375:2375 - - 6443:6443 - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" + # services: + # docker: + # image: docker:23.0.1-dind + # options: --privileged + # ports: + # - 2375:2375 + # - 6443:6443 + # volumes: + # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged - ports: - - 2375:2375 - - 6443:6443 steps: - uses: actions/checkout@v4 - name: Set Git config From 1ec2e5d16d3523978d9446ec7a50c12a412cb24f Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 17:23:21 +0900 Subject: [PATCH 23/35] fix: update settings Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 8 +++++--- .github/workflows/e2e.yml | 18 +++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 1fcc0dcc46..6e238d6834 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -94,11 +94,13 @@ runs: id: start_k3d run: | # k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} - k3d cluster create vald --api-port 127.0.0.1:6443 --port '8080:80@loadbalancer' + # k3d cluster create vald --api-port 127.0.0.1:6443 --port '8080:80@loadbalancer' + k3d cluster create vald + export KUBECONFIG=$(k3d kubeconfig write my-cluster) + echo "${KUBECONFIG}" + kubectl config view --minify kubectl cluster-info - k3d kubeconfig get vald > kubeconfig - export KUBECONFIG=$(pwd)/kubeconfig - name: Ready k8s cluster shell: bash run: | diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c6200f5c7b..f925dedbe7 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -38,15 +38,15 @@ jobs: name: "E2E test (Stream CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 - # services: - # docker: - # image: docker:23.0.1-dind - # options: --privileged - # ports: - # - 2375:2375 - # - 6443:6443 - # volumes: - # - "/var/run/docker.sock:/var/run/docker.sock:ro" + services: + docker: + image: docker:23.0.1-dind + options: --privileged + ports: + - 2375:2375 + - 6443:6443 + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged From 364dbe4944117c46e1a30a5037bbe2182fe4a86e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 17:29:34 +0900 Subject: [PATCH 24/35] fix: revert all changes Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 2 +- .github/workflows/e2e.yml | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 6e238d6834..43423c0de2 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -95,7 +95,7 @@ runs: run: | # k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} # k3d cluster create vald --api-port 127.0.0.1:6443 --port '8080:80@loadbalancer' - k3d cluster create vald + k3d cluster create vald --network host export KUBECONFIG=$(k3d kubeconfig write my-cluster) echo "${KUBECONFIG}" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f925dedbe7..64f92db613 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -38,15 +38,15 @@ jobs: name: "E2E test (Stream CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 - services: - docker: - image: docker:23.0.1-dind - options: --privileged - ports: - - 2375:2375 - - 6443:6443 - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" + # services: + # docker: + # image: docker:23.0.1-dind + # options: --privileged + # ports: + # - 2375:2375 + # - 6443:6443 + # volumes: + # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: --privileged From 56e1a068b3dbf4fb6ebf7bf535194584bfa51e9b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 17:36:20 +0900 Subject: [PATCH 25/35] fix Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 64f92db613..dcda010413 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -49,7 +49,7 @@ jobs: # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly - options: --privileged + options: "--privileged --net=host" steps: - uses: actions/checkout@v4 - name: Set Git config From 300f9cd6ae6086c87b41ffec7c2187f2d9d0e90a Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 17:36:50 +0900 Subject: [PATCH 26/35] fix: Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 43423c0de2..6e238d6834 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -95,7 +95,7 @@ runs: run: | # k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} # k3d cluster create vald --api-port 127.0.0.1:6443 --port '8080:80@loadbalancer' - k3d cluster create vald --network host + k3d cluster create vald export KUBECONFIG=$(k3d kubeconfig write my-cluster) echo "${KUBECONFIG}" From c0238b6abc8e35dc860cb73fb95763ffbca83162 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 17:39:18 +0900 Subject: [PATCH 27/35] fix: Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index dcda010413..902fcef9d1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -49,7 +49,7 @@ jobs: # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly - options: "--privileged --net=host" + options: "--privileged" steps: - uses: actions/checkout@v4 - name: Set Git config From f3a4551cfda3efb9d6e1890b5b2ab19353b66413 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 17:42:49 +0900 Subject: [PATCH 28/35] fix Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 902fcef9d1..359942ef0d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -48,7 +48,7 @@ jobs: # volumes: # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:pr-2549 options: "--privileged" steps: - uses: actions/checkout@v4 From 59cee1d45d0910b29aa48555e56fca52e1e94325 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 18:02:07 +0900 Subject: [PATCH 29/35] fix Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 1 + .github/workflows/e2e.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 6e238d6834..73e8949e14 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -89,6 +89,7 @@ runs: INGRESS_PORT: ${{ inputs.ingress_port }} OPTIONS: ${{ inputs.options }} K3S_VERSION: ${{ steps.k3s_version.outputs.tag }} + - uses: azure/setup-kubectl@v3 - name: Create k8s cluster shell: bash id: start_k3d diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 359942ef0d..b1a248f752 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -48,7 +48,7 @@ jobs: # volumes: # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: - image: ghcr.io/vdaas/vald/vald-ci-container:pr-2549 + image: docker:23.0.1-dind options: "--privileged" steps: - uses: actions/checkout@v4 From ce1e3b61bad619dd1a291052420ee3faa06a9ed7 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 30 Jul 2024 18:08:19 +0900 Subject: [PATCH 30/35] Revert "fix" This reverts commit 59cee1d45d0910b29aa48555e56fca52e1e94325. --- .github/actions/setup-k3d/action.yaml | 1 - .github/workflows/e2e.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 73e8949e14..6e238d6834 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -89,7 +89,6 @@ runs: INGRESS_PORT: ${{ inputs.ingress_port }} OPTIONS: ${{ inputs.options }} K3S_VERSION: ${{ steps.k3s_version.outputs.tag }} - - uses: azure/setup-kubectl@v3 - name: Create k8s cluster shell: bash id: start_k3d diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b1a248f752..359942ef0d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -48,7 +48,7 @@ jobs: # volumes: # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: - image: docker:23.0.1-dind + image: ghcr.io/vdaas/vald/vald-ci-container:pr-2549 options: "--privileged" steps: - uses: actions/checkout@v4 From 2795b05c488169f2a02dd8dc6242cb678766e810 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 31 Jul 2024 11:21:49 +0900 Subject: [PATCH 31/35] fix: Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 359942ef0d..ab6368a1d8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -48,13 +48,25 @@ jobs: # volumes: # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: - image: ghcr.io/vdaas/vald/vald-ci-container:pr-2549 + image: cruizba/ubuntu-dind options: "--privileged" steps: - uses: actions/checkout@v4 - - name: Set Git config + - name: Update run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/* \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + git + - uses: azure/setup-kubectl@v3 - name: Setup K3d environment uses: ./.github/actions/setup-k3d # env: From 227cd19bff388b732d4ff61075add15237f8e1b1 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 31 Jul 2024 12:04:00 +0900 Subject: [PATCH 32/35] fix: Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 3 ++- .github/workflows/e2e.yml | 34 +++++++++++++-------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 6e238d6834..6c87c9d8f9 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -95,7 +95,8 @@ runs: run: | # k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} # k3d cluster create vald --api-port 127.0.0.1:6443 --port '8080:80@loadbalancer' - k3d cluster create vald + # k3d cluster create --api-port 6550 -p "8081:80@loadbalancer" + k3d cluster create --api-port host.docker.internal:6550 -p "8081:80@loadbalancer" export KUBECONFIG=$(k3d kubeconfig write my-cluster) echo "${KUBECONFIG}" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ab6368a1d8..899a4d5b53 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -48,25 +48,25 @@ jobs: # volumes: # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: - image: cruizba/ubuntu-dind - options: "--privileged" + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - - name: Update - run: | - apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /var/cache/* \ - && apt-get update -y \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends --fix-missing \ - ca-certificates \ - curl \ - gnupg \ - lsb-release \ - git - - uses: azure/setup-kubectl@v3 + # - name: Update + # run: | + # apt-get clean \ + # && rm -rf \ + # /var/lib/apt/lists/* \ + # /var/cache/* \ + # && apt-get update -y \ + # && apt-get upgrade -y \ + # && apt-get install -y --no-install-recommends --fix-missing \ + # ca-certificates \ + # curl \ + # gnupg \ + # lsb-release \ + # git + # - uses: azure/setup-kubectl@v3 - name: Setup K3d environment uses: ./.github/actions/setup-k3d # env: From 3557a32f28fbf06a5fec59f949d2c1b76929cf47 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 31 Jul 2024 15:21:38 +0900 Subject: [PATCH 33/35] feat: add e2e job Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 899a4d5b53..048fbf44c4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -76,3 +76,46 @@ jobs: kubectl get pods -A # env: # DOCKER_HOST: tcp://0.0.0.0:2375 + + e2e-crud: + name: "E2E test (Stream CRUD)" + runs-on: ubuntu-latest + timeout-minutes: 60 + # services: + # docker: + # image: docker:23.0.1-dind + # options: --privileged + # ports: + # - 2375:2375 + # - 6443:6443 + # volumes: + # - "/var/run/docker.sock:/var/run/docker.sock:ro" + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" + steps: + - uses: actions/checkout@v4 + # - name: Update + # run: | + # apt-get clean \ + # && rm -rf \ + # /var/lib/apt/lists/* \ + # /var/cache/* \ + # && apt-get update -y \ + # && apt-get upgrade -y \ + # && apt-get install -y --no-install-recommends --fix-missing \ + # ca-certificates \ + # curl \ + # gnupg \ + # lsb-release \ + # git + # - uses: azure/setup-kubectl@v3 + - name: Setup K3d environment + uses: ./.github/actions/setup-k3d + # env: + # DOCKER_HOST: tcp://0.0.0.0:2375 + - name: Chec + run: | + kubectl get pods -A + # env: + # DOCKER_HOST: tcp://0.0.0.0:2375 From d15517ba941ea1910ea57f20fff07fb3232ff435 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 31 Jul 2024 15:27:53 +0900 Subject: [PATCH 34/35] fix: update e2e and chaos Signed-off-by: hlts2 --- .github/workflows/e2e-chaos.yaml | 38 +-- .github/workflows/e2e.yml | 432 +++++++++++++++++++++++++------ 2 files changed, 369 insertions(+), 101 deletions(-) diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index ed93101f73..d41c8d3467 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -22,8 +22,7 @@ on: - "*.*.*-*" - "v*.*.*-*" pull_request: - types: - - "labeled" + env: VALUES: .github/helm/values/values-chaos.yaml DATASET: fashion-mnist-784-euclidean.hdf5 @@ -35,15 +34,15 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context - detect-ci-container: - uses: ./.github/workflows/_detect-ci-container.yml + # detect-ci-container: + # uses: ./.github/workflows/_detect-ci-container.yml agent-failure: name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)" - needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - name: Set Git config @@ -79,11 +78,11 @@ jobs: e2e/insert/search random-pod-failure: name: "E2E chaos test (random Pod failure: to test redundancy)" - needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - name: Set Git config @@ -119,11 +118,11 @@ jobs: e2e/insert/search agent-network-partition: name: "E2E chaos test (agent network partition: to test retries)" - needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - name: Set Git config @@ -159,11 +158,11 @@ jobs: e2e/insert/search clusterwide-network-bandwidth: name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)" - needs: [detect-ci-container] runs-on: ubuntu-latest timeout-minutes: 60 container: - image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - name: Set Git config @@ -196,18 +195,3 @@ jobs: E2E_INSERT_COUNT=5000 \ E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ e2e/insert/search - slack-notification: - name: "Slack notification" - if: startsWith( github.ref, 'refs/tags/') - runs-on: ubuntu-latest - needs: - - agent-failure - - agent-network-partition - - clusterwide-network-bandwidth - - random-pod-failure - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/notify-slack - with: - author_name: E2E Chaos test - slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 048fbf44c4..a40b43372d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -26,96 +26,380 @@ on: env: DATASET: fashion-mnist-784-euclidean.hdf5 jobs: - # dump-contexts-to-log: - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - uses: ./.github/actions/dump-context + dump-contexts-to-log: + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/dump-context # detect-ci-container: # uses: ./.github/workflows/_detect-ci-container.yml e2e-stream-crud: name: "E2E test (Stream CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 - # services: - # docker: - # image: docker:23.0.1-dind - # options: --privileged - # ports: - # - 2375:2375 - # - 6443:6443 - # volumes: - # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: "--privileged --add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - # - name: Update - # run: | - # apt-get clean \ - # && rm -rf \ - # /var/lib/apt/lists/* \ - # /var/cache/* \ - # && apt-get update -y \ - # && apt-get upgrade -y \ - # && apt-get install -y --no-install-recommends --fix-missing \ - # ca-certificates \ - # curl \ - # gnupg \ - # lsb-release \ - # git - # - uses: azure/setup-kubectl@v3 - - name: Setup K3d environment - uses: ./.github/actions/setup-k3d - # env: - # DOCKER_HOST: tcp://0.0.0.0:2375 - - name: Chec - run: | - kubectl get pods -A - # env: - # DOCKER_HOST: tcp://0.0.0.0:2375 + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=10000\ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e + env: + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - e2e-crud: - name: "E2E test (Stream CRUD)" + e2e-stream-crud-for-operator: + name: "E2E test (Stream CRUD) for operator" + needs: [dump-contexts-to-log] + runs-on: ubuntu-latest + timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" + steps: + - uses: actions/checkout@v4 + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + - name: Create Helm options for vald-helm-operator + uses: ./.github/actions/setup-e2e + id: vald_helm_operator + with: + target_images: "vdaas/vald-helm-operator" + require_libhdf5: false + require_go: false + require_helm: false + require_k3d: false + - name: Merge Docker image tag + run: | + IMAGE_TAGS=(${{ steps.setup_e2e.outputs.IMAGE_TAGS }}) + + for IMAGE_TAG in "${IMAGE_TAGS[@]}" + do + echo "IMAGE_TAG:${IMAGE_TAG}" + + ARR=(${IMAGE_TAG//=/ }) + FIELD=${ARR[0]} + TAG="\"${ARR[1]}\"" + yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml + done + - name: Deploy Vald using Vald Helm Operator + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald-helm-operator + with: + helm_extra_options: ${{ steps.vald_helm_operator.outputs.HELM_EXTRA_OPTIONS }} + valdrelease: ./.github/valdrelease/valdrelease.yaml + wait_for_selector: app=vald-lb-gateway + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e + env: + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-stream-crud-under-index-management-jobs: + name: "E2E test (Stream CRUD) under index management jobs" + needs: [dump-contexts-to-log] + runs-on: ubuntu-latest + timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" + steps: + - uses: actions/checkout@v4 + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + with: + target_images: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-index-creation vdaas/vald-index-save" + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + env: + HELM_EXTRA_OPTIONS: "--values .github/helm/values/values-index-management-jobs.yaml" + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} ${{ env.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=10000\ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e + env: + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-stream-crud-skip-exist-check: + name: "E2E test (Stream CRUD: skip strict exist check)" + needs: [dump-contexts-to-log] + runs-on: ubuntu-latest + timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" + steps: + - uses: actions/checkout@v4 + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=10 \ + E2E_SEARCH_COUNT=10 \ + E2E_SEARCH_BY_ID_COUNT=10 \ + E2E_GET_OBJECT_COUNT=10 \ + E2E_UPDATE_COUNT=10 \ + E2E_UPSERT_COUNT=10 \ + E2E_REMOVE_COUNT=10 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/skip + env: + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-multiapis-crud: + name: "E2E test (Multi-APIs CRUD)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - # services: - # docker: - # image: docker:23.0.1-dind - # options: --privileged - # ports: - # - 2375:2375 - # - 6443:6443 - # volumes: - # - "/var/run/docker.sock:/var/run/docker.sock:ro" container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly options: "--privileged --add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - # - name: Update - # run: | - # apt-get clean \ - # && rm -rf \ - # /var/lib/apt/lists/* \ - # /var/cache/* \ - # && apt-get update -y \ - # && apt-get upgrade -y \ - # && apt-get install -y --no-install-recommends --fix-missing \ - # ca-certificates \ - # curl \ - # gnupg \ - # lsb-release \ - # git - # - uses: azure/setup-kubectl@v3 - - name: Setup K3d environment - uses: ./.github/actions/setup-k3d - # env: - # DOCKER_HOST: tcp://0.0.0.0:2375 - - name: Chec - run: | - kubectl get pods -A - # env: - # DOCKER_HOST: tcp://0.0.0.0:2375 + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=100 \ + E2E_SEARCH_COUNT=10 \ + E2E_SEARCH_BY_ID_COUNT=10 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/multi + env: + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-jobs: + name: "E2E test (Jobs)" + needs: [dump-contexts-to-log] + runs-on: ubuntu-latest + timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" + steps: + - uses: actions/checkout@v4 + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + - name: Run E2E Jobs + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_INSERT_COUNT=10000\ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/index/job/correction + env: + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-stream-crud-with-readreplica: + name: "E2E test (Stream CRUD) with read replica" + needs: [dump-contexts-to-log] + runs-on: ubuntu-latest + timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" + steps: + - uses: actions/checkout@v4 + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + with: + require_k3d: "false" + require_minikube: "true" + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-readreplica.yaml + wait_for_selector: app=vald-agent + - name: Deploy Vald Read Replica + id: deploy_vald_readreplica + uses: ./.github/actions/e2e-deploy-vald-readreplica + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-readreplica.yaml + wait_for_selector: app=vald-lb-gateway + - name: Run E2E CRUD with read replica rotation + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=1000 \ + E2E_SEARCH_COUNT=1000 \ + E2E_SEARCH_BY_ID_COUNT=1000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/readreplica + env: + POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }} + e2e-stream-crud-with-mirror: + name: "E2E test (Stream CRUD) with mirror" + needs: [dump-contexts-to-log] + runs-on: ubuntu-latest + timeout-minutes: 60 + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + options: "--privileged --add-host host.docker.internal:host-gateway" + steps: + - uses: actions/checkout@v4 + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + with: + target_images: "vdaas/vald-agent-ngt vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-manager-index vdaas/vald-mirror-gateway" + - name: Deploy Vald-01 + id: deploy_vald_01 + uses: ./.github/actions/e2e-deploy-vald + with: + namespace: vald-01 + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-mirror-01.yaml + wait_for_selector: app=vald-mirror-gateway + - name: Deploy Vald-02 + id: deploy_vald_02 + uses: ./.github/actions/e2e-deploy-vald + with: + namespace: vald-02 + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-mirror-02.yaml + wait_for_selector: app=vald-mirror-gateway + - name: Deploy Mirror Target + run: | + kubectl apply -f .github/helm/values/vald-mirror-target.yaml -n vald-01 + sleep 5s + kubectl get pods -A && kubectl get vmt -o wide -A + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=10000\ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=vald-01 \ + e2e + env: + POD_NAME: ${{ steps.deploy_vald_01.outputs.POD_NAME }} From 5ea726978dc0462f33b0ad676f969a5bd59bd8b3 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 31 Jul 2024 15:39:00 +0900 Subject: [PATCH 35/35] fix Signed-off-by: hlts2 --- .github/actions/setup-k3d/action.yaml | 2 +- .github/workflows/e2e.yml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 6c87c9d8f9..d83ba87ecf 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -96,7 +96,7 @@ runs: # k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} # k3d cluster create vald --api-port 127.0.0.1:6443 --port '8080:80@loadbalancer' # k3d cluster create --api-port 6550 -p "8081:80@loadbalancer" - k3d cluster create --api-port host.docker.internal:6550 -p "8081:80@loadbalancer" + k3d cluster create --api-port host.docker.internal:6550 -p "8081:80@loadbalancer" --image rancher/k3s:v1.29.4-k3s1 export KUBECONFIG=$(k3d kubeconfig write my-cluster) echo "${KUBECONFIG}" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a40b43372d..3d3b61ccc3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -53,7 +53,7 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - name: Run E2E CRUD @@ -117,7 +117,7 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald-helm-operator with: - helm_extra_options: ${{ steps.vald_helm_operator.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.vald_helm_operator.outputs.HELM_EXTRA_OPTIONS }} valdrelease: ./.github/valdrelease/valdrelease.yaml wait_for_selector: app=vald-lb-gateway - name: Run E2E CRUD @@ -162,7 +162,7 @@ jobs: env: HELM_EXTRA_OPTIONS: "--values .github/helm/values/values-index-management-jobs.yaml" with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} ${{ env.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} ${{ env.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - name: Run E2E CRUD @@ -203,7 +203,7 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - name: Run E2E CRUD @@ -244,7 +244,7 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - name: Run E2E CRUD @@ -281,7 +281,7 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - name: Run E2E Jobs @@ -318,14 +318,14 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-readreplica.yaml wait_for_selector: app=vald-agent - name: Deploy Vald Read Replica id: deploy_vald_readreplica uses: ./.github/actions/e2e-deploy-vald-readreplica with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-readreplica.yaml wait_for_selector: app=vald-lb-gateway - name: Run E2E CRUD with read replica rotation @@ -369,7 +369,7 @@ jobs: uses: ./.github/actions/e2e-deploy-vald with: namespace: vald-01 - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-mirror-01.yaml wait_for_selector: app=vald-mirror-gateway - name: Deploy Vald-02 @@ -377,7 +377,7 @@ jobs: uses: ./.github/actions/e2e-deploy-vald with: namespace: vald-02 - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-mirror-02.yaml wait_for_selector: app=vald-mirror-gateway - name: Deploy Mirror Target