Kun lu20 patch 6

.github/workflows/ci-build-all-in-one-multiarch.yml

@@ -12,12 +12,10 @@ jobs:
     strategy:
       matrix:
         platform:
-        - name: linux
+        - name: amd64
           task: GOOS=linux GOARCH=amd64 $(MAKE) build-all-in-one
-        - name: system/390
+        - name: s390x
           task: GOOS=linux GOARCH=s390x $(MAKE) build-all-in-one
-        - name: arm
-          task: GOOS=linux GOARCH=arm64 $(MAKE) build-all-in-one
     name: build all-in-one for ${{ matrix.platform.name }}
     steps:
     - uses: actions/checkout@v2
@@ -40,3 +38,41 @@ jobs:
 
     - name: Build All-in-one
       run: make ${{ matrix.platform.task }}
+
+    - uses: actions/upload-artifact@v2
+      with: 
+        name: all-in-one-linux
+        path: ./cmd/all-in-one/all-in-one-linux-*
+
+
+  build-docker-image:
+    needs: build-binaries
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2.3.4
+      with:
+        submodules: true
+
+    - name: Fetch git tags
+      run: |
+        git fetch --prune --unshallow --tags
+
+    - name: Export BRANCH variable
+      uses: ./.github/actions/setup-branch
+
+    - uses: docker/setup-qemu-action@v1
+
+    - uses: docker/setup-buildx-action@v1
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: all-in-one-linux
+        path: ./cmd/all-in-one/
+
+    - name: "build and upload the all-in-one image"
+      run: bash scripts/build-all-in-one-image-multiarch.sh
+      env:
+        DOCKERHUB_USERNAME: kunlu20
+        DOCKERHUB_TOKEN: 4f850c08-489a-4b71-9deb-eb75138258ad
+        QUAY_USERNAME: kunlu20
+        QUAY_TOKEN: DfB2mjgo0fL0wFHsxBiG9nkvq0CDy/7i9mnQ1ye6/ARu0sqsrIJlllTts6GO+XSy

.github/workflows/ci-build-other-images-multiarch.yml

@@ -0,0 +1,48 @@
+name: Build other multi-arch docker images
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  multiarch-other-images:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Fetch git tags
+      run: |
+        git fetch --prune --unshallow --tags
+
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ^1.16
+
+    - uses: actions/setup-node@v2.1.5
+      with:
+        node-version: '10'
+
+    - name: Export BRANCH variable
+      uses: ./.github/actions/setup-branch
+
+    - name: Install tools
+      run: make install-ci
+
+    - uses: docker/setup-qemu-action@v1
+
+    - uses: docker/setup-buildx-action@v1
+      with:
+        driver-opts: network=host
+
+    - name: Build multiarch other images
+      run: bash scripts/build-or-upload-other-images.sh
+      env:
+        DOCKERHUB_USERNAME: kunlu20
+        DOCKERHUB_TOKEN: 4f850c08-489a-4b71-9deb-eb75138258ad
+        QUAY_USERNAME: kunlu20
+        QUAY_TOKEN: DfB2mjgo0fL0wFHsxBiG9nkvq0CDy/7i9mnQ1ye6/ARu0sqsrIJlllTts6GO+XSy

.github/workflows/ci-docker-build-multiarch.yml

@@ -0,0 +1,58 @@
+name: Build and upload multi-arch docker images
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  multiarch-docker-images:
+    runs-on: ubuntu-latest
+
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Fetch git tags
+      run: |
+        git fetch --prune --unshallow --tags
+
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ^1.16
+
+    - uses: actions/setup-node@v2.1.5
+      with:
+        node-version: '10'
+
+    - name: Export BRANCH variable
+      uses: ./.github/actions/setup-branch
+
+    - name: Install tools
+      run: make install-ci
+
+    - name: Build binaries
+      id: build-binaries
+      run: make build-all-platforms
+
+    - uses: docker/setup-qemu-action@v1
+
+    - uses: docker/setup-buildx-action@v1
+      with:
+        driver-opts: network=host
+
+    - name: Build and upload multiarch docker images
+      run: bash scripts/build-or-upload-docker-images-multiarch.sh
+      env:
+        DOCKERHUB_USERNAME: kunlu20
+        DOCKERHUB_TOKEN: 4f850c08-489a-4b71-9deb-eb75138258ad
+        QUAY_USERNAME: kunlu20
+        QUAY_TOKEN: DfB2mjgo0fL0wFHsxBiG9nkvq0CDy/7i9mnQ1ye6/ARu0sqsrIJlllTts6GO+XSy

Makefile

@@ -72,6 +72,20 @@ MOCKERY=mockery
 
 .DEFAULT_GOAL := test-and-lint
 
+
+BASE_IMAGE_MULTIARCH := localhost:5000/baseimg:$(VERSION)-$(shell echo $(ROOT_IMAGE) | tr : -)
+PLATFORMS=linux/amd64,linux/arm64,linux/s390x,linux/ppc64le
+repo_multiarch_prefix=kunlu20/jaeger-
+CASSANDRA_TAG=$(subst JAGERCOMP,cassandra-schema,$(IMAGE_TAGS))
+INGESTER_TAG=$(subst JAGERCOMP,ingester,$(IMAGE_TAGS))
+AGENT_TAG=$(subst JAGERCOMP,agent,$(IMAGE_TAGS))
+COLLECTOR_TAG=$(subst JAGERCOMP,collector,$(IMAGE_TAGS))
+QUERY_TAG=$(subst JAGERCOMP,query,$(IMAGE_TAGS))
+ESINDEX_TAG=$(subst JAGERCOMP,es-index-cleaner,$(IMAGE_TAGS))
+ESROLLOVER_TAG=$(subst JAGERCOMP,es-rollover,$(IMAGE_TAGS))
+TRACEGEN_TAG=$(subst JAGERCOMP,tracegen,$(IMAGE_TAGS))
+ANONYMIZER_TAG=$(subst JAGERCOMP,anonymizer,$(IMAGE_TAGS))
+
 .PHONY: test-and-lint
 test-and-lint: test fmt lint
 
@@ -298,7 +312,8 @@ build-platform-binaries: build-agent \
 	build-all-in-one \
 	build-examples \
 	build-tracegen \
-	build-anonymizer
+	build-anonymizer \
+	build-esmapping-generator
 
 .PHONY: build-all-platforms
 build-all-platforms: build-binaries-linux build-binaries-windows build-binaries-darwin build-binaries-s390x build-binaries-arm64 build-binaries-ppc64le
@@ -348,6 +363,89 @@ docker-images-only: docker-images-cassandra \
 	docker-images-tracegen \
 	docker-images-anonymizer
 
+.PHONY: multiarch-docker-images-only
+multiarch-docker-images-only: docker-images-jaeger-backend-multiarch \
+	docker-images-cassandra-multiarch \
+	docker-images-elastic-multiarch \
+	docker-images-tracegen-multiarch \
+	docker-images-anonymizer-multiarch
+
+.PHONY: create-baseimage-multiarch
+create-baseimage-multiarch:
+	docker buildx build -t $(BASE_IMAGE_MULTIARCH) --push \
+		--build-arg root_image=$(ROOT_IMAGE) \
+		--build-arg cert_image=$(CERT_IMAGE) \
+		--platform=$(PLATFORMS) \
+		docker/base
+	echo "Finished building multiarch base image =============="
+
+.PHONY: docker-images-jaeger-backend-multiarch
+docker-images-jaeger-backend-multiarch: create-baseimage-multiarch
+	for component in agent collector query ingester ; do \
+		docker buildx build --output "$(PUSHTAG)" \
+    		--progress=plain --target release \
+    		--build-arg base_image=$(BASE_IMAGE_MULTIARCH) \
+			--build-arg debug_image=$(GOLANG_IMAGE) \
+    		--platform=$(PLATFORMS) \
+    		--file cmd/$$component/Dockerfile \
+    		$(subst JAGERCOMP,$$component,$(IMAGE_TAGS)) \
+			cmd/$$component; \
+		echo "Finished building $$component ==============" ; \
+	done
+
+.PHONY: docker-images-cassandra-multiarch
+docker-images-cassandra-multiarch:
+	docker buildx build --output "$(PUSHTAG)" \
+		--progress=plain \
+		--platform=$(PLATFORMS) \
+		--file plugin/storage/cassandra/Dockerfile.multiarch \
+		${CASSANDRA_TAG} \
+		plugin/storage/cassandra/
+	echo "Finished building multiarch jaeger-cassandra-schema =============="
+
+.PHONY: docker-images-elastic-multiarch
+docker-images-elastic-multiarch:
+	docker buildx build --output "$(PUSHTAG)" \
+		--progress=plain \
+		--platform=$(PLATFORMS) \
+		${ESINDEX_TAG} \
+		plugin/storage/es
+	docker buildx build --output "$(PUSHTAG)" \
+		--progress=plain \
+		--platform=$(PLATFORMS) \
+		--file plugin/storage/es/Dockerfile.rollover.multiarch \
+		${ESROLLOVER_TAG} \
+		plugin/storage/es
+	@echo "Finished building multiarch jaeger-es-indices-clean =============="
+
+.PHONY: docker-images-tracegen-multiarch
+docker-images-tracegen-multiarch:
+	docker buildx build --output "$(PUSHTAG)" \
+    	--progress=plain \
+    	--platform=$(PLATFORMS) \
+    	${TRACEGEN_TAG} \
+		cmd/tracegen/
+	@echo "Finished building multiarch jaeger-tracegen =============="
+
+.PHONY: docker-images-anonymizer-multiarch
+docker-images-anonymizer-multiarch:
+	docker buildx build --output "$(PUSHTAG)" \
+    	--progress=plain \
+    	--platform=$(PLATFORMS) \
+    	$(ANONYMIZER_TAG) \
+		cmd/anonymizer/
+	@echo "Finished building multiarch jaeger-anonymizer =============="
+
+.PHONY: docker-images-cassandra-multiarch-nopush
+docker-images-cassandra-multiarch-nopush:
+	docker buildx build --output "$(PUSHTAG)" \
+		--progress=plain \
+		--platform=$(PLATFORMS) \
+		--file plugin/storage/cassandra/Dockerfile.multiarch \
+		$(CASSANDRA_TAG) \
+		plugin/storage/cassandra/
+	echo "Finished building multiarch jaeger-cassandra-schema =============="
+
 .PHONY: docker-push
 docker-push:
 	@while [ -z "$$CONFIRM" ]; do \

cmd/agent/Dockerfile

@@ -2,7 +2,7 @@ ARG base_image
 ARG debug_image
 
 FROM $base_image AS release
-ARG TARGETARCH=amd64
+ARG TARGETARCH
 ARG USER_UID=10001
 COPY agent-linux-$TARGETARCH /go/bin/agent-linux
 EXPOSE 5775/udp 6831/udp 6832/udp 5778/tcp

cmd/all-in-one/Dockerfile

@@ -2,7 +2,7 @@ ARG base_image
 ARG debug_image
 
 FROM $base_image AS release
-ARG TARGETARCH=amd64
+ARG TARGETARCH
 
 # Agent zipkin.thrift compact
 EXPOSE 5775/udp
@@ -28,6 +28,8 @@ EXPOSE 16686
 COPY all-in-one-linux-$TARGETARCH /go/bin/all-in-one-linux
 COPY sampling_strategies.json /etc/jaeger/
 
+RUN chmod +x /go/bin/all-in-one-linux
+
 VOLUME ["/tmp"]
 ENTRYPOINT ["/go/bin/all-in-one-linux"]
 CMD ["--sampling.strategies-file=/etc/jaeger/sampling_strategies.json"]

cmd/anonymizer/Dockerfile

@@ -1,5 +1,5 @@
 FROM scratch
-ARG TARGETARCH=amd64
+ARG TARGETARCH
 
 COPY anonymizer-linux-$TARGETARCH /go/bin/anonymizer-linux
 ENTRYPOINT ["/go/bin/anonymizer-linux"]

cmd/collector/Dockerfile

@@ -2,7 +2,7 @@ ARG base_image
 ARG debug_image
 
 FROM $base_image AS release
-ARG TARGETARCH=amd64
+ARG TARGETARCH
 COPY collector-linux-$TARGETARCH /go/bin/collector-linux
 EXPOSE 14250/tcp
 ENTRYPOINT ["/go/bin/collector-linux"]

cmd/ingester/Dockerfile

@@ -2,7 +2,7 @@ ARG base_image
 ARG debug_image
 
 FROM $base_image AS release
-ARG TARGETARCH=amd64
+ARG TARGETARCH
 COPY ingester-linux-$TARGETARCH /go/bin/ingester-linux
 EXPOSE 14270/tcp 14271/tcp
 ENTRYPOINT ["/go/bin/ingester-linux"]

cmd/query/Dockerfile

@@ -2,7 +2,7 @@ ARG base_image
 ARG debug_image
 
 FROM $base_image AS release
-ARG TARGETARCH=amd64
+ARG TARGETARCH
 COPY query-linux-$TARGETARCH /go/bin/query-linux
 EXPOSE 16686/tcp
 ENTRYPOINT ["/go/bin/query-linux"]

cmd/tracegen/Dockerfile

@@ -1,5 +1,5 @@
 FROM scratch
-ARG TARGETARCH=amd64
+ARG TARGETARCH
 
 COPY tracegen-linux-$TARGETARCH /go/bin/tracegen-linux
 ENTRYPOINT ["/go/bin/tracegen-linux"]

plugin/storage/cassandra/Dockerfile

@@ -3,4 +3,4 @@ FROM cassandra:3.11
 COPY schema/* /cassandra-schema/
 
 ENV CQLSH_HOST=cassandra
-ENTRYPOINT ["/cassandra-schema/docker.sh"]
+ENTRYPOINT ["/cassandra-schema/docker.sh"]

plugin/storage/cassandra/Dockerfile.multiarch

@@ -0,0 +1,14 @@
+FROM cassandra:3.11 AS base-amd64
+
+FROM cassandra:4.0 AS base-arm64
+
+FROM cassandra:3.11 AS base-ppc64le
+
+FROM ibmcom/apachecassandra-s390x:3.11.3 AS base-s390x
+
+FROM base-$TARGETARCH AS final
+
+COPY schema/* /cassandra-schema/
+
+ENV CQLSH_HOST=cassandra
+ENTRYPOINT ["/cassandra-schema/docker.sh"]

plugin/storage/es/Dockerfile.rollover

@@ -8,4 +8,4 @@ COPY ./mappings/* /mappings/
 COPY esRollover.py /es-rollover/
 COPY esmapping-generator /usr/bin/
 
-ENTRYPOINT ["python3", "/es-rollover/esRollover.py"]
+ENTRYPOINT ["python3", "/es-rollover/esRollover.py"]

plugin/storage/es/Dockerfile.rollover.multiarch

@@ -0,0 +1,12 @@
+FROM python:3-alpine3.11
+ARG TARGETARCH
+
+# Temporary fix for https://github.com/jaegertracing/jaeger/issues/1494
+RUN pip install urllib3==1.24.3
+
+RUN pip install elasticsearch elasticsearch-curator
+COPY ./mappings/* /mappings/
+COPY esRollover.py /es-rollover/
+COPY esmapping-generator-linux-$TARGETARCH /usr/bin/esmapping-generator
+
+ENTRYPOINT ["python3", "/es-rollover/esRollover.py"]