Set Up Publishing-Bot

publishing-bot/Makefile

@@ -0,0 +1,85 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+all: build
+.PHONY: all
+
+-include $(CONFIG)
+-include $(CONFIG)-token
+
+DOCKER_REPO ?= k8s-publishing-bot
+NAMESPACE ?=
+KUBECTL ?= kubectl
+SCHEDULE ?= 0 5 * * *
+INTERVAL ?= 86400
+MEMORY_REQUESTS ?= 200Mi
+MEMORY_LIMITS ?= 1.6Gi
+GOOS ?= linux
+
+build_cmd = GO111MODULE=on mkdir -p _output && GOOS=$(GOOS) go build -o _output/$(1) ./cmd/$(1)
+prepare_spec = sed 's,DOCKER_IMAGE,$(DOCKER_REPO),g;s,MEMORY_REQUESTS,$(MEMORY_REQUESTS),g;s,MEMORY_LIMITS,$(MEMORY_LIMITS),g'
+
+SHELL := /bin/bash
+
+build:
+	$(call build_cmd,collapsed-kube-commit-mapper)
+	$(call build_cmd,publishing-bot)
+	$(call build_cmd,sync-tags)
+	$(call build_cmd,init-repo)
+	$(call build_cmd,godeps-gen)
+	$(call build_cmd,gomod-zip)
+.PHONY: build
+
+build-image: build
+	docker build -t $(DOCKER_REPO) .
+.PHONY: build-image
+
+push-image:
+	docker push $(DOCKER_REPO):latest
+
+clean:
+	rm -rf _output
+.PHONY: clean
+
+update-deps:
+	glide update --strip-vendor
+.PHONY: update-deps
+
+validate:
+	if [ -f $(CONFIG)-rules-configmap.yaml ]; then \
+		go run ./cmd/validate-rules <(sed '1,/config: /d;s/^    //' $(CONFIG)-rules-configmap.yaml); \
+	else \
+		go run ./cmd/validate-rules $$(grep "rules-file: " $(CONFIG)-configmap.yaml | sed 's/.*rules-file: //'); \
+	fi
+.PHONY: validate
+
+init-deploy: validate
+	$(KUBECTL) delete -n "$(NAMESPACE)" --ignore-not-found=true replicaset publisher
+	$(KUBECTL) delete -n "$(NAMESPACE)" --ignore-not-found=true pod publisher
+	while $(KUBECTL) get pod -n "$(NAMESPACE)" publisher -a &>/dev/null; do echo -n .; sleep 1; done
+	$(KUBECTL) apply -n "$(NAMESPACE)" -f manifests/storage-class.yaml || true
+	$(KUBECTL) get StorageClass ssd
+	$(KUBECTL) apply -n "$(NAMESPACE)" -f $(CONFIG)-configmap.yaml
+	$(KUBECTL) apply -n "$(NAMESPACE)" -f $(CONFIG)-rules-configmap.yaml; \
+	$(KUBECTL) apply -n "$(NAMESPACE)" -f manifests/pvc.yaml
+
+run: init-deploy
+	{ cat manifests/pod.yaml && sed 's/^/  /' manifests/podspec.yaml; } | \
+	$(call prepare_spec) | $(KUBECTL) apply -n "$(NAMESPACE)" -f -
+
+deploy: init-deploy
+	$(KUBECTL) apply -n "$(NAMESPACE)" -f manifests/service.yaml
+	{ cat manifests/rs.yaml && sed 's/^/      /' manifests/podspec.yaml; } | \
+	$(call prepare_spec) | sed 's/-interval=0/-interval=$(INTERVAL)/g' | \
+	$(KUBECTL) apply -n "$(NAMESPACE)" -f -

publishing-bot/README.md

@@ -0,0 +1,36 @@
+# Kubernetes Publishing Bot
+
+The publishing-bot for the Kubernetes project is running on a CNCF sponsored
+GKE cluster `aaa` in the `kubernetes-public` project.
+
+The bot is responsible for updating `go.mod`/`Godeps` and `vendor` for target repos.
+To support both godeps for releases <= v1.14 and go modules for the master branch
+and releases >= v1.14, we run two instances of the bot today.
+
+The instance of the bot responsible for syncing releases <= v1.14 runs in the
+`k8s-publishing-bot-godeps` namespace and the instance of the bot responsible
+for syncing the master branch and releases >= v1.14 runs in the `k8s-publishing-bot`
+namespace.
+
+The code for the former can be found in the [godeps branch] and the latter in the master
+branch of the publishing-bot repo.
+
+## Permissions
+
+The cluster can be accessed by members of the [k8s-infra-cluster-admins] and the [k8s-infra-rbac-publishing-bot]
+google groups. Members can be added to the groups by updating [groups.yaml].
+
+## Running the bot
+
+Make sure you are at the root of the publishing-bot repo before running these commands.
+
+### Deploying the bot
+
+```sh
+make deploy CONFIG=configs/kubernetes
+```
+
+[godeps branch]: https://github.com/kubernetes/publishing-bot/tree/godeps
+[k8s-infra-cluster-admins]: https://groups.google.com/forum/#!forum/k8s-infra-cluster-admins
+[k8s-infra-rbac-publishing-bot]: https://groups.google.com/forum/#!forum/k8s-infra-rbac-publishing-bot
+[groups.yaml]: https://git.k8s.io/k8s.io/groups/groups.yaml

publishing-bot/configs/kubernetes

@@ -0,0 +1,6 @@
+DOCKER_REPO = gcr.io/k8s-staging-publishing-bot/k8s-publishing-bot
+NAMESPACE = publishing-bot
+SCHEDULE = * */4 * * *
+INTERVAL = 14400
+MEMORY_REQUESTS = 2Gi
+MEMORY_LIMITS = 3Gi

publishing-bot/configs/kubernetes-configmap.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: publisher-config
+data:
+  config: |
+    source-org: kubernetes
+    source-repo: kubernetes
+    target-org: kubernetes
+    rules-file: https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml
+    github-issue: 56876
+    dry-run: false

publishing-bot/configs/kubernetes-nightly

@@ -0,0 +1,6 @@
+DOCKER_REPO = gcr.io/k8s-staging-publishing-bot/k8s-publishing-bot-nightly
+NAMESPACE = k8s-publishing-bot-nightly
+SCHEDULE = * */4 * * *
+INTERVAL = 14400
+MEMORY_REQUESTS = 2Gi
+MEMORY_LIMITS = 3Gi

publishing-bot/configs/kubernetes-nightly-configmap.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: publisher-config
+data:
+  config: |
+    source-org: kubernetes
+    source-repo: kubernetes
+    target-org: kubernetes-nightly
+    rules-file: https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml
+    # github-issue: 56876
+    dry-run: false

publishing-bot/configs/kubernetes-nightly-rules-configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: publisher-rules
+data:
+  config: |

publishing-bot/configs/kubernetes-rules-configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: publisher-rules
+data:
+  config: |

publishing-bot/manifests/.gitignore

@@ -0,0 +1 @@
+local.*

publishing-bot/manifests/cronjob.yaml

@@ -0,0 +1,18 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: publisher
+  namespace: publishing-bot
+spec:
+  schedule: "SCHEDULE"
+  startingDeadlineSeconds: 36000
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 14
+  jobTemplate:
+    metadata:
+      labels:
+        app: publisher
+    template:
+      spec:
+        restartPolicy: Never

publishing-bot/manifests/pod.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: publisher
+  namespace: publishing-bot
+spec:
+  restartPolicy: Never

publishing-bot/manifests/podspec.yaml

@@ -0,0 +1,84 @@
+initContainers:
+- name: initialize-repos
+  command:
+  - /init-repo
+  - --alsologtostderr
+  - --config=/etc/munge-config/config
+  - --rules-file=/etc/publisher-rules/config
+  - 2>&1
+  image: DOCKER_IMAGE
+  imagePullPolicy: Always
+  resources:
+    requests:
+      cpu: 300m
+      memory: 200Mi
+    limits:
+      cpu: 2
+      memory: 1.6Gi
+  volumeMounts:
+  - mountPath: /etc/munge-config
+    name: munge-config
+  - mountPath: /go-workspace
+    name: publisher-gopath
+  - mountPath: /etc/publisher-rules
+    name: publisher-rules
+  - mountPath: /.cache
+    name: cache
+containers:
+- name: publisher
+  command:
+  - /publishing-bot
+  - --alsologtostderr
+  - --config=/etc/munge-config/config
+  - --token-file=/etc/secret-volume/token
+  - --interval=0
+  - --server-port=8080
+  - 2>&1
+  image: DOCKER_IMAGE
+  imagePullPolicy: Always
+  livenessProbe:
+    httpGet:
+      path: /healthz
+      port: 80
+  resources:
+    requests:
+      cpu: 300m
+      memory: MEMORY_REQUESTS
+    limits:
+      cpu: 2
+      memory: MEMORY_LIMITS
+  volumeMounts:
+  - mountPath: /etc/munge-config
+    name: munge-config
+  - mountPath: /etc/publisher-rules
+    name: publisher-rules
+  - mountPath: /etc/secret-volume
+    name: secret-volume
+  - mountPath: /netrc
+    name: netrc
+  - mountPath: /gitrepos
+    name: repo
+  - mountPath: /go-workspace
+    name: publisher-gopath
+  - mountPath: /.cache
+    name: cache
+volumes:
+- name: munge-config
+  configMap:
+    name: publisher-config
+- name: publisher-rules
+  configMap:
+    name: publisher-rules
+- name: secret-volume
+  secret:
+    secretName: github-token
+- name: repo
+  emptyDir: {}
+- name: cache
+  emptyDir: {}
+- name: netrc
+  emptyDir:
+    medium: Memory
+- name: publisher-gopath
+  persistentVolumeClaim:
+    claimName: publisher-gopath

publishing-bot/manifests/pvc.yaml

@@ -0,0 +1,14 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  labels:
+    app: publisher
+  name: publisher-gopath
+  namespace: publishing-bot
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Gi
+  storageClassName: ssd

publishing-bot/manifests/rs.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Replicaset
+metadata:
+  name: publisher
+  namespace: publishing-bot
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: publisher
+  template:
+    metadata:
+      labels:
+        name: publisher
+    spec:

publishing-bot/manifests/service.yaml

@@ -0,0 +1,13 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: health
+  namespace: publishing-bot
+spec:
+  selector:
+    name: publisher
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+      targetPort: 8080

publishing-bot/manifests/storage-class.yaml

@@ -0,0 +1,7 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1beta1
+metadata:
+  name: ssd
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-ssd