From 0bf0aa3fc86f7a89d6ebe735944b52bc0b69ffd3 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Mon, 16 Dec 2019 20:22:47 -0800 Subject: [PATCH] Move the CD scripts and Tekton pipelines into kubeflow/testing * Delete all the Tekton pipelines and scripts for continuous delivery of Kubeflow applications because they are moving into kubeflow/testing * kubeflow/testing#551 is the PR moving the code into kubeflow/testing Related to: kubeflow/testing#544 redo how we use kustomize and Tekton to parameterize the pipelines --- .../admission-webhook/ci/kustomization.yaml | 50 ------- components/admission-webhook/ci/params.env | 5 - components/admission-webhook/ci/params.yaml | 33 ----- .../admission-webhook/ci/pipeline-run.yaml | 18 --- components/base/README.md | 133 ------------------ components/base/kustomization.yaml | 66 --------- components/base/params.env | 7 - components/base/params.env.save | 7 - components/base/params.yaml | 49 ------- components/base/pipeline-resource.yaml | 33 ----- components/base/pipeline.yaml | 60 -------- components/base/role-binding.yaml | 12 -- components/base/service-account.yaml | 12 -- components/base/task.yaml | 79 ----------- .../centraldashboard/ci/kustomization.yaml | 50 ------- components/centraldashboard/ci/params.env | 5 - components/centraldashboard/ci/params.yaml | 33 ----- .../centraldashboard/ci/pipeline-run.yaml | 18 --- .../notebook-controller/ci/kustomization.yaml | 50 ------- components/notebook-controller/ci/params.env | 5 - components/notebook-controller/ci/params.yaml | 33 ----- .../notebook-controller/ci/pipeline-run.yaml | 18 --- .../profile-controller/ci/kustomization.yaml | 50 ------- components/profile-controller/ci/params.env | 8 -- components/profile-controller/ci/params.yaml | 33 ----- .../profile-controller/ci/pipeline-run.yaml | 18 --- py/kubeflow/kubeflow/ci/rebuild-manifests.sh | 99 ------------- 27 files changed, 984 deletions(-) delete mode 100644 components/admission-webhook/ci/kustomization.yaml delete mode 100644 components/admission-webhook/ci/params.env delete mode 100644 components/admission-webhook/ci/params.yaml delete mode 100644 components/admission-webhook/ci/pipeline-run.yaml delete mode 100644 components/base/README.md delete mode 100644 components/base/kustomization.yaml delete mode 100644 components/base/params.env delete mode 100644 components/base/params.env.save delete mode 100644 components/base/params.yaml delete mode 100644 components/base/pipeline-resource.yaml delete mode 100644 components/base/pipeline.yaml delete mode 100644 components/base/role-binding.yaml delete mode 100644 components/base/service-account.yaml delete mode 100644 components/base/task.yaml delete mode 100644 components/centraldashboard/ci/kustomization.yaml delete mode 100644 components/centraldashboard/ci/params.env delete mode 100644 components/centraldashboard/ci/params.yaml delete mode 100644 components/centraldashboard/ci/pipeline-run.yaml delete mode 100644 components/notebook-controller/ci/kustomization.yaml delete mode 100644 components/notebook-controller/ci/params.env delete mode 100644 components/notebook-controller/ci/params.yaml delete mode 100644 components/notebook-controller/ci/pipeline-run.yaml delete mode 100644 components/profile-controller/ci/kustomization.yaml delete mode 100644 components/profile-controller/ci/params.env delete mode 100644 components/profile-controller/ci/params.yaml delete mode 100644 components/profile-controller/ci/pipeline-run.yaml delete mode 100755 py/kubeflow/kubeflow/ci/rebuild-manifests.sh diff --git a/components/admission-webhook/ci/kustomization.yaml b/components/admission-webhook/ci/kustomization.yaml deleted file mode 100644 index 51a107764c0..00000000000 --- a/components/admission-webhook/ci/kustomization.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../base -- pipeline-run.yaml -configMapGenerator: -- name: ci-pipeline-run-parameters - behavior: merge - env: params.env -generatorOptions: - disableNameSuffixHash: true -vars: -- name: docker_target - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.docker_target -- name: image_name - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.image_name -- name: path_to_context - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_context -- name: path_to_docker_file - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_docker_file -- name: path_to_manifests_dir - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_manifests_dir -configurations: -- params.yaml -namespace: $(namespace) diff --git a/components/admission-webhook/ci/params.env b/components/admission-webhook/ci/params.env deleted file mode 100644 index e1d8200bbe5..00000000000 --- a/components/admission-webhook/ci/params.env +++ /dev/null @@ -1,5 +0,0 @@ -docker_target=serve -image_name=admission-webhook -path_to_context=components/admission-webhook -path_to_docker_file=components/admission-webhook/Dockerfile -path_to_manifests_dir=admission-webhook/webhook/base diff --git a/components/admission-webhook/ci/params.yaml b/components/admission-webhook/ci/params.yaml deleted file mode 100644 index f42a86a115c..00000000000 --- a/components/admission-webhook/ci/params.yaml +++ /dev/null @@ -1,33 +0,0 @@ -varReference: -- path: metadata/name - kind: PipelineRun -- path: spec/params/value - kind: PipelineRun -- path: spec/resources/name - kind: PipelineRun -- path: spec/resources/resourceRef/name - kind: PipelineRun -- path: spec/resources/name - kind: Pipeline -- path: spec/tasks/resources/inputs/name - kind: Pipeline -- path: spec/tasks/resources/inputs/resource - kind: Pipeline -- path: spec/tasks/resources/outputs/name - kind: Pipeline -- path: spec/tasks/resources/outputs/resource - kind: Pipeline -- path: spec/tasks/params/value - kind: Pipeline -- path: spec/inputs/resources/name - kind: Task -- path: spec/outputs/resources/name - kind: Task -- path: spec/steps/args - kind: Task -- path: spec/params/value - kind: PipelineResource -- path: metadata/name - kind: PipelineResource -- path: metadata/namespace - kind: PipelineRun diff --git a/components/admission-webhook/ci/pipeline-run.yaml b/components/admission-webhook/ci/pipeline-run.yaml deleted file mode 100644 index c48d4b1eee0..00000000000 --- a/components/admission-webhook/ci/pipeline-run.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: tekton.dev/v1alpha1 -kind: PipelineRun -metadata: - name: ci-admission-webhook -spec: - pipelineRef: - name: ci-pipeline - resources: - - name: kubeflow - resourceRef: - name: kubeflow - - name: manifests - resourceRef: - name: manifests - - name: $(image_name) - resourceRef: - name: $(image_name) - serviceAccount: ci-pipeline-run-service-account diff --git a/components/base/README.md b/components/base/README.md deleted file mode 100644 index 9858475feb4..00000000000 --- a/components/base/README.md +++ /dev/null @@ -1,133 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Kubeflow CI with tektoncd pipelines](#kubeflow-ci-with-tektoncd-pipelines) - - [Use Cases](#use-cases) - - [Background information on TektonCD pipelineruns, pipelines and tasks](#background-information-on-tektoncd-pipelineruns-pipelines-and-tasks) - - [Parameterization](#parameterization) - - [Secrets](#secrets) - - - -## Kubeflow CD with tektoncd pipelines - -This directory contains Tekton pipelines intended to rebuild Kubeflow docker images -and open PRs to update Kubeflow kustomize manifests to use the newly built images. - - -### Use Cases - -The following use cases can be run on the following components (should be run from the components directory): -- `kustomize build --reorder none `*centraldashboard*`/ci | kubectl apply -f -` -- `kustomize build --reorder none `*jupyter-web-app*`/ci | kubectl apply -f -` -- `kustomize build --reorder none `*notebook-controller*`/ci | kubectl apply -f -` -- `kustomize build --reorder none `*profile-controller*`/ci | kubectl apply -f - - -This uses TektonCD [pipelinerun](https://github.com/tektoncd/pipeline/blob/master/docs/pipelineruns.md) to enable the following use case: - -1. A PR is merged into kubeflow/kubeflow updating the component -1. The merged commit is 1234 -1. This tekton pipelinerun is triggered to build the component image from commit @1234. -1. The pipelinerun edits manifests/common/centraldashboard/base/kustomization.yaml (using kubeflow-bot repo) and adds the new image tag -1. The pipelinerun calls `make generate; make test` -1. If successful then -1. The pipeline checks in the changes -1. Opens a PR with the updated kubeflow/manifests that uses the newly built image -1. Approvers LGTM the PR to kubeflow/manifests and it gets merged - -### Background information on TektonCD pipelineruns, pipelines and tasks - -A TektonCD PipelineRun takes 1 Pipeline and N PipelineResources. -The PipelineResources can be git repos, git pull requests, docker images. -These resources are made available to the Pipeline via PipelineRun. - -The general relationship between TektonCD resources is shown below: - -``` -── PipelineRun - ├── PipelineResources - └── Pipeline -    └── Tasks -``` - -In this use case the following instance is created: - -``` -── ci-centraldashboard-pipeline-run - ├── resources - │   ├── image - │ │   └── component - │ └── git - │    ├── kubeflow+revision - │    └── manifests+revision - └── pipeline -    └── tasks -    ├── build-push   -``` - -The PipelineRun includes a Pipeline that has 1 tasks and 3 PipelineResources of type image (component) and git (kubeflow, manifests). The Tasks reference these resources in their inputs or outputs. - -### Parameterization - -The PipelineRun uses parameterized PipelineResources which are passed down to the the Pipeline and Tasks. -The Pipeline uses parameterized Tasks. -Reusing this pipeline only requires changing parameters in params.env in the target component - -The parameters are noted below, those with an asterix should change per component: -Those parameters without an asterix allow different gcr.io locations and namespace. - -``` - container_image=gcr.io/kubeflow-ci/test-worker:latest -* docker_target=serve -* image_name=centraldashboard - image_url=gcr.io/kubeflow_public_images -* kubeflow_repo_revision=1234 -* kubeflow_repo_url=git@github.com:kubeflow/kubeflow.git -* manifests_repo_revision=master -* manifests_repo_url=git@github.com:kubeflow/manifests.git - namespace=kubeflow-test-infra -* path_to_context=components/centraldashboard -* path_to_docker_file=components/centraldashboard/Dockerfile -* path_to_manifests_dir=common/centraldashboard/base - pvc_mount_path=/kubeflow -``` - -### Setting up a cluster to run the pipelines - -The kustomize manifests are currently written so as to run in a Kubeflow releasing cluster. - -The current release cluster is - -* **project**: **kf-releasing** -* **cluster**: **kf-releasing-0-6-2** -* **namespace**: **kf-releasing** - -This is a Kubeflow cluster (v0.6.2) and we rely on that to configure certain things like the secrets and service accounts. - -1. Follow [Tektons' instructions](https://github.com/tektoncd/pipeline/blob/master/docs/auth.md#ssh-authentication-git) for - creating a secret containing ssh credentials for use with GitHub - - * We are currently using the secret named **kubeflow-bot-github-ssh** - - -1. Ensure the GCP service account used with Kaniko has storage admin permissions for the project - where the images are pushed. - - * most likely **gcr.io/kubeflow-images-public** - -1. Create a secret named **github-token** containing a github token to be used by the hub CLI to create PRs. - -### Run a pipeline - -1. Modify `base/params.env` - - * set namespace to the namespace where it will run - -1. Run - - ``` - kustomize build --reorder none `*profile-controller*`/ci | kubectl apply -f - - ``` - - diff --git a/components/base/kustomization.yaml b/components/base/kustomization.yaml deleted file mode 100644 index 6e0037fbea9..00000000000 --- a/components/base/kustomization.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- service-account.yaml -- role-binding.yaml -- pipeline-resource.yaml -- task.yaml -- pipeline.yaml -namespace: $(namespace) -configMapGenerator: -- name: ci-pipeline-run-parameters - env: params.env -generatorOptions: - disableNameSuffixHash: true -vars: -- name: container_image - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.container_image -- name: image_url - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.image_url -- name: kubeflow_repo_revision - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.kubeflow_repo_revision -- name: kubeflow_repo_url - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.kubeflow_repo_url -- name: manifests_repo_revision - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.manifests_repo_revision -- name: manifests_repo_url - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.manifests_repo_url -- name: namespace - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.namespace -configurations: -- params.yaml diff --git a/components/base/params.env b/components/base/params.env deleted file mode 100644 index 6436a4e8a6c..00000000000 --- a/components/base/params.env +++ /dev/null @@ -1,7 +0,0 @@ -namespace=kf-releasing -container_image=gcr.io/kubeflow-releasing/test-worker@sha256:35138a42b57160a078e802b7d69aec3c3e79a3e2e55518af7798275ebcc84d25 -image_url=gcr.io/kubeflow-images-public -kubeflow_repo_revision=master -kubeflow_repo_url=git@github.com:kubeflow/kubeflow.git -manifests_repo_revision=master -manifests_repo_url=git@github.com:kubeflow/manifests.git diff --git a/components/base/params.env.save b/components/base/params.env.save deleted file mode 100644 index 79df1ceab65..00000000000 --- a/components/base/params.env.save +++ /dev/null @@ -1,7 +0,0 @@ -namespace=kubeflow-test-infra -container_image=gcr.io/kubeflow-ci/test-worker:latest -image_url=gcr.io/kubeflow-images-public -kubeflow_repo_revision=1234 -kubeflow_repo_url=git@github.com:kubeflow/kubeflow.git -manifests_repo_revision=master -manifests_repo_url=git@github.com:kubeflow/manifests.git diff --git a/components/base/params.yaml b/components/base/params.yaml deleted file mode 100644 index 1122fe6dff6..00000000000 --- a/components/base/params.yaml +++ /dev/null @@ -1,49 +0,0 @@ -varReference: -- path: metadata/name - kind: Namespace -- path: spec/params/value - kind: PipelineRun -- path: spec/resources/name - kind: PipelineRun -- path: spec/resources/resourceSpec/params/value - kind: PipelineRun -- path: spec/resources/name - kind: Pipeline -- path: spec/tasks/resources/inputs/name - kind: Pipeline -- path: spec/tasks/resources/inputs/resource - kind: Pipeline -- path: spec/tasks/resources/outputs/name - kind: Pipeline -- path: spec/tasks/resources/outputs/resource - kind: Pipeline -- path: spec/tasks/params/value - kind: Pipeline -- path: spec/inputs/resources/name - kind: Task -- path: spec/outputs/resources/name - kind: Task -- path: spec/steps/args - kind: Task -- path: spec/params/value - kind: PipelineResource -- path: metadata/name - kind: PipelineResource -- path: metadata/namespace - kind: Task -- path: metadata/namespace - kind: ServiceAccount -- path: metadata/namespace - kind: RoleBinding -- path: subjects/namespace - kind: RoleBinding -- path: metadata/namespace - kind: ConfigMap -- path: metadata/namespace - kind: Secret -- path: metadata/namespace - kind: Pipeline -- path: metadata/namespace - kind: PipelineResource -- path: metadata/namespace - kind: PersistentVolumeClaim diff --git a/components/base/pipeline-resource.yaml b/components/base/pipeline-resource.yaml deleted file mode 100644 index 08fdfa1d570..00000000000 --- a/components/base/pipeline-resource.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: tekton.dev/v1alpha1 -kind: PipelineResource -metadata: - name: manifests -spec: - type: git - params: - - name: revision - value: $(manifests_repo_revision) - - name: url - value: $(manifests_repo_url) ---- -apiVersion: tekton.dev/v1alpha1 -kind: PipelineResource -metadata: - name: kubeflow -spec: - type: git - params: - - name: revision - value: $(kubeflow_repo_revision) - - name: url - value: $(kubeflow_repo_url) ---- -apiVersion: tekton.dev/v1alpha1 -kind: PipelineResource -metadata: - name: $(image_name) -spec: - type: image - params: - - name: url - value: $(image_url)/$(image_name) diff --git a/components/base/pipeline.yaml b/components/base/pipeline.yaml deleted file mode 100644 index 556d10d8cb0..00000000000 --- a/components/base/pipeline.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: tekton.dev/v1alpha1 -kind: Pipeline -metadata: - name: ci-pipeline -spec: - resources: - - name: kubeflow - type: git - - name: manifests - type: git - - name: $(image_name) - type: image - tasks: - - name: build-push - params: - - name: docker_target - value: $(docker_target) - - name: image_name - value: $(image_name) - - name: path_to_context - value: $(path_to_context) - - name: path_to_docker_file - value: $(path_to_docker_file) - - name: container_image - value: "$(container_image)" - - name: path_to_manifests_dir - value: "$(path_to_manifests_dir)" - resources: - inputs: - - name: kubeflow - resource: kubeflow - - name: manifests - resource: manifests - outputs: - - name: $(image_name) - resource: $(image_name) - taskRef: - name: build-push - kind: namespaced - #- name: update-manifests - # runAfter: - # - build-push - # params: - # - name: container_image - # value: "$(container_image)" - # - name: path_to_manifests_dir - # value: "$(path_to_manifests_dir)" - # resources: - # inputs: - # - name: kubeflow - # resource: kubeflow - # - name: manifests - # resource: manifests - # - name: $(image_name) - # resource: $(image_name) - # from: - # - build-push - # taskRef: - # name: update-manifests - # kind: namespaced diff --git a/components/base/role-binding.yaml b/components/base/role-binding.yaml deleted file mode 100644 index 2306987f125..00000000000 --- a/components/base/role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: ci-pipeline-run-role-binding -roleRef: - # TODO use a weaker clusterrole or just a role if possible - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- kind: ServiceAccount - name: ci-pipeline-run-service-account diff --git a/components/base/service-account.yaml b/components/base/service-account.yaml deleted file mode 100644 index 66146de0771..00000000000 --- a/components/base/service-account.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# TODO(jlewi): When we switch to workload identity should we continue to use this service account -# or use the default Kubeflow service account which is already bound to a GCP SA. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ci-pipeline-run-service-account -secrets: -# This is the name of the secret containing the ssh secret for the kubeflow-bot -# This is used to create pull requests updating the manifests -# For more info see the Tekton authentication docs -# https://github.com/tektoncd/pipeline/blob/master/docs/auth.md#ssh-authentication-git -- name: kubeflow-bot-github-ssh diff --git a/components/base/task.yaml b/components/base/task.yaml deleted file mode 100644 index ef881148314..00000000000 --- a/components/base/task.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: tekton.dev/v1alpha1 -kind: Task -metadata: - name: build-push - annotations: - # This gets passed down to the individual pods - sidecar.istio.io/inject: "false" -spec: - inputs: - params: - - description: docker target arg - name: docker_target - type: string - - description: image target name - name: image_name - type: string - - description: The build context used by Kaniko - name: path_to_context - type: string - - description: The path to the dockerfile to build - name: path_to_docker_file - type: string - - description: pod container image - name: container_image - type: string - - description: Where the components manifest dir is - name: path_to_manifests_dir - type: string - resources: - - name: kubeflow - type: git - - name: manifests - type: git - outputs: - resources: - - name: $(image_name) - type: image - steps: - - name: build-push - image: gcr.io/kaniko-project/executor:v0.11.0 - command: - - /kaniko/executor - - --dockerfile=/workspace/$(inputs.resources.kubeflow.name)/$(inputs.params.path_to_docker_file) - - --target=$(inputs.params.docker_target) - - --destination=$(outputs.resources.$(inputs.params.image_name).url) - - --context=/workspace/$(inputs.resources.kubeflow.name)/$(inputs.params.path_to_context) - - --digest-file=/workspace/$(inputs.params.image_name)-digest - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/user-gcp-sa.json - volumeMounts: - - mountPath: /secret - name: gcp-credentials - - name: update-manifests - workingDir: /workspace/$(inputs.resources.manifests.name)/$(inputs.params.path_to_manifests_dir) - image: $(inputs.params.container_image) - command: - - /workspace/$(inputs.resources.kubeflow.name)/py/kubeflow/kubeflow/ci/rebuild-manifests.sh - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - - name: GITHUB_TOKEN - valueFrom: - secretKeyRef: - name: github-token - key: github_token - envFrom: - - configMapRef: - name: ci-pipeline-run-parameters - volumeMounts: - - mountPath: /secret - name: gcp-credentials - volumes: - - name: gcp-credentials - secret: - secretName: user-gcp-sa - - name: github-token - secret: - secretName: github-token \ No newline at end of file diff --git a/components/centraldashboard/ci/kustomization.yaml b/components/centraldashboard/ci/kustomization.yaml deleted file mode 100644 index 51a107764c0..00000000000 --- a/components/centraldashboard/ci/kustomization.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../base -- pipeline-run.yaml -configMapGenerator: -- name: ci-pipeline-run-parameters - behavior: merge - env: params.env -generatorOptions: - disableNameSuffixHash: true -vars: -- name: docker_target - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.docker_target -- name: image_name - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.image_name -- name: path_to_context - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_context -- name: path_to_docker_file - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_docker_file -- name: path_to_manifests_dir - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_manifests_dir -configurations: -- params.yaml -namespace: $(namespace) diff --git a/components/centraldashboard/ci/params.env b/components/centraldashboard/ci/params.env deleted file mode 100644 index 082fc641c53..00000000000 --- a/components/centraldashboard/ci/params.env +++ /dev/null @@ -1,5 +0,0 @@ -docker_target=serve -image_name=centraldashboard -path_to_context=components/centraldashboard -path_to_docker_file=components/centraldashboard/Dockerfile -path_to_manifests_dir=common/centraldashboard/base diff --git a/components/centraldashboard/ci/params.yaml b/components/centraldashboard/ci/params.yaml deleted file mode 100644 index f42a86a115c..00000000000 --- a/components/centraldashboard/ci/params.yaml +++ /dev/null @@ -1,33 +0,0 @@ -varReference: -- path: metadata/name - kind: PipelineRun -- path: spec/params/value - kind: PipelineRun -- path: spec/resources/name - kind: PipelineRun -- path: spec/resources/resourceRef/name - kind: PipelineRun -- path: spec/resources/name - kind: Pipeline -- path: spec/tasks/resources/inputs/name - kind: Pipeline -- path: spec/tasks/resources/inputs/resource - kind: Pipeline -- path: spec/tasks/resources/outputs/name - kind: Pipeline -- path: spec/tasks/resources/outputs/resource - kind: Pipeline -- path: spec/tasks/params/value - kind: Pipeline -- path: spec/inputs/resources/name - kind: Task -- path: spec/outputs/resources/name - kind: Task -- path: spec/steps/args - kind: Task -- path: spec/params/value - kind: PipelineResource -- path: metadata/name - kind: PipelineResource -- path: metadata/namespace - kind: PipelineRun diff --git a/components/centraldashboard/ci/pipeline-run.yaml b/components/centraldashboard/ci/pipeline-run.yaml deleted file mode 100644 index cdbbf1ced77..00000000000 --- a/components/centraldashboard/ci/pipeline-run.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: tekton.dev/v1alpha1 -kind: PipelineRun -metadata: - name: ci-centraldashboard -spec: - pipelineRef: - name: ci-pipeline - resources: - - name: kubeflow - resourceRef: - name: kubeflow - - name: manifests - resourceRef: - name: manifests - - name: $(image_name) - resourceRef: - name: $(image_name) - serviceAccount: ci-pipeline-run-service-account diff --git a/components/notebook-controller/ci/kustomization.yaml b/components/notebook-controller/ci/kustomization.yaml deleted file mode 100644 index 51a107764c0..00000000000 --- a/components/notebook-controller/ci/kustomization.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../base -- pipeline-run.yaml -configMapGenerator: -- name: ci-pipeline-run-parameters - behavior: merge - env: params.env -generatorOptions: - disableNameSuffixHash: true -vars: -- name: docker_target - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.docker_target -- name: image_name - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.image_name -- name: path_to_context - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_context -- name: path_to_docker_file - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_docker_file -- name: path_to_manifests_dir - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_manifests_dir -configurations: -- params.yaml -namespace: $(namespace) diff --git a/components/notebook-controller/ci/params.env b/components/notebook-controller/ci/params.env deleted file mode 100644 index 4c14a58a7aa..00000000000 --- a/components/notebook-controller/ci/params.env +++ /dev/null @@ -1,5 +0,0 @@ -docker_target=serve -image_name=notebook-controller -path_to_context=components/notebook-controller -path_to_docker_file=components/notebook-controller/Dockerfile -path_to_manifests_dir=jupyter/notebook-controller/base diff --git a/components/notebook-controller/ci/params.yaml b/components/notebook-controller/ci/params.yaml deleted file mode 100644 index f42a86a115c..00000000000 --- a/components/notebook-controller/ci/params.yaml +++ /dev/null @@ -1,33 +0,0 @@ -varReference: -- path: metadata/name - kind: PipelineRun -- path: spec/params/value - kind: PipelineRun -- path: spec/resources/name - kind: PipelineRun -- path: spec/resources/resourceRef/name - kind: PipelineRun -- path: spec/resources/name - kind: Pipeline -- path: spec/tasks/resources/inputs/name - kind: Pipeline -- path: spec/tasks/resources/inputs/resource - kind: Pipeline -- path: spec/tasks/resources/outputs/name - kind: Pipeline -- path: spec/tasks/resources/outputs/resource - kind: Pipeline -- path: spec/tasks/params/value - kind: Pipeline -- path: spec/inputs/resources/name - kind: Task -- path: spec/outputs/resources/name - kind: Task -- path: spec/steps/args - kind: Task -- path: spec/params/value - kind: PipelineResource -- path: metadata/name - kind: PipelineResource -- path: metadata/namespace - kind: PipelineRun diff --git a/components/notebook-controller/ci/pipeline-run.yaml b/components/notebook-controller/ci/pipeline-run.yaml deleted file mode 100644 index c47605cf0f1..00000000000 --- a/components/notebook-controller/ci/pipeline-run.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: tekton.dev/v1alpha1 -kind: PipelineRun -metadata: - name: ci-notebook-controller -spec: - pipelineRef: - name: ci-pipeline - resources: - - name: kubeflow - resourceRef: - name: kubeflow - - name: manifests - resourceRef: - name: manifests - - name: $(image_name) - resourceRef: - name: $(image_name) - serviceAccount: ci-pipeline-run-service-account diff --git a/components/profile-controller/ci/kustomization.yaml b/components/profile-controller/ci/kustomization.yaml deleted file mode 100644 index 51a107764c0..00000000000 --- a/components/profile-controller/ci/kustomization.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../base -- pipeline-run.yaml -configMapGenerator: -- name: ci-pipeline-run-parameters - behavior: merge - env: params.env -generatorOptions: - disableNameSuffixHash: true -vars: -- name: docker_target - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.docker_target -- name: image_name - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.image_name -- name: path_to_context - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_context -- name: path_to_docker_file - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_docker_file -- name: path_to_manifests_dir - objref: - kind: ConfigMap - name: ci-pipeline-run-parameters - apiVersion: v1 - fieldref: - fieldpath: data.path_to_manifests_dir -configurations: -- params.yaml -namespace: $(namespace) diff --git a/components/profile-controller/ci/params.env b/components/profile-controller/ci/params.env deleted file mode 100644 index 4a24f5259d1..00000000000 --- a/components/profile-controller/ci/params.env +++ /dev/null @@ -1,8 +0,0 @@ -docker_target=serve -image_name=profile-controller -path_to_context=components/profile-controller -path_to_docker_file=components/profile-controller/Dockerfile -path_to_manifests_dir=profiles/base -do_not_submit=deletelinesbelow -kubeflow_repo_revision=tekton_cicd -kubeflow_repo_url=git@github.com:jlewi/kubeflow.git \ No newline at end of file diff --git a/components/profile-controller/ci/params.yaml b/components/profile-controller/ci/params.yaml deleted file mode 100644 index f42a86a115c..00000000000 --- a/components/profile-controller/ci/params.yaml +++ /dev/null @@ -1,33 +0,0 @@ -varReference: -- path: metadata/name - kind: PipelineRun -- path: spec/params/value - kind: PipelineRun -- path: spec/resources/name - kind: PipelineRun -- path: spec/resources/resourceRef/name - kind: PipelineRun -- path: spec/resources/name - kind: Pipeline -- path: spec/tasks/resources/inputs/name - kind: Pipeline -- path: spec/tasks/resources/inputs/resource - kind: Pipeline -- path: spec/tasks/resources/outputs/name - kind: Pipeline -- path: spec/tasks/resources/outputs/resource - kind: Pipeline -- path: spec/tasks/params/value - kind: Pipeline -- path: spec/inputs/resources/name - kind: Task -- path: spec/outputs/resources/name - kind: Task -- path: spec/steps/args - kind: Task -- path: spec/params/value - kind: PipelineResource -- path: metadata/name - kind: PipelineResource -- path: metadata/namespace - kind: PipelineRun diff --git a/components/profile-controller/ci/pipeline-run.yaml b/components/profile-controller/ci/pipeline-run.yaml deleted file mode 100644 index 1f73e333bd1..00000000000 --- a/components/profile-controller/ci/pipeline-run.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: tekton.dev/v1alpha1 -kind: PipelineRun -metadata: - name: ci-profile-controller -spec: - pipelineRef: - name: ci-pipeline - resources: - - name: kubeflow - resourceRef: - name: kubeflow - - name: manifests - resourceRef: - name: manifests - - name: $(image_name) - resourceRef: - name: $(image_name) - serviceAccountName: ci-pipeline-run-service-account diff --git a/py/kubeflow/kubeflow/ci/rebuild-manifests.sh b/py/kubeflow/kubeflow/ci/rebuild-manifests.sh deleted file mode 100755 index 908d436b2b2..00000000000 --- a/py/kubeflow/kubeflow/ci/rebuild-manifests.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# -# this script assumes the following -# - PipelineResource of type git for kubeflow is mounted at /workspace/kubeflow -# - PipelineResource of type git for manifests is mounted at /workspace/manifests -# -# and expects the following: -# env-vars: -# namespace -# image_name (eg centraldashboard) -# kubeflow_repo_revision -# kubeflow_repo_url -# secrets: -# gcp-credentials -# github-token -# github-ssh -# kubeflow-oauth -# -# The script does the following in the kubeflow manifests repo -# - edits the image tag in the kustomization.yaml (its workingdir is where the component's manifest is) -# - calls `make generate; make test` under manifests/tests -# - if successful -# - commits the changes -# - creates a PR. -# -# how to set env vars from configmap if debugging -# for i in $(kubectl get cm ci-pipeline-run-parameters -ojson | jq -r '.data | keys[] as $k | "\($k)=\(.[$k])"'); do echo export $i; export $i; done -# -set -ex - -# This is for debug -echo '--env--' -env | sort -echo '--env--' - -# GitHub user to store the fork -fork_user=kubeflow-bot - -# Get the commit for the kubeflow repository -cd /workspace/kubeflow -kubeflow_commit=$(git rev-parse HEAD) -kubeflow_commit=${digest:0:8} - -# We use a unique branch name based on the digest of the image -digest=$(cat /workspace/${image_name}-digest) -# Shorten the digest. The digest will start with sha256: -short_digest=${digest:7:8} -new_branch_name='update_'$image_name'_'${short_digest} - -# TODO(jlewi): We should mount the GITHUB_TOKEN as a secret -#export GITHUB_TOKEN=$(kubectl get secrets github-token -ojson | jq '. | .data.token' | xargs | base64 -d) - -# Tekton will automatically mount the ssh private key and known hosts stored in the git secret -# in /tekton/home/.ssh -# however since this scriptt runs in our test worker image it ends up using /root/.sssh -ln -sf /tekton/home/.ssh /root/.ssh -ssh-keyscan -t rsa github.com > /root/.ssh/known_hosts -cd /workspace/manifests - -# Do a full fetch to unshallow the clone -# it looks like Tekton might do a shallow checkout -git fetch --unshallow - -# Create a new branch for the pull request -git checkout -b $new_branch_name origin/${manifests_repo_revision} - -# Add the kubeflow-bot repo -git remote add ${fork_user} git@github.com:${fork_user}/manifests.git - -cd /workspace/manifests/${path_to_manifests_dir} -kustomize edit set image gcr.io/kubeflow-images-public/${image_name}=gcr.io/kubeflow-images-public/${image_name}@$(cat /workspace/${image_name}-digest) -cd /workspace/manifests/tests - -# TODO(jlewi): Changed to make generate-changed-only once https://github.com/kubeflow/manifests/pull/665 -# is submitted -# make generate-changed-only -make generate -make test -if (( $? == 0 )); then - git config --global user.email "kubeflow-bot@kubflow.org" - git config --global user.name "kubeflow-bot" - - tmpfile=$(mktemp) - - echo "[auto PR] Update the ${image_name} image to commit ${kubeflow_commit}" > $tmpfile - echo "" >> $tmpfile - echo "* Use image digest $(cat /workspace/${image_name}-digest)" >> $tmpfile - - echo "" >> $tmpfile - echo "" >> $tmpfile - echo "* Image built from repo:$manifests_repo_url branch:$new_branch_name commit:$(kubeflow_commit)" >> $tmpfile - - git commit -a -F ${tmpfile} - - git push ${fork_user} $new_branch_name -f - hub pull-request -f -b 'kubeflow:master' -F $tmpfile -else - echo 'make generate && make test' failed -fi