diff --git a/apps-cd/Dockerfile b/apps-cd/Dockerfile index d2dc61319..8bd8ceed7 100644 --- a/apps-cd/Dockerfile +++ b/apps-cd/Dockerfile @@ -1,3 +1,7 @@ +# Build the docker image used to run the scripts +# to continuously update our docker files. +# +# The context for this docker file should be the root of the kubeflow/testing repository. FROM ubuntu:18.04 RUN apt-get update -y && \ @@ -22,13 +26,9 @@ RUN export KUSTOMIZE_VERSION=3.2.0 && \ mv kustomize_${KUSTOMIZE_VERSION}_linux_amd64 /usr/local/bin/kustomize && \ chmod a+x /usr/local/bin/kustomize - -RUN python -m pip install fire \ - google-api-python-client \ - google-cloud-storage \ - kubernetes \ - watchdog - +COPY apps-cd/requirements.txt /tmp +RUN python -m pip install \ + -r /tmp/requirements.txt # Create go symlinks RUN ln -sf /usr/local/go/bin/go /usr/local/bin && \ @@ -36,8 +36,18 @@ RUN ln -sf /usr/local/go/bin/go /usr/local/bin && \ ln -sf /usr/local/go/bin/godoc /usr/local/bin RUN mkdir -p /app -COPY update_launcher.py /app -COPY run_with_auto_restart.py /app + +RUN cd /app && \ + mkdir -p /app/src/kubeflow && \ + cd /app/src/kubeflow && \ + git clone https://github.com/kubeflow/code-intelligence code-intelligence && \ + cd code-intelligence && \ + git checkout db1230d + +COPY apps-cd/run_with_auto_restart.py /app +COPY py /app/src/kubeflow/testing/py + +ENV PYTHONPATH /app/src/kubeflow/testing/py:/app/src/kubeflow/code-intelligence/py:$PYTHONPATH # See(https://github.com/tektoncd/pipeline/issues/1271): Tekton will put ssh # credentials in /tekton/home. We can't change the home directory diff --git a/apps-cd/README.md b/apps-cd/README.md index 5123a9eb8..8c74ae184 100644 --- a/apps-cd/README.md +++ b/apps-cd/README.md @@ -43,13 +43,7 @@ and open PRs to update Kubeflow kustomize manifests to use the newly built image * The kubeflow-bot GitHub account is used to create the PRs - * Continuous building is achieved by running `update_launcher.py` in a Kubernetes deployment - - * This script periodically fetches the latest code in `kubeflow/testing` to pick up any changes - to the code or config - - * It then launches `update_kf_apps.py` to create Tekton PipelineRuns for any applications that - need to be updated. + * Continuous building is achieved by running `update_kf_apps.py` in a Kubernetes deployment ### Adding Applications to continuous delivery @@ -150,12 +144,21 @@ This is a Kubeflow cluster (v0.6.2) and we rely on that to configure certain thi ## Developer Guide -You can use skaffold to build a docker image and auto update the deployment running `update_launcher.py` +You can use skaffold to build a docker image and auto update the deployment running `update_kf_apps.py` + +* The namespace `kf-releasing-dev` is intended for trying out your local changes +* You can update `apps-cd/pipelines/base/config/app-pipeline.template` to pull in your branch of changes to the CD tooling + + * The template defines a PipelineRun that is used to generate the PipelineRun's to update each application + * The template defines a Git resource which points to the kubeflow/testing repository to obtain the scripts to update the manifests + * You can change this to point to your fork of kubeflow/testing to test your changes +* You can also deploy any changes to tekton resources to that namespace before trying out + in prod. 1. Run skaffold ``` - skaffold dev -v info --cleanup=false --trigger=polling + skaffold dev -p dev -v info --cleanup=false --trigger=polling ``` 1. During development you can take advantage of skaffold's continuous file sync mode to update diff --git a/apps-cd/applications.yaml b/apps-cd/applications.yaml index 749ab13f6..13fb5b930 100644 --- a/apps-cd/applications.yaml +++ b/apps-cd/applications.yaml @@ -119,26 +119,3 @@ versions: value: v1.0-branch - name: url value: git@github.com:kubeflow/manifests.git - # Define a v0-8 release - # This is primarily so we can test that the CI/CD infrastructure is workin - # with release branches. We don't plan on actually releasing 0.8 - - name: v0-8 - # A tag to prefix image names with - tag: "v0.8.0" - repos: - - name: kubeflow - resourceSpec: - type: git - params: - - name: revision - value: v0.8-branch - - name: url - value: git@github.com:kubeflow/kubeflow.git - - name: manifests - resourceSpec: - type: git - params: - - name: revision - value: v0.8-branch - - name: url - value: git@github.com:kubeflow/manifests.git \ No newline at end of file diff --git a/apps-cd/runs/app-pipeline.template.yaml b/apps-cd/pipelines/base/config/app-pipeline.template.yaml similarity index 89% rename from apps-cd/runs/app-pipeline.template.yaml rename to apps-cd/pipelines/base/config/app-pipeline.template.yaml index 225a29149..1090e0398 100644 --- a/apps-cd/runs/app-pipeline.template.yaml +++ b/apps-cd/pipelines/base/config/app-pipeline.template.yaml @@ -45,9 +45,13 @@ spec: # https://github.com/kubeflow/testing/pull/572 # is merged - name: revision - value: master + value: cicd_delete_old_prs - name: url - value: git@github.com:kubeflow/testing.git + value: git@github.com:jlewi/testing.git + #- name: revision + # value: master + #- name: url + # value: git@github.com:kubeflow/testing.git # The image we want to build - name: image resourceSpec: diff --git a/apps-cd/pipelines/base/deployment.yaml b/apps-cd/pipelines/base/deployment.yaml index fbcf7ff37..1d019e673 100644 --- a/apps-cd/pipelines/base/deployment.yaml +++ b/apps-cd/pipelines/base/deployment.yaml @@ -17,41 +17,47 @@ spec: - name: app image: gcr.io/kubeflow-releasing/update_kf_apps command: - # Uncomment the following lines when running with skaffold - # and you want to use skaffold's auto-sync feature wto pick up changes - # to update_launcher. - # Begin skaffold - python - - run_with_auto_restart.py - - --dir=/app - - -- - # End skaffold - # - - python - - update_launcher.py - - run - # repo_dir is the directory where kubeflow testing should be checked out to - - --repo_dir=/launcher_src/kubeflow/testing - # repo is the code to check out to get the configuration as well as code for - # launching the applications - # - # To test changes before they are checked in you can set this to the repo containing your PR - # e.g. - --repo=https://github.com/jlewi/testing.git?ref=cicd_0.8 - - --repo=https://github.com/kubeflow/testing.git + - -m + - kubeflow.testing.cd.update_kf_apps + - sync # Extra arguments to be passed to update_kf_apps.py - - --namespace=kf-releasing - - --config=/launcher_src/kubeflow/testing/apps-cd/applications.yaml + - --namespace=kf-releasing + # TODO(jlewi): Should we just put this in a config map as well and then check in into source control? + # Would a tool like Anthos CM be smart enough to detect changes in that file and trigger + # updates with kustomize? + - --config=https://raw.githubusercontent.com/kubeflow/testing/master/apps-cd/applications.yaml - --output_dir=/tmp/runs - --src_dir=/src - - --template=/launcher_src/kubeflow/testing/apps-cd/runs/app-pipeline.template.yaml + - --template=/app/config/app-pipeline.template.yaml env: + # TODO(jlewi): We should stop using a GITHUB_TOKEN and switch to using + # the github APP. Is this only used by the Hub CLI? - name: GITHUB_TOKEN valueFrom: secretKeyRef: name: github-token key: github_token + - name: GITHUB_APP_PEM_KEY + value: /var/secrets/github/kubeflow-auto-bot.2020-01-24.private-key.pem resources: requests: cpu: 4 memory: 8Gi workingDir: /app + volumeMounts: + - name: github-app + mountPath: /var/secrets/github + volumeMounts: + - name: pipelinerun-template + mountPath: /app/config + volumes: + # Secret containing the PEM key for the GitHub app used to close PRs. + - name: github-app + secret: + secretName: github-app-pem + - name: pipelinerun-template + configMap: + # Kustomize will automatically replace the name with the unique name given + # to the configmap based on the config contents. + name: pipelinerun-template \ No newline at end of file diff --git a/apps-cd/pipelines/base/kustomization.yaml b/apps-cd/pipelines/base/kustomization.yaml index 476ea03e6..0300f3c41 100644 --- a/apps-cd/pipelines/base/kustomization.yaml +++ b/apps-cd/pipelines/base/kustomization.yaml @@ -11,3 +11,9 @@ resources: - task.yaml - pipeline.yaml namespace: kf-releasing +# Create a configMap containing the template for the pipeline run +configMapGenerator: +- name: pipelinerun-template + files: + # key will be name of the file + - ./config/app-pipeline.template.yaml \ No newline at end of file diff --git a/apps-cd/pipelines/base/task.yaml b/apps-cd/pipelines/base/task.yaml index 06dabc656..0af11aa57 100644 --- a/apps-cd/pipelines/base/task.yaml +++ b/apps-cd/pipelines/base/task.yaml @@ -37,7 +37,11 @@ spec: - --digest-file=/workspace/image-digest env: - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/user-gcp-sa.json + value: /secret/user-gcp-sa.json + resources: + requests: + cpu: 7 + memory: 16Gi volumeMounts: - mountPath: /secret name: gcp-credentials @@ -88,8 +92,8 @@ spec: - -m - kubeflow.testing.cd.create_manifests_pr - apply - - --image_url=${IMAGE_URL} - - --src_image_url=${SRC_IMAGE_URL} + - --image_url=$(inputs.resources.image.url) + - --src_image_url=$(inputs.params.src_image_url) - --manifests_dir=/workspace/$(inputs.resources.manifests.name)/$(inputs.params.path_to_manifests_dir) - --manifests_base=$(inputs.resources.manifests.revision) env: @@ -102,14 +106,10 @@ spec: secretKeyRef: name: github-token key: github_token - - name: SRC_IMAGE_URL - value: $(inputs.params.src_image_url) - - name: IMAGE_URL - value: $(inputs.resources.image.url) - - name: MANIFESTS_REPO_REVISION - value: $(inputs.resources.manifests.revision) - - name: PATH_TO_MANIFESTS_DIR - value: $(inputs.params.path_to_manifests_dir) + resources: + requests: + cpu: 4 + memory: 4Gi volumeMounts: - mountPath: /secret name: gcp-credentials diff --git a/apps-cd/pipelines/overlays/dev/deployment.yaml b/apps-cd/pipelines/overlays/dev/deployment.yaml new file mode 100644 index 000000000..582fac580 --- /dev/null +++ b/apps-cd/pipelines/overlays/dev/deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: server +spec: + template: + spec: + containers: + - name: app + image: gcr.io/kubeflow-releasing/update_kf_apps + command: + # In the dev environment we run with auto_restart for compatibility with + # skaffold + - python + - run_with_auto_restart.py + - --dir=/app + - -- + - python + - -m + - kubeflow.testing.cd.update_kf_apps + - sync + # repo is the code to check out to get the configuration as well as code for + # launching the applications + # + # To test changes before they are checked in you can set this to the repo containing your PR + # e.g. + # Extra arguments to be passed to update_kf_apps.py + - --namespace=kf-releasing-dev + - --config=https://raw.githubusercontent.com/jlewi/testing/cicd_delete_old_prs/apps-cd/applications.yaml + - --output_dir=/tmp/runs + - --src_dir=/src + - --template=/app/config/app-pipeline.template.yaml + \ No newline at end of file diff --git a/apps-cd/pipelines/overlays/dev/kustomization.yaml b/apps-cd/pipelines/overlays/dev/kustomization.yaml new file mode 100644 index 000000000..23f1fa86c --- /dev/null +++ b/apps-cd/pipelines/overlays/dev/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../base +commonLabels: + environment: dev +namespace: kf-releasing-dev +patchesStrategicMerge: +- deployment.yaml diff --git a/apps-cd/pipelines/overlays/prod/kustomization.yaml b/apps-cd/pipelines/overlays/prod/kustomization.yaml new file mode 100644 index 000000000..3d3e97d4c --- /dev/null +++ b/apps-cd/pipelines/overlays/prod/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../base +commonLabels: + environment: prod +namespace: kf-releasing \ No newline at end of file diff --git a/apps-cd/requirements.txt b/apps-cd/requirements.txt new file mode 100644 index 000000000..91e7d50a4 --- /dev/null +++ b/apps-cd/requirements.txt @@ -0,0 +1,20 @@ +-i https://pypi.org/simple +asn1crypto==0.24.0 +cffi==1.13.2 +fire==0.2.1 +github3.py==1.3.0 +google-api-core==1.14.2 +google-api-python-client==1.7.10 +google-auth==1.6.3 +google-auth-httplib2==0.0.3 +google-cloud-core==1.0.3 +google-cloud-storage==1.17.0 +json-log-formatter==0.2.0 +jwcrypto==0.6.0 +kubernetes==9.0.0 +pyjwt==1.7.1 +requests==2.22.0 +requests-oauthlib==1.2.0 +requests-toolbelt==0.9.1 +tqdm==4.42.0 +watchdog==0.9.0 diff --git a/apps-cd/runs/README.md b/apps-cd/runs/README.md deleted file mode 100644 index 7be66c625..000000000 --- a/apps-cd/runs/README.md +++ /dev/null @@ -1,4 +0,0 @@ -This directory contains Tekton PipelineRuns to build kubeflow applications -and update the Kubeflow kustomize manifests to use the newly built application. - -Each PipelineRun builds a specific application at a specific commit. \ No newline at end of file diff --git a/apps-cd/runs/profile_controller_v1795828.yaml b/apps-cd/runs/profile_controller_v1795828.yaml deleted file mode 100644 index b23c8694a..000000000 --- a/apps-cd/runs/profile_controller_v1795828.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: tekton.dev/v1alpha1 -kind: PipelineRun -metadata: - # Generate a unique name for each run - generateName: ci-profile-controller- -spec: - pipelineRef: - name: ci-pipeline - params: - - name: "path_to_context" - value: "components/profile-controller" - - name: "path_to_docker_file" - value: "components/profile-controller/Dockerfile" - - name: "path_to_manifests_dir" - value: "profiles/base" - - name: "src_image_url" - value: "gcr.io/kubeflow-images-public/profile-controller" - - name: "container_image" - value: "gcr.io/kubeflow-releasing/test-worker@sha256:35138a42b57160a078e802b7d69aec3c3e79a3e2e55518af7798275ebcc84d25" - resources: - # The git resources that will be used - - name: kubeflow - resourceSpec: - type: git - params: - - name: revision - value: "1795828" - - name: url - value: git@github.com:kubeflow/kubeflow.git - - name: manifests - resourceSpec: - type: git - params: - - name: revision - value: master - - name: url - value: git@github.com:kubeflow/manifests.git - # TODO(jlewi): Replace with kubeflw/kubeflow once the PR is checked in. - - name: ci-tools - resourceSpec: - type: git - params: - - name: revision - value: tekton - - name: url - value: git@github.com:jlewi/testing.git - # The image we want to build - - name: image - resourceSpec: - type: image - params: - - name: url - value: gcr.io/kubeflow-images-public/profile-controller:vmaster-g1795828 - serviceAccountName: ci-pipeline-run-service-account diff --git a/apps-cd/skaffold.yaml b/apps-cd/skaffold.yaml index 3de853abe..25643c5ee 100644 --- a/apps-cd/skaffold.yaml +++ b/apps-cd/skaffold.yaml @@ -3,49 +3,72 @@ apiVersion: skaffold/v2alpha1 kind: Config metadata: name: kf-apps-cd +build: + artifacts: + - image: gcr.io/kubeflow-releasing/update_kf_apps + # Set the context to the root directory. + # All paths in the Dockerfile should be relative to this one. + context: .. + # Automatically sync python files to the container. This should avoid + # the need to rebuild and redeploy when the files change. + # TODO(https://github.com/GoogleContainerTools/skaffold/issues/3448): We use manual sync + # because inferred sync doesn't work + # + # This only works if we autorestart the program on changes. + # + # Important: Make sure you current context has the namespace + # set to the namespace where your pods are deployed otherwise + # the sync doesn't appear to work. + sync: + manual: + # See https://skaffold.dev/docs/pipeline-stages/filesync/ + # src is relative to context + # dest should be location inside the container where files should be placed + - src: 'py/**/*.py' + dest: '/app/src/kubeflow/testing' + kaniko: + dockerfile: ./apps-cd/Dockerfile + buildContext: + gcsBucket: kubeflow-releasing_skaffold + env: + # TODO(GoogleContainerTools/skaffold#3468) skaffold doesn't + # appear to work with workload identity + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/user-gcp-sa.json + cache: {} + cluster: + pullSecretName: user-gcp-sa + resources: + requests: + cpu: 8 + memory: 16Gi profiles: # Build on the kubeflow releasing cluster - - name: kf-releasing - build: - artifacts: - - image: gcr.io/kubeflow-releasing/update_kf_apps - # Set the context to the root directory. - # All paths in the Dockerfile should be relative to this one. - context: . - # Automatically sync python files to the container. This should avoid - # the need to rebuild and redeploy when the files change. - # TODO(https://github.com/GoogleContainerTools/skaffold/issues/3448): We use manual sync - # because inferred sync doesn't work - # - # This only works if we autorestart the program on changes. - # - # Important: Make sure you current context has the namespace - # set to the namespace where your pods are deployed otherwise - # the sync doesn't appear to work. - sync: - manual: - - src: 'update_launcher.py' - dest: '/app' - kaniko: - dockerfile: Dockerfile - buildContext: - gcsBucket: kubeflow-releasing_skaffold - env: - # TODO(GoogleContainerTools/skaffold#3468) skaffold doesn't - # appear to work with workload identity - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/user-gcp-sa.json - cache: {} + - name: prod + build: cluster: - pullSecretName: user-gcp-sa # Build in a namespace with ISTIO sidecar injection disabled # see GoogleContainerTools/skaffold#3442 namespace: kf-releasing + pullSecretName: user-gcp-sa + resources: + requests: + cpu: 8 + memory: 16Gi + deploy: + kustomize: + path: pipelines/overlays/prod + - name: dev + build: + cluster: + # Build in a namespace with ISTIO sidecar injection disabled + # see GoogleContainerTools/skaffold#3442 + namespace: kf-releasing-dev + pullSecretName: user-gcp-sa resources: requests: cpu: 8 memory: 16Gi - deploy: kustomize: - path: pipelines/base \ No newline at end of file + path: pipelines/overlays/dev \ No newline at end of file diff --git a/apps-cd/update_launcher.py b/apps-cd/update_launcher.py deleted file mode 100644 index c61ba3158..000000000 --- a/apps-cd/update_launcher.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/python - -"""A program to continuously launch Tekton pipelines. - -This is a simple script to run update_kf_apps.py. The script does the following - -1. Fetch the latest code from GitHub -2. Run update_kf_apps.py to submit Tekton PipelineRuns to update any apps - that need it. -3. Sleep -4. Go back to step 1. - - -The reason for not calling update_kf_apps.py directly is that we want to -run a git sync to pick up the latest code. - -The reason the code is not in py is because we don't want this script to -have any dependencies on the code in py since its just intended to be a -simple shell script -""" -import fire -import logging -import os -import subprocess -import time -from urllib import parse - -DEFAULT_REPO = "https://github.com/kubeflow/testing.git" - -class Launcher: - @staticmethod - def run(repo=DEFAULT_REPO, repo_dir= "/app/src/kubeflow/testing", - sync_time_seconds=600, **update_args): - """Run the program. - - Args: - repo: Git URL to fetch the code. This should be a kubeflow/testing repo - (or fork) and container the update_kf_apps.py code as well as the - configs used with update_kf_appss.py. To specify a particular branch - add a query arg "?ref= - app_src_dir: Directory where repo should be checked out. - sync_time_seconds: Time in seconds to wait between launches. - update_args: Arguments for update_kf_apps - """ - - parent_dir = os.path.dirname(repo_dir) - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - - # Parse out the query args to look for a branch - p = parse.urlparse(repo) - query = p.query - - repo_url = p._replace(query="").geturl() - - ref = "master" - if query: - logging.info(f"URL has query string {query}") - q = parse.parse_qs(p.query) - logging.info(f"Parsed query {q}") - if "ref" in q: - ref = q["ref"][-1] - - logging.info(f"Using ref {ref}") - while True: - if not os.path.exists(repo_dir): - logging.info(f"Cloning {repo}") - subprocess.check_call(["git", "clone", repo_url, repo_dir]) - logging.info(f"Fetching latest code") - subprocess.check_call(["git", "fetch", "origin"], cwd=repo_dir) - - logging.info(f"Checking out origin/{ref}") - subprocess.check_call(["git", "checkout", f"origin/{ref}"], cwd=repo_dir) - - commit = subprocess.check_output(["git", "describe", "--tags", - "--always", "--dirty"], cwd=repo_dir) - - logging.info(f"using update_kf_apps.py from {p.geturl()} " - f"at commit: {commit}") - - logging.info("Launching update_kf_apps") - - command = ["python", "-m", "kubeflow.testing.cd.update_kf_apps", "apply"] - - extra = [f"--{k}={v}" for k,v in update_args.items()] - command.extend(extra) - py_dir = os.path.join(repo_dir, "py") - subprocess.check_call(command, cwd=py_dir) - - logging.info("Wait before rerunning") - time.sleep(sync_time_seconds) - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, - format=('%(levelname)s|%(asctime)s' - '|%(pathname)s|%(lineno)d| %(message)s'), - datefmt='%Y-%m-%dT%H:%M:%S', - ) - logging.getLogger().setLevel(logging.INFO) - fire.Fire(Launcher) diff --git a/images/skaffold.yaml b/images/skaffold.yaml deleted file mode 100644 index 7ac16b81b..000000000 --- a/images/skaffold.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Reference: https://skaffold.dev/docs/references/yaml/ -# -# TODO(jlewi): This is just a rudimentary skaffold config. -# Users probably need to create their own profile based on the resources (e.g. GCR) -# They have access too. -apiVersion: skaffold/v2alpha1 -kind: Config -metadata: - name: notebook-controller -profiles: -- name: kf-releasing - build: - artifacts: - # TODO(jlewi): We should probably use skaffold profiles to build this for the - # test cluster vs release cluster - - image: gcr.io/kubeflow-releasing/test-worker - # Context should be ${KUBEFLOW_REPO}/components - context: . - kaniko: - dockerfile: Dockerfile - buildContext: - gcsBucket: kubeflow-releasing_skaffold - env: - # TODO(GoogleContainerTools/skaffold#3468) skaffold doesn't - # appear to work with workload identity and the kubeflow-testing - # cluster isn't using it yet. - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/user-gcp-sa.json - cache: {} - cluster: - pullSecretName: user-gcp-sa - # Build in the kaniko namespace because we need to disable ISTIO sidecar injection - # see GoogleContainerTools/skaffold#3442 - namespace: kf-releasing - resources: - requests: - # TODO(https://github.com/kubeflow/testing/issues/565): Right now the nodes in our cluster - # have a max of 8 CPU so we need to set the requests small enough so the pods get scheduled. - cpu: 6 - memory: 16Gi -# TODO(jlewi): We should add a deploy section to actually deploy the controller. Assuming -# kubeflow/manifests is checked out we should be able to just point to he kustomize manifest in that -# directory \ No newline at end of file diff --git a/py/kubeflow/testing/cd/close_old_prs.py b/py/kubeflow/testing/cd/close_old_prs.py index aa40f2bc6..875515703 100644 --- a/py/kubeflow/testing/cd/close_old_prs.py +++ b/py/kubeflow/testing/cd/close_old_prs.py @@ -7,6 +7,7 @@ import logging import re +from code_intelligence import github_app from code_intelligence import graphql # The name of the GitHub user under which kubeflow-bot branches exist @@ -23,6 +24,28 @@ class PRCloser: def __init__(self): self._client = graphql.GraphQLClient() + self._headers = None + self._token_refresher = None + # Try various methods to obtain credentials + try: + self._token_refresher = github_app.FixedAccessTokenGenerator.from_env() + + except ValueError: + logging.info("Could not create a FixedAccessTokenGenerator; will try " + "other methods for obtaining credentials") + + if not self._token_refresher: + app = github_app.GitHubApp.create_from_env() + self._token_refresher = github_app.GitHubAppTokenGenerator( + app, "kubeflow/manifests") + + + def _run_query(self, *args, **kwargs): + if not kwargs: + kwargs = {} + kwargs["headers"] = self._token_refresher.auth_headers + return self._client.run_query(*args, **kwargs) + def apply(self): app_prs = collections.defaultdict(lambda: []) @@ -48,7 +71,7 @@ def apply(self): # We sort the PRs by URL since this will correspond to PR number # We will keep the most recent one. - sorted_prs = sorted(prs, key=lambda pr: pr["url"]) + sorted_prs = sorted(prs, key=lambda pr: int(pr["number"])) latest = sorted_prs[-1] logging.info(f"For app_tag={app} newest pr is {latest['url']}") @@ -73,7 +96,8 @@ def apply(self): } } - results = self._client.run_query(add_comment, variables=add_variables) + results = self._client.run_query(add_comment, variables=add_variables, + headers=self._headers) if results.get("errors"): message = json.dumps(results.get("errors")) @@ -94,7 +118,8 @@ def apply(self): } } - results = self._client.run_query(close_pr, variables=close_variables) + results = self._run_query(close_pr, variables=close_variables, + headers=self._headers) if results.get("errors"): message = json.dumps(results.get("errors")) @@ -134,6 +159,7 @@ def _iter_prs(self, org, repo): } } id + number title url state @@ -169,7 +195,7 @@ def _iter_prs(self, org, repo): "pageSize": num_prs_per_page, "issueCursor": prs_cursor, } - results = self._client.run_query(query, variables=variables) + results = self._run_query(query, variables=variables) if results.get("errors"): message = json.dumps(results.get("errors")) diff --git a/py/kubeflow/testing/cd/create_manifests_pr.py b/py/kubeflow/testing/cd/create_manifests_pr.py index 4fcbbf4dc..39e6c7798 100644 --- a/py/kubeflow/testing/cd/create_manifests_pr.py +++ b/py/kubeflow/testing/cd/create_manifests_pr.py @@ -74,10 +74,11 @@ def apply(image_url, src_image_url, logging.info(f"git config user.email not set; defaulting to " f"{default_email}") email = default_email + user_name = default_name util.run(["git", "config", "--global", "user.email", - default_email]) + email]) util.run(["git", "config", "--global", "user.name", - default_name]) + user_name]) try: util.run(["git", "fetch", "--unshallow"], cwd=manifests_repo) diff --git a/py/kubeflow/testing/cd/update_kf_apps.py b/py/kubeflow/testing/cd/update_kf_apps.py index 7fd4a13b3..492e99cd4 100644 --- a/py/kubeflow/testing/cd/update_kf_apps.py +++ b/py/kubeflow/testing/cd/update_kf_apps.py @@ -9,10 +9,13 @@ import os import subprocess import traceback +import urllib import re import yaml from kubeflow.testing import util +from kubeflow.testing import yaml_util +from kubeflow.testing.cd import close_old_prs from kubernetes import client as k8s_client from kubernetes import config as k8s_config from kubernetes.client import rest @@ -373,7 +376,6 @@ def _branch_for_app(app, image_tag): src_image_url = _get_param(app["params"], "src_image_url")["value"] image_name = src_image_url.split("/")[-1] - return f"update_{image_name}_{image_tag}" class UpdateKfApps: @@ -389,8 +391,7 @@ def create_runs(config, output_dir, template, src_dir): src_dir: Directory where source should be checked out """ - with open(config) as hf: - run_config = yaml.load(hf) + run_config = yaml_util.load_file(config) failures = [] @@ -407,9 +408,7 @@ def create_runs(config, output_dir, template, src_dir): for app in run_config["applications"]: pair = APP_VERSION_TUPLE(app["name"], version["name"]) # Load a fresh copy of the template - with open(template) as hf: - run = yaml.load(hf) - + run = yaml_util.load_file(template) # Make copies of app and version so that we don't end up modifying them try: @@ -444,7 +443,18 @@ def create_runs(config, output_dir, template, src_dir): @staticmethod def apply(config, output_dir, template, src_dir, namespace): - """Create PipelineRuns for any applications that need to be updated.""" + """Create PipelineRuns for any applications that need to be updated. + + Args: + config: The path to the configuration; can be local or http file + output_dir: Directory where pipeline runs should be written + template: The path to the YAML file to act as a template + src_dir: Directory where source should be checked out + """ + + logging.info("Closing old PRs") + closer = close_old_prs.PRCloser() + closer.apply() service_account_path = "/var/run/secrets/kubernetes.io" if os.path.exists("/var/run/secrets/kubernetes.io"): @@ -463,55 +473,74 @@ def apply(config, output_dir, template, src_dir, namespace): if not pipelines_to_run: logging.info("No pipelines need to be run") - return + else: + logging.info("Submitting pipeline runs to update applications") + for p in pipelines_to_run: + with open(p) as hf: + run = yaml.load(hf) - for p in pipelines_to_run: - with open(p) as hf: - run = yaml.load(hf) + group, version = run["apiVersion"].split("/", 1) + kind = run["kind"] + plural = kind.lower() + "s" - group, version = run["apiVersion"].split("/", 1) - kind = run["kind"] - plural = kind.lower() + "s" + # Check if there are any pipelines running for the same application + label_filter = {} + for k in ["app", "version", "image_tag"]: + label_filter[k] = run["metadata"]["labels"][k] - # Check if there are any pipelines running for the same application - label_filter = {} - for k in ["app", "version", "image_tag"]: - label_filter[k] = run["metadata"]["labels"][k] + items = [f"{k}={v}" for k,v in label_filter.items()] + selector = ",".join(items) - items = [f"{k}={v}" for k,v in label_filter.items()] - selector = ",".join(items) + # TODO(https://github.com/tektoncd/pipeline/issues/1302): We should + # probably do some garbage collection of old runs. + current_runs = crd_api.list_namespaced_custom_object( + group, version, namespace, plural, label_selector=selector) - # TODO(https://github.com/tektoncd/pipeline/issues/1302): We should - # probably do some garbage collection of old runs. - current_runs = crd_api.list_namespaced_custom_object( - group, version, namespace, plural, label_selector=selector) + active_run = None + for r in current_runs["items"]: + conditions = r["status"].get("conditions", []) - active_run = None - for r in current_runs["items"]: - conditions = r["status"].get("conditions", []) + running = True - running = True + for c in conditions[::-1]: + if c.get("type", "").lower() == "succeeded": + if c.get("status", "").lower() in ["true", "false"]: + running = False + break - for c in conditions[::-1]: - if c.get("type", "").lower() == "succeeded": - if c.get("status", "").lower() in ["true", "false"]: - running = False + if running: + active_run = r['metadata']['name'] break - if running: - active_run = r['metadata']['name'] - break + if active_run: + logging.info(f"Found pipeline run {active_run} " + f"already running for {p}; not rerunning") + continue - if active_run: - logging.info(f"Found pipeline run {active_run} " - f"already running for {p}; not rerunning") - continue + logging.info(f"Creating run from file {p}") + result = crd_api.create_namespaced_custom_object(group, version, namespace, plural, + run) + logging.info(f"Created run " + f"{result['metadata']['namespace']}" + f".{result['metadata']['name']}") - logging.info(f"Creating run from file {p}") - result = crd_api.create_namespaced_custom_object(group, version, namespace, plural, - run) - logging.info(f"Created run {result['metadata']['name']}") + @staticmethod + def sync(config, output_dir, template, src_dir, namespace, + sync_time_seconds=600): + """Perioridically fire off tekton pipelines to update the manifests. + + Args: + config: The path to the configuration + output_dir: Directory where pipeline runs should be written + template: The path to the YAML file to act as a template + src_dir: Directory where source should be checked out + sync_time_seconds: Time in seconds to wait between launches. + """ + while True: + UpdateKfApps.apply(config, output_dir, template, src_dir, namespace) + logging.info("Wait before rerunning") + time.sleep(sync_time_seconds) class AppVersion: """App version is a wrapper around a combination of application and version. diff --git a/py/kubeflow/testing/tools/secret_creator.py b/py/kubeflow/testing/tools/secret_creator.py new file mode 100644 index 000000000..6ddfce08c --- /dev/null +++ b/py/kubeflow/testing/tools/secret_creator.py @@ -0,0 +1,113 @@ +#!/usr/bin/python +"""A script to copy kubernetes secrets from one namespace to another +""" + +import base64 +import fire +from google.cloud import storage +from kubernetes import client as k8s_client +from kubernetes import config as k8s_config +from kubernetes.client import rest +import logging +import yaml +import os +import re +import subprocess + +GCS_REGEX = re.compile("gs://([^/]*)(/.*)?") + +def split_gcs_uri(gcs_uri): + """Split a GCS URI into bucket and path.""" + m = GCS_REGEX.match(gcs_uri) + bucket = m.group(1) + path = "" + if m.group(2): + path = m.group(2).lstrip("/") + return bucket, path + + +def _read_gcs_path(gcs_path): + bucket_name, blob_name = split_gcs_uri(gcs_path) + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + contents = blob.download_as_string().decode() + + return contents + +class SecretCreator: + + def __init__(self): + k8s_config.load_kube_config(persist_config=False) + + self._k8s_client = k8s_client.ApiClient() + + + + @staticmethod + def from_gcs(secret_name, gcs_path): + """Create a secret from a GCS. + + Args: + secret_name: {namespace}.{secret_name} of the secret to create + gcs_path: The path of the GCS file to create the secret from. + """ + bucket_name, blob_name = split_gcs_uri(gcs_path) + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + contents = blob.download_as_string().decode() + + file_name = os.path.basename(blob_name) + namespace, name = secret_name.split("/", 1) + subprocess.check_call(["kubectl", "-n", namespace, "create", + "secret", "generic", + name, + f"--from-literal=f{file_name}={contents}"]) + + def copy_secret(self, source, dest): + """Create a secret from one namespace to another. + + Args: + source: {namespace}.{secret name} + dest: {namespace}.{secret name} + """ + src_namespace, src_name = source.split(".", 1) + dest_namespace, dest_name = dest.split(".", 1) + + client = k8s_client.ApiClient() + api = k8s_client.CoreV1Api(client) + + try: + source_secret = api.read_namespaced_secret(src_name, src_namespace) + except rest.ApiException as e: + if e.status != 404: + raise + + if not source_secret: + raise ValueError(f"Secret {source} doesn't exist") + + # delete metadata fields + for f in ["creation_timestamp", "owner_references", "resource_version", + "self_link", "uid"]: + setattr(source_secret.metadata, f, None) + + source_secret.metadata.name = dest_name + source_secret.metadata.namespace = dest_namespace + + api.create_namespaced_secret(dest_namespace, source_secret) + logging.info(f"Created secret {dest}") + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, + format=('%(levelname)s|%(asctime)s' + '|%(message)s|%(pathname)s|%(lineno)d|'), + datefmt='%Y-%m-%dT%H:%M:%S', + ) + + fire.Fire(SecretCreator) + diff --git a/py/kubeflow/testing/yaml_util.py b/py/kubeflow/testing/yaml_util.py new file mode 100644 index 000000000..44d706c63 --- /dev/null +++ b/py/kubeflow/testing/yaml_util.py @@ -0,0 +1,24 @@ +"""YAML utilities +""" +import requests +import urllib +import yaml + +def load_file(path): + """Load a YAML file. + + Args: + path: Path to the YAML file; can be a local path or http URL + + Returs: + data: The parsed YAML. + """ + url_for_spec = urllib.parse.urlparse(path) + + if url_for_spec.scheme in ["http", "https"]: + data = requests.get(path) + return yaml.load(data.content) + else: + with open(path, 'r') as f: + config_spec = yaml.load(f) + return config_spec