verify-k8s.yml

#
# Copyright 2021 The Sigstore Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Verify-K8s

on: [push, pull_request]

jobs:
  verify-k8s-manifests:
    name: k8s manifest check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 #v2.4.0
      - name: Extract version of Go to use
        run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV
      - uses: actions/setup-go@424fc82d43fa5a37540bae62709ddcc23d9520d4 #v2.1.5
        with:
          go-version: ${{ env.GOVERSION }}
      - name: Install kubeval
        run: go get github.com/instrumenta/kubeval@v0.16.1
      - run: kubeval config/*.yaml

  verify-k8s-deployment:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false # Keep running if one leg fails.
      matrix:
        issuer:
        - "OIDC Issuer"
        - "Meta Issuer"

        include:
        - issuer: "OIDC Issuer"
          issuer-config: |
            "OIDCIssuers": {"https://kubernetes.default.svc": {"IssuerURL": "https://kubernetes.default.svc","ClientID": "sigstore","Type": "kubernetes"}}
        - issuer: "Meta Issuer"
          issuer-config: |
            "MetaIssuers": {"https://kubernetes.*.svc": {"ClientID": "sigstore","Type": "kubernetes"}}

    env:
      # https://github.com/google/go-containerregistry/pull/125 allows insecure registry for
      # '*.local' hostnames. This works both for `ko` and our own tag-to-digest resolution logic,
      # thus allowing us to test without bypassing tag-to-digest resolution.
      REGISTRY_NAME: registry.local
      REGISTRY_PORT: 5000
      KO_PREFIX: registry.local:5000/fulcio
      GIT_HASH: ${{ github.sha }}
      GIT_VERSION: test

    steps:
      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 #v2.4.0
      - name: Extract version of Go to use
        run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV
      - uses: actions/setup-go@424fc82d43fa5a37540bae62709ddcc23d9520d4 #v2.1.5
        with:
          go-version: ${{ env.GOVERSION }}

      - uses: imjasonh/setup-ko@2c3450ca27f6e6f2b02e72a40f2163c281a1f675 #v0.4

      - name: Install KinD
        run: |
          # Disable swap otherwise memory enforcement doesn't work
          # See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600009955324200
          sudo swapoff -a
          sudo rm -f /swapfile
          # Use in-memory storage to avoid etcd server timeouts.
          # https://kubernetes.slack.com/archives/CEKK1KTN2/p1615134111016300
          # https://github.com/kubernetes-sigs/kind/issues/845
          sudo mkdir -p /tmp/etcd
          sudo mount -t tmpfs tmpfs /tmp/etcd
          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.11.1/kind-$(uname)-amd64
          chmod +x ./kind
          sudo mv kind /usr/local/bin

      - name: Configure KinD Cluster
        run: |
          set -x
          # KinD configuration.
          cat > kind.yaml <<EOF
          apiVersion: kind.x-k8s.io/v1alpha4
          kind: Cluster
          # Configure registry for KinD.
          containerdConfigPatches:
          - |-
            [plugins."io.containerd.grpc.v1.cri".registry.mirrors."$REGISTRY_NAME:$REGISTRY_PORT"]
              endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"]
          # This is needed in order to support projected volumes with service account tokens.
          # See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600
          kubeadmConfigPatches:
            - |
              apiVersion: kubeadm.k8s.io/v1beta2
              kind: ClusterConfiguration
              metadata:
                name: config
              apiServer:
                extraArgs:
                  "service-account-issuer": "https://kubernetes.default.svc"
                  "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
                  "service-account-jwks-uri": "https://kubernetes.default.svc/openid/v1/jwks"
                  "service-account-key-file": "/etc/kubernetes/pki/sa.pub"
              networking:
                dnsDomain: "cluster.local"
          nodes:
          - role: control-plane
            image: kindest/node:v1.22.0@sha256:f97edf7f7ed53c57762b24f90a34fad101386c5bd4d93baeb45449557148c717
            extraMounts:
            - containerPath: /var/lib/etcd
              hostPath: /tmp/etcd
          - role: worker
            image: kindest/node:v1.22.0@sha256:f97edf7f7ed53c57762b24f90a34fad101386c5bd4d93baeb45449557148c717
          EOF

      - name: Create KinD Cluster
        run: |
          kind create cluster --config kind.yaml

      - name: Setup local registry
        run: |
          # Run a registry.
          docker run -d --restart=always \
            -p $REGISTRY_PORT:$REGISTRY_PORT --name $REGISTRY_NAME registry:2
          # Connect the registry to the KinD network.
          docker network connect "kind" $REGISTRY_NAME
          # Make the $REGISTRY_NAME -> 127.0.0.1, to tell `ko` to publish to
          # local reigstry, even when pushing $REGISTRY_NAME:$REGISTRY_PORT/some/image
          sudo echo "127.0.0.1 $REGISTRY_NAME" | sudo tee -a /etc/hosts

      - name: Generate temporary CA files
        run: |
          openssl req -x509 \
              -newkey ed25519 \
              -sha256 \
              -keyout "${{ github.run_id }}-key.pem" \
              -out "${{ github.run_id }}-cert.pem" \
              -subj "/CN=ed25519" \
              -days 36500 \
              -addext basicConstraints=critical,CA:TRUE,pathlen:1 \
              -passout pass:"${{ github.run_id }}"

      - name: Deploy fulcio-dev
        run: |
          # Reduce the resource requests of Fulcio
          sed -i -e 's,memory: "1G",memory: "100Mi",g' ${{ github.workspace }}/config/deployment.yaml
          sed -i -e 's,cpu: ".5",cpu: "50m",g' ${{ github.workspace }}/config/deployment.yaml
          # Switch to one replica to make it easier to test the scraping of
          # metrics since we know all the requests then go to the same server.
          sed -i -E 's,replicas: 3,replicas: 1,g' ${{ github.workspace }}/config/deployment.yaml
          # Expose the prometheus port as a service so tests can grab it
          # without hitting the k8s API
          cat <<EOF >> ${{ github.workspace }}/config/deployment.yaml
              - name: prometheus
                protocol: TCP
                port: 2112
                targetPort: 2112
          EOF

          # From: https://banzaicloud.com/blog/kubernetes-oidc/
          # To be able to fetch the public keys and validate the JWT tokens against
          # the Kubernetes cluster’s issuer we have to allow external unauthenticated
          # requests. To do this, we bind this special role with a ClusterRoleBinding
          # to unauthenticated users (make sure that this is safe in your environment,
          # but only public keys are visible on this URL)
          kubectl create clusterrolebinding oidc-reviewer \
            --clusterrole=system:service-account-issuer-discovery \
            --group=system:unauthenticated

          # Overwrite the configuration to be what we need for KinD.
          cat <<EOF > config/fulcio-config.yaml
          apiVersion: v1
          kind: ConfigMap
          metadata:
            name: fulcio-config
            namespace: fulcio-dev
          data:
            config.json: |-
              {
                ${{ matrix.issuer-config }}
              }
            server.yaml: |-
              host: 0.0.0.0
              port: 5555
              ca: fileca
              fileca-cert: /etc/fulcio-secret/cert.pem
              fileca-key: /etc/fulcio-secret/key.pem
              fileca-key-passwd: "${{ github.run_id }}"
              ct-log-url: ""
              log_type: prod
          EOF

          # Create secret needed to use fileca
          cat <<EOF > config/fulcio-secret.yaml
          apiVersion: v1
          kind: Secret
          metadata:
            name: fulcio-secret
            namespace: fulcio-dev
          data:
            cert.pem: $(cat ${{ github.run_id }}-cert.pem | base64 -w 0)
            key.pem: $(cat ${{ github.run_id }}-key.pem | base64 -w 0)
          EOF

          kubectl create ns fulcio-dev

          make ko-apply

          kubectl wait --for=condition=Available --timeout=5m -n fulcio-dev deployment/fulcio-server

          kubectl get po -n fulcio-dev

      - name: Run signing job
        run: |
          DIGEST=$(make ko-publish | sed '1d')

          cat <<EOF | kubectl apply -f -
          apiVersion: batch/v1
          kind: Job
          metadata:
            name: check-oidc
          spec:
            template:
              spec:
                restartPolicy: Never
                automountServiceAccountToken: false
                containers:
                - name: check-oidc
                  image: gcr.io/projectsigstore/cosign:v1.4.1
                  args: [
                    "sign",
                    "--fulcio-url=http://fulcio-server.fulcio-dev.svc",
                    "--insecure-skip-verify=true",
                    "--upload=false",
                    "${DIGEST}",
                  ]

                  env:
                  - name: COSIGN_EXPERIMENTAL
                    value: "true"
                  volumeMounts:
                  - name: oidc-info
                    mountPath: /var/run/sigstore/cosign
                volumes:
                  - name: oidc-info
                    projected:
                      sources:
                        - serviceAccountToken:
                            path: oidc-token
                            expirationSeconds: 600 # Use as short-lived as possible.
                            audience: sigstore
          EOF

          kubectl wait --for=condition=Complete --timeout=90s job/check-oidc

      - name: Validate prometheus metrics exported and look correct
        run: |
          cat <<EOF | ko apply -f -
          apiVersion: batch/v1
          kind: Job
          metadata:
            name: check-prometheus-metrics
          spec:
            template:
              spec:
                restartPolicy: Never
                automountServiceAccountToken: false
                containers:
                - name: check-metrics
                  image: ko://github.com/sigstore/fulcio/test/prometheus/
          EOF

          kubectl wait --for=condition=Complete --timeout=90s job/check-prometheus-metrics
        env:
          KO_DOCKER_REPO: registry.local:5000/fulcio

      - name: Collect logs
        if: ${{ always() }}
        run: |
          mkdir -p /tmp/logs
          kind export logs /tmp/logs

      - name: Upload artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 #v2.3.1
        with:
          name: logs
          path: /tmp/logs