diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..ce2ec3d3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "harbor-helm"] + path = harbor-helm + url = https://github.com/goharbor/harbor-helm.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 71fd1506..f3493dbc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,38 +17,57 @@ 1. Install Go. - The project requires [Go 1.17][go-download] or later. We also assume that you're familiar with + The project requires [Go 1.21][go-download] or later. We also assume that you're familiar with Go's [GOPATH workspace][go-code] convention, and have the appropriate environment variables set. -2. Install Docker, Docker Compose, and Make. +2. Install Docker, Make, and Skaffold. 3. Get the source code. ``` git clone https://github.com/aquasecurity/harbor-scanner-trivy.git cd harbor-scanner-trivy ``` + +**NOTE**: Podman is [not supported](https://github.com/GoogleContainerTools/skaffold/issues/8430) by Skaffold. -## Setup Development Environment with Vagrant +## Set up Development Environment with Kubernetes 1. Get the source code. ``` - git clone https://github.com/aquasecurity/harbor-scanner-trivy.git + git clone --recursive https://github.com/aquasecurity/harbor-scanner-trivy.git cd harbor-scanner-trivy ``` -2. Create and configure a guest development machine, which is based on Ubuntu 20.4 LTS and has Go, Docker, Docker Compose, - Make, and Harbor v2.5.1 preinstalled. Harbor is installed in the `/opt/harbor` directory. +2. Launch a Kubernetes cluster + + We recommend setting up a Kubernetes cluster with: + + - [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) + - [minikube](https://minikube.sigs.k8s.io/docs/start/) + - [Docker Desktop](https://docs.docker.com/desktop/) + - [Rancher Desktop](https://docs.rancherdesktop.io/ui/preferences/kubernetes/) + + However, you can use any Kubernetes cluster you want. + + In case of kind, you can use the following command to create a cluster: ``` - vagrant up + kind create cluster --name harbor-scanner-trivy ``` - If everything goes well Harbor will be accessible at http://localhost:8181 (admin/Harbor12345). - To SSH into a running Vagrant machine. +3. Run Skaffold + + The following command will build the image and deploy Harbor with the scanner adapter to the Kubernetes cluster: ``` - vagrant ssh + make dev ``` - The `/vagrant` directory in the development machine is shared between host and guest. This, for example, allows you - to rebuild a container image for testing. + +4. Access Harbor UI + + After the Harbor chart is deployed, you can access `https://core.harbor.domain`. + It depends on how you configure the Kubernetes cluster, but you may need to add a host entry to `/etc/hosts` file. + ``` - vagrant@ubuntu-focal:/vagrant$ make docker-build + echo "127.0.0.1\tcore.harbor.domain" | sudo tee -a /etc/hosts ``` + + username: admin, password: Harbor12345 ## Build Binaries diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 00000000..beb3a2be --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,33 @@ +# That's the only place where you're supposed to specify version of Trivy. +ARG TRIVY_VERSION=0.46.0 +ARG SKAFFOLD_GO_GCFLAGS + +FROM golang:1.21-alpine AS builder + +WORKDIR /go/src/github.com/aquasecurity/harbor-scanner-trivy + +# Download Go dependencies first +COPY go.mod go.sum ./ +RUN go mod download + +# Copy the entire project and build it. +COPY cmd/ ./cmd +COPY pkg/ ./pkg +RUN CGO_ENABLED=0 go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o scanner-trivy cmd/scanner-trivy/main.go + +FROM aquasec/trivy:${TRIVY_VERSION} + +ENV GOTRACEBACK=all + +# An ARG declared before a FROM is outside of a build stage, so it can't be used in any +# instruction after a FROM. To use the default value of an ARG declared before the first +# FROM use an ARG instruction without a value inside of a build stage. +ARG TRIVY_VERSION + +RUN adduser -u 10000 -D -g '' scanner scanner + +COPY --from=builder /go/src/github.com/aquasecurity/harbor-scanner-trivy/scanner-trivy /home/scanner/bin/scanner-trivy + +ENV TRIVY_VERSION=${TRIVY_VERSION} + +ENTRYPOINT ["/home/scanner/bin/scanner-trivy"] diff --git a/Makefile b/Makefile index 74c3f689..f237375c 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,13 @@ BINARY := scanner-trivy IMAGE_TAG := dev IMAGE := aquasec/harbor-scanner-trivy:$(IMAGE_TAG) +.PHONY: build test test-integration test-component docker-build setup dev debug run + build: $(BINARY) test: build GO111MODULE=on go test -v -short -race -coverprofile=coverage.txt -covermode=atomic ./... -.PHONY: test-integration test-integration: build GO111MODULE=on go test -count=1 -v -tags=integration ./test/integration/... @@ -26,11 +27,18 @@ docker-build: build lint: ./bin/golangci-lint --build-tags component,integration run -v -.PHONY: setup setup: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.21.0 -.PHONY: run +submodule: + git submodule update --init --recursive + +dev: + skaffold dev --tolerate-failures-until-deadline=true + +debug: + skaffold debug --tolerate-failures-until-deadline=true + run: export SCANNER_TRIVY_CACHE_DIR = $(TMPDIR)harbor-scanner-trivy/.cache/trivy run: export SCANNER_TRIVY_REPORTS_DIR=$(TMPDIR)harbor-scanner-trivy/.cache/reports run: export SCANNER_LOG_LEVEL=debug diff --git a/harbor-helm b/harbor-helm new file mode 160000 index 00000000..fd3a4b7b --- /dev/null +++ b/harbor-helm @@ -0,0 +1 @@ +Subproject commit fd3a4b7bcbe5e99cc799287e7dc79c401c5cf24f diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 00000000..bf76c1ee --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,22 @@ +apiVersion: skaffold/v4beta7 +kind: Config +metadata: + name: harbor-scanner-trivy +build: + local: + push: false # Use kind, minikube, Docker Desktop, Rancher Desktop, etc. + useDockerCLI: true + artifacts: + - image: aquasecurity/harbor-scanner-trivy + docker: + dockerfile: Dockerfile.dev +deploy: + helm: + releases: + - name: my-harbor + chartPath: ./harbor-helm # remoteChart doesn't work for some reason + valuesFiles: + - ./harbor-helm/values.yaml + setValueTemplates: + trivy.image.repository: "{{.IMAGE_REPO_aquasecurity_harbor_scanner_trivy}}" + trivy.image.tag: "{{.IMAGE_TAG_aquasecurity_harbor_scanner_trivy}}" diff --git a/vagrant/harbor.yml b/vagrant/harbor.yml deleted file mode 100644 index ae7e29d9..00000000 --- a/vagrant/harbor.yml +++ /dev/null @@ -1,115 +0,0 @@ ---- -# Configuration file of Harbor - -# The IP address or hostname to access admin UI and registry service. -# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -hostname: harbor.dev.io - -# http related config -http: - # port for http, default is 80. If https enabled, this port will redirect to https port - port: 80 - -# The initial password of Harbor admin -# It only works in first time to install harbor -# Remember Change the admin password from UI after launching Harbor. -harbor_admin_password: Harbor12345 - -# Harbor DB configuration -database: - # The password for the root user of Harbor DB. Change this before any production use. - password: root123 - # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. - max_idle_conns: 100 - # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. - # Note: the default number of connections is 1024 for postgres of harbor. - max_open_conns: 900 - -# The default data volume -data_volume: /data - -# Trivy configuration -# -# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. -# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached -# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it -# should download a newer version from the Internet or use the cached one. Currently, the database is updated every -# 12 hours and published as a new release to GitHub. -trivy: - # ignoreUnfixed The flag to display only fixed vulnerabilities - ignore_unfixed: true - # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub - # - # You might want to enable this flag in test or CI/CD environments to avoid GitHub rate limiting issues. - # If the flag is enabled you have to download the `trivy-offline.tar.gz` archive manually, extract `trivy.db` and - # `metadata.json` files and mount them in the `/home/scanner/.cache/trivy/db` path. - skip_update: false - # - # The offline_scan option prevents Trivy from sending API requests to identify dependencies. - # Scanning JAR files and pom.xml may require Internet access for better detection, but this option tries to avoid it. - # For example, the offline mode will not try to resolve transitive dependencies in pom.xml when the dependency doesn't - # exist in the local repositories. It means a number of detected vulnerabilities might be fewer in offline mode. - # It would work if all the dependencies are in local. - # This option doesn’t affect DB download. You need to specify "skip-update" as well as "offline-scan" in an air-gapped environment. - offline_scan: false - # - # insecure The flag to skip verifying registry certificate - insecure: false - # github_token The GitHub access token to download Trivy DB - # - # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough - # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000 - # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult - # https://developer.github.com/v3/#rate-limiting - # - # You can create a GitHub token by following the instructions in - # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line - # - # github_token: xxx - -jobservice: - # Maximum number of job workers in job service - max_job_workers: 10 - -notification: - # Maximum retry count for webhook job - webhook_job_max_retry: 10 - -chart: - # Change the value of absolute_url to enabled can enable absolute url in chart - absolute_url: disabled - -# Log configurations -log: - # options are debug, info, warning, error, fatal - level: info - # configs for logs in local storage - local: - # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. - rotate_count: 50 - # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. - # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G - # are all valid. - rotate_size: 200M - # The directory on your host that store log - location: /var/log/harbor - -# This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 2.5.0 - -# Global proxy -# Config http proxy for components, e.g. http://my.proxy.com:3128 -# Components doesn't need to connect to each others via http proxy. -# Remove component from `components` array if want disable proxy -# for it. If you want use proxy for replication, MUST enable proxy -# for core and jobservice, and set `http_proxy` and `https_proxy`. -# Add domain to the `no_proxy` field, when you want disable proxy -# for some special registry. -proxy: - http_proxy: - https_proxy: - no_proxy: - components: - - core - - jobservice - - trivy diff --git a/vagrant/install-docker.sh b/vagrant/install-docker.sh deleted file mode 100755 index 14f423ce..00000000 --- a/vagrant/install-docker.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash - -# This script installs Docker and Docker Compose on Ubuntu according to the official Docker documentation on -# https://docs.docker.com/engine/install/ubuntu/ and https://docs.docker.com/compose/install/. - -# To list the available versions in the repo: -# apt-cache madison containerd.io docker-ce -CONTAINERD_VERSION="1.4.9-1" -DOCKER_VERSION="5:20.10.8~3-0~ubuntu-focal" -DOCKER_COMPOSE_VERSION="1.29.2" - -sudo apt-get update -sudo apt-get install --yes apt-transport-https ca-certificates curl gnupg lsb-release - -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - -echo \ - "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - -sudo apt-get update -sudo apt-get install --yes containerd.io=$CONTAINERD_VERSION docker-ce=$DOCKER_VERSION docker-ce-cli=$DOCKER_VERSION - -# Download the current stable release of Docker Compose: -sudo curl -L "https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - -# Apply executable permissions to the binary: -sudo chmod +x /usr/local/bin/docker-compose - -# Add vagrant user to the docker group: -sudo usermod -aG docker vagrant - -# Allow Trivy scanner adapter to resolve api.github.com. -echo '{"dns": ["192.168.1.1", "8.8.8.8"]}' > /etc/docker/daemon.json -service docker restart diff --git a/vagrant/install-go.sh b/vagrant/install-go.sh deleted file mode 100755 index 17df2a28..00000000 --- a/vagrant/install-go.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash - -wget --quiet https://golang.org/dl/go1.17.2.linux-amd64.tar.gz -tar -C /usr/local -xzf go1.17.2.linux-amd64.tar.gz -rm go1.17.2.linux-amd64.tar.gz -echo 'export PATH=$PATH:/usr/local/go/bin' >> /home/vagrant/.profile - -sudo apt-get update -sudo apt-get install --yes build-essential diff --git a/vagrant/install-harbor.sh b/vagrant/install-harbor.sh deleted file mode 100755 index d3ad888b..00000000 --- a/vagrant/install-harbor.sh +++ /dev/null @@ -1,36 +0,0 @@ -#! /bin/bash - -HARBOR_VERSION="v2.5.1" - -# Keep in sync with vagrant/harbor.yml. -HARBOR_HOSTNAME="harbor.dev.io" -HARBOR_USERNAME="admin" -HARBOR_PASSWORD="Harbor12345" - -# Download the Harbor installer. -# The online installer downloads the Harbor images from DockerHub. For this reason, the installer is very small in size. -wget https://github.com/goharbor/harbor/releases/download/$HARBOR_VERSION/harbor-online-installer-$HARBOR_VERSION.tgz - -# Download the corresponding *.asc file to verify that the package is genuine: -wget https://github.com/goharbor/harbor/releases/download/$HARBOR_VERSION/harbor-online-installer-$HARBOR_VERSION.tgz.asc - -# Obtain the public key for the *.asc file: -gpg --keyserver hkps://keyserver.ubuntu.com --receive-keys 644FF454C0B4115C - -# Verify that the package is genuine: -gpg --verbose --keyserver hkps://keyserver.ubuntu.com --verify harbor-online-installer-$HARBOR_VERSION.tgz.asc - -tar -C /opt -xzf harbor-online-installer-$HARBOR_VERSION.tgz -rm harbor-online-installer-$HARBOR_VERSION.tgz -rm harbor-online-installer-$HARBOR_VERSION.tgz.asc - -rm /opt/harbor/harbor.yml.tmpl -cp /vagrant/vagrant/harbor.yml /opt/harbor/harbor.yml - -cat << EOF >> /etc/hosts -127.0.0.1 $HARBOR_HOSTNAME -EOF - -cd /opt/harbor - -./install.sh --with-trivy diff --git a/vagrant/push-images.sh b/vagrant/push-images.sh deleted file mode 100755 index 60dc7574..00000000 --- a/vagrant/push-images.sh +++ /dev/null @@ -1,15 +0,0 @@ -#! /bin/bash - -# Keep in sync with vagrant/harbor.yml. -HARBOR_HOSTNAME="harbor.dev.io" -HARBOR_USERNAME="admin" -HARBOR_PASSWORD="Harbor12345" - -echo "$HARBOR_PASSWORD" | docker login --username=$HARBOR_USERNAME --password-stdin $HARBOR_HOSTNAME - -for image in "alpine:3.14" "photon:3.0-20200202" "gcr.io/distroless/java:11" "debian:9" "amazonlinux:2.0.20200406.0" -do - docker image pull $image - docker image tag $image "$HARBOR_HOSTNAME/library/$image" - docker image push "$HARBOR_HOSTNAME/library/$image" -done