Add sharding e2e test to Github Actions

Signed-off-by: Priya Wadhwa <priya@chainguard.dev>

.github/workflows/main.yml

@@ -88,3 +88,31 @@ jobs:
         with:
           name: Docker Compose logs
           path: /tmp/docker-compose.log
+
+  sharding-e2e:
+    runs-on: ubuntu-20.04
+    needs: build
+
+    steps:
+      - name: download minisign
+        run: sudo add-apt-repository ppa:dysfunctionalprogramming/minisign && sudo apt-get update && sudo apt-get install minisign
+      - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # v3
+      - name: Docker Build
+        run: docker-compose build
+      - name: Extract version of Go to use
+        run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV
+      - uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v3.0.0
+        with:
+          go-version: ${{ env.GOVERSION }}
+      - name: Install jq
+        run: sudo apt-get update && sudo apt-get install jq
+      - name: Install trillian createtree
+        run: go install github.com/google/trillian/cmd/createtree@latest
+      - name: CLI
+        run: ./tests/sharding-e2e-test.sh
+      - name: Upload logs if they exist
+        uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 # v3
+        if: failure()
+        with:
+          name: Docker Compose logs
+          path: /tmp/docker-compose.log

.gitignore

@@ -11,3 +11,5 @@ swagger
 dist/*
 hack/tools/bin/*
 *fuzz.zip
+docker-compose-sharding.yaml
+

cmd/rekor-cli/app/log_info.go

@@ -43,7 +43,7 @@ type logInfoCmdOutput struct {
 	TreeSize       int64
 	RootHash       string
 	TimestampNanos uint64
-	TreeID         int64
+	TreeID         string
 }
 
 func (l *logInfoCmdOutput) String() string {
@@ -53,7 +53,7 @@ func (l *logInfoCmdOutput) String() string {
 Tree Size: %v
 Root Hash: %s
 Timestamp: %s
-TreeID:    %v
+TreeID:    %s
 `, l.TreeSize, l.RootHash, ts, l.TreeID)
 }
 

openapi.yaml

@@ -582,8 +582,9 @@ definitions:
         format: signedCheckpoint
         description: The current signed tree head
       treeID:
-        type: integer
+        type: string
         description: The current treeID
+        pattern: '^[0-9]+$'
     required:
       - rootHash
       - treeSize

pkg/api/tlog.go

@@ -76,12 +76,16 @@ func GetLogInfoHandler(params tlog.GetLogInfoParams) middleware.Responder {
 		RootHash:       &hashString,
 		TreeSize:       &treeSize,
 		SignedTreeHead: &scString,
-		TreeID:         &tc.logID,
+		TreeID:         stringPointer(fmt.Sprintf("%d", tc.logID)),
 	}
 
 	return tlog.NewGetLogInfoOK().WithPayload(&logInfo)
 }
 
+func stringPointer(s string) *string {
+	return &s
+}
+
 // GetLogProofHandler returns information required to compute a consistency proof between two snapshots of log
 func GetLogProofHandler(params tlog.GetLogProofParams) middleware.Responder {
 	if *params.FirstSize > params.LastSize {

pkg/api/trillian_client.go

@@ -24,6 +24,7 @@ import (
 	"github.com/google/trillian/merkle/logverifier"
 	"github.com/google/trillian/merkle/rfc6962"
 	"github.com/pkg/errors"
+	"github.com/sigstore/rekor/pkg/log"
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -324,6 +325,7 @@ func createAndInitTree(ctx context.Context, adminClient trillian.TrillianAdminCl
 
 	for _, t := range trees.Tree {
 		if t.TreeType == trillian.TreeType_LOG {
+			log.Logger.Infof("Found existing tree with ID: %v", t.TreeId)
 			return t, nil
 		}
 	}

tests/sharding-e2e-test.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+#
+# Copyright 2021 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Things to install first:
+# - jq, createtree
+
+# Spin up services as usual
+
+echo "starting services"
+docker-compose up -d
+rm ~/.rekor/state.json || true
+
+echo "building CLI and server"
+go build -o rekor-cli ./cmd/rekor-cli
+go build -o rekor-server ./cmd/rekor-server
+
+count=0
+
+echo -n "waiting up to 60 sec for system to start"
+until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
+do
+    if [ $count -eq 6 ]; then
+       echo "! timeout reached"
+       exit 1
+    else
+       echo -n "."
+       sleep 10
+       let 'count+=1'
+    fi
+done
+
+echo
+
+# rekor-cli loginfo should work
+rekor-cli loginfo --rekor_server http://localhost:3000 --store_tree_state=false
+CURRENT_TREE_ID=$(rekor-cli loginfo --rekor_server http://localhost:3000  --format json --store_tree_state=false | jq -r .TreeID)
+echo "current Tree ID is $CURRENT_TREE_ID"
+
+
+# Add some things to the tlog :)
+cd tests
+rekor-cli upload --artifact test_file.txt --signature test_file.sig --public-key test_public_key.key --rekor_server http://localhost:3000
+cd sharding-testdata
+rekor-cli upload --artifact file1 --signature file1.sig --pki-format=x509 --public-key=ec_public.pem --rekor_server http://localhost:3000
+rekor-cli upload --artifact file2 --signature file2.sig --pki-format=x509 --public-key=ec_public.pem --rekor_server http://localhost:3000
+cd ../..
+
+# Make sure we have three entries in the log
+rekor-cli get --log-index 2 --rekor_server http://localhost:3000
+
+# Now, we want to shard the log.
+# Create a new tree
+echo "creating a new Tree ID...."
+SHARD_TREE_ID=$(createtree --admin_server localhost:8090)
+echo "the new shard ID is $SHARD_TREE_ID"
+
+# Once more
+rekor-cli loginfo --rekor_server http://localhost:3000 --store_tree_state=false
+
+# Spin down the rekor server
+echo "stopping the rekor server..."
+docker stop rekor-rekor-server-1
+
+
+# Now we want to spin up the Rekor server again, but this time point
+# to the new tree
+
+COMPOSE_FILE=docker-compose-sharding.yaml
+cat << EOF > $COMPOSE_FILE
+version: '3.4'
+services:
+  rekor-server:
+    build:
+      context: .
+      target: "deploy"
+    command: [
+      "rekor-server",
+      "serve",
+      "--trillian_log_server.address=trillian-log-server",
+      "--trillian_log_server.port=8090",
+      "--redis_server.address=redis-server",
+      "--redis_server.port=6379",
+      "--rekor_server.address=0.0.0.0",
+      "--rekor_server.signer=memory",
+      "--enable_attestation_storage",
+      "--attestation_storage_bucket=file:///var/run/attestations",
+      "--trillian_log_server.tlog_id=$SHARD_TREE_ID",
+      "--trillian_log_server.log_id_ranges=$CURRENT_TREE_ID=3,$SHARD_TREE_ID"
+      # Uncomment this for production logging
+      # "--log_type=prod",
+      ]
+    volumes:
+    - "/var/run/attestations:/var/run/attestations:z"
+    restart: always # keep the server running
+    ports:
+      - "3000:3000"
+      - "2112:2112"
+    depends_on:
+      - mysql
+      - redis-server
+      - trillian-log-server
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+      start_period: 5s
+EOF
+
+# Spin up the new Rekor
+
+docker-compose -f $COMPOSE_FILE up  -d
+sleep 15
+# TODO: priyawadhwa@ remove --store_tree_state=false once rekor-cli loginfo is aware of shards
+rekor-cli loginfo --rekor_server http://localhost:3000 --store_tree_state=false
+
+# Make sure we are pointing to the new tree now
+TREE_ID=$(rekor-cli loginfo --rekor_server http://localhost:3000  --format json --store_tree_state=false)
+# Check that the SHARD_TREE_ID is a substring of the `rekor-cli loginfo` output
+if [[ "$TREE_ID" == *"$SHARD_TREE_ID"* ]]; then
+  echo "Rekor server is now pointing to the new shard"
+else
+  echo "Rekor server is not pointing to the new shard"
+  exit 1
+fi
+
+# Now, if we run rekor-cli get --log_index 2 again, it should grab the log index
+# from Shard 0
+rekor-cli get --log-index 2 --rekor_server http://localhost:3000
+
+# TODO: Try to get the entry via Entry ID (Tree ID in hex + UUID)
+UUID=$(rekor-cli get --log-index 2 --rekor_server http://localhost:3000 --format json | jq -r .UUID)
+
+echo "Test passed successfully :)"

tests/sharding-testdata/ec_public.pem

@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEMOcTfRBS9jiXM81FZ8gm/1+omeMw
+mn/347/556g/lriS72uMhY9LcT+5UJ6fGBglr5Z8L0JNSuasyed9OtaRvw==
+-----END PUBLIC KEY-----

tests/sharding-testdata/file1

@@ -0,0 +1 @@
+file1

tests/sharding-testdata/file2

@@ -0,0 +1,2 @@
+file2
+

tests/sharding-testdata/file2.sig

@@ -0,0 +1,2 @@
+0D F9βΤ/;Α»O19 Τα!|+˜Ι­‹*©bό)0§
ΩΣ3 7Κ)Ψt©”!Γ•"
+Ρ·ϋ3I7}ίβΣFǟρµΠΡΛ2