vote extensions (#242)

* Support vote extensions

* Update single signer test

* Lint

* add additional checks

* Fix leader election test

* fix e2e

* lint

* use grpc timeout in nonce overallocation

* lint

* Bump horcrux-proxy

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Jille/raft-grpc-transport v1.4.0
 	github.com/Jille/raftadmin v1.2.1
 	github.com/armon/go-metrics v0.4.1
-	github.com/cometbft/cometbft v0.38.0
+	github.com/cometbft/cometbft v0.38.2
 	github.com/cosmos/cosmos-sdk v0.50.1
 	github.com/cosmos/gogoproto v1.4.11
 	github.com/ethereum/go-ethereum v1.13.5

go.sum

@@ -132,8 +132,8 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
-github.com/cometbft/cometbft v0.38.0 h1:ogKnpiPX7gxCvqTEF4ly25/wAxUqf181t30P3vqdpdc=
-github.com/cometbft/cometbft v0.38.0/go.mod h1:5Jz0Z8YsHSf0ZaAqGvi/ifioSdVFPtEGrm8Y9T/993k=
+github.com/cometbft/cometbft v0.38.2 h1:io0JCh5EPxINKN5ZMI5hCdpW3QVZRy+o8qWe3mlJa/8=
+github.com/cometbft/cometbft v0.38.2/go.mod h1:PIi48BpzwlHqtV3mzwPyQgOyOnU94BNBimLS2ebBHOg=
 github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo=
 github.com/cometbft/cometbft-db v0.7.0/go.mod h1:yiKJIm2WKrt6x8Cyxtq9YTEcIMPcEe4XPxhgX59Fzf0=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

proto/strangelove/horcrux/cosigner.proto

@@ -17,7 +17,8 @@ message Block {
 	int64 round = 2;
 	int32 step = 3;
 	bytes signBytes = 4;
-	int64 timestamp = 5;
+	bytes voteExtSignBytes = 5;
+	int64 timestamp = 6;
 }
 
 message SignBlockRequest {
@@ -27,7 +28,8 @@ message SignBlockRequest {
 
 message SignBlockResponse {
 	bytes signature = 1;
-	int64 timestamp = 2;
+	bytes vote_ext_signature = 2;
+	int64 timestamp = 3;
 }
 
 message Nonce {
@@ -55,13 +57,18 @@ message SetNoncesAndSignRequest {
 	repeated Nonce nonces = 2;
 	HRST hrst = 3;
 	bytes signBytes = 4;
-	string chainID = 5;
+	bytes voteExtUuid = 5;
+	repeated Nonce voteExtNonces = 6;
+	bytes voteExtSignBytes = 7;
+	string chainID = 8;
 }
 
 message SetNoncesAndSignResponse {
-	bytes noncePublic = 1;
-	int64 timestamp = 2;
+	int64 timestamp = 1;
+	bytes noncePublic = 2;
 	bytes signature = 3;
+	bytes voteExtNoncePublic = 4;
+	bytes voteExtSignature = 5;
 }
 
 message GetNoncesRequest {

scripts/protocgen.sh

@@ -11,5 +11,5 @@ for dir in $proto_dirs; do
   done
 done
 
-cp -r github.com/strangelove-ventures/horcrux/signer ./
+cp -r github.com/strangelove-ventures/horcrux/v3/signer ./
 rm -rf github.com

signer/cosigner.go

@@ -2,9 +2,13 @@ package signer
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"time"
 
 	cometcrypto "github.com/cometbft/cometbft/crypto"
+	"github.com/cometbft/cometbft/libs/protoio"
+	cometproto "github.com/cometbft/cometbft/proto/tendermint/types"
 	"github.com/google/uuid"
 	"github.com/strangelove-ventures/horcrux/v3/signer/proto"
 )
@@ -45,15 +49,19 @@ func (cosigners Cosigners) GetByID(id int) Cosigner {
 // CosignerSignRequest is sent to a co-signer to obtain their signature for the SignBytes
 // The SignBytes should be a serialized block
 type CosignerSignRequest struct {
-	ChainID   string
-	SignBytes []byte
-	UUID      uuid.UUID
+	ChainID                string
+	SignBytes              []byte
+	UUID                   uuid.UUID
+	VoteExtensionSignBytes []byte
+	VoteExtUUID            uuid.UUID
 }
 
 type CosignerSignResponse struct {
-	NoncePublic []byte
-	Timestamp   time.Time
-	Signature   []byte
+	Timestamp                time.Time
+	NoncePublic              []byte
+	Signature                []byte
+	VoteExtensionNoncePublic []byte
+	VoteExtensionSignature   []byte
 }
 
 type CosignerNonce struct {
@@ -107,7 +115,8 @@ type CosignerSignBlockRequest struct {
 }
 
 type CosignerSignBlockResponse struct {
-	Signature []byte
+	Signature              []byte
+	VoteExtensionSignature []byte
 }
 type CosignerUUIDNonces struct {
 	UUID   uuid.UUID
@@ -138,8 +147,60 @@ func (n CosignerUUIDNoncesMultiple) toProto() []*proto.UUIDNonce {
 }
 
 type CosignerSetNoncesAndSignRequest struct {
-	ChainID   string
+	ChainID string
+	HRST    HRSTKey
+
 	Nonces    *CosignerUUIDNonces
-	HRST      HRSTKey
 	SignBytes []byte
+
+	VoteExtensionNonces    *CosignerUUIDNonces
+	VoteExtensionSignBytes []byte
+}
+
+func verifySignPayload(chainID string, signBytes, voteExtensionSignBytes []byte) (HRSTKey, bool, error) {
+	var vote cometproto.CanonicalVote
+	voteErr := protoio.UnmarshalDelimited(signBytes, &vote)
+	if voteErr == nil && (vote.Type == cometproto.PrevoteType || vote.Type == cometproto.PrecommitType) {
+		hrstKey := HRSTKey{
+			Height:    vote.Height,
+			Round:     vote.Round,
+			Step:      CanonicalVoteToStep(&vote),
+			Timestamp: vote.Timestamp.UnixNano(),
+		}
+
+		if hrstKey.Step == stepPrecommit && len(voteExtensionSignBytes) > 0 && vote.BlockID != nil {
+			var voteExt cometproto.CanonicalVoteExtension
+			if err := protoio.UnmarshalDelimited(voteExtensionSignBytes, &voteExt); err != nil {
+				return hrstKey, false, fmt.Errorf("failed to unmarshal vote extension: %w", err)
+			}
+			if voteExt.ChainId != chainID {
+				return hrstKey, false, fmt.Errorf("vote extension chain ID %s does not match chain ID %s", voteExt.ChainId, chainID)
+			}
+			if voteExt.Height != hrstKey.Height {
+				return hrstKey, false,
+					fmt.Errorf("vote extension height %d does not match block height %d", voteExt.Height, hrstKey.Height)
+			}
+			if voteExt.Round != hrstKey.Round {
+				return hrstKey, false,
+					fmt.Errorf("vote extension round %d does not match block round %d", voteExt.Round, hrstKey.Round)
+			}
+			return hrstKey, true, nil
+		}
+
+		return hrstKey, false, nil
+	}
+
+	var proposal cometproto.CanonicalProposal
+	proposalErr := protoio.UnmarshalDelimited(signBytes, &proposal)
+	if proposalErr == nil {
+		return HRSTKey{
+			Height:    proposal.Height,
+			Round:     proposal.Round,
+			Step:      stepPropose,
+			Timestamp: proposal.Timestamp.UnixNano(),
+		}, false, nil
+	}
+
+	return HRSTKey{}, false,
+		fmt.Errorf("failed to unmarshal sign bytes into vote or proposal: %w", errors.Join(voteErr, proposalErr))
 }

signer/cosigner_grpc_server.go

@@ -34,28 +34,41 @@ func (rpc *CosignerGRPCServer) SignBlock(
 	ctx context.Context,
 	req *proto.SignBlockRequest,
 ) (*proto.SignBlockResponse, error) {
-	res, _, err := rpc.thresholdValidator.Sign(ctx, req.ChainID, BlockFromProto(req.Block))
+	sig, voteExtSig, _, err := rpc.thresholdValidator.Sign(ctx, req.ChainID, BlockFromProto(req.Block))
 	if err != nil {
 		return nil, err
 	}
 	return &proto.SignBlockResponse{
-		Signature: res,
+		Signature:        sig,
+		VoteExtSignature: voteExtSig,
 	}, nil
 }
 
 func (rpc *CosignerGRPCServer) SetNoncesAndSign(
 	ctx context.Context,
 	req *proto.SetNoncesAndSignRequest,
 ) (*proto.SetNoncesAndSignResponse, error) {
-	res, err := rpc.cosigner.SetNoncesAndSign(ctx, CosignerSetNoncesAndSignRequest{
+	cosignerReq := CosignerSetNoncesAndSignRequest{
 		ChainID: req.ChainID,
+
+		HRST: HRSTKeyFromProto(req.Hrst),
+
 		Nonces: &CosignerUUIDNonces{
 			UUID:   uuid.UUID(req.Uuid),
-			Nonces: CosignerNoncesFromProto(req.GetNonces()),
+			Nonces: CosignerNoncesFromProto(req.Nonces),
 		},
-		HRST:      HRSTKeyFromProto(req.GetHrst()),
-		SignBytes: req.GetSignBytes(),
-	})
+		SignBytes: req.SignBytes,
+	}
+
+	if len(req.VoteExtSignBytes) > 0 && len(req.VoteExtUuid) == 16 {
+		cosignerReq.VoteExtensionNonces = &CosignerUUIDNonces{
+			UUID:   uuid.UUID(req.VoteExtUuid),
+			Nonces: CosignerNoncesFromProto(req.VoteExtNonces),
+		}
+		cosignerReq.VoteExtensionSignBytes = req.VoteExtSignBytes
+	}
+
+	res, err := rpc.cosigner.SetNoncesAndSign(ctx, cosignerReq)
 	if err != nil {
 		rpc.raftStore.logger.Error(
 			"Failed to sign with shard",
@@ -75,9 +88,11 @@ func (rpc *CosignerGRPCServer) SetNoncesAndSign(
 		"step", req.Hrst.Step,
 	)
 	return &proto.SetNoncesAndSignResponse{
-		NoncePublic: res.NoncePublic,
-		Timestamp:   res.Timestamp.UnixNano(),
-		Signature:   res.Signature,
+		NoncePublic:        res.NoncePublic,
+		Timestamp:          res.Timestamp.UnixNano(),
+		Signature:          res.Signature,
+		VoteExtNoncePublic: res.VoteExtensionNoncePublic,
+		VoteExtSignature:   res.VoteExtensionSignature,
 	}, nil
 }
 

signer/cosigner_nonce_cache.go

@@ -15,6 +15,7 @@ const (
 	defaultGetNoncesInterval = 3 * time.Second
 	defaultGetNoncesTimeout  = 4 * time.Second
 	defaultNonceExpiration   = 10 * time.Second // half of the local cosigner cache expiration
+	nonceOverallocation      = 1.5
 )
 
 type CosignerNonceCache struct {
@@ -198,7 +199,8 @@ func (cnc *CosignerNonceCache) getUuids(n int) []uuid.UUID {
 }
 
 func (cnc *CosignerNonceCache) target(noncesPerMinute float64) int {
-	t := int((noncesPerMinute / 60) * ((cnc.getNoncesInterval.Seconds() * 1.2) + 0.5))
+	t := int((noncesPerMinute / 60) *
+		((cnc.getNoncesInterval.Seconds() * nonceOverallocation) + cnc.getNoncesTimeout.Seconds()))
 	if t <= 0 {
 		return 1 // always target at least one nonce ready
 	}
@@ -332,20 +334,20 @@ func (cnc *CosignerNonceCache) Start(ctx context.Context) {
 	cnc.lastReconcileNonces.Store(uint64(cnc.cache.Size()))
 	cnc.lastReconcileTime = time.Now()
 
-	ticker := time.NewTimer(cnc.getNoncesInterval)
+	timer := time.NewTimer(cnc.getNoncesInterval)
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
+		case <-timer.C:
 		case <-cnc.empty:
 			// clear out channel
 			for len(cnc.empty) > 0 {
 				<-cnc.empty
 			}
 		}
 		cnc.reconcile(ctx)
-		ticker.Reset(cnc.getNoncesInterval)
+		timer.Reset(cnc.getNoncesInterval)
 	}
 }
 

signer/file.go

@@ -200,14 +200,33 @@ func (pv *FilePV) GetPubKey() (crypto.PubKey, error) {
 	return pv.Key.PubKey, nil
 }
 
-func (pv *FilePV) Sign(block Block) ([]byte, time.Time, error) {
-	height, round, step, signBytes := block.Height, int32(block.Round), block.Step, block.SignBytes
+func (pv *FilePV) Sign(chainID string, block Block) ([]byte, []byte, time.Time, error) {
+	height, round, step := block.Height, int32(block.Round), block.Step
+	signBytes, voteExtensionSignBytes := block.SignBytes, block.VoteExtensionSignBytes
 
 	lss := pv.LastSignState
 
 	sameHRS, err := lss.CheckHRS(height, round, step)
 	if err != nil {
-		return nil, block.Timestamp, err
+		return nil, nil, block.Timestamp, err
+	}
+
+	_, hasVoteExtensions, err := verifySignPayload(chainID, signBytes, voteExtensionSignBytes)
+	if err != nil {
+		return nil, nil, block.Timestamp, err
+	}
+
+	// Vote extensions are non-deterministic, so it is possible that an
+	// application may have created a different extension. We therefore always
+	// re-sign the vote extensions of precommits. For prevotes and nil
+	// precommits, the extension signature will always be empty.
+	// Even if the signed over data is empty, we still add the signature
+	var extSig []byte
+	if hasVoteExtensions {
+		extSig, err = pv.Key.PrivKey.Sign(voteExtensionSignBytes)
+		if err != nil {
+			return nil, nil, block.Timestamp, err
+		}
 	}
 
 	// We might crash before writing to the wal,
@@ -218,28 +237,28 @@ func (pv *FilePV) Sign(block Block) ([]byte, time.Time, error) {
 	if sameHRS {
 		switch {
 		case bytes.Equal(signBytes, lss.SignBytes):
-			return lss.Signature, block.Timestamp, nil
+			return lss.Signature, nil, block.Timestamp, nil
 		case block.Step == stepPropose:
 			if timestamp, ok := checkProposalsOnlyDifferByTimestamp(lss.SignBytes, signBytes); ok {
-				return lss.Signature, timestamp, nil
+				return lss.Signature, nil, timestamp, nil
 			}
 		case block.Step == stepPrevote || block.Step == stepPrecommit:
 			if timestamp, ok := checkVotesOnlyDifferByTimestamp(lss.SignBytes, signBytes); ok {
-				return lss.Signature, timestamp, nil
+				return lss.Signature, extSig, timestamp, nil
 			}
 		}
 
-		return nil, block.Timestamp, fmt.Errorf("conflicting data")
+		return nil, extSig, block.Timestamp, fmt.Errorf("conflicting data")
 	}
 
 	// It passed the checks. Sign the vote
 	sig, err := pv.Key.PrivKey.Sign(signBytes)
 	if err != nil {
-		return nil, block.Timestamp, err
+		return nil, nil, block.Timestamp, err
 	}
 	pv.saveSigned(height, round, step, signBytes, sig)
 
-	return sig, block.Timestamp, nil
+	return sig, extSig, block.Timestamp, nil
 }
 
 // Save persists the FilePV to disk.

signer/io.go

@@ -0,0 +1,23 @@
+package signer
+
+import (
+	"io"
+
+	"github.com/cometbft/cometbft/libs/protoio"
+	cometprotoprivval "github.com/cometbft/cometbft/proto/tendermint/privval"
+)
+
+// ReadMsg reads a message from an io.Reader
+func ReadMsg(reader io.Reader) (msg cometprotoprivval.Message, err error) {
+	const maxRemoteSignerMsgSize = 1024 * 10
+	protoReader := protoio.NewDelimitedReader(reader, maxRemoteSignerMsgSize)
+	_, err = protoReader.ReadMsg(&msg)
+	return msg, err
+}
+
+// WriteMsg writes a message to an io.Writer
+func WriteMsg(writer io.Writer, msg cometprotoprivval.Message) (err error) {
+	protoWriter := protoio.NewDelimitedWriter(writer)
+	_, err = protoWriter.WriteMsg(&msg)
+	return err
+}