From 58e010348ff709bf162a8d748ea41e05fa903cec Mon Sep 17 00:00:00 2001 From: sweexordious Date: Fri, 24 May 2024 13:09:43 +0400 Subject: [PATCH 1/3] feat: initial draft of inner proofs --- nmt.go | 103 ++++++++++++++++++++++++++++++++++++++--- nmt_test.go | 2 +- proof.go | 126 +++++++++++++++++++++++++++++++++++++++++++++++++- proof_test.go | 14 ++++-- 4 files changed, 233 insertions(+), 12 deletions(-) diff --git a/nmt.go b/nmt.go index e9c318a..f381fb3 100644 --- a/nmt.go +++ b/nmt.go @@ -190,13 +190,86 @@ func (n *NamespacedMerkleTree) ProveRange(start, end int) (Proof, error) { if err := n.validateRange(start, end); err != nil { return NewEmptyRangeProof(isMaxNsIgnored), err } - proof, err := n.buildRangeProof(start, end) + proof, _, err := n.buildRangeProof(start, end) if err != nil { return Proof{}, err } return NewInclusionProof(start, end, proof, isMaxNsIgnored), nil } +// Coordinate identifies a tree node using the depth and position +// +// Depth Position +// 0 0 +// / \ +// / \ +// 1 0 1 +// /\ /\ +// 2 0 1 2 3 +// /\ /\ /\ /\ +// 3 0 1 2 3 4 5 6 7 +type Coordinate struct { + // depth is the typical depth of a tree, 0 being the root + depth int + // position is the index of a node of a given depth, 0 being the left most + // node + position int +} + +// ProveInner +// TODO: range is consecutive +func (n *NamespacedMerkleTree) ProveInner(coordinates []Coordinate) (InnerProof, error) { + isMaxNsIgnored := n.treeHasher.IsMaxNamespaceIDIgnored() + start, end := toRange(coordinates, n.Size()) + proof, coordinates, err := n.buildRangeProof(start, end) + if err != nil { + return InnerProof{}, err + } + return NewInnerInclusionProof(proof, coordinates, n.Size(), isMaxNsIgnored), nil +} + +// toRange +// makes the range consecutive +func toRange(coordinates []Coordinate, treeSize int) (int, int) { + //if err := validateRange(coordinates, treeSize); err != nil { + // return -1, -1, err // TODO is -1 a good return? or 0? or maybe remove this from here and keep it in ProveInner? + //} + start := 0 + end := 0 + maxDepth := maxDepth(treeSize) + for _, coord := range coordinates { + currentStart := startLeafIndex(coord, maxDepth) + currentEnd := endLeafIndex(coord, maxDepth) + if currentEnd < start { + start = currentStart + } + if currentEnd > end { + end = currentEnd + } + } + return start, end +} + +func maxDepth(treeSize int) int { + return bits.Len(uint(treeSize)) - 1 +} + +func endLeafIndex(coordinate Coordinate, maxDepth int) int { + height := maxDepth - coordinate.depth + subtreeSize := 1 << height + return (coordinate.position + 1) * subtreeSize +} + +func startLeafIndex(coordinate Coordinate, maxDepth int) int { + // since the coordinates are expressed in depth. We need to calculate the height + // using ... + height := maxDepth - coordinate.depth + // In a merkle tree, the tree height grows with every number of leaves multiple of 2. + // For example, for all the trees of size 4 to 7, the RFC 6962 tree will have a height of 3. + subtreeSize := 1 << height + return coordinate.position * subtreeSize +} + // ProveNamespace returns a range proof for the given NamespaceID. // // case 1) If the namespace nID is out of the range of the tree's min and max @@ -265,7 +338,7 @@ func (n *NamespacedMerkleTree) ProveNamespace(nID namespace.ID) (Proof, error) { // the tree or calculated the range it would be in (to generate a proof of // absence and to return the corresponding leaf hashes). - proof, err := n.buildRangeProof(proofStart, proofEnd) + proof, _, err := n.buildRangeProof(proofStart, proofEnd) if err != nil { return Proof{}, err } @@ -290,13 +363,14 @@ func (n *NamespacedMerkleTree) validateRange(start, end int) error { // supplied range i.e., [proofStart, proofEnd) where proofEnd is non-inclusive. // The nodes are ordered according to in order traversal of the namespaced tree. // Any errors returned by this method are irrecoverable and indicate an illegal state of the tree (n). -func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]byte, error) { - proof := [][]byte{} // it is the list of nodes hashes (as byte slices) with no index +func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]byte, []Coordinate, error) { + var proof [][]byte // it is the list of nodes hashes (as byte slices) with no index + var coordinates []Coordinate var recurse func(start, end int, includeNode bool) ([]byte, error) // validate the range if err := n.validateRange(proofStart, proofEnd); err != nil { - return nil, err + return nil, nil, err } // start, end are indices of leaves in the tree hence they should be within @@ -318,6 +392,10 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by if (start < proofStart || start >= proofEnd) && includeNode { // add the leafHash to the proof proof = append(proof, leafHash) + coordinates = append(coordinates, Coordinate{ + depth: maxDepth(n.Size()), + position: start, + }) } // if the index of the leaf is within the queried range i.e., // [proofStart, proofEnd] OR if the leaf is not required as part of @@ -368,6 +446,7 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by // of the proof but not its left and right subtrees if includeNode && !newIncludeNode { proof = append(proof, hash) + coordinates = append(coordinates, ToCoordinate(start, end, n.Size())) } return hash, nil @@ -378,9 +457,19 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by fullTreeSize = 1 } if _, err := recurse(0, fullTreeSize, true); err != nil { - return nil, err + return nil, nil, err + } + return proof, coordinates, nil +} + +func ToCoordinate(start, end, treeSize int) Coordinate { + height := bits.Len(uint(end-start)) - 1 + maxDepth := maxDepth(treeSize) + position := start / (1 << height) + return Coordinate{ + depth: maxDepth - height, + position: position, } - return proof, nil } // Get returns leaves for the given namespace.ID. diff --git a/nmt_test.go b/nmt_test.go index 6e0565e..594d1a8 100644 --- a/nmt_test.go +++ b/nmt_test.go @@ -906,7 +906,7 @@ func Test_buildRangeProof_Err(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := tt.tree.buildRangeProof(tt.proofStart, tt.proofEnd) + _, _, err := tt.tree.buildRangeProof(tt.proofStart, tt.proofEnd) assert.Equal(t, tt.wantErr, err != nil) if tt.wantErr { assert.True(t, errors.Is(err, tt.errType)) diff --git a/proof.go b/proof.go index 998b9a8..02ea113 100644 --- a/proof.go +++ b/proof.go @@ -9,7 +9,7 @@ import ( "math/bits" "github.com/celestiaorg/nmt/namespace" - pb "github.com/celestiaorg/nmt/pb" + "github.com/celestiaorg/nmt/pb" ) var ( @@ -404,6 +404,130 @@ func (proof Proof) VerifyLeafHashes(nth *NmtHasher, verifyCompleteness bool, nID return bytes.Equal(rootHash, root), nil } +type InnerProof struct { + nodes [][]byte + coordinates []Coordinate + treeSize int + isMaxNamespaceIDIgnored bool +} + +// TODO add marshallers and protobuf definitions + +// NewInnerInclusionProof constructs a proof that proves that a set of inner is +// included in an NMT. +func NewInnerInclusionProof(proofNodes [][]byte, coordinates []Coordinate, treeSize int, ignoreMaxNamespace bool) InnerProof { + return InnerProof{ + nodes: proofNodes, + coordinates: coordinates, + treeSize: treeSize, + isMaxNamespaceIDIgnored: ignoreMaxNamespace, + } +} + +// VerifyInnerNodes +// coordinates should be in the same order as inner nodes +func (proof InnerProof) VerifyInnerNodes(nth *NmtHasher, nID namespace.ID, innerNodes [][]byte, coordinates []Coordinate, root []byte) (bool, error) { + // validate the inner proof: same number of nodes and coordinates + + // perform some consistency checks: + if nID.Size() != nth.NamespaceSize() { + return false, fmt.Errorf("namespace ID size (%d) does not match the namespace size of the NMT hasher (%d)", nID.Size(), nth.NamespaceSize()) + } + // check that the root is valid w.r.t the NMT hasher + if err := nth.ValidateNodeFormat(root); err != nil { + return false, fmt.Errorf("root does not match the NMT hasher's hash format: %w", err) + } + // check that all the proof.nodes are valid w.r.t the NMT hasher + for _, node := range proof.nodes { + if err := nth.ValidateNodeFormat(node); err != nil { + return false, fmt.Errorf("proof nodes do not match the NMT hasher's hash format: %w", err) + } + } + // check that all the leafHashes are valid w.r.t the NMT hasher + for _, leafHash := range innerNodes { + if err := nth.ValidateNodeFormat(leafHash); err != nil { + return false, fmt.Errorf("leaf hash does not match the NMT hasher's hash format: %w", err) + } + } + // validate that nID is included in the inner nodes + + _, proofEnd := toRange(coordinates, proof.treeSize) + + allNodes := append(proof.nodes, innerNodes...) + allCoordinates := append(proof.coordinates, coordinates...) + + var computeRoot func(start, end int) ([]byte, error) + // computeRoot can return error iff the HashNode function fails while calculating the root + computeRoot = func(start, end int) ([]byte, error) { + innerNode, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start, end) + if found { + return innerNode, nil + } + + // Recursively get left and right subtree + k := getSplitPoint(end - start) + left, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start, start+k) + var err error + if found { + // TODO: do we want to remove the node and coordinates from allNodes and allCoordinates? Or it's just a premature optimisation + } else { + left, err = computeRoot(start, start+k) + if err != nil { + return nil, fmt.Errorf("failed to compute subtree root [%d, %d): %w", start, start+k, err) + } + } + right, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start+k, end) + if found { + // TODO: do we want to remove the node and coordinates from allNodes and allCoordinates? Or it's just a premature optimisation + } else { + right, err = computeRoot(start+k, end) + if err != nil { + return nil, fmt.Errorf("failed to compute subtree root [%d, %d): %w", start+k, end, err) + } + } + + // only right leaf/subtree can be non-existent + if right == nil { + return left, nil + } + hash, err := nth.HashNode(left, right) + if err != nil { + return nil, fmt.Errorf("failed to hash node: %w", err) + } + return hash, nil + } + + // estimate the leaf size of the subtree containing the proof range + proofRangeSubtreeEstimate := getSplitPoint(proofEnd) * 2 + if proofRangeSubtreeEstimate < 1 { + proofRangeSubtreeEstimate = 1 + } + rootHash, err := computeRoot(0, proofRangeSubtreeEstimate) + if err != nil { + return false, fmt.Errorf("failed to compute root [%d, %d): %w", 0, proofRangeSubtreeEstimate, err) + } + for i := 0; i < len(proof.nodes); i++ { + rootHash, err = nth.HashNode(rootHash, proof.nodes[i]) + if err != nil { + return false, fmt.Errorf("failed to hash node: %w", err) + } + } + + return bytes.Equal(rootHash, root), nil +} + +// getInnerNode +// expect the number of nodes and coordinates to be the same +func getInnerNode(nodes [][]byte, coordinates []Coordinate, treeSize int, start int, end int) ([]byte, bool) { + for index, coordinate := range coordinates { + startLeaf, endLeaf := toRange([]Coordinate{coordinate}, treeSize) + if startLeaf == start && endLeaf == end { + return nodes[index], true + } + } + return nil, false +} + // VerifyInclusion checks that the inclusion proof is valid by using leaf data // and the provided proof to regenerate and compare the root. Note that the leavesWithoutNamespace data should not contain the prefixed namespace, unlike the tree.Push method, // which takes prefixed data. All leaves implicitly have the same namespace ID: diff --git a/proof_test.go b/proof_test.go index 235a403..b6e177c 100644 --- a/proof_test.go +++ b/proof_test.go @@ -122,7 +122,7 @@ func TestProof_VerifyNamespace_False(t *testing.T) { t.Fatalf("invalid test setup: error on ProveNamespace(): %v", err) } // inclusion proof of the leaf index 0 - incProof0, err := n.buildRangeProof(0, 1) + incProof0, _, err := n.buildRangeProof(0, 1) require.NoError(t, err) incompleteFirstNs := NewInclusionProof(0, 1, incProof0, false) type args struct { @@ -135,13 +135,13 @@ func TestProof_VerifyNamespace_False(t *testing.T) { // an invalid absence proof for an existing namespace ID (2) in the constructed tree leafIndex := 3 - inclusionProofOfLeafIndex, err := n.buildRangeProof(leafIndex, leafIndex+1) + inclusionProofOfLeafIndex, _, err := n.buildRangeProof(leafIndex, leafIndex+1) require.NoError(t, err) leafHash := n.leafHashes[leafIndex] // the only data item with namespace ID = 2 in the constructed tree is at index 3 invalidAbsenceProof := NewAbsenceProof(leafIndex, leafIndex+1, inclusionProofOfLeafIndex, leafHash, false) // inclusion proof of the leaf index 10 - incProof10, err := n.buildRangeProof(10, 11) + incProof10, _, err := n.buildRangeProof(10, 11) require.NoError(t, err) // root @@ -229,6 +229,14 @@ func TestProof_VerifyNamespace_False(t *testing.T) { } } +func TestInnerProofs(t *testing.T) { + n := exampleNMT(1, true, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + proof, err := n.ProveInner([]Coordinate{ + {1, 0}, {3, 4}, {4, 10}, + }) + assert.NoError(t, err) + assert.NotNil(t, proof) // this is just to stop the debugger here and see if the proof is valid +} func TestProof_MultipleLeaves(t *testing.T) { n := New(sha256.New()) ns := []byte{1, 2, 3, 4, 5, 6, 7, 8} From e3c6e35fc52e41189385aedd2b8e77acdec81159 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Sat, 25 May 2024 12:00:29 +0400 Subject: [PATCH 2/3] chore: add tests and docs --- nmt.go | 29 +++++++++++++++++-- nmt_test.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/nmt.go b/nmt.go index f381fb3..ae9e512 100644 --- a/nmt.go +++ b/nmt.go @@ -362,10 +362,12 @@ func (n *NamespacedMerkleTree) validateRange(start, end int) error { // buildRangeProof returns the nodes (as byte slices) in the range proof of the // supplied range i.e., [proofStart, proofEnd) where proofEnd is non-inclusive. // The nodes are ordered according to in order traversal of the namespaced tree. +// Also, it returns the coordinates of the nodes of the range proof in the same +// order as the nodes. These can be used for creating inner nodes proofs. // Any errors returned by this method are irrecoverable and indicate an illegal state of the tree (n). func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]byte, []Coordinate, error) { - var proof [][]byte // it is the list of nodes hashes (as byte slices) with no index - var coordinates []Coordinate + var proof [][]byte // it is the list of nodes hashes (as byte slices) with no index + var coordinates []Coordinate // the list of the proof nodes coordinates var recurse func(start, end int, includeNode bool) ([]byte, error) // validate the range @@ -462,9 +464,32 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by return proof, coordinates, nil } +// ToCoordinate takes a start leaf index, an end exclusive leaf index +// and a tree size and returns the coordinates of the node +// that covers that whole range. +// The target node can either be a leaf node if the range contains +// a single element, i.e. (end-start == 1), or an inner node. +// The coordinate calculation follows the RFC-6962 standard. +// This means that leaves get elevated in trees that have +// a size that is not a power of 2. +// Important: Expects the range to be pre-validated: +// - start >= 0 +// - end > start +// - treeSize >= end +// Note: the formula used is based on: +// - start_leaf = position * (2 ** height) +// - end_leaf = start_leaf + (2 ** height) +// with position being the index of the inner node inside the tree +// and the height being the traditional height of a tree, i.e. bottom -> top. func ToCoordinate(start, end, treeSize int) Coordinate { + // calculates the height of the smallest subtree + // that can contain the [start, end) range. + // bits.Len() - 1 is used as a fast alternative to compute + // the integer part of the result of log2(end-start). height := bits.Len(uint(end-start)) - 1 maxDepth := maxDepth(treeSize) + // 1 << height == 2 ** height. This result is based + // on the formula documented above. position := start / (1 << height) return Coordinate{ depth: maxDepth - height, diff --git a/nmt_test.go b/nmt_test.go index 594d1a8..a45bafb 100644 --- a/nmt_test.go +++ b/nmt_test.go @@ -1175,3 +1175,84 @@ func TestForcedOutOfOrderNamespacedMerkleTree(t *testing.T) { assert.NoError(t, err) } } + +func TestToCoordinate(t *testing.T) { + tests := []struct { + start, end, treeSize int + expectedCoordinate Coordinate + }{ + {start: 0, end: 1, treeSize: 1, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + {start: 0, end: 2, treeSize: 2, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + {start: 0, end: 3, treeSize: 4, expectedCoordinate: Coordinate{depth: 1, position: 0}}, + {start: 1, end: 2, treeSize: 4, expectedCoordinate: Coordinate{depth: 2, position: 1}}, + {start: 0, end: 4, treeSize: 4, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + {start: 0, end: 3, treeSize: 5, expectedCoordinate: Coordinate{depth: 1, position: 0}}, + {start: 0, end: 3, treeSize: 6, expectedCoordinate: Coordinate{depth: 1, position: 0}}, + {start: 0, end: 3, treeSize: 7, expectedCoordinate: Coordinate{depth: 1, position: 0}}, + {start: 2, end: 4, treeSize: 8, expectedCoordinate: Coordinate{depth: 2, position: 1}}, + {start: 3, end: 4, treeSize: 8, expectedCoordinate: Coordinate{depth: 3, position: 3}}, + {start: 0, end: 8, treeSize: 8, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + } + + for _, test := range tests { + result := ToCoordinate(test.start, test.end, test.treeSize) + if result != test.expectedCoordinate { + t.Errorf("ToCoordinate(%d, %d, %d) = %+v, want %+v", test.start, test.end, test.treeSize, result, test.expectedCoordinate) + } + } +} + +func TestBuildRangeProofCoordinates(t *testing.T) { + tests := []struct { + leavesNID []byte + proofStart int + proofEnd int + expected []Coordinate + expectError bool + }{ + { + leavesNID: []byte{1, 2, 3, 4, 5, 6, 7, 8}, + proofStart: 2, + proofEnd: 4, + expected: []Coordinate{{depth: 2, position: 0}, {depth: 1, position: 1}}, + expectError: false, + }, + { + leavesNID: []byte{1, 2, 3, 4, 5, 6, 7, 8}, + proofStart: 1, + proofEnd: 3, + expected: []Coordinate{{depth: 3, position: 0}, {depth: 3, position: 3}, {depth: 1, position: 1}}, + expectError: false, + }, + { + leavesNID: []byte{1, 2, 3, 4, 5, 6, 7, 8}, + proofStart: 0, + proofEnd: 8, + expected: []Coordinate{}, + expectError: false, + }, + } + + for _, test := range tests { + tree := exampleNMT(1, true, test.leavesNID...) + _, coords, err := tree.buildRangeProof(test.proofStart, test.proofEnd) + if (err != nil) != test.expectError { + t.Fatalf("expected error: %v, got: %v", test.expectError, err) + } + if !test.expectError && !equalCoordinates(coords, test.expected) { + t.Fatalf("expected: %+v, got: %+v", test.expected, coords) + } + } +} + +func equalCoordinates(a, b []Coordinate) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} From 54400416bf0df3c3673951c5e790be6136911def Mon Sep 17 00:00:00 2001 From: sweexordious Date: Wed, 29 May 2024 17:44:32 +0400 Subject: [PATCH 3/3] feat: more coordinates but with limitation of tree where the inner nodes are not consecutive --- nmt.go | 206 +++++++++++++++++++++++++++++++++++++++++----------- nmt_test.go | 33 +++++---- proof.go | 94 +++++++++++++++++------- 3 files changed, 248 insertions(+), 85 deletions(-) diff --git a/nmt.go b/nmt.go index ae9e512..a2b8c1f 100644 --- a/nmt.go +++ b/nmt.go @@ -190,7 +190,7 @@ func (n *NamespacedMerkleTree) ProveRange(start, end int) (Proof, error) { if err := n.validateRange(start, end); err != nil { return NewEmptyRangeProof(isMaxNsIgnored), err } - proof, _, err := n.buildRangeProof(start, end) + proof, _, err := n.buildRangeProof(start, end, false) if err != nil { return Proof{}, err } @@ -209,37 +209,90 @@ func (n *NamespacedMerkleTree) ProveRange(start, end int) (Proof, error) { // /\ /\ /\ /\ // 3 0 1 2 3 4 5 6 7 type Coordinate struct { - // depth is the typical depth of a tree, 0 being the root + // depth is the typical depth of a tree, 0 being the root. + // TODO test 0 case and 7 leaves all nodes depth int - // position is the index of a node of a given depth, 0 being the left most - // node + // position is the index of the node at the provided depth, 0 being the left most + // node. position int } -// ProveInner +func (coordinate Coordinate) Validate() error { + if coordinate.depth < 0 { + // TODO: test this case + return fmt.Errorf("depth cannot be negative: %d", coordinate.depth) + } + if coordinate.position < 0 { + // TODO: test this case + return fmt.Errorf("position cannot be negative: %d", coordinate.position) + } + return nil +} + +// ProveInner takes a list of inner nodes coordinates and returns the corresponding +// inner proof. +// The range used to build the proof // TODO: range is consecutive +// TODO investigate the range in drawIO +// TODO: write test for that case and see if it works func (n *NamespacedMerkleTree) ProveInner(coordinates []Coordinate) (InnerProof, error) { isMaxNsIgnored := n.treeHasher.IsMaxNamespaceIDIgnored() - start, end := toRange(coordinates, n.Size()) - proof, coordinates, err := n.buildRangeProof(start, end) + start, end, err := toRange(coordinates, n.Size()) + if err != nil { + return InnerProof{}, err + } + proof, coordinates, err := n.buildRangeProof(start, end, true) if err != nil { return InnerProof{}, err } return NewInnerInclusionProof(proof, coordinates, n.Size(), isMaxNsIgnored), nil } -// toRange -// makes the range consecutive -func toRange(coordinates []Coordinate, treeSize int) (int, int) { - //if err := validateRange(coordinates, treeSize); err != nil { - // return -1, -1, err // TODO is -1 a good return? or 0? or maybe remove this from here and keep it in ProveInner? - //} +// toRange takes a list of coordinates and a tree size, then converts +// that range to a list of leaves. +// The returned range is consecutive even if the coordinates refer to +// a disjoint range. +// For example, in an eight leaves tree: +// +// Depth Position +// 0 0 +// / \ +// / \ +// 1 0 1 +// /\ /\ +// 2 0 1 2 3 +// /\ /\ /\ /\ +// 3 0 1 2 3 4 5 6 7 +// +// If the provided coordinates are {2, 0}, which cover the range [0, 2) +// and {2, 3}, which cover the range [6, 8), the returned range will be [0, 8). +func toRange(coordinates []Coordinate, treeSize int) (int, int, error) { + for _, coordinate := range coordinates { + // TODO: test this case + if err := coordinate.Validate(); err != nil { + return 0, 0, fmt.Errorf("coordinate {%d, %d} is invalid: %w", coordinate.depth, coordinate.position, err) + } + } + if treeSize < 0 { + // TODO: test this case + return 0, 0, fmt.Errorf("tree size %d cannot be stricly negative", treeSize) + } + // TODO check the case where treeSize == 0 and multiple coordinates, and what are the possibilities. start := 0 end := 0 - maxDepth := maxDepth(treeSize) - for _, coord := range coordinates { - currentStart := startLeafIndex(coord, maxDepth) - currentEnd := endLeafIndex(coord, maxDepth) + maxDepth, err := maxDepth(treeSize) + if err != nil { + return 0, 0, err + } + for _, coordinate := range coordinates { + currentStart, err := startLeafIndex(coordinate, maxDepth) + if err != nil { + return 0, 0, err + } + currentEnd, err := endLeafIndex(coordinate, maxDepth) + if err != nil { + return 0, 0, err + } if currentEnd < start { start = currentStart } @@ -247,27 +300,64 @@ func toRange(coordinates []Coordinate, treeSize int) (int, int) { end = currentEnd } } - return start, end + return start, end, nil } -func maxDepth(treeSize int) int { - return bits.Len(uint(treeSize)) - 1 +// maxDepth returns the maximum depth of a tree with treeSize +// number of leaves. +func maxDepth(treeSize int) (int, error) { + if treeSize < 0 { + // TODO: test this case + return 0, fmt.Errorf("tree size %d cannot be stricly negative", treeSize) + } + return bits.Len(uint(treeSize)) - 1, nil } -func endLeafIndex(coordinate Coordinate, maxDepth int) int { +// endLeafIndex returns the index of range's end leaf covered by the provided +// inner node coordinates. +// The max depth is provided to know at which level to stop. +// Note: the formula used is based on: +// - end_leaf = start_leaf + (2 ** height) +// with position being the index of the inner node inside the tree +// and the height being the traditional height of a tree, i.e., bottom -> top. +func endLeafIndex(coordinate Coordinate, maxDepth int) (int, error) { + if err := coordinate.Validate(); err != nil { + return 0, err + } + if maxDepth < coordinate.depth { + return 0, fmt.Errorf("max depth %d cannot be stricly smaller than the coordinates depth %d", maxDepth, coordinate.depth) + } + // since the coordinates are expressed in depth, we need to calculate the height + // using: maxDepth = height + depth height := maxDepth - coordinate.depth + // the bit shift is used to compute 2 ** height. subtreeSize := 1 << height - return (coordinate.position + 1) * subtreeSize + return (coordinate.position + 1) * subtreeSize, nil } -func startLeafIndex(coordinate Coordinate, maxDepth int) int { - // since the coordinates are expressed in depth. We need to calculate the height - // using ... +// startLeafIndex returns the index of the range's start leaf covered by +// the provided inner node coordinates. +// The max depth is provided to know at which level to stop. +// Note: the formula used is based on: +// - start_leaf = position * (2 ** height) +// with position being the index of the inner node inside the tree +// and the height being the traditional height of a tree, i.e., bottom -> top. +func startLeafIndex(coordinate Coordinate, maxDepth int) (int, error) { + if err := coordinate.Validate(); err != nil { + // TODO: test this + return 0, err + } + if maxDepth < coordinate.depth { + // TODO: test this + return 0, fmt.Errorf("max depth %d cannot be stricly smaller than the coordinates depth %d", maxDepth, coordinate.depth) + } + // since the coordinates are expressed in depth, we need to calculate the height + // using: maxDepth = height + depth height := maxDepth - coordinate.depth - // In a merkle tree, the tree height grows with every number of leaves multiple of 2. - // For example, for all the trees of size 4 to 7, the RFC 6962 tree will have a height of 3. + // In an RFC-6962 merkle tree, the tree height increases with every multiple of 2. + // For example, for all the trees of size 4 to 7, the RFC-6962 tree will have a height of 3. subtreeSize := 1 << height - return coordinate.position * subtreeSize + return coordinate.position * subtreeSize, nil } // ProveNamespace returns a range proof for the given NamespaceID. @@ -338,7 +428,7 @@ func (n *NamespacedMerkleTree) ProveNamespace(nID namespace.ID) (Proof, error) { // the tree or calculated the range it would be in (to generate a proof of // absence and to return the corresponding leaf hashes). - proof, _, err := n.buildRangeProof(proofStart, proofEnd) + proof, _, err := n.buildRangeProof(proofStart, proofEnd, false) if err != nil { return Proof{}, err } @@ -362,12 +452,15 @@ func (n *NamespacedMerkleTree) validateRange(start, end int) error { // buildRangeProof returns the nodes (as byte slices) in the range proof of the // supplied range i.e., [proofStart, proofEnd) where proofEnd is non-inclusive. // The nodes are ordered according to in order traversal of the namespaced tree. -// Also, it returns the coordinates of the nodes of the range proof in the same -// order as the nodes. These can be used for creating inner nodes proofs. +// If the saveInnerNodesCoordinates flag is set to true, the method also returns +// the coordinates of the range proof's nodes in the same +// order. These can be used for creating inner nodes proofs. // Any errors returned by this method are irrecoverable and indicate an illegal state of the tree (n). -func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]byte, []Coordinate, error) { - var proof [][]byte // it is the list of nodes hashes (as byte slices) with no index - var coordinates []Coordinate // the list of the proof nodes coordinates +func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int, saveProofNodesCoordinates bool) ([][]byte, []Coordinate, error) { + var proof [][]byte // it is the list of nodes hashes (as byte slices) with no index + // the list of the proof nodes coordinates. + // gets populated if the saveProofNodesCoordinates flag is set + var coordinates []Coordinate var recurse func(start, end int, includeNode bool) ([]byte, error) // validate the range @@ -394,10 +487,16 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by if (start < proofStart || start >= proofEnd) && includeNode { // add the leafHash to the proof proof = append(proof, leafHash) - coordinates = append(coordinates, Coordinate{ - depth: maxDepth(n.Size()), - position: start, - }) + if saveProofNodesCoordinates { + maxDepth, err := maxDepth(n.Size()) + if err != nil { + return nil, err + } + coordinates = append(coordinates, Coordinate{ + depth: maxDepth, + position: start, + }) + } } // if the index of the leaf is within the queried range i.e., // [proofStart, proofEnd] OR if the leaf is not required as part of @@ -448,7 +547,13 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by // of the proof but not its left and right subtrees if includeNode && !newIncludeNode { proof = append(proof, hash) - coordinates = append(coordinates, ToCoordinate(start, end, n.Size())) + coordinate, err := ToCoordinate(start, end, n.Size()) + if err != nil { + return nil, err + } + if saveProofNodesCoordinates { + coordinates = append(coordinates, coordinate) + } } return hash, nil @@ -472,29 +577,44 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by // The coordinate calculation follows the RFC-6962 standard. // This means that leaves get elevated in trees that have // a size that is not a power of 2. -// Important: Expects the range to be pre-validated: +// Important: The inputs need to satisfy the following criteria: // - start >= 0 // - end > start // - treeSize >= end +// Otherwise, a sensible error is returned. // Note: the formula used is based on: // - start_leaf = position * (2 ** height) // - end_leaf = start_leaf + (2 ** height) // with position being the index of the inner node inside the tree // and the height being the traditional height of a tree, i.e. bottom -> top. -func ToCoordinate(start, end, treeSize int) Coordinate { +func ToCoordinate(start, end, treeSize int) (Coordinate, error) { + if start < 0 { + return Coordinate{}, fmt.Errorf("start cannot be stricly negative: %d", start) + } + if end <= start { + return Coordinate{}, fmt.Errorf("end %d cannot be smaller than start %d", end, start) + } + if treeSize < end { + return Coordinate{}, fmt.Errorf("tree size %d cannot be smaller than end %d", treeSize, end) + } + // calculates the height of the smallest subtree // that can contain the [start, end) range. // bits.Len() - 1 is used as a fast alternative to compute // the integer part of the result of log2(end-start). height := bits.Len(uint(end-start)) - 1 - maxDepth := maxDepth(treeSize) + maxDepth, err := maxDepth(treeSize) + if err != nil { + // TODO test this case + return Coordinate{}, err + } // 1 << height == 2 ** height. This result is based // on the formula documented above. position := start / (1 << height) return Coordinate{ depth: maxDepth - height, position: position, - } + }, nil } // Get returns leaves for the given namespace.ID. diff --git a/nmt_test.go b/nmt_test.go index a45bafb..79b1504 100644 --- a/nmt_test.go +++ b/nmt_test.go @@ -1179,25 +1179,29 @@ func TestForcedOutOfOrderNamespacedMerkleTree(t *testing.T) { func TestToCoordinate(t *testing.T) { tests := []struct { start, end, treeSize int + expectError bool expectedCoordinate Coordinate }{ - {start: 0, end: 1, treeSize: 1, expectedCoordinate: Coordinate{depth: 0, position: 0}}, - {start: 0, end: 2, treeSize: 2, expectedCoordinate: Coordinate{depth: 0, position: 0}}, - {start: 0, end: 3, treeSize: 4, expectedCoordinate: Coordinate{depth: 1, position: 0}}, - {start: 1, end: 2, treeSize: 4, expectedCoordinate: Coordinate{depth: 2, position: 1}}, - {start: 0, end: 4, treeSize: 4, expectedCoordinate: Coordinate{depth: 0, position: 0}}, - {start: 0, end: 3, treeSize: 5, expectedCoordinate: Coordinate{depth: 1, position: 0}}, - {start: 0, end: 3, treeSize: 6, expectedCoordinate: Coordinate{depth: 1, position: 0}}, - {start: 0, end: 3, treeSize: 7, expectedCoordinate: Coordinate{depth: 1, position: 0}}, - {start: 2, end: 4, treeSize: 8, expectedCoordinate: Coordinate{depth: 2, position: 1}}, - {start: 3, end: 4, treeSize: 8, expectedCoordinate: Coordinate{depth: 3, position: 3}}, - {start: 0, end: 8, treeSize: 8, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + {start: 0, end: 1, treeSize: 1, expectError: false, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + {start: 0, end: 2, treeSize: 2, expectError: false, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + {start: 0, end: 3, treeSize: 4, expectError: false, expectedCoordinate: Coordinate{depth: 1, position: 0}}, + {start: 1, end: 2, treeSize: 4, expectError: false, expectedCoordinate: Coordinate{depth: 2, position: 1}}, + {start: 0, end: 4, treeSize: 4, expectError: false, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + {start: 0, end: 3, treeSize: 5, expectError: false, expectedCoordinate: Coordinate{depth: 1, position: 0}}, + {start: 0, end: 3, treeSize: 6, expectError: false, expectedCoordinate: Coordinate{depth: 1, position: 0}}, + {start: 0, end: 3, treeSize: 7, expectError: false, expectedCoordinate: Coordinate{depth: 1, position: 0}}, + {start: 2, end: 4, treeSize: 8, expectError: false, expectedCoordinate: Coordinate{depth: 2, position: 1}}, + {start: 3, end: 4, treeSize: 8, expectError: false, expectedCoordinate: Coordinate{depth: 3, position: 3}}, + {start: 0, end: 8, treeSize: 8, expectError: false, expectedCoordinate: Coordinate{depth: 0, position: 0}}, + // TODO add false cases and add all cases for 7 leaves tree } for _, test := range tests { - result := ToCoordinate(test.start, test.end, test.treeSize) - if result != test.expectedCoordinate { - t.Errorf("ToCoordinate(%d, %d, %d) = %+v, want %+v", test.start, test.end, test.treeSize, result, test.expectedCoordinate) + result, err := ToCoordinate(test.start, test.end, test.treeSize) + if test.expectError { + assert.Error(t, err) + } else { + assert.Equal(t, test.expectedCoordinate, result) } } } @@ -1231,6 +1235,7 @@ func TestBuildRangeProofCoordinates(t *testing.T) { expected: []Coordinate{}, expectError: false, }, + // TODO add all cases for 7 leaves tree } for _, test := range tests { diff --git a/proof.go b/proof.go index 02ea113..798e4bc 100644 --- a/proof.go +++ b/proof.go @@ -404,17 +404,30 @@ func (proof Proof) VerifyLeafHashes(nth *NmtHasher, verifyCompleteness bool, nID return bytes.Equal(rootHash, root), nil } +// InnerProof contains an inclusion proof for a set of inner nodes to the NMT. +// Currently, the inner proof generation only supports adjacent ranges even if the +// provided coordinates cover a disjoint range. +// For example, if the inner nodes of the proof represent the ranges [1, 3), [6, 10), +// the generated proof will be targeting the range [1, 10). +// However, the inner proof verification can take any inner proof, even if the represented +// range is disjointed, and will verify it accordingly. type InnerProof struct { - nodes [][]byte - coordinates []Coordinate - treeSize int + // nodes the proof inner nodes needed to verify inclusion. + nodes [][]byte + // coordinates the coordinates of the above nodes in the + // same order. + coordinates []Coordinate + // treeSize the size of the tree, i.e., the number of leaves. + treeSize int + // isMaxNamespaceIDIgnored whether to ignore the maximum namespace IDs. isMaxNamespaceIDIgnored bool } // TODO add marshallers and protobuf definitions -// NewInnerInclusionProof constructs a proof that proves that a set of inner is +// NewInnerInclusionProof constructs a proof proving that a set of inner nodes is // included in an NMT. +// Does not validate the inputs. func NewInnerInclusionProof(proofNodes [][]byte, coordinates []Coordinate, treeSize int, ignoreMaxNamespace bool) InnerProof { return InnerProof{ nodes: proofNodes, @@ -426,13 +439,11 @@ func NewInnerInclusionProof(proofNodes [][]byte, coordinates []Coordinate, treeS // VerifyInnerNodes // coordinates should be in the same order as inner nodes -func (proof InnerProof) VerifyInnerNodes(nth *NmtHasher, nID namespace.ID, innerNodes [][]byte, coordinates []Coordinate, root []byte) (bool, error) { - // validate the inner proof: same number of nodes and coordinates - - // perform some consistency checks: - if nID.Size() != nth.NamespaceSize() { - return false, fmt.Errorf("namespace ID size (%d) does not match the namespace size of the NMT hasher (%d)", nID.Size(), nth.NamespaceSize()) +func (proof InnerProof) VerifyInnerNodes(nth *NmtHasher, innerNodes [][]byte, coordinates []Coordinate, root []byte) (bool, error) { + if len(innerNodes) != len(coordinates) { + return false, fmt.Errorf("the number of inner nodes %d is different than the number of coordinates %d", len(innerNodes), len(coordinates)) } + // check that the root is valid w.r.t the NMT hasher if err := nth.ValidateNodeFormat(root); err != nil { return false, fmt.Errorf("root does not match the NMT hasher's hash format: %w", err) @@ -449,9 +460,11 @@ func (proof InnerProof) VerifyInnerNodes(nth *NmtHasher, nID namespace.ID, inner return false, fmt.Errorf("leaf hash does not match the NMT hasher's hash format: %w", err) } } - // validate that nID is included in the inner nodes - _, proofEnd := toRange(coordinates, proof.treeSize) + _, proofEnd, err := toRange(coordinates, proof.treeSize) + if err != nil { + return false, err + } allNodes := append(proof.nodes, innerNodes...) allCoordinates := append(proof.coordinates, coordinates...) @@ -459,27 +472,33 @@ func (proof InnerProof) VerifyInnerNodes(nth *NmtHasher, nID namespace.ID, inner var computeRoot func(start, end int) ([]byte, error) // computeRoot can return error iff the HashNode function fails while calculating the root computeRoot = func(start, end int) ([]byte, error) { - innerNode, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start, end) + innerNode, found, err := getInnerNode(allNodes, allCoordinates, proof.treeSize, start, end) + if err != nil { + return nil, err + } if found { return innerNode, nil } // Recursively get left and right subtree k := getSplitPoint(end - start) - left, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start, start+k) - var err error - if found { - // TODO: do we want to remove the node and coordinates from allNodes and allCoordinates? Or it's just a premature optimisation - } else { + left, found, err := getInnerNode(allNodes, allCoordinates, proof.treeSize, start, start+k) + if err != nil { + return nil, err + } + // if a node is found, we could optimize by removing it from the list of nodes. + if !found { left, err = computeRoot(start, start+k) if err != nil { return nil, fmt.Errorf("failed to compute subtree root [%d, %d): %w", start, start+k, err) } } - right, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start+k, end) - if found { - // TODO: do we want to remove the node and coordinates from allNodes and allCoordinates? Or it's just a premature optimisation - } else { + right, found, err := getInnerNode(allNodes, allCoordinates, proof.treeSize, start+k, end) + if err != nil { + return nil, err + } + // Similarly, if a node is found, we could optimize by removing it from the list of nodes. + if !found { right, err = computeRoot(start+k, end) if err != nil { return nil, fmt.Errorf("failed to compute subtree root [%d, %d): %w", start+k, end, err) @@ -488,6 +507,7 @@ func (proof InnerProof) VerifyInnerNodes(nth *NmtHasher, nID namespace.ID, inner // only right leaf/subtree can be non-existent if right == nil { + // TODO test with coordinates return left, nil } hash, err := nth.HashNode(left, right) @@ -516,16 +536,34 @@ func (proof InnerProof) VerifyInnerNodes(nth *NmtHasher, nID namespace.ID, inner return bytes.Equal(rootHash, root), nil } -// getInnerNode -// expect the number of nodes and coordinates to be the same -func getInnerNode(nodes [][]byte, coordinates []Coordinate, treeSize int, start int, end int) ([]byte, bool) { +// getInnerNode takes a list of nodes and coordinates and returns the inner node +// corresponding to the [start, end) range. +// Expects the number of nodes and coordinates to be in the same order. +// Otherwise, the returned node might not be the correct one. +func getInnerNode(nodes [][]byte, coordinates []Coordinate, treeSize int, start int, end int) ([]byte, bool, error) { + if start < 0 { + return nil, false, fmt.Errorf("range start %d cannot be strictly negative", start) + } + if end <= start { + return nil, false, fmt.Errorf("range end %d cannot be smaller than start %d", end, start) + } + if treeSize < end { + return nil, false, fmt.Errorf("tree size %d cannot be strictly smaller than the end of range %d", treeSize, end) + } + // TODO: test these validates for index, coordinate := range coordinates { - startLeaf, endLeaf := toRange([]Coordinate{coordinate}, treeSize) + if err := coordinate.Validate(); err != nil { + return nil, false, err + } + startLeaf, endLeaf, err := toRange([]Coordinate{coordinate}, treeSize) + if err != nil { + return nil, false, err + } if startLeaf == start && endLeaf == end { - return nodes[index], true + return nodes[index], true, nil } } - return nil, false + return nil, false, nil } // VerifyInclusion checks that the inclusion proof is valid by using leaf data