From b8c3e717a93ece26516ca7b843b25127c7207f00 Mon Sep 17 00:00:00 2001 From: Darren Kellenschwiler Date: Wed, 3 Nov 2021 12:28:48 -0500 Subject: [PATCH] Kellogg wars Signed-off-by: Darren Kellenschwiler --- go.sum | 1 - mapicallback.go | 10 ++ spv/ancestry_json_test.go | 4 +- spv/envelope.go | 282 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 280 insertions(+), 17 deletions(-) diff --git a/go.sum b/go.sum index 22396a8..1da0ec1 100644 --- a/go.sum +++ b/go.sum @@ -21,7 +21,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/mapicallback.go b/mapicallback.go index 49c9340..fc5fa58 100644 --- a/mapicallback.go +++ b/mapicallback.go @@ -16,6 +16,16 @@ type MapiCallback struct { CallbackReason string `json:"callbackReason"` } +// NewMapiCallbackFromBytes is a glorified json unmarshaller, but might be more sophisticated in future. +func NewMapiCallbackFromBytes(b []byte) (*MapiCallback, error) { + var mapiCallback MapiCallback + err := json.Unmarshal(b, &mapiCallback) + if err != nil { + return nil, err + } + return &mapiCallback, nil +} + // Bytes converts the MapiCallback struct into a binary format. // We are not going to parse anything out but rather take the whole json object as a binary blob. // The reason behind this approach is that the whole callback is signed by the mapi server, diff --git a/spv/ancestry_json_test.go b/spv/ancestry_json_test.go index 96f87f3..c47db2c 100644 --- a/spv/ancestry_json_test.go +++ b/spv/ancestry_json_test.go @@ -251,12 +251,12 @@ func TestEnvelope_Bytes_IsValid(t *testing.T) { assert.Error(t, err, "Couldn't decode hexString") } - efromB, err := NewEnvelopeFromBytes(b) + efromB, err := NewCrunchyNutEnvelopeFromBytes(b) if err != nil { assert.Error(t, err, "Couldn't create envelope from bytes") } - bFromE, err := e.Bytes() + bFromE, err := e.CrunchyNutBytes() if err != nil { assert.Error(t, err, "Couldn't convert envelope to bytes") } diff --git a/spv/envelope.go b/spv/envelope.go index 8c7c9eb..1e1d036 100644 --- a/spv/envelope.go +++ b/spv/envelope.go @@ -3,6 +3,7 @@ package spv import ( "encoding/hex" "fmt" + "sync" "github.com/libsv/go-bt/v2" "github.com/pkg/errors" @@ -46,8 +47,8 @@ func (e *Envelope) ParentTX(txID string) (*bt.Tx, error) { return bt.NewTxFromString(env.RawTx) } -// Bytes takes an spvEnvelope struct and returns a pointer to the serialised bytes. -func (e *Envelope) Bytes() (*[]byte, error) { +// CrunchyNutBytes takes an spvEnvelope struct and returns a pointer to the serialised bytes. +func (e *Envelope) CrunchyNutBytes() (*[]byte, error) { flake := make([]byte, 0) // Binary format version 1 @@ -63,15 +64,15 @@ func (e *Envelope) Bytes() (*[]byte, error) { }, } - err := serialiseInputs(initialTx, &flake) + err := serialiseCrunchyNutInputs(initialTx, &flake) if err != nil { fmt.Println(err) } return &flake, nil } -// serialiseInputs is a recursive input serialiser for spv Envelopes. -func serialiseInputs(parents map[string]*Envelope, flake *[]byte) error { +// serialiseCrunchyNutInputs is a recursive input serialiser for spv Envelopes. +func serialiseCrunchyNutInputs(parents map[string]*Envelope, flake *[]byte) error { for _, input := range parents { currentTx, err := hex.DecodeString(input.RawTx) if err != nil { @@ -103,14 +104,14 @@ func serialiseInputs(parents map[string]*Envelope, flake *[]byte) error { *flake = append(*flake, proofLength...) // of this length. *flake = append(*flake, proof...) // the data. } else if input.HasParents() { - return serialiseInputs(input.Parents, flake) + return serialiseCrunchyNutInputs(input.Parents, flake) } } return nil } -// NewEnvelopeFromBytes will encode an spv envelope byte slice into the Envelope structure. -func NewEnvelopeFromBytes(b []byte) (*Envelope, error) { +// NewCrunchyNutEnvelopeFromBytes will encode an spv envelope byte slice into the Envelope structure. +func NewCrunchyNutEnvelopeFromBytes(b []byte) (*Envelope, error) { var envelope Envelope var offset uint64 @@ -120,13 +121,13 @@ func NewEnvelopeFromBytes(b []byte) (*Envelope, error) { return nil, errors.New("We can only handle version 1 of the SPV Envelope Binary format") } offset++ - parseChunksRecursively(b, &offset, &envelope) + parseCrunchyNutFlakesRecursively(b, &offset, &envelope) return &envelope, nil } -// ParseChunksRecursively will identify the next chunk of data's type and length, +// parseCrunchyNutChunksRecursively will identify the next chunk of data's type and length, // and pull out the stream into the appropriate struct. -func parseChunksRecursively(b []byte, offset *uint64, eCurrent *Envelope) { +func parseCrunchyNutFlakesRecursively(b []byte, offset *uint64, eCurrent *Envelope) { typeOfNextData := b[*offset] *offset++ l, size := bt.DecodeVarInt(b[*offset:]) @@ -149,13 +150,13 @@ func parseChunksRecursively(b []byte, offset *uint64, eCurrent *Envelope) { fmt.Println("tx1", tx) *offset += l if uint64(len(b)) > *offset && b[*offset] != flagTx { - parseChunksRecursively(b, offset, eCurrent) + parseCrunchyNutFlakesRecursively(b, offset, eCurrent) } else { eCurrent.Parents = inputs } for _, input := range inputs { if uint64(len(b)) > *offset { - parseChunksRecursively(b, offset, input) + parseCrunchyNutFlakesRecursively(b, offset, input) } } case flagProof: @@ -174,12 +175,23 @@ func parseChunksRecursively(b []byte, offset *uint64, eCurrent *Envelope) { } eCurrent.Proof = &proof *offset += l + case flagMapi: + mapiResponse, err := bc.NewMapiCallbackFromBytes(b[*offset : *offset+l]) + if err != nil { + fmt.Println(err) + } + if eCurrent.MapiResponses != nil { + eCurrent.MapiResponses = append(eCurrent.MapiResponses, *mapiResponse) + } else { + eCurrent.MapiResponses = []bc.MapiCallback{*mapiResponse} + } + *offset += l default: fmt.Printf("Unknown data type: %v, used for: %v", typeOfNextData, b[*offset:*offset+l]) *offset += l } if uint64(len(b)) > *offset { - parseChunksRecursively(b, offset, eCurrent) + parseCrunchyNutFlakesRecursively(b, offset, eCurrent) } } @@ -196,3 +208,245 @@ func flagType(flags byte) string { return "" } } + +/* From Mark Smith +txs := map[string]*bt.Tx{} +go func() { + select { + case tx <- txChan: + txs[tx.TxID()] = tx + case <-done: + break + } +}() + +wg := sync.WaitGroup{} + +for i := 0; i < len(txs);i++{ + wg.Add(1) + go func(){ + defer wg.Done() + txChan <- deserialise(blob) + } +} + +wg.Wait() +done <- struct{}{} +// iterate map and build tree + +*/ + +// SpecialKBytes takes an spvEnvelope struct and returns a pointer to the serialised bytes. +func (e *Envelope) SpecialKBytes() (*[]byte, error) { + flake := make([]byte, 0) + + // Binary format version 1 + flake = append(flake, 1) + + initialTx := map[string]*Envelope{ + e.TxID: { + TxID: e.TxID, + RawTx: e.RawTx, + Proof: e.Proof, + MapiResponses: e.MapiResponses, + Parents: e.Parents, + }, + } + + err := serialiseSpecialKInputs(initialTx, &flake) + if err != nil { + fmt.Println(err) + } + return &flake, nil +} + +// serialiseSpecialKInputs is a recursive input serialiser for spv Envelopes. +func serialiseSpecialKInputs(parents map[string]*Envelope, flake *[]byte) error { + for _, input := range parents { + currentTx, err := hex.DecodeString(input.RawTx) + if err != nil { + fmt.Print(err) + } + // the transaction itself + dataLength := bt.VarInt(uint64(len(currentTx))) + *flake = append(*flake, dataLength...) // of this length. + *flake = append(*flake, currentTx...) // the data. + + // proof or zero + if input.Proof == nil { + *flake = append(*flake, bt.VarInt(0)...) + } else { + proof, err := input.Proof.ToBytes() + if err != nil { + return errors.Wrap(err, "Failed to serialise this input's proof struct") + } + proofLength := bt.VarInt(uint64(len(proof))) + *flake = append(*flake, proofLength...) // of this length. + *flake = append(*flake, proof...) // the data. + } + + if input.MapiResponses == nil || len(input.MapiResponses) == 0 { + *flake = append(*flake, bt.VarInt(0)...) + } else { + for _, mapiResponse := range input.MapiResponses { + mapiR, err := mapiResponse.Bytes() + if err != nil { + return err + } + dataLength := bt.VarInt(uint64(len(mapiR))) + *flake = append(*flake, dataLength...) // of this length. + *flake = append(*flake, mapiR...) // the data. + } + } + + if input.Proof == nil && input.HasParents() { + return serialiseSpecialKInputs(input.Parents, flake) + } + } + return nil +} + +// NewSpecialKEnvelopeFromBytes will encode an spv envelope byte slice into the Envelope structure. +func NewSpecialKEnvelopeFromBytes(b []byte) (*Envelope, error) { + allBinary := uint64(len(b)) + var envelope Envelope + var offset uint64 + + // the first byte is the version number. + version := b[offset] + if version != 1 { + return nil, errors.New("We can only handle version 1 of the SPV Envelope Binary format") + } + offset++ + + // split up the binary into flakes where each one is to be processed concurrently. + var flakes = [][]byte{} + for ok := true; ok; ok = allBinary > offset { + l, size := bt.DecodeVarInt(b[offset:]) + offset += uint64(size) + flake := b[offset : offset+l] + offset += l + flakes = append(flakes, flake) + } + + mapiCallbackChan := make(chan bc.MapiCallback) + proofChan := make(chan bc.MerkleProof) + txChan := make(chan bt.Tx) + done := make(chan bool) + txs := map[string]*bt.Tx{} + go func() { + select { + case tx := <-txChan: + txs[tx.TxID()] = &tx + case <-done: + break + } + }() + + wg := sync.WaitGroup{} + + for idx, flake := range flakes { + wg.Add(1) + go func() { + defer wg.Done() + switch idx % 3 { + case 2: + mapiCallbackChan <- parseSpecialKMapi(flake) + case 1: + proofChan <- parseSpecialKProof(flake) + case 0: + default: + txChan <- parseSpecialKFlake(flake) + } + }() + } + + wg.Wait() + done <- true + + // construct something useful + + return &envelope, nil +} + +// parseSpecialKFlakesRecursively will identify the next chunk of data's type and length, +// and pull out the stream into the appropriate struct. +func parseSpecialKFlakeTx(b []byte) *bt.Tx { + tx, err := bt.NewTxFromBytes(b) + if err != nil { + fmt.Println(err) + return &bt.Tx{} + } + return tx +} + +func parseSpecialKFlake(b []byte) bt.Tx { + txid := tx.TxID() + inputs := map[string]*Envelope{} + for _, input := range tx.Inputs { + inputs[hex.EncodeToString(input.PreviousTxID())] = &Envelope{} + } + eCurrent.TxID = txid + eCurrent.RawTx = tx.String() + howManyInputs := len(tx.Inputs) + fmt.Println("txid", txid, " has ", howManyInputs, " inputs") + fmt.Println("tx1", tx) + *offset += l + + // size of next blob of data (proof) + l, size = bt.DecodeVarInt(b[*offset:]) + *offset += uint64(size) + if l != 0 { + binaryProof, err := parseBinaryMerkleProof(b[*offset : *offset+l]) + fmt.Println(hex.EncodeToString(b[*offset : *offset+l])) + if err != nil { + fmt.Println(err) + } + proof := bc.MerkleProof{ + Index: binaryProof.index, + TxOrID: binaryProof.txOrID, + Target: binaryProof.target, + Nodes: binaryProof.nodes, + TargetType: flagType(binaryProof.flags), + // ignoring proofType and compositeType for this version. + } + eCurrent.Proof = &proof + *offset += l + } + + // number of mapiCallbacks + l, size = bt.DecodeVarInt(b[*offset:]) + *offset += uint64(size) + if l != 0 { + for i := uint64(0); i < l; i++ { + // size of next blob of data (mapiCallback) + l, size = bt.DecodeVarInt(b[*offset:]) + *offset += uint64(size) + mapiResponse, err := bc.NewMapiCallbackFromBytes(b[*offset : *offset+l]) + if err != nil { + fmt.Println(err) + } + if eCurrent.MapiResponses != nil { + eCurrent.MapiResponses = append(eCurrent.MapiResponses, *mapiResponse) + } else { + eCurrent.MapiResponses = []bc.MapiCallback{*mapiResponse} + } + *offset += l + } + } + + if uint64(len(b)) > *offset && b[*offset] != flagTx { + parseSpecialKFlakesRecursively(b, offset, eCurrent) + } else { + eCurrent.Parents = inputs + } + for _, input := range inputs { + if uint64(len(b)) > *offset { + parseSpecialKFlakesRecursively(b, offset, input) + } + } + + if uint64(len(b)) > *offset { + parseSpecialKFlakesRecursively(b, offset, eCurrent) + } +}