Adding NewTxFromReader to handle large blocks (#56)

input.go

@@ -1,9 +1,11 @@
 package bt
 
 import (
+	"bytes"
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"io"
 
 	"github.com/libsv/go-bt/bscript"
 )
@@ -46,28 +48,55 @@ func NewInput() *Input {
 }
 
 // NewInputFromBytes returns a transaction input from the bytes provided.
-func NewInputFromBytes(bytes []byte) (*Input, int, error) {
-	if len(bytes) < 36 {
+func NewInputFromBytes(b []byte) (*Input, int, error) {
+	if len(b) < 36 {
 		return nil, 0, fmt.Errorf("input length too short < 36")
 	}
 
-	offset := 36
-	l, size := DecodeVarInt(bytes[offset:])
-	offset += size
+	r := bytes.NewReader(b)
 
-	totalLength := offset + int(l) + 4 // 4 bytes for nSeq
+	i, err := NewInputFromReader(r)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return i, len(i.ToBytes(false)), nil
+}
+
+// NewInputFromReader returns a transaction input from the io.Reader provided.
+func NewInputFromReader(r io.Reader) (*Input, error) {
+	previousTxID := make([]byte, 32)
+	if n, err := io.ReadFull(r, previousTxID); n != 32 || err != nil {
+		return nil, fmt.Errorf("Could not read previousTxID(32), got %d bytes and err: %w", n, err)
+	}
+
+	prevIndex := make([]byte, 4)
+	if n, err := io.ReadFull(r, prevIndex); n != 4 || err != nil {
+		return nil, fmt.Errorf("Could not read prevIndex(4), got %d bytes and err: %w", n, err)
+	}
+
+	l, _, err := DecodeVarIntFromReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("Could not read varint: %w", err)
+	}
 
-	if len(bytes) < totalLength {
-		return nil, 0, fmt.Errorf("input length too short < 36 + script + 4")
+	script := make([]byte, l)
+	if n, err := io.ReadFull(r, script); uint64(n) != l || err != nil {
+		return nil, fmt.Errorf("Could not read script(%d), got %d bytes and err: %w", l, n, err)
+	}
+
+	sequence := make([]byte, 4)
+	if n, err := io.ReadFull(r, sequence); n != 4 || err != nil {
+		return nil, fmt.Errorf("Could not read sequence(4), got %d bytes and err: %w", n, err)
 	}
 
 	return &Input{
-		PreviousTxIDBytes:  ReverseBytes(bytes[0:32]),
-		PreviousTxID:       hex.EncodeToString(ReverseBytes(bytes[0:32])),
-		PreviousTxOutIndex: binary.LittleEndian.Uint32(bytes[32:36]),
-		SequenceNumber:     binary.LittleEndian.Uint32(bytes[offset+int(l):]),
-		UnlockingScript:    bscript.NewFromBytes(bytes[offset : offset+int(l)]),
-	}, totalLength, nil
+		PreviousTxIDBytes:  ReverseBytes(previousTxID),
+		PreviousTxID:       hex.EncodeToString(ReverseBytes(previousTxID)),
+		PreviousTxOutIndex: binary.LittleEndian.Uint32(prevIndex),
+		UnlockingScript:    bscript.NewFromBytes(script),
+		SequenceNumber:     binary.LittleEndian.Uint32(sequence),
+	}, nil
 }
 
 // NewInputFromUTXO returns a transaction input from the UTXO fields provided.

output.go

@@ -1,9 +1,11 @@
 package bt
 
 import (
+	"bytes"
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"io"
 
 	"github.com/libsv/go-bt/bscript"
 	"github.com/libsv/go-bt/crypto"
@@ -28,27 +30,53 @@ type Output struct {
 }
 
 // NewOutputFromBytes returns a transaction Output from the bytes provided
-func NewOutputFromBytes(bytes []byte) (*Output, int, error) {
-	if len(bytes) < 8 {
+func NewOutputFromBytes(b []byte) (*Output, int, error) {
+	if len(b) < 8 {
 		return nil, 0, fmt.Errorf("output length too short < 8")
 	}
 
 	offset := 8
-	l, size := DecodeVarInt(bytes[offset:])
+	l, size := DecodeVarInt(b[offset:])
 	offset += size
 
 	totalLength := offset + int(l)
 
-	if len(bytes) < totalLength {
+	if len(b) < totalLength {
 		return nil, 0, fmt.Errorf("output length too short < 8 + script")
 	}
 
-	s := bscript.Script(bytes[offset:totalLength])
+	r := bytes.NewReader(b)
+
+	o, err := NewOutputFromReader(r)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return o, len(o.ToBytes()), nil
+}
+
+// NewOutputFromReader returns a transaction Output from the io.Reader provided
+func NewOutputFromReader(r io.Reader) (*Output, error) {
+	satoshis := make([]byte, 8)
+	if n, err := io.ReadFull(r, satoshis); n != 8 || err != nil {
+		return nil, fmt.Errorf("Could not read satoshis(8), got %d bytes and err: %w", n, err)
+	}
+
+	l, _, err := DecodeVarIntFromReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("Could not read varint: %w", err)
+	}
+
+	script := make([]byte, l)
+	if n, err := io.ReadFull(r, script); uint64(n) != l || err != nil {
+		return nil, fmt.Errorf("Could not read LockingScript(%d), got %d bytes and err: %w", l, n, err)
+	}
+	s := bscript.Script(script)
 
 	return &Output{
-		Satoshis:      binary.LittleEndian.Uint64(bytes[0:8]),
+		Satoshis:      binary.LittleEndian.Uint64(satoshis),
 		LockingScript: &s,
-	}, totalLength, nil
+	}, nil
 }
 
 // NewP2PKHOutputFromPubKeyHashStr makes an output to a PKH with a value.

tx.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
 
 	"github.com/libsv/go-bt/bscript"
 	"github.com/libsv/go-bt/crypto"
@@ -90,47 +91,69 @@ func NewTxFromStream(b []byte) (*Tx, int, error) {
 		return nil, 0, fmt.Errorf("too short to be a tx - even an empty tx has 10 bytes")
 	}
 
-	var offset int
-	t := Tx{
-		Version: binary.LittleEndian.Uint32(b[offset:4]),
+	r := bytes.NewReader(b)
+
+	tx, err := NewTxFromReader(r)
+	if err != nil {
+		return nil, 0, err
 	}
-	offset += 4
 
-	inputCount, size := DecodeVarInt(b[offset:])
-	offset += size
+	return tx, len(tx.ToBytes()), nil
+}
+
+// NewTxFromReader creates a transaction from an io.Reader
+func NewTxFromReader(r io.Reader) (*Tx, error) {
+	t := Tx{}
+
+	version := make([]byte, 4)
+	if n, err := io.ReadFull(r, version); n != 4 || err != nil {
+		return nil, err
+	}
+	t.Version = binary.LittleEndian.Uint32(version)
 
-	// create inputs
-	var i uint64
 	var err error
+
+	inputCount, _, err := DecodeVarIntFromReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// create Inputs
+	var i uint64
 	var input *Input
+
 	for ; i < inputCount; i++ {
-		input, size, err = NewInputFromBytes(b[offset:])
+		input, err = NewInputFromReader(r)
 		if err != nil {
-			return nil, 0, err
+			return nil, err
 		}
-		offset += size
-
 		t.Inputs = append(t.Inputs, input)
 	}
 
-	// create outputs
-	var outputCount uint64
+	outputCount, _, err := DecodeVarIntFromReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// create Outputs
 	var output *Output
-	outputCount, size = DecodeVarInt(b[offset:])
-	offset += size
+
 	for i = 0; i < outputCount; i++ {
-		output, size, err = NewOutputFromBytes(b[offset:])
+		output, err = NewOutputFromReader(r)
 		if err != nil {
-			return nil, 0, err
+			return nil, err
 		}
-		offset += size
+
 		t.Outputs = append(t.Outputs, output)
 	}
 
-	t.LockTime = binary.LittleEndian.Uint32(b[offset:])
-	offset += 4
+	locktime := make([]byte, 4)
+	if n, err := io.ReadFull(r, version); n != 4 || err != nil {
+		return nil, err
+	}
+	t.LockTime = binary.LittleEndian.Uint32(locktime)
 
-	return &t, offset, nil
+	return &t, nil
 }
 
 // AddInput adds a new input to the transaction.

tx_test.go

@@ -1,8 +1,11 @@
 package bt_test
 
 import (
+	"bufio"
 	"encoding/hex"
 	"errors"
+	"io"
+	"os"
 	"reflect"
 	"testing"
 
@@ -139,6 +142,69 @@ func TestAddInputFromTx(t *testing.T) {
 	assert.Equal(t, newTx.GetTotalInputSatoshis(), uint64(200000))
 }
 
+// This test reads a sample block from a file, but normally
+// we would get the block directly from the node via a REST GET...
+/*
+	resp, err := http.Get(fmt.Sprintf("%s/rest/block/%s.bin", b.client.serverAddr, blockHash))
+	if err != nil {
+		return nil, fmt.Errorf("Could not GET block: %v", err)
+	}
+	if resp.StatusCode != 200 {
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read response body: %w", err)
+		}
+		return nil, fmt.Errorf("ERROR: code %d: %s", resp.StatusCode, data)
+	}
+	Then use resp.Body as in the test below
+*/
+func TestNewTxFromReader(t *testing.T) {
+	var err error
+	var n int
+
+	f, err := os.Open("./block.bin")
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+	defer f.Close()
+
+	r := bufio.NewReader(f)
+
+	header := make([]byte, 80)
+	if n, err = io.ReadFull(f, header); n != 80 || err != nil {
+		t.Errorf("Read %d bytes, err: %v", n, err)
+	}
+
+	txCount, _, err := bt.DecodeVarIntFromReader(r)
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+
+	if txCount != 648 {
+		t.Errorf("Expected %d transactions, got %d", 648, txCount)
+		t.FailNow()
+	}
+
+	var tx *bt.Tx
+
+	for i := uint64(0); i < txCount; i++ {
+		tx, err = bt.NewTxFromReader(r)
+		if err != nil {
+			t.Error(err)
+			t.FailNow()
+		}
+		// t.Log(tx.TxID())
+		// t.Logf("%x", tx.Bytes())
+	}
+	if tx.GetTxID() != "b7c59d7fa17a74bbe0a05e5381f42b9ac7fe23b8a1ca40005a74802fe5b8bb5a" {
+		t.Errorf("Expected %q, got %q", "b7c59d7fa17a74bbe0a05e5381f42b9ac7fe23b8a1ca40005a74802fe5b8bb5a", tx.GetTxID())
+		t.FailNow()
+	}
+}
+
 func TestTx_GetTxID(t *testing.T) {
 	t.Parallel()