From b8dd127eab290faa83177931a1bb79188396d5da Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Thu, 7 Mar 2024 08:48:23 +0100 Subject: [PATCH 1/5] Add ability to add/replace headers when doing header re-encryption Allow passing of zero or more EncryptedHeaderPacket that will be added to the reencrypted header by ReEncryptHeader. If a packet of the same type exists already it will be replaced. --- model/headers/headers.go | 26 +++++++++- model/headers/headers_test.go | 96 +++++++++++++++++++++++++++++++++++ streaming/streaming_test.go | 80 +++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) diff --git a/model/headers/headers.go b/model/headers/headers.go index 3558965..a7a1e3f 100644 --- a/model/headers/headers.go +++ b/model/headers/headers.go @@ -508,8 +508,9 @@ func (delhp DataEditListHeaderPacket) MarshalBinary() (data []byte, err error) { } // ReEncryptHeader takes an old header, decrypts it and using a list of receivers public keys -// and re-encrypts the header for those keys while keeping the dataEditList packets -func ReEncryptHeader(oldHeader []byte, readerPrivateKey [chacha20poly1305.KeySize]byte, readerPublicKeyList [][chacha20poly1305.KeySize]byte) (newBinaryHeader []byte, err error) { +// re-encrypts the header for those keys while keeping the dataEditList packets. +// Optionally adds additional headers, replacing any previous headers of the same type +func ReEncryptHeader(oldHeader []byte, readerPrivateKey [chacha20poly1305.KeySize]byte, readerPublicKeyList [][chacha20poly1305.KeySize]byte, additionalEncryptedHeaderPackets ...EncryptedHeaderPacket) (newBinaryHeader []byte, err error) { buffer := bytes.NewBuffer(oldHeader) decryptedHeader, err := NewHeader(buffer, readerPrivateKey) @@ -560,6 +561,27 @@ func ReEncryptHeader(oldHeader []byte, readerPrivateKey [chacha20poly1305.KeySiz EncryptedHeaderPacket: dataEditList, }) } + + for _, packet := range additionalEncryptedHeaderPackets { + found := false + packetType := packet.GetPacketType() + + for i := range headerPackets { + if headerPackets[i].EncryptedHeaderPacket.GetPacketType() == packetType { + headerPackets[i].EncryptedHeaderPacket = packet + found = true + } + } + if !found { + headerPackets = append(headerPackets, HeaderPacket{ + WriterPrivateKey: privateKey, + ReaderPublicKey: readerPublicKey, + HeaderEncryptionMethod: X25519ChaCha20IETFPoly1305, + EncryptedHeaderPacket: packet, + }) + + } + } } var magicNumber [8]byte diff --git a/model/headers/headers_test.go b/model/headers/headers_test.go index 0ba973c..e35d8cd 100644 --- a/model/headers/headers_test.go +++ b/model/headers/headers_test.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "os" + "reflect" "strings" "testing" @@ -253,6 +254,101 @@ func TestHeader_GetDataEditListHeaderPacket(t *testing.T) { } } +func TestReEncryptedHeaderReplacementAndAddition(t *testing.T) { + inFile, err := os.Open("../../test/sample.txt.enc") + if err != nil { + t.Error(err) + } + readerSecretKey, err := keys.ReadPrivateKey(strings.NewReader(crypt4ghX25519Sec), []byte("password")) + if err != nil { + t.Error(err) + } + oldHeader, err := ReadHeader(inFile) + if err != nil { + t.Error(err) + } + + newReaderPublicKey, err := keys.ReadPublicKey(strings.NewReader(newRecipientPub)) + if err != nil { + t.Error(err) + } + newReaderPublicKeyList := [][chacha20poly1305.KeySize]byte{} + newReaderPublicKeyList = append(newReaderPublicKeyList, newReaderPublicKey) + + del := DataEditListHeaderPacket{PacketType: PacketType{DataEditList}, NumberLengths: 2, Lengths: []uint64{10, 100}} + anotherDel := DataEditListHeaderPacket{PacketType: PacketType{DataEditList}, NumberLengths: 4, Lengths: []uint64{0, 5, 10, 15}} + + newHeader, err := ReEncryptHeader(oldHeader, readerSecretKey, newReaderPublicKeyList, del, anotherDel) + if err != nil { + panic(err) + } + t.Logf("Header: %v", newHeader) + + // if the headers are similar then that is not ok + if fmt.Sprintf("%v", oldHeader) == fmt.Sprintf("%v", newHeader) { + t.Fail() + } + + // check the header contents is what we expect + newReaderSecretKey, err := keys.ReadPrivateKey(strings.NewReader(newRecipientSec), []byte("password")) + if err != nil { + t.Error(err) + } + buffer := bytes.NewBuffer(newHeader) + header, err := NewHeader(buffer, newReaderSecretKey) + if err != nil { + panic(err) + } + + newDel, ok := header.HeaderPackets[1].EncryptedHeaderPacket.(DataEditListHeaderPacket) + + if !ok { + t.Logf("Not DEL as expected: %v", header.HeaderPackets[1].EncryptedHeaderPacket) + t.Fail() + } + + if newDel.NumberLengths != 4 || !reflect.DeepEqual(newDel.Lengths, []uint64{0, 5, 10, 15}) { + t.Logf("Unexpected length (%d vs 4) or content in overriden DEL: %v vs {0, 5, 10, 15}", newDel.NumberLengths, newDel.Lengths) + t.Fail() + } + + // Test DEL copying when reencryption, i.e. when the DEL is not replaced. Encrypt back for the original recipient + + newRecipientSecretKey, err := keys.ReadPrivateKey(strings.NewReader(newRecipientSec), []byte("password")) + + newerReaderPublicKey, err := keys.ReadPublicKey(strings.NewReader(crypt4ghX25519Pub)) + if err != nil { + t.Error(err) + } + + newerReaderPublicKeyList := [][chacha20poly1305.KeySize]byte{} + newerReaderPublicKeyList = append(newerReaderPublicKeyList, newerReaderPublicKey) + + newerHeader, err := ReEncryptHeader(newHeader, newRecipientSecretKey, newerReaderPublicKeyList) + if err != nil { + t.Errorf("Reencryption back to original recipient failed: %v", err) + } + + buffer = bytes.NewBuffer(newerHeader) + header, err = NewHeader(buffer, readerSecretKey) + if err != nil { + panic(err) + } + + newDel, ok = header.HeaderPackets[1].EncryptedHeaderPacket.(DataEditListHeaderPacket) + + if !ok { + t.Logf("Not DEL as expected: %v", header.HeaderPackets[1].EncryptedHeaderPacket) + t.Fail() + } + + if newDel.NumberLengths != 4 || !reflect.DeepEqual(newDel.Lengths, []uint64{0, 5, 10, 15}) { + t.Logf("Unexpected length (%d vs 4) or content in copied DEL: %v vs {0, 5, 10, 15}", newDel.NumberLengths, newDel.Lengths) + t.Fail() + } + +} + func TestReEncryptedHeader(t *testing.T) { inFile, err := os.Open("../../test/sample.txt.enc") if err != nil { diff --git a/streaming/streaming_test.go b/streaming/streaming_test.go index 25f51fd..2cd58ba 100644 --- a/streaming/streaming_test.go +++ b/streaming/streaming_test.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/hex" + "fmt" "io" "os" "strings" @@ -1397,3 +1398,82 @@ func TestUnencryptedPrivate(t *testing.T) { } } } + +// More of a headers test, but avoid cyclic import +func TestReEncryptedHeaderReplacementAndAdditionFileRead(t *testing.T) { + inFile, err := os.Open("../test/sample.txt.enc") + if err != nil { + t.Error(err) + } + readerSecretKey, err := keys.ReadPrivateKey(strings.NewReader(crypt4ghX25519Sec), []byte("password")) + if err != nil { + t.Error(err) + } + oldHeader, err := headers.ReadHeader(inFile) + if err != nil { + t.Error(err) + } + + newReaderPublicKey, err := keys.ReadPublicKey(strings.NewReader(newRecipientPub)) + if err != nil { + t.Error(err) + } + newReaderPublicKeyList := [][chacha20poly1305.KeySize]byte{} + newReaderPublicKeyList = append(newReaderPublicKeyList, newReaderPublicKey) + + // create a new data edit list, pass over a segment boundary for sports + del := headers.DataEditListHeaderPacket{PacketType: headers.PacketType{headers.DataEditList}, NumberLengths: 2, Lengths: []uint64{65500, 100}} + + newHeader, err := headers.ReEncryptHeader(oldHeader, readerSecretKey, newReaderPublicKeyList, del) + if err != nil { + panic(err) + } + t.Logf("Header: %v", newHeader) + + // if the headers are similar then that is not ok + if fmt.Sprintf("%v", oldHeader) == fmt.Sprintf("%v", newHeader) { + t.Fail() + } + + // check the header contents is what we expect + newReaderSecretKey, err := keys.ReadPrivateKey(strings.NewReader(newRecipientSec), []byte("password")) + if err != nil { + t.Error(err) + } + + buffer := bytes.NewBuffer(newHeader) + + newFile := io.MultiReader(buffer, inFile) + c4ghReader, err := NewCrypt4GHReader(newFile, newReaderSecretKey, nil) + + readBackWithDel := make([]byte, 100000) + delRead, err := c4ghReader.Read(readBackWithDel) + t.Logf("Read %d bytes", delRead) + + if delRead != 100 { + t.Errorf("Expected 100 bytes read but got %d bytes", delRead) + } + + if err != nil && err != io.EOF { + t.Errorf("Unexpected error: %v", err) + } + + sourceFile, err := os.Open("../test/sample.txt") + if err != nil { + t.Errorf("Opening of unencrypted source file failed: %v", err) + } + + sourceFile.Seek(65500, 0) + sourceBytes := make([]byte, 400) + sourceRead, err := sourceFile.Read(sourceBytes) + if err != nil || sourceRead != 400 { + t.Errorf("Reading unencrypted source file failed: %v", err) + } + + if !bytes.Equal(readBackWithDel[:delRead], sourceBytes[:100]) { + t.Logf("Observed: %v", readBackWithDel[:delRead]) + t.Logf("Expected: %v", sourceBytes[:100]) + t.Errorf("Unexpected data after re-encryption and del added") + } + +} From 54c7a16d8dad4dc47d2f744ff9c89997225de9bb Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Thu, 7 Mar 2024 08:49:27 +0100 Subject: [PATCH 2/5] Bump version --- internal/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version/version.go b/internal/version/version.go index 703f549..6277ede 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -7,7 +7,7 @@ import ( ) // The version in the current branch -var Version = "1.9.1" +var Version = "1.10.0" // If this is "" (empty string) then it means that it is a final release. // Otherwise, this is a pre-release e.g. "dev", "beta", "rc1", etc. From cc4ac02ebf1c16607f2bbc7021200e6f8f08595d Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Thu, 7 Mar 2024 08:58:22 +0100 Subject: [PATCH 3/5] Linter fixes --- model/headers/headers_test.go | 3 +++ streaming/streaming_test.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/model/headers/headers_test.go b/model/headers/headers_test.go index e35d8cd..f7367b3 100644 --- a/model/headers/headers_test.go +++ b/model/headers/headers_test.go @@ -315,6 +315,9 @@ func TestReEncryptedHeaderReplacementAndAddition(t *testing.T) { // Test DEL copying when reencryption, i.e. when the DEL is not replaced. Encrypt back for the original recipient newRecipientSecretKey, err := keys.ReadPrivateKey(strings.NewReader(newRecipientSec), []byte("password")) + if err != nil { + t.Errorf("Failed creating new recipient secret key: %v", err) + } newerReaderPublicKey, err := keys.ReadPublicKey(strings.NewReader(crypt4ghX25519Pub)) if err != nil { diff --git a/streaming/streaming_test.go b/streaming/streaming_test.go index 2cd58ba..025d335 100644 --- a/streaming/streaming_test.go +++ b/streaming/streaming_test.go @@ -1422,7 +1422,7 @@ func TestReEncryptedHeaderReplacementAndAdditionFileRead(t *testing.T) { newReaderPublicKeyList = append(newReaderPublicKeyList, newReaderPublicKey) // create a new data edit list, pass over a segment boundary for sports - del := headers.DataEditListHeaderPacket{PacketType: headers.PacketType{headers.DataEditList}, NumberLengths: 2, Lengths: []uint64{65500, 100}} + del := headers.DataEditListHeaderPacket{PacketType: headers.PacketType{PacketType: headers.DataEditList}, NumberLengths: 2, Lengths: []uint64{65500, 100}} newHeader, err := headers.ReEncryptHeader(oldHeader, readerSecretKey, newReaderPublicKeyList, del) if err != nil { @@ -1445,6 +1445,9 @@ func TestReEncryptedHeaderReplacementAndAdditionFileRead(t *testing.T) { newFile := io.MultiReader(buffer, inFile) c4ghReader, err := NewCrypt4GHReader(newFile, newReaderSecretKey, nil) + if err != nil { + t.Errorf("Opening of reencrypted stream failed: %v", err) + } readBackWithDel := make([]byte, 100000) delRead, err := c4ghReader.Read(readBackWithDel) @@ -1463,7 +1466,11 @@ func TestReEncryptedHeaderReplacementAndAdditionFileRead(t *testing.T) { t.Errorf("Opening of unencrypted source file failed: %v", err) } - sourceFile.Seek(65500, 0) + pos, err := sourceFile.Seek(65500, 0) + if err != nil || pos != 65500 { + t.Errorf("Seeking in unencrypted source file failed (%v) or returned wrong position (%d vs 65500)", err, pos) + } + sourceBytes := make([]byte, 400) sourceRead, err := sourceFile.Read(sourceBytes) if err != nil || sourceRead != 400 { From 5ebb396cfc88c23fe099ad11cef494269256b43c Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Mon, 11 Mar 2024 10:15:08 +0100 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- model/headers/headers_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/model/headers/headers_test.go b/model/headers/headers_test.go index f7367b3..97662e9 100644 --- a/model/headers/headers_test.go +++ b/model/headers/headers_test.go @@ -339,12 +339,10 @@ func TestReEncryptedHeaderReplacementAndAddition(t *testing.T) { } newDel, ok = header.HeaderPackets[1].EncryptedHeaderPacket.(DataEditListHeaderPacket) - if !ok { t.Logf("Not DEL as expected: %v", header.HeaderPackets[1].EncryptedHeaderPacket) t.Fail() } - if newDel.NumberLengths != 4 || !reflect.DeepEqual(newDel.Lengths, []uint64{0, 5, 10, 15}) { t.Logf("Unexpected length (%d vs 4) or content in copied DEL: %v vs {0, 5, 10, 15}", newDel.NumberLengths, newDel.Lengths) t.Fail() From 5bd7dd8b5b85402ab8d4eb783e4e23e5bba41d8a Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Mon, 11 Mar 2024 10:23:46 +0100 Subject: [PATCH 5/5] Change panic to Errorf --- model/headers/headers_test.go | 23 +++++++++++------------ streaming/streaming_test.go | 3 +-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/model/headers/headers_test.go b/model/headers/headers_test.go index 97662e9..3ee1608 100644 --- a/model/headers/headers_test.go +++ b/model/headers/headers_test.go @@ -46,12 +46,12 @@ func TestHeaderMarshallingWithNonce(t *testing.T) { writerPrivateKey, err := keys.ReadPrivateKey(strings.NewReader(sshEd25519SecEnc), []byte("123123")) if err != nil { - panic(err) + t.Errorf("Reading private key from string failed: %v", err) } readerPublicKey, err := keys.ReadPublicKey(strings.NewReader(crypt4ghX25519Pub)) if err != nil { - panic(err) + t.Errorf("Reading public key from string failed: %v", err) } var nonce = [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} magic := [8]byte{} @@ -104,11 +104,11 @@ func TestNewHeader(t *testing.T) { buffer := bytes.NewBuffer(decodedHeader) readerSecretKey, err := keys.ReadPrivateKey(strings.NewReader(crypt4ghX25519Sec), []byte("password")) if err != nil { - panic(err) + t.Errorf("Reading private key from string failed: %v", err) } header, err := NewHeader(buffer, readerSecretKey) if err != nil { - panic(err) + t.Errorf("NewHeader failed unexpectedly: %v", err) } if fmt.Sprintf("%v", header) != "&{[99 114 121 112 116 52 103 104] 1 2 [{[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] 108 0 {65564 {0} 0 [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]}} {[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] 100 0 {{1} 3 [1 2 3]}}]}" { t.Fail() @@ -154,12 +154,12 @@ func TestHeaderMarshallingWithoutNonce(t *testing.T) { writerPrivateKey, err := keys.ReadPrivateKey(strings.NewReader(sshEd25519SecEnc), []byte("123123")) if err != nil { - panic(err) + t.Errorf("Reading private key from string failed: %v", err) } readerPublicKey, err := keys.ReadPublicKey(strings.NewReader(crypt4ghX25519Pub)) if err != nil { - panic(err) + t.Errorf("Reading public key from string failed: %v", err) } magic := [8]byte{} copy(magic[:], MagicNumber) @@ -280,7 +280,7 @@ func TestReEncryptedHeaderReplacementAndAddition(t *testing.T) { newHeader, err := ReEncryptHeader(oldHeader, readerSecretKey, newReaderPublicKeyList, del, anotherDel) if err != nil { - panic(err) + t.Errorf("Reencrypting header gave unexpected failure: %v", err) } t.Logf("Header: %v", newHeader) @@ -297,7 +297,7 @@ func TestReEncryptedHeaderReplacementAndAddition(t *testing.T) { buffer := bytes.NewBuffer(newHeader) header, err := NewHeader(buffer, newReaderSecretKey) if err != nil { - panic(err) + t.Errorf("NewHeader gave unexpected failure: %v", err) } newDel, ok := header.HeaderPackets[1].EncryptedHeaderPacket.(DataEditListHeaderPacket) @@ -335,7 +335,7 @@ func TestReEncryptedHeaderReplacementAndAddition(t *testing.T) { buffer = bytes.NewBuffer(newerHeader) header, err = NewHeader(buffer, readerSecretKey) if err != nil { - panic(err) + t.Errorf("NewHeader gave unexpected failure: %v", err) } newDel, ok = header.HeaderPackets[1].EncryptedHeaderPacket.(DataEditListHeaderPacket) @@ -373,7 +373,7 @@ func TestReEncryptedHeader(t *testing.T) { newHeader, err := ReEncryptHeader(oldHeader, readerSecretKey, newReaderPublicKeyList) if err != nil { - panic(err) + t.Errorf("ReEncryptHeader gave unexpected failure: %v", err) } // if the headers are similar then that is not ok @@ -389,7 +389,7 @@ func TestReEncryptedHeader(t *testing.T) { buffer := bytes.NewBuffer(newHeader) header, err := NewHeader(buffer, newReaderSecretKey) if err != nil { - panic(err) + t.Errorf("NewHeader gave unexpected failure: %v", err) } if fmt.Sprintf("%v", header) != "&{[99 114 121 112 116 52 103 104] 1 1 [{[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] 108 0 {65564 {0} 0 [111 194 187 210 222 31 213 211 134 204 70 51 56 197 11 150 188 141 28 253 188 188 76 243 7 143 50 179 45 172 135 132]}}]}" { t.Error(header) @@ -428,5 +428,4 @@ func TestEncryptedSegmentSize(t *testing.T) { if err == nil { t.Errorf("EncryptedSegmentSize worked where it should fail: %v", err) } - } diff --git a/streaming/streaming_test.go b/streaming/streaming_test.go index 025d335..83ab875 100644 --- a/streaming/streaming_test.go +++ b/streaming/streaming_test.go @@ -1426,7 +1426,7 @@ func TestReEncryptedHeaderReplacementAndAdditionFileRead(t *testing.T) { newHeader, err := headers.ReEncryptHeader(oldHeader, readerSecretKey, newReaderPublicKeyList, del) if err != nil { - panic(err) + t.Errorf("NewHeader failed unexpectedly: %v", err) } t.Logf("Header: %v", newHeader) @@ -1482,5 +1482,4 @@ func TestReEncryptedHeaderReplacementAndAdditionFileRead(t *testing.T) { t.Logf("Expected: %v", sourceBytes[:100]) t.Errorf("Unexpected data after re-encryption and del added") } - }