Skip to content

Latest commit

 

History

History
80 lines (57 loc) · 7.14 KB

go_offset_notes.md

File metadata and controls

80 lines (57 loc) · 7.14 KB
Raw

Notes

What do we need to build?

  • Really, all we need is a unidirecitonal iterator
    • TODO: check which bits of the flume api patchql uses

How would we iterate over a go offset?

  • Open all three files.

  • Check the journal

    • (how do we respect locking?)
    • the journal gives us the number or entries in the log

  • Start iterating the offset file

    • map the offset file number to size.
    • map the size to a slice of bytes by reading that size from the file.

Actually, that's not even needed right?

  • all we do is open the data file, read a u64 as the size, then read that many bytes. Repeat.

Testing

cryptix tried to add tests for ssb messages but failed to understand the .map(type..) stuff on the iterator and got a panic:

$ cargo test

...

failures:

---- go_offset_log::test::ssb_messages stdout ----
thread 'go_offset_log::test::ssb_messages' panicked at 'called `Result::unwrap()` on an `Err` value: FromUtf8Error { bytes: [134, 166, 65, 117, 116, 104, 111, 114, 130, 164, 65, 108, 103, 111, 167, 101, 100, 50, 53, 53, 49, 57, 162, 73, 68, 218, 0, 32, 252, 86, 210, 174, 217, 193, 106, 79, 195, 74, 31, 175, 157, 175, 85, 245, 187, 143, 225, 99, 213, 34, 94, 251, 142, 150, 30, 254, 168, 193, 223, 82, 163, 75, 101, 121, 130, 164, 65, 108, 103, 111, 166, 115, 104, 97, 50, 53, 54, 164, 72, 97, 115, 104, 218, 0, 32, 187, 110, 22, 14, 1, 75, 74, 100, 92, 125, 185, 28, 161, 22, 127, 164, 235, 84, 144, 227, 238, 69, 81, 5, 59, 181, 32, 35, 20, 75, 150, 173, 168, 80, 114, 101, 118, 105, 111, 117, 115, 192, 163, 82, 97, 119, 218, 1, 156, 123, 10, 32, 32, 34, 112, 114, 101, 118, 105, 111, 117, 115, 34, 58, 32, 110, 117, 108, 108, 44, 10, 32, 32, 34, 97, 117, 116, 104, 111, 114, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 34, 115, 101, 113, 117, 101, 110, 99, 101, 34, 58, 32, 49, 44, 10, 32, 32, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 32, 49, 53, 54, 53, 55, 56, 48, 51, 56, 56, 56, 49, 50, 44, 10, 32, 32, 34, 104, 97, 115, 104, 34, 58, 32, 34, 115, 104, 97, 50, 53, 54, 34, 44, 10, 32, 32, 34, 99, 111, 110, 116, 101, 110, 116, 34, 58, 32, 123, 10, 32, 32, 32, 32, 34, 97, 98, 111, 117, 116, 34, 58
, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 116, 101, 115, 116, 32, 117, 115, 101, 114, 34, 44, 10, 32, 32, 32, 32, 34, 116, 121, 112, 101, 34, 58, 32, 34, 97, 98, 111, 117, 116, 34, 10, 32, 32, 125, 44, 10, 32, 32, 34, 115, 105, 103, 110, 97, 116, 117, 114, 101, 34, 58, 32, 34, 51, 73, 99, 99, 71, 56, 116, 83, 54, 83, 70, 106, 80, 98, 114, 76, 122, 105, 81, 53, 98, 121, 69, 47, 110, 80, 101, 43, 84, 98, 105, 52, 98, 115, 121, 99, 48, 49, 82, 84, 53, 98, 69, 114, 52, 79, 122, 76, 113, 65, 114, 55, 121, 112, 111, 48, 104, 113, 74, 114, 79, 120, 116, 79, 67, 88, 106, 80, 120, 97, 88, 104, 121, 87, 117, 85, 72, 47, 106, 116, 48, 69, 80, 77, 67, 103, 61, 61, 46, 115, 105, 103, 46, 101, 100, 50, 53, 53, 49, 57, 34, 10, 125, 168, 83, 101, 113, 117, 101, 110, 99, 101, 1, 169, 84, 105, 109, 101, 115, 116, 97, 109, 112, 168, 193, 244, 94, 88, 93, 83, 233, 164], error: Utf8Error { valid_up_to: 0, error_len: Some(1) } }', src/libcore/result.rs:999:5
note: Run with `RUST_BACKTRACE=1` environment variable to display a backtrace.

docoding those bytes in go works though:

func TestDecode(t *testing.T) {
	r := require.New(t)
	var input = []byte{134, 166, 65, 117, 116, 104, 111, 114, 130, 164, 65, 108, 103, 111, 167, 101, 100, 50, 53, 53, 49, 57, 162, 73, 68, 218, 0, 32, 252, 86, 210, 174, 217, 193, 106, 79, 195, 74, 31, 175, 157, 175, 85, 245, 187, 143, 225, 99, 213, 34, 94, 251, 142, 150, 30, 254, 168, 193, 223, 82, 163, 75, 101, 121, 130, 164, 65, 108, 103, 111, 166, 115, 104, 97, 50, 53, 54, 164, 72, 97, 115, 104, 218, 0, 32, 187, 110, 22, 14, 1, 75, 74, 100, 92, 125, 185, 28, 161, 22, 127, 164, 235, 84, 144, 227, 238, 69, 81, 5, 59, 181, 32, 35, 20, 75, 150, 173, 168, 80, 114, 101, 118, 105, 111, 117, 115, 192, 163, 82, 97, 119, 218, 1, 156, 123, 10, 32, 32, 34, 112, 114, 101, 118, 105, 111, 117, 115, 34, 58, 32, 110, 117, 108, 108, 44, 10, 32, 32, 34, 97, 117, 116, 104, 111, 114, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 34, 115, 101, 113, 117, 101, 110, 99, 101, 34, 58, 32, 49, 44, 10, 32, 32, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 32, 49, 53, 54, 53, 55, 56, 48, 51, 56, 56, 56, 49, 50, 44, 10, 32, 32, 34, 104, 97, 115, 104, 34, 58, 32, 34, 115, 104, 97, 50, 53, 54, 34, 44, 10, 32, 32, 34, 99, 111, 110, 116, 101, 110, 116, 34, 58, 32, 123, 10, 32, 32, 32, 32, 34, 97, 98, 111, 117, 116, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 116, 101, 115, 116, 32, 117, 115, 101, 114, 34, 44, 10, 32, 32, 32, 32, 34, 116, 121, 112, 101, 34, 58, 32, 34, 97, 98, 111, 117, 116, 34, 10, 32, 32, 125, 44, 10, 32, 32, 34, 115, 105, 103, 110, 97, 116, 117, 114, 101, 34, 58, 32, 34, 51, 73, 99, 99, 71, 56, 116, 83, 54, 83, 70, 106, 80, 98, 114, 76, 122, 105, 81, 53, 98, 121, 69, 47, 110, 80, 101, 43, 84, 98, 105, 52, 98, 115, 121, 99, 48, 49, 82, 84, 53, 98, 69, 114, 52, 79, 122, 76, 113, 65, 114, 55, 121, 112, 111, 48, 104, 113, 74, 114, 79, 120, 116, 79, 67, 88, 106, 80, 120, 97, 88, 104, 121, 87, 117, 85, 72, 47, 106, 116, 48, 69, 80, 77, 67, 103, 61, 61, 46, 115, 105, 103, 46, 101, 100, 50, 53, 53, 49, 57, 34, 10, 125, 168, 83, 101, 113, 117, 101, 110, 99, 101, 1, 169, 84, 105, 109, 101, 115, 116, 97, 109, 112, 168, 193, 244, 94, 88, 93, 83, 233, 164}

	var msgt message.StoredMessage
	c := New(&msgt)

	v, err := c.Unmarshal(input)
	r.NoError(err)

	msg, ok := v.(message.StoredMessage)
	r.True(ok)
	t.Log(msg)

    // Author and Key are not stored a a string in msgpack but a struct with the fields Algo and ID and Algo and Hash (for key)
    r.Equal("@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519", msg.Author.Ref())
    r.Equal("%u24WDgFLSmRcfbkcoRZ/pOtUkOPuRVEFO7UgIxRLlq0=.sha256", msg.Key.Ref())
    // Raw should be an opaque byte array
	rawMessage := `{
  "previous": null,
  "author": "@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519",
  "sequence": 1,
  "timestamp": 1565780388812,
  "hash": "sha256",
  "content": {
    "about": "@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519",
    "name": "test user",
    "type": "about"
  },
  "signature": "3IccG8tS6SFjPbrLziQ5byE/nPe+Tbi4bsyc01RT5bEr4OzLqAr7ypo0hqJrOxtOCXjPxaXhyWuUH/jt0EPMCg==.sig.ed25519"
}`
	r.Equal(rawMessage, string(msg.Raw))

}