Feature branch: storage & sync revamp (bluesky-social#446)

* wip

* fleshing out repo storage

* fleshing out sql storage

* cleaning things up

* fix up tests

* dumb bug - commit log reversed

* rm staging in favor of commiting diffs to blockstore

* clean up benches

* fixing up sql storage

* some caching for sql repo store

* pr feedback

* migration

* wip

* migraiton test

* unclear param

* sql repo storage tests

* rm unused code

* fix up some diff code

* pr feedback

* enum for action types

* missed some

* wip

* ripping out auth lib

* more auth cleanup

* another lurker

* wip better sync primitives

* wip

* improving diffs & sync

* tests working!

* actually implemented checkout lol

* simplify interface & improve error handling

* writing sql storage code

* fixing up tests

* testing & bugfixes

* checkouts return records instead of cids

* one last refactor lol

* missed one

* handle other cid codecs on incoming car verification

* tests + tricky bugs

* unneeded blockstore method

* trim mst on del instead of save

* cleanup comment

* dont resolve did for every commit

* use "commit" instead of "root"

* getRoot -> getHead

* pr feedback

* very silly bug fix

* improve sync output

* reorging + sync of particular records

* serve & verify proofs. also rename some ipld methods

* fix up sync issue in mst

* find reachable records form carfile

* getRecord xrpc method

* pr feedback

* better migration test

* check migraiton result

* fixing up a couple things for pg

* explicit migrateTo

* async exceptions

* ipld car mimetype + remove updateRepo

* Update module publish scripts (bluesky-social#478)

* Update pds package publishing scripts

* Update auth package publishing scripts

* Update crypto package publishing scripts

* Update did-resolver package publishing scripts

* Update handle package publishing scripts

* Update xrpc-server package publishing scripts

* Update common package publishing scripts

* Update plc package publishing scripts

* Update uri package publishing scripts

* Update repo package publishing scripts

* Sort "suggested follows" by number of posts (bluesky-social#477)

* return suggestions by post count

* pr feedback

* fix up PG pagination issue

* partiion commit-history & commit-blocks by user did

* fix migration type

* switch pk indexes

Co-authored-by: Paul Frazee <pfrazee@gmail.com>

lexicons/com/atproto/sync/getCheckout.json

@@ -0,0 +1,21 @@
+{
+  "lexicon": 1,
+  "id": "com.atproto.sync.getCheckout",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Gets the repo state.",
+      "parameters": {
+        "type": "params",
+        "required": ["did"],
+        "properties": {
+          "did": {"type": "string", "description": "The DID of the repo."},
+          "commit": {"type": "string", "description": "The commit to get the checkout from. Defaults to current HEAD."}
+        }
+      },
+      "output": {
+        "encoding": "application/vnd.ipld.car"
+      }
+    }
+  }
+}

lexicons/com/atproto/sync/getCommitPath.json

@@ -0,0 +1,32 @@
+{
+  "lexicon": 1,
+  "id": "com.atproto.sync.getCommitPath",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Gets the path of repo commits",
+      "parameters": {
+        "type": "params",
+        "required": ["did"],
+        "properties": {
+          "did": {"type": "string", "description": "The DID of the repo."},
+          "latest": { "type": "string", "description": "The most recent commit"},
+          "earliest": { "type": "string", "description": "The earliest commit to start from"}
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": ["commits"],
+          "properties": {
+            "commits": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          }
+        }
+      }
+    }
+  }
+}

lexicons/com/atproto/sync/getRoot.json → lexicons/com/atproto/sync/getHead.json

@@ -1,10 +1,10 @@
 {
   "lexicon": 1,
-  "id": "com.atproto.sync.getRoot",
+  "id": "com.atproto.sync.getHead",
   "defs": {
     "main": {
       "type": "query",
-      "description": "Gets the current root CID of a repo.",
+      "description": "Gets the current HEAD CID of a repo.",
       "parameters": {
         "type": "params",
         "required": ["did"],

lexicons/com/atproto/sync/getRecord.json

@@ -0,0 +1,23 @@
+{
+  "lexicon": 1,
+  "id": "com.atproto.sync.getRecord",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Gets blocks needed for existence or non-existence of record.",
+      "parameters": {
+        "type": "params",
+        "required": ["did", "collection", "rkey"],
+        "properties": {
+          "did": {"type": "string", "description": "The DID of the repo."},
+          "collection": {"type": "string" },
+          "rkey": {"type": "string" },
+          "commit": {"type": "string", "description": "An optional past commit CID."}
+        }
+      },
+      "output": {
+        "encoding": "application/vnd.ipld.car"
+      }
+    }
+  }
+}

lexicons/com/atproto/sync/getRepo.json

@@ -14,7 +14,7 @@
         }
       },
       "output": {
-        "encoding": "application/cbor"
+        "encoding": "application/vnd.ipld.car"
       }
     }
   }

packages/README.md

@@ -9,7 +9,6 @@
 ## Libraries
 
 - [API](./api): A library for communicating with ATP servers.
-- [Auth](./auth): ATP's core permissioning library (based on UCANs).
 - [Common](./common): A library containing code which is shared between ATP packages.
 - [Crypto](./crypto): ATP's common cryptographic operations.
 - [DID Resolver](./did-resolver): A library for resolving ATP's Decentralized ID methods.

packages/api/src/client/index.ts

@@ -46,9 +46,11 @@ import * as ComAtprotoSessionCreate from './types/com/atproto/session/create'
 import * as ComAtprotoSessionDelete from './types/com/atproto/session/delete'
 import * as ComAtprotoSessionGet from './types/com/atproto/session/get'
 import * as ComAtprotoSessionRefresh from './types/com/atproto/session/refresh'
+import * as ComAtprotoSyncGetCheckout from './types/com/atproto/sync/getCheckout'
+import * as ComAtprotoSyncGetCommitPath from './types/com/atproto/sync/getCommitPath'
+import * as ComAtprotoSyncGetHead from './types/com/atproto/sync/getHead'
+import * as ComAtprotoSyncGetRecord from './types/com/atproto/sync/getRecord'
 import * as ComAtprotoSyncGetRepo from './types/com/atproto/sync/getRepo'
-import * as ComAtprotoSyncGetRoot from './types/com/atproto/sync/getRoot'
-import * as ComAtprotoSyncUpdateRepo from './types/com/atproto/sync/updateRepo'
 import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile'
 import * as AppBskyActorGetSuggestions from './types/app/bsky/actor/getSuggestions'
 import * as AppBskyActorProfile from './types/app/bsky/actor/profile'
@@ -125,9 +127,11 @@ export * as ComAtprotoSessionCreate from './types/com/atproto/session/create'
 export * as ComAtprotoSessionDelete from './types/com/atproto/session/delete'
 export * as ComAtprotoSessionGet from './types/com/atproto/session/get'
 export * as ComAtprotoSessionRefresh from './types/com/atproto/session/refresh'
+export * as ComAtprotoSyncGetCheckout from './types/com/atproto/sync/getCheckout'
+export * as ComAtprotoSyncGetCommitPath from './types/com/atproto/sync/getCommitPath'
+export * as ComAtprotoSyncGetHead from './types/com/atproto/sync/getHead'
+export * as ComAtprotoSyncGetRecord from './types/com/atproto/sync/getRecord'
 export * as ComAtprotoSyncGetRepo from './types/com/atproto/sync/getRepo'
-export * as ComAtprotoSyncGetRoot from './types/com/atproto/sync/getRoot'
-export * as ComAtprotoSyncUpdateRepo from './types/com/atproto/sync/updateRepo'
 export * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile'
 export * as AppBskyActorGetSuggestions from './types/app/bsky/actor/getSuggestions'
 export * as AppBskyActorProfile from './types/app/bsky/actor/profile'
@@ -662,36 +666,58 @@ export class SyncNS {
     this._service = service
   }
 
-  getRepo(
-    params?: ComAtprotoSyncGetRepo.QueryParams,
-    opts?: ComAtprotoSyncGetRepo.CallOptions,
-  ): Promise<ComAtprotoSyncGetRepo.Response> {
+  getCheckout(
+    params?: ComAtprotoSyncGetCheckout.QueryParams,
+    opts?: ComAtprotoSyncGetCheckout.CallOptions,
+  ): Promise<ComAtprotoSyncGetCheckout.Response> {
     return this._service.xrpc
-      .call('com.atproto.sync.getRepo', params, undefined, opts)
+      .call('com.atproto.sync.getCheckout', params, undefined, opts)
       .catch((e) => {
-        throw ComAtprotoSyncGetRepo.toKnownErr(e)
+        throw ComAtprotoSyncGetCheckout.toKnownErr(e)
       })
   }
 
-  getRoot(
-    params?: ComAtprotoSyncGetRoot.QueryParams,
-    opts?: ComAtprotoSyncGetRoot.CallOptions,
-  ): Promise<ComAtprotoSyncGetRoot.Response> {
+  getCommitPath(
+    params?: ComAtprotoSyncGetCommitPath.QueryParams,
+    opts?: ComAtprotoSyncGetCommitPath.CallOptions,
+  ): Promise<ComAtprotoSyncGetCommitPath.Response> {
     return this._service.xrpc
-      .call('com.atproto.sync.getRoot', params, undefined, opts)
+      .call('com.atproto.sync.getCommitPath', params, undefined, opts)
       .catch((e) => {
-        throw ComAtprotoSyncGetRoot.toKnownErr(e)
+        throw ComAtprotoSyncGetCommitPath.toKnownErr(e)
       })
   }
 
-  updateRepo(
-    data?: ComAtprotoSyncUpdateRepo.InputSchema,
-    opts?: ComAtprotoSyncUpdateRepo.CallOptions,
-  ): Promise<ComAtprotoSyncUpdateRepo.Response> {
+  getHead(
+    params?: ComAtprotoSyncGetHead.QueryParams,
+    opts?: ComAtprotoSyncGetHead.CallOptions,
+  ): Promise<ComAtprotoSyncGetHead.Response> {
+    return this._service.xrpc
+      .call('com.atproto.sync.getHead', params, undefined, opts)
+      .catch((e) => {
+        throw ComAtprotoSyncGetHead.toKnownErr(e)
+      })
+  }
+
+  getRecord(
+    params?: ComAtprotoSyncGetRecord.QueryParams,
+    opts?: ComAtprotoSyncGetRecord.CallOptions,
+  ): Promise<ComAtprotoSyncGetRecord.Response> {
     return this._service.xrpc
-      .call('com.atproto.sync.updateRepo', opts?.qp, data, opts)
+      .call('com.atproto.sync.getRecord', params, undefined, opts)
       .catch((e) => {
-        throw ComAtprotoSyncUpdateRepo.toKnownErr(e)
+        throw ComAtprotoSyncGetRecord.toKnownErr(e)
+      })
+  }
+
+  getRepo(
+    params?: ComAtprotoSyncGetRepo.QueryParams,
+    opts?: ComAtprotoSyncGetRepo.CallOptions,
+  ): Promise<ComAtprotoSyncGetRepo.Response> {
+    return this._service.xrpc
+      .call('com.atproto.sync.getRepo', params, undefined, opts)
+      .catch((e) => {
+        throw ComAtprotoSyncGetRepo.toKnownErr(e)
       })
   }
 }