feat: offchain & onchain metadata management

packages/auto-drive/package.json

@@ -4,18 +4,26 @@
   "version": "0.4.0",
   "main": "dist/index.js",
   "type": "module",
+  "installConfig": {
+    "hoistingLimits": "dependencies"
+  },
   "scripts": {
     "build": "tsc",
+    "pb": "BASE_DIR='./src/metadata/onchain/protobuf/' && npx pbjs -t static-module -w es6 -o $BASE_DIR/compiled.js $BASE_DIR/onchainMetadata.proto && npx pbts -o $BASE_DIR/compiled.d.ts $BASE_DIR/compiled.js",
     "clean": "rm -rf dist",
     "format": "prettier --write \"src/**/*.ts\"",
     "test": "jest"
   },
   "devDependencies": {
+    "protobufjs": "^7.4.0",
     "typescript": "^5.6.2"
   },
   "dependencies": {
     "@ipld/dag-pb": "^4.1.2",
     "blake3": "1.1.0",
-    "multiformats": "^13.2.2"
+    "multiformats": "^13.2.2",
+    "protobufjs": "^7.4.0",
+    "protons": "^7.6.0",
+    "protons-runtime": "^5.5.0"
   }
 }

packages/auto-drive/src/index.ts

@@ -1,2 +1,3 @@
 export * from './cid/index.js'
-export * from './fileChunker/index.js'
+export * from './ipld/index.js'
+export * from './metadata/index.js'

packages/auto-drive/src/ipld/chunker.ts

@@ -0,0 +1,66 @@
+import { encode, PBNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats'
+import { cidOfNode } from '../cid/index.js'
+import {
+  createChunkedFileIpldNode,
+  createChunkIpldNode,
+  createSingleFileIpldNode,
+} from './nodes.js'
+import { chunkBuffer, encodeNode } from './utils.js'
+
+const MAX_CHUNK_SIZE = 1024 * 64
+
+export interface IPLDDag {
+  headCID: CID
+  nodes: Map<CID, PBNode>
+}
+
+export const createFileIPLDDag = (
+  file: Buffer,
+  filename?: string,
+  chunkSize: number = MAX_CHUNK_SIZE,
+): IPLDDag => {
+  if (file.length <= chunkSize) {
+    const head = createSingleFileIpldNode(file, BigInt(file.length), filename)
+    const headCID = cidOfNode(head)
+    return {
+      headCID,
+      nodes: new Map([[headCID, head]]),
+    }
+  }
+
+  const bufferChunks = chunkBuffer(file, chunkSize)
+
+  const nodes = new Map<CID, PBNode>()
+
+  const CIDs = bufferChunks.map((chunk) => {
+    const node = createChunkIpldNode(chunk, BigInt(chunk.length))
+    const cid = cidOfNode(node)
+    nodes.set(cid, node)
+
+    return cid
+  })
+
+  const head = createChunkedFileIpldNode(CIDs, BigInt(file.length), filename)
+
+  if (encode(head).length <= chunkSize) {
+    const headCID = cidOfNode(head)
+    nodes.set(headCID, head)
+
+    return {
+      headCID,
+      nodes,
+    }
+  }
+
+  throw new Error('Not implemented support for large chunks')
+}
+
+export const ensureNodeMaxSize = (node: PBNode, maxSize: number = MAX_CHUNK_SIZE): PBNode => {
+  const nodeSize = encodeNode(node).byteLength
+  if (nodeSize > maxSize) {
+    throw new Error(`Node is too large to fit in a single chunk: ${nodeSize} > ${maxSize}`)
+  }
+
+  return node
+}

packages/auto-drive/src/ipld/index.ts

@@ -0,0 +1,3 @@
+export * from './chunker.js'
+export * from './nodes.js'
+export { encodeNode } from './utils.js'

packages/auto-drive/src/ipld/nodes.ts

@@ -0,0 +1,89 @@
+import { createNode, PBNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats/cid'
+import { OffchainMetadata } from '../metadata/index.js'
+import { encodeIPLDNodeData, IPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
+import { ensureNodeMaxSize } from './chunker.js'
+
+/// Creates a chunk ipld node
+export const createChunkIpldNode = (data: Buffer, size: bigint): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FileChunk,
+        size,
+        linkDepth: 0,
+        data,
+      }),
+      [],
+    ),
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createChunkedFileIpldNode = (links: CID[], size: bigint, name?: string): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.File,
+        name,
+        size,
+        linkDepth: 1,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createSingleFileIpldNode = (
+  data: Buffer,
+  size: bigint,
+  name?: string,
+  // fileMetadata: Omit<OffchainFileMetadata, 'size'>,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.File,
+        name,
+        size,
+        linkDepth: 0,
+        data,
+      }),
+      [],
+    ),
+  )
+
+// Creates a folder ipld node
+// links: the CIDs of the folder's contents
+// @todo: add the folder's metadata
+export const createFolderIpldNode = (links: CID[], name: string, size: bigint): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Folder,
+        name,
+        size,
+        linkDepth: 0,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+  )
+
+/// Creates a metadata ipld node
+export const createMetadataNode = (metadata: OffchainMetadata): PBNode => {
+  const data = Buffer.from(JSON.stringify(metadata))
+
+  return ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Metadata,
+        name: metadata.name,
+        linkDepth: 0,
+        data,
+      }),
+    ),
+  )
+}

packages/auto-drive/src/fileChunker/utils.ts → packages/auto-drive/src/ipld/utils.ts

@@ -1,3 +1,5 @@
+import { encode, PBNode } from '@ipld/dag-pb'
+
 export const chunkBuffer = (buffer: Buffer, chunkSize: number) => {
   const chunks: Buffer[] = []
 
@@ -7,3 +9,5 @@ export const chunkBuffer = (buffer: Buffer, chunkSize: number) => {
 
   return chunks
 }
+
+export const encodeNode = (node: PBNode): Buffer => Buffer.from(encode(node))

packages/auto-drive/src/metadata/index.ts

@@ -0,0 +1,2 @@
+export * from './offchain/index.js'
+export * from './onchain/index.js'

packages/auto-drive/src/metadata/offchain/base.ts

@@ -0,0 +1,4 @@
+import { OffchainFileMetadata } from './file.js'
+import { OffchainFolderMetadata } from './folder.js'
+
+export type OffchainMetadata = OffchainFolderMetadata | OffchainFileMetadata

packages/auto-drive/src/metadata/offchain/file.ts

@@ -0,0 +1,36 @@
+import { cidOfNode, cidToString, IPLDDag } from '../../index.js'
+
+export type OffchainFileMetadata = {
+  type: 'file'
+  dataCid: string
+  name?: string
+  mimeType?: string
+  totalSize: number
+  totalChunks: number
+  chunks: ChunkInfo[]
+}
+
+export interface ChunkInfo {
+  size: number
+  cid: string
+}
+
+export const fileMetadata = (
+  dag: IPLDDag,
+  totalSize: number,
+  name?: string,
+  mimeType?: string,
+): OffchainFileMetadata => {
+  return {
+    type: 'file',
+    dataCid: cidToString(dag.headCID),
+    name,
+    mimeType,
+    totalSize,
+    totalChunks: dag.nodes.size,
+    chunks: Array.from(dag.nodes.values()).map((chunk) => ({
+      cid: cidToString(cidOfNode(chunk)),
+      size: chunk.Data?.length ?? 0,
+    })),
+  }
+}

packages/auto-drive/src/metadata/offchain/folder.ts

@@ -0,0 +1,28 @@
+interface ChildrenMetadata {
+  type: 'folder' | 'file'
+  name?: string
+  cid: string
+  totalSize: number
+}
+
+export type OffchainFolderMetadata = {
+  type: 'folder'
+  dataCid: string
+  name?: string
+  totalSize: number
+  totalFiles: number
+  children: ChildrenMetadata[]
+}
+
+export const folderMetadata = (
+  cid: string,
+  children: ChildrenMetadata[],
+): OffchainFolderMetadata => {
+  return {
+    dataCid: cid,
+    totalSize: children.reduce((acc, child) => acc + child.totalSize, 0),
+    totalFiles: children.length,
+    children,
+    type: 'folder',
+  }
+}

packages/auto-drive/src/metadata/offchain/index.ts

@@ -0,0 +1,3 @@
+export * from './base.js'
+export * from './file.js'
+export * from './folder.js'

packages/auto-drive/src/metadata/onchain/index.ts

@@ -0,0 +1,2 @@
+export * from './protobuf/onchainMetadata.js'
+export * from './utils.js'