Merge pull request #103 from autonomys/feat/auto-drive

feat: add auto-drive package

packages/auto-drive/README.md

@@ -0,0 +1 @@
+# auto-drive

packages/auto-drive/jest.config.ts

@@ -0,0 +1,17 @@
+const { createDefaultEsmPreset } = require('ts-jest')
+
+module.exports = {
+  ...createDefaultEsmPreset(),
+  extensionsToTreatAsEsm: ['.ts'],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        useESM: true,
+      },
+    ],
+  },
+}

packages/auto-drive/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@autonomys/auto-drive",
+  "packageManager": "yarn@4.1.1",
+  "version": "0.4.0",
+  "main": "dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/autonomys/auto-sdk"
+  },
+  "author": {
+    "name": "Autonomys",
+    "url": "https://www.autonomys.net"
+  },
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "pb": "yarn protons src/metadata/onchain/protobuf/OnchainMetadata.proto -o src/metadata/onchain/protobuf",
+    "clean": "rm -rf dist",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "test": "yarn node --experimental-vm-modules $(yarn bin jest)"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./protobuf": "./dist/metadata/onchain/protobuf/onchainMetadata.js"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.13",
+    "jest": "^29.7.0",
+    "protobufjs": "^7.4.0",
+    "ts-jest": "^29.2.5",
+    "typescript": "^5.6.2"
+  },
+  "dependencies": {
+    "@ipld/dag-pb": "^4.1.2",
+    "blake3": "1.1.0",
+    "multiformats": "^13.2.2",
+    "protobufjs": "^7.4.0",
+    "protons": "^7.6.0",
+    "protons-runtime": "^5.5.0"
+  }
+}

packages/auto-drive/src/cid/index.ts

@@ -0,0 +1,26 @@
+import { encode, PBNode } from '@ipld/dag-pb'
+import { hash } from 'blake3'
+import { CID } from 'multiformats/cid'
+import * as base32 from 'multiformats/bases/base32'
+import * as raw from 'multiformats/codecs/raw'
+import { create } from 'multiformats/hashes/digest'
+
+export const BLAKE3_CODE = 0x1f
+
+export const cidOfNode = (node: PBNode) => {
+  return cidFromBlakeHash(hash(encode(node)))
+}
+
+export const cidToString = (cid: CID) => {
+  return cid.toString(base32.base32)
+}
+
+export const stringToCid = (str: string) => {
+  return CID.parse(str, base32.base32)
+}
+
+export const cidFromBlakeHash = (hash: Buffer) => {
+  return CID.create(1, raw.code, create(BLAKE3_CODE, hash))
+}
+
+export const blake3HashFromCid = (cid: CID) => cid.multihash.digest

packages/auto-drive/src/index.ts

@@ -0,0 +1,3 @@
+export * from './cid/index.js'
+export * from './ipld/index.js'
+export * from './metadata/index.js'

packages/auto-drive/src/ipld/chunker.ts

@@ -0,0 +1,117 @@
+import { PBNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats'
+import { cidOfNode } from '../cid/index.js'
+import {
+  createChunkedFileIpldNode,
+  createChunkIpldNode,
+  createFileInlinkIpldNode,
+  createFolderInlinkIpldNode,
+  createFolderIpldNode,
+  createSingleFileIpldNode,
+} from './nodes.js'
+import { chunkBuffer, encodeNode } from './utils.js'
+
+export const DEFAULT_MAX_CHUNK_SIZE = 1024 * 64
+export const DEFAULT_MAX_LINK_PER_NODE = DEFAULT_MAX_CHUNK_SIZE / 64
+
+export interface IPLDDag {
+  headCID: CID
+  nodes: Map<CID, PBNode>
+}
+
+export const createFileIPLDDag = (
+  file: Buffer,
+  filename?: string,
+  { chunkSize, maxLinkPerNode }: { chunkSize: number; maxLinkPerNode: number } = {
+    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+  },
+): IPLDDag => {
+  if (file.length <= chunkSize) {
+    const head = createSingleFileIpldNode(file, filename)
+    const headCID = cidOfNode(head)
+    return {
+      headCID,
+      nodes: new Map([[headCID, head]]),
+    }
+  }
+
+  const bufferChunks = chunkBuffer(file, chunkSize)
+
+  const nodes = new Map<CID, PBNode>()
+
+  let CIDs: CID[] = bufferChunks.map((chunk) => {
+    const node = createChunkIpldNode(chunk)
+    const cid = cidOfNode(node)
+    nodes.set(cid, node)
+
+    return cid
+  })
+
+  let depth = 1
+  while (CIDs.length > maxLinkPerNode) {
+    const newCIDs: CID[] = []
+    for (let i = 0; i < CIDs.length; i += maxLinkPerNode) {
+      const chunk = CIDs.slice(i, i + maxLinkPerNode)
+
+      const node = createFileInlinkIpldNode(chunk, chunk.length, depth, chunkSize)
+      const cid = cidOfNode(node)
+      nodes.set(cid, node)
+      newCIDs.push(cid)
+    }
+    depth++
+    CIDs = newCIDs
+  }
+  const head = createChunkedFileIpldNode(CIDs, file.length, depth, filename, chunkSize)
+  const headCID = cidOfNode(head)
+  nodes.set(headCID, head)
+
+  return {
+    headCID,
+    nodes,
+  }
+}
+
+export const createFolderIPLDDag = (
+  children: CID[],
+  name: string,
+  size: number,
+  { maxLinkPerNode }: { maxLinkPerNode: number } = { maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE },
+): IPLDDag => {
+  const nodes = new Map<CID, PBNode>()
+  let cids = children
+  let depth = 0
+  while (cids.length > maxLinkPerNode) {
+    const newCIDs: CID[] = []
+    for (let i = 0; i < cids.length; i += maxLinkPerNode) {
+      const chunk = cids.slice(i, i + maxLinkPerNode)
+      const node = createFolderInlinkIpldNode(chunk, depth)
+      const cid = cidOfNode(node)
+      nodes.set(cid, node)
+      newCIDs.push(cid)
+    }
+    cids = newCIDs
+    depth++
+  }
+
+  const node = createFolderIpldNode(cids, name, depth, size)
+  const cid = cidOfNode(node)
+  nodes.set(cid, node)
+
+  return {
+    headCID: cid,
+    nodes,
+  }
+}
+
+export const ensureNodeMaxSize = (
+  node: PBNode,
+  maxSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode => {
+  const nodeSize = encodeNode(node).byteLength
+  if (nodeSize > maxSize) {
+    throw new Error(`Node is too large to fit in a single chunk: ${nodeSize} > ${maxSize}`)
+  }
+
+  return node
+}

packages/auto-drive/src/ipld/index.ts

@@ -0,0 +1,3 @@
+export * from './chunker.js'
+export * from './nodes.js'
+export { encodeNode } from './utils.js'

packages/auto-drive/src/ipld/nodes.ts

@@ -0,0 +1,134 @@
+import { createNode, PBNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats/cid'
+import { OffchainMetadata } from '../metadata/index.js'
+import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
+import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'
+
+/// Creates a chunk ipld node
+export const createChunkIpldNode = (data: Buffer): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.FileChunk,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+    [],
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createChunkedFileIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  name?: string,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.File,
+        name,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createFileInlinkIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FileInlink,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createSingleFileIpldNode = (data: Buffer, name?: string): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.File,
+      name,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+    [],
+  )
+
+// Creates a folder ipld node
+// links: the CIDs of the folder's contents
+// @todo: add the folder's metadata
+export const createFolderIpldNode = (
+  links: CID[],
+  name: string,
+  linkDepth: number,
+  size: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Folder,
+        name,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+export const createFolderInlinkIpldNode = (
+  links: CID[],
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FolderInlink,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+/// Creates a metadata ipld node
+export const createMetadataNode = (
+  metadata: OffchainMetadata,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode => {
+  const data = Buffer.from(JSON.stringify(metadata))
+
+  return ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Metadata,
+        name: metadata.name,
+        linkDepth: 0,
+        data,
+      }),
+    ),
+    maxNodeSize,
+  )
+}

packages/auto-drive/src/ipld/utils.ts

@@ -0,0 +1,15 @@
+import { decode, encode, PBNode } from '@ipld/dag-pb'
+
+export const chunkBuffer = (buffer: Buffer, chunkSize: number) => {
+  const chunks: Buffer[] = []
+
+  for (let i = 0; i < buffer.length; i += chunkSize) {
+    chunks.push(Buffer.from(buffer.buffer.slice(i, i + chunkSize)))
+  }
+
+  return chunks
+}
+
+export const encodeNode = (node: PBNode): Buffer => Buffer.from(encode(node))
+
+export const decodeNode = (data: Uint8Array): PBNode => decode(data)

packages/auto-drive/src/metadata/index.ts

@@ -0,0 +1,2 @@
+export * from './offchain/index.js'
+export * from './onchain/index.js'

packages/auto-drive/src/metadata/offchain/base.ts

@@ -0,0 +1,4 @@
+import { OffchainFileMetadata } from './file.js'
+import { OffchainFolderMetadata } from './folder.js'
+
+export type OffchainMetadata = OffchainFolderMetadata | OffchainFileMetadata