feat: versioned protobufs

examples/schema_test.js

@@ -4,18 +4,90 @@ import Hypercore from 'hypercore'
 import ram from 'random-access-memory'
 import { randomBytes } from 'node:crypto'
 
+// FILTER_1
+// const obj = {
+//   id: randomBytes(32).toString('hex'),
+//   type: 'filter',
+//   schemaVersion: 1,
+//   created_at: new Date().toJSON(),
+//   filter: ['observation'],
+//   name: 'john',
+// }
+
+// DEVICE
+// const obj = {
+//   type: 'Device',
+//   schemaVersion: 1,
+//   id: randomBytes(32).toString('hex'),
+//   action: 'device:add',
+//   authorId: randomBytes(32).toString('hex'),
+//   projectId: randomBytes(32).toString('hex'),
+//   signature: 'hi',
+//   authorIndex: 10,
+//   deviceIndex: 10,
+// }
+
+// ROLE
 const obj = {
   id: randomBytes(32).toString('hex'),
-  type: 'Observation',
-  schemaVersion: 5,
-  links: [],
-  created_at: new Date().toJSON(),
-  refs: [],
-  attachments: [],
-  metadata: {
-    manual_location: true,
-  },
+  type: 'Role',
+  schemaVersion: 1,
+  role: 'project-creator',
+  created_at: new Date(),
+  projectId: randomBytes(32).toString('hex'),
+  action: 'role:set',
+  signature: 'hi',
+  authorIndex: 10,
+  deviceIndex: 10,
 }
+
+// CORE OWNERSHIP
+// const obj = {
+//   type: 'coreOwnership',
+//   schemaVersion: 1,
+//   id: randomBytes(32).toString('hex'),
+//   coreId: randomBytes(32).toString('hex'),
+//   projectId: randomBytes(32).toString('hex'),
+//   storeType: 'blob',
+//   authorIndex: 10,
+//   deviceIndex: 10,
+//   action: 'core:owner',
+// }
+
+// PRESET_1
+// const obj = {
+//   id: randomBytes(32).toString('hex'),
+//   type: 'Preset',
+//   schemaVersion: 1,
+//   tags: { nature: 'tree' },
+//   geometry: ['point'],
+//   name: 'john',
+// }
+
+// FIELD_1
+// const obj = {
+//   id: randomBytes(32).toString('hex'),
+//   type: 'Field',
+//   schemaVersion: 1,
+//   key: 'hi',
+// }
+
+// OBSERVATION 4
+// const obj = {
+//   id: randomBytes(32).toString('hex'),
+//   type: 'observation',
+//   schemaVersion: 4,
+//   created_at: new Date().toJSON(),
+// }
+
+// OBSERVATION 5
+// const obj = {
+//   id: randomBytes(32).toString('hex'),
+//   type: 'Observation',
+//   schemaVersion: 5,
+//   created_at: new Date().toJSON(),
+// }
+
 const record = encode(obj)
 
 const core = new Hypercore(ram, { valueEncoding: 'binary' })
@@ -26,7 +98,7 @@ try {
   const index = 0
   const data = await core.get(index)
   const decodedData = decode(data, { coreId: core.key, seq: index })
-  console.log('decoded data', decodedData)
+  console.log('decoded', decodedData)
   console.log('VALID?', validate(decodedData))
   if (Buffer.compare(data, record) !== 0) {
     throw new Error(`data doesn't match: ${data} != ${record}`)

index.js

@@ -6,6 +6,7 @@ import * as cenc from 'compact-encoding'
 import * as JSONSchemas from './dist/schemas.js'
 import * as ProtobufSchemas from './types/proto/index.js'
 import schemasPrefix from './schemasPrefix.js'
+import { inheritsFromCommon, formatSchemaKey } from './utils.js'
 
 const dataTypeIdSize = 6
 const schemaVersionSize = 2
@@ -27,25 +28,24 @@ const jsonSchemaToProto = (obj) => {
   /** @type {Object} */
   const uncommon = Object.keys(obj)
     .filter((k) => !commonKeys.includes(k))
-    .reduce((acc, k) => {
-      acc[k] = obj[k]
-      return acc
-    }, {})
-
-  const common = commonKeys.reduce((common, field) => {
-    if (obj[field]) {
-      if (field === 'id') {
-        common[field] = Buffer.from(obj[field], 'hex')
-      } else {
-        common[field] = obj[field]
-      }
-    }
-    return common
-  }, {})
-  return {
-    ...uncommon,
-    common,
-  }
+    .reduce((uncommon, field) => ({ ...uncommon, [field]: obj[field] }), {})
+
+  /** @type {Object} */
+  const common = commonKeys
+    .filter((field) => obj[field])
+    .reduce((common, field) => ({ ...common, [field]: obj[field] }), {})
+
+  common.id = Buffer.from(obj['id'], 'hex')
+  // turn date represented as string to Date
+  common.created_at = new Date(common.created_at)
+  common.timestamp = new Date(common.timestamp)
+
+  const key = formatSchemaKey(obj.type, obj.schemaVersion)
+  // when we inherit from common, common is actually a field inside the protobuf object,
+  // so we don't destructure it
+  return inheritsFromCommon(key)
+    ? { ...uncommon, common }
+    : { ...uncommon, ...common }
 }
 
 /**
@@ -57,23 +57,31 @@ const jsonSchemaToProto = (obj) => {
  * @returns {import('./types/schema/index').MapeoRecord}
  */
 const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => {
-  const common = protobufObj.common
-  delete protobufObj.common
-  return {
-    ...protobufObj,
-    ...common,
-    schemaVersion,
-    type,
-    version,
-    id: common ? common.id.toString('hex') : '',
+  /** @type {Object} */
+  let obj = { ...protobufObj, schemaVersion, type }
+  if (obj.common) {
+    obj = { ...obj, ...obj.common }
+    delete obj.common
+  }
+
+  // Preset_1 and Field_1 don't have a version field and don't accept additional fields
+  const key = formatSchemaKey(type, schemaVersion)
+  if (key !== 'Preset_1' && key !== 'Field_1') {
+    obj.version = version
   }
+
+  obj.id = obj.id.toString('hex')
+  // turn date represented as Date to string
+  if (obj.created_at) obj.created_at = obj.created_at.toJSON()
+  if (obj.timestamp) obj.timestamp = obj.timestamp.toJSON()
+  return obj
 }
 
 /**
  * given a schemaVersion and type, return a buffer with the corresponding data
  * @param {Object} obj
  * @param {string} obj.dataTypeId hex encoded string of a 6-byte buffer indicating type
- * @param {number} obj.schemaVersion number to indicate version. Gets converted to a padded 4-byte hex string
+ * @param {number | undefined} obj.schemaVersion number to indicate version. Gets converted to a padded 4-byte hex string
  * @returns {Buffer} blockPrefix for corresponding schema
  */
 export const encodeBlockPrefix = ({ dataTypeId, schemaVersion }) => {
@@ -95,7 +103,7 @@ export const decodeBlockPrefix = (buf) => {
   state.buffer = buf
   state.start = 0
   state.end = dataTypeIdSize
-  const dataTypeId = cenc.hex.fixed(6).decode(state)
+  const dataTypeId = cenc.hex.fixed(dataTypeIdSize).decode(state)
 
   state.start = dataTypeIdSize
   state.end = dataTypeIdSize + schemaVersionSize
@@ -110,7 +118,14 @@ export const decodeBlockPrefix = (buf) => {
  * @returns {Boolean} indicating if the object is valid
  */
 export const validate = (obj) => {
-  const key = `${obj.type.toLowerCase()}_${obj.schemaVersion}`
+  const key = formatSchemaKey(obj.type, obj.schemaVersion)
+
+  // Preset_1 doesn't have a type field, so validation won't pass
+  // but we still need it to now which schema to validate, so we delete it after grabbing the key
+  if (key === 'Preset_1') delete obj['type']
+  // Field_1 doesn't have a schemaVersion field, so validation won't pass
+  // but we still need it to now which schema to validate, so we delete it after grabbing the key
+  if (key === 'Field_1') delete obj['schemaVersion']
 
   const validatefn = JSONSchemas[key]
   const isValid = validatefn(obj)
@@ -123,12 +138,22 @@ export const validate = (obj) => {
  * @param {import('./types/schema/index').MapeoRecord} obj - Object to be encoded
  * @returns {Buffer} protobuf encoded buffer with dataTypeIdSize + schemaVersionSize bytes prepended, one for the type of record and the other for the version of the schema */
 export const encode = (obj) => {
+  const key = formatSchemaKey(obj.type, obj.schemaVersion)
+  // some schemas don't have type field so it can be undefined
+  const type = obj.type || ''
+  if (!ProtobufSchemas[key]) {
+    throw new Error(
+      `Invalid schemaVersion for ${type} version ${obj.schemaVersion}`
+    )
+  }
+
   const blockPrefix = encodeBlockPrefix({
-    dataTypeId: schemasPrefix[formatSchemaType(obj.type)],
-    schemaVersion: obj.schemaVersion === undefined ? 0 : obj.schemaVersion,
+    dataTypeId: schemasPrefix[type].dataTypeId,
+    schemaVersion: obj.schemaVersion,
   })
   const record = jsonSchemaToProto(obj)
-  const protobuf = ProtobufSchemas[formatSchemaType(obj.type)].encode(record).finish()
+  const partial = ProtobufSchemas[key].fromPartial(record)
+  const protobuf = ProtobufSchemas[key].encode(partial).finish()
   return Buffer.concat([blockPrefix, protobuf])
 }
 
@@ -139,20 +164,20 @@ export const encode = (obj) => {
  * */
 export const decode = (buf, { coreId, seq }) => {
   const { dataTypeId, schemaVersion } = decodeBlockPrefix(buf)
-  const type = Object.keys(schemasPrefix).filter(
-    (key) => schemasPrefix[key] === dataTypeId
-  )[0]
+  const type = Object.keys(schemasPrefix).reduce(
+    (type, key) => (schemasPrefix[key].dataTypeId === dataTypeId ? key : type),
+    ''
+  )
+  const key = formatSchemaKey(type, schemaVersion)
+  if (!ProtobufSchemas[key]) {
+    throw new Error(
+      `Invalid schemaVersion for ${type} version ${schemaVersion}`
+    )
+  }
+
   const version = `${coreId.toString('hex')}/${seq.toString()}`
   const record = buf.subarray(dataTypeIdSize + schemaVersionSize, buf.length)
-  const protobufObj = ProtobufSchemas[type].decode(record)
-  return protoToJsonSchema(protobufObj, { schemaVersion, type, version })
-}
 
-/**
- * Format schema type string to match protobuf/schema prefix type lookups
- * @param {String} text
- * @returns {String} First letter capitalized, the rest lowercased
- */ 
-function formatSchemaType(text) {
-  return text.charAt(0).toUpperCase() + text.slice(1).toLowerCase()
+  const protobufObj = ProtobufSchemas[key].decode(record)
+  return protoToJsonSchema(protobufObj, { schemaVersion, type, version })
 }

proto/common.proto → proto/common/v1.proto

@@ -1,13 +1,15 @@
 syntax = "proto3";
 package mapeo;
 
-message Common {
-  string created_at = 1;
+import "google/protobuf/timestamp.proto";
+
+message Common_1 {
+  google.protobuf.Timestamp created_at = 1;
   optional string deviceId = 2;
   // 32-byte random generated number
   bytes id = 3;
   repeated string links = 4;
-  optional string timestamp = 5;
+  optional google.protobuf.Timestamp timestamp = 5;
   optional string userId = 6;
 }
 /* ignored fields and differences from common.json jsonSchema

proto/coreOwnership/v1.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+package mapeo;
+
+import "google/protobuf/struct.proto";
+import "common/v1.proto";
+
+message CoreOwnership_1 {
+  Common_1 common = 1;
+  string action = 2;
+  string coreId = 3;
+  string projectId = 4;
+  string storeType = 5;
+  string signature = 6;
+  int32 authorIndex = 7;
+  int32 deviceIndex = 8;
+}

proto/device/v1.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+package mapeo;
+
+import "google/protobuf/struct.proto";
+import "common/v1.proto";
+
+message Device_1 {
+  Common_1 common = 1;
+  string action = 2;
+  string authorId = 3;
+  string projectId = 4;
+  string signature = 5;
+  int32 authorIndex = 6;
+  int32 deviceIndex = 7;
+}

proto/field/v1.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+package mapeo;
+
+import "google/protobuf/any.proto";
+
+message Field_1 {
+  bytes id = 1;
+  // keys can be an array of strings or a string
+  google.protobuf.Any key = 2;
+}

proto/filter/v1.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+package mapeo;
+
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+message Filter_1 {
+  bytes id = 1;
+  google.protobuf.Timestamp created_at = 2;
+  repeated string  filter = 3;
+  string name = 4;
+  optional google.protobuf.Timestamp timestamp = 5;
+  optional string userId = 6;
+  optional string deviceId = 7;
+  repeated string links = 8;
+}