From daa21185d899a10224a616fe70fa91083ce5d0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Tue, 14 Feb 2023 11:19:10 -0300 Subject: [PATCH 01/18] feat: minor refactoring. move formatSchemaType to utils.js --- index.js | 28 ++++++++++++---------------- scripts/generate.js | 15 +++++---------- utils.js | 7 +++++++ 3 files changed, 24 insertions(+), 26 deletions(-) create mode 100644 utils.js diff --git a/index.js b/index.js index be0b1140..e8c052e3 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,7 @@ import * as cenc from 'compact-encoding' import * as JSONSchemas from './dist/schemas.js' import * as ProtobufSchemas from './types/proto/index.js' import schemasPrefix from './schemasPrefix.js' +import { formatSchemaType } from './utils.js' const dataTypeIdSize = 6 const schemaVersionSize = 2 @@ -95,7 +96,7 @@ export const decodeBlockPrefix = (buf) => { state.buffer = buf state.start = 0 state.end = dataTypeIdSize - const dataTypeId = cenc.hex.fixed(6).decode(state) + const dataTypeId = cenc.hex.fixed(dataTypeIdSize).decode(state) state.start = dataTypeIdSize state.end = dataTypeIdSize + schemaVersionSize @@ -124,11 +125,14 @@ export const validate = (obj) => { * @returns {Buffer} protobuf encoded buffer with dataTypeIdSize + schemaVersionSize bytes prepended, one for the type of record and the other for the version of the schema */ export const encode = (obj) => { const blockPrefix = encodeBlockPrefix({ - dataTypeId: schemasPrefix[formatSchemaType(obj.type)], - schemaVersion: obj.schemaVersion === undefined ? 0 : obj.schemaVersion, + dataTypeId: schemasPrefix[formatSchemaType(obj.type)].dataTypeId, + // TODO: how to handle wrong/missing schemaVersions? + schemaVersion: obj.schemaVersion === undefined ? 1 : obj.schemaVersion, }) const record = jsonSchemaToProto(obj) - const protobuf = ProtobufSchemas[formatSchemaType(obj.type)].encode(record).finish() + const protobuf = ProtobufSchemas[formatSchemaType(obj.type)] + .encode(record) + .finish() return Buffer.concat([blockPrefix, protobuf]) } @@ -139,20 +143,12 @@ export const encode = (obj) => { * */ export const decode = (buf, { coreId, seq }) => { const { dataTypeId, schemaVersion } = decodeBlockPrefix(buf) - const type = Object.keys(schemasPrefix).filter( - (key) => schemasPrefix[key] === dataTypeId - )[0] + const type = Object.keys(schemasPrefix).reduce( + (type, key) => (schemasPrefix[key].dataTypeId === dataTypeId ? key : type), + '' + ) const version = `${coreId.toString('hex')}/${seq.toString()}` const record = buf.subarray(dataTypeIdSize + schemaVersionSize, buf.length) const protobufObj = ProtobufSchemas[type].decode(record) return protoToJsonSchema(protobufObj, { schemaVersion, type, version }) } - -/** - * Format schema type string to match protobuf/schema prefix type lookups - * @param {String} text - * @returns {String} First letter capitalized, the rest lowercased - */ -function formatSchemaType(text) { - return text.charAt(0).toUpperCase() + text.slice(1).toLowerCase() -} diff --git a/scripts/generate.js b/scripts/generate.js index eb7732b0..5ec6d601 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -11,15 +11,10 @@ import { URL } from 'url' import Ajv from 'ajv' import standaloneCode from 'ajv/dist/standalone/index.js' import glob from 'glob-promise' +import { formatSchemaType } from '../utils.js' const __dirname = new URL('.', import.meta.url).pathname -/** - * @param {String} str - * @returns {String} - */ -const capitalize = (str) => str.charAt(0).toUpperCase() + str.slice(1) - /** * @param {string} path * @returns {Object} @@ -69,7 +64,7 @@ ${schemas (schema) => { const version = schema.properties.schemaVersion.enum const varName = `${schema.title.toLowerCase()}_${version ? version : 0}` - return `import { ${capitalize( + return `import { ${formatSchemaType( schema.title )} as ${varName} } from './${schema.title.toLowerCase()}/v${ version || 1 @@ -105,15 +100,15 @@ const linesdts = [] const union = protobufFiles .filter((f) => !f.match(/.d/)) .filter((f) => f !== 'index.js') - .map((f) => capitalize(path.parse(f).name)) + .map((f) => formatSchemaType(path.parse(f).name)) .join(' & ') protobufFiles .filter((f) => !f.match(/.d/)) .map((f) => { const name = path.parse(f).name - const linejs = `export { ${capitalize(name)} } from './${name}.js'` - const linets = `import { ${capitalize(name)} } from './${name}'` + const linejs = `export { ${formatSchemaType(name)} } from './${name}.js'` + const linets = `import { ${formatSchemaType(name)} } from './${name}'` linesdts.push(linets) linesjs.push(linejs) }) diff --git a/utils.js b/utils.js new file mode 100644 index 00000000..6b812b0f --- /dev/null +++ b/utils.js @@ -0,0 +1,7 @@ +/** + * Format schema type string to match protobuf/schema prefix type lookups + * @param {String} str + * @returns {String} First letter capitalized, the rest lowercased + */ +export const formatSchemaType = (str) => + str.charAt(0).toUpperCase() + str.slice(1).toLowerCase() From d2ae5375ff5b1b438621f52d8788330ae7825d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Thu, 16 Feb 2023 14:04:37 -0300 Subject: [PATCH 02/18] chore: update schemasPrefix.js --- schemasPrefix.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schemasPrefix.js b/schemasPrefix.js index c652d176..245c8165 100644 --- a/schemasPrefix.js +++ b/schemasPrefix.js @@ -1,4 +1,4 @@ -const schemasDataTypeId = { - Observation: '71f85084cb94', +const schemasPrefix = { + Observation: { dataTypeId: '71f85084cb94', schemaVersions: [4, 5] }, } -export default schemasDataTypeId +export default schemasPrefix From cae214d46730bb7b75871b762445c90b765ec4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Tue, 21 Feb 2023 13:44:35 -0300 Subject: [PATCH 03/18] feat: version protobufs. contemplate invalid schemaVersion on tests --- examples/schema_test.js | 5 +- index.js | 46 ++++++++++++++---- proto/{common.proto => common/v1.proto} | 2 +- proto/observation/v4.proto | 47 +++++++++++++++++++ .../v5.proto} | 6 +-- schema/observation/v5.json | 2 +- scripts/generate.js | 41 ++++++++++------ test/index.js | 14 +++--- 8 files changed, 126 insertions(+), 37 deletions(-) rename proto/{common.proto => common/v1.proto} (96%) create mode 100644 proto/observation/v4.proto rename proto/{observation.proto => observation/v5.proto} (87%) diff --git a/examples/schema_test.js b/examples/schema_test.js index 253e44b3..8aad24ec 100644 --- a/examples/schema_test.js +++ b/examples/schema_test.js @@ -8,10 +8,11 @@ const obj = { id: randomBytes(32).toString('hex'), type: 'Observation', schemaVersion: 5, - links: [], created_at: new Date().toJSON(), + links: [], refs: [], attachments: [], + tags: {}, metadata: { manual_location: true, }, @@ -26,7 +27,7 @@ try { const index = 0 const data = await core.get(index) const decodedData = decode(data, { coreId: core.key, seq: index }) - console.log('decoded data', decodedData) + console.log('decoded', decodedData) console.log('VALID?', validate(decodedData)) if (Buffer.compare(data, record) !== 0) { throw new Error(`data doesn't match: ${data} != ${record}`) diff --git a/index.js b/index.js index e8c052e3..74423f46 100644 --- a/index.js +++ b/index.js @@ -7,6 +7,7 @@ import * as JSONSchemas from './dist/schemas.js' import * as ProtobufSchemas from './types/proto/index.js' import schemasPrefix from './schemasPrefix.js' import { formatSchemaType } from './utils.js' +import { and } from 'ajv/dist/compile/codegen/index.js' const dataTypeIdSize = 6 const schemaVersionSize = 2 @@ -43,6 +44,15 @@ const jsonSchemaToProto = (obj) => { } return common }, {}) + + const key = `${formatSchemaType(obj.type)}_${obj.schemaVersion}` + // TODO, match for every schema that doesn't inherit common/v1.json + if (key === 'Observation_4') { + return { + ...uncommon, + ...common, + } + } return { ...uncommon, common, @@ -58,14 +68,26 @@ const jsonSchemaToProto = (obj) => { * @returns {import('./types/schema/index').MapeoRecord} */ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { - const common = protobufObj.common - delete protobufObj.common - return { + const key = `${formatSchemaType(type)}_${schemaVersion}` + const obj = { ...protobufObj, - ...common, schemaVersion, type, version, + } + // TODO, match for every schema that doesn't inherit common/v1.json + if (key === 'Observation_4') { + return { + ...obj, + id: obj.id.toString('hex'), + type: obj.type.toLowerCase(), + } + } + const common = protobufObj.common + delete obj.common + return { + ...obj, + ...common, id: common ? common.id.toString('hex') : '', } } @@ -124,13 +146,20 @@ export const validate = (obj) => { * @param {import('./types/schema/index').MapeoRecord} obj - Object to be encoded * @returns {Buffer} protobuf encoded buffer with dataTypeIdSize + schemaVersionSize bytes prepended, one for the type of record and the other for the version of the schema */ export const encode = (obj) => { + const key = `${formatSchemaType(obj.type)}_${obj.schemaVersion}` + if (!ProtobufSchemas[key]) { + throw new Error( + `Invalid schemaVersion for ${obj.type} version ${obj.schemaVersion}` + ) + } const blockPrefix = encodeBlockPrefix({ dataTypeId: schemasPrefix[formatSchemaType(obj.type)].dataTypeId, - // TODO: how to handle wrong/missing schemaVersions? - schemaVersion: obj.schemaVersion === undefined ? 1 : obj.schemaVersion, + schemaVersion: obj.schemaVersion, }) const record = jsonSchemaToProto(obj) - const protobuf = ProtobufSchemas[formatSchemaType(obj.type)] + const protobuf = ProtobufSchemas[ + `${formatSchemaType(obj.type)}_${obj.schemaVersion}` + ] .encode(record) .finish() return Buffer.concat([blockPrefix, protobuf]) @@ -149,6 +178,7 @@ export const decode = (buf, { coreId, seq }) => { ) const version = `${coreId.toString('hex')}/${seq.toString()}` const record = buf.subarray(dataTypeIdSize + schemaVersionSize, buf.length) - const protobufObj = ProtobufSchemas[type].decode(record) + const key = `${formatSchemaType(type)}_${schemaVersion}` + const protobufObj = ProtobufSchemas[key].decode(record) return protoToJsonSchema(protobufObj, { schemaVersion, type, version }) } diff --git a/proto/common.proto b/proto/common/v1.proto similarity index 96% rename from proto/common.proto rename to proto/common/v1.proto index 2191c441..504b5d07 100644 --- a/proto/common.proto +++ b/proto/common/v1.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package mapeo; -message Common { +message Common_1 { string created_at = 1; optional string deviceId = 2; // 32-byte random generated number diff --git a/proto/observation/v4.proto b/proto/observation/v4.proto new file mode 100644 index 00000000..658a8223 --- /dev/null +++ b/proto/observation/v4.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; +package mapeo; + +import "google/protobuf/struct.proto"; + +message Observation_4 { + bytes id = 1; + string created_at = 4; + optional string timestamp = 5; + optional string userId = 6; + optional string deviceId = 7; + repeated string links = 8; + optional float lat = 9; + optional float lon = 10; + repeated google.protobuf.Struct refs = 11; + repeated google.protobuf.Struct attachments = 12; + google.protobuf.Struct tags = 13; + message Metadata { + optional bool manual_location = 1; + + message Position { + float timestamp = 1; + bool mocked = 2; + + message Coords { + float altitude = 1; + float heading = 2; + float longitude = 3; + float latitude = 4; + float speed = 5; + float acurracy = 6; + } + optional Coords coords = 3; + } + + message PositionProvider { + bool gpsAvailable = 1; + bool passiveAvailable = 2; + bool locationServicesEnabled = 3; + bool networkAvailable = 4; + } + optional Position position = 3; + optional Position lastSavedPosition = 4; + optional PositionProvider positionProvider = 5; + } + optional Metadata metadata = 14; +} diff --git a/proto/observation.proto b/proto/observation/v5.proto similarity index 87% rename from proto/observation.proto rename to proto/observation/v5.proto index 0b596da0..bfa9e77d 100644 --- a/proto/observation.proto +++ b/proto/observation/v5.proto @@ -2,10 +2,10 @@ syntax = "proto3"; package mapeo; import "google/protobuf/struct.proto"; -import "common.proto"; +import "common/v1.proto"; -message Observation { - Common common = 1; +message Observation_5 { + Common_1 common = 1; optional float lat = 2; optional float lon = 3; diff --git a/schema/observation/v5.json b/schema/observation/v5.json index 12e04c9f..0a392d19 100644 --- a/schema/observation/v5.json +++ b/schema/observation/v5.json @@ -51,7 +51,7 @@ "type": { "description": "Must be `observation`", "type": "string", - "enum": ["observation"] + "enum": ["Observation"] }, "schemaVersion": { "description": "Version of this schema. Should increment for breaking changes to the schema", diff --git a/scripts/generate.js b/scripts/generate.js index 5ec6d601..686452c3 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -94,24 +94,37 @@ fs.writeFileSync( ) // generate index.js for protobuf schemas and index.d.ts -const protobufFiles = glob.sync('*.ts', { cwd: 'types/proto' }) +const protobufFiles = glob.sync('../types/proto/*/*.ts', { cwd: 'scripts' }) +const obj = protobufFiles + .filter((f) => !f.match(/.d/)) + .map((p) => { + const arr = p.split('/') + return { + type: arr[arr.length - 2], + version: arr[arr.length - 1].split('.')[0], + } + }) + const linesjs = [] const linesdts = [] -const union = protobufFiles - .filter((f) => !f.match(/.d/)) - .filter((f) => f !== 'index.js') - .map((f) => formatSchemaType(path.parse(f).name)) +const union = obj + .map((t) => `${formatSchemaType(t.type)}_${t.version.replace('v', '')}`) .join(' & ') -protobufFiles - .filter((f) => !f.match(/.d/)) - .map((f) => { - const name = path.parse(f).name - const linejs = `export { ${formatSchemaType(name)} } from './${name}.js'` - const linets = `import { ${formatSchemaType(name)} } from './${name}'` - linesdts.push(linets) - linesjs.push(linejs) - }) +obj.forEach((f) => { + const linejs = `export { ${formatSchemaType(f.type)}_${f.version.replace( + 'v', + '' + )} } from './${f.type}/${f.version}.js'` + + const linedts = `import { ${formatSchemaType(f.type)}_${f.version.replace( + 'v', + '' + )} } from './${f.type}/${f.version}'` + linesdts.push(linedts) + linesjs.push(linejs) +}) + fs.writeFileSync( path.join(__dirname, '../types/proto/index.js'), linesjs.join('\n') diff --git a/test/index.js b/test/index.js index 11a22c6b..c8114293 100644 --- a/test/index.js +++ b/test/index.js @@ -34,14 +34,12 @@ test('test encoding of wrong record type', async (t) => { }) }) -// TODO: handle bad schema versions in encode function -// test('test encoding of record with wrong schema version', async (t) => { -// t.plan(1) -// console.log(docs.badSchemaVersion, encode(docs.badSchemaVersion)) -// t.throws(() => { -// encode(docs.badSchemaVersion) -// }) -// }) +test('test encoding of record with wrong schema version', async (t) => { + t.plan(1) + t.throws(() => { + encode(docs.badSchemaVersion) + }) +}) test('test encoding of rightfully formated record', async (t) => { t.plan(1) From 3e33597fb9577ded75f5daab3cb171d31a2865d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Tue, 21 Feb 2023 13:54:29 -0300 Subject: [PATCH 04/18] feat: fromPartial on encoding to allow optional lists on input --- examples/schema_test.js | 8 ++------ index.js | 7 ++----- proto/observation/v4.proto | 2 +- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/examples/schema_test.js b/examples/schema_test.js index 8aad24ec..9b3c28c7 100644 --- a/examples/schema_test.js +++ b/examples/schema_test.js @@ -6,13 +6,9 @@ import { randomBytes } from 'node:crypto' const obj = { id: randomBytes(32).toString('hex'), - type: 'Observation', - schemaVersion: 5, + type: 'observation', + schemaVersion: 4, created_at: new Date().toJSON(), - links: [], - refs: [], - attachments: [], - tags: {}, metadata: { manual_location: true, }, diff --git a/index.js b/index.js index 74423f46..6dd2dd33 100644 --- a/index.js +++ b/index.js @@ -157,11 +157,8 @@ export const encode = (obj) => { schemaVersion: obj.schemaVersion, }) const record = jsonSchemaToProto(obj) - const protobuf = ProtobufSchemas[ - `${formatSchemaType(obj.type)}_${obj.schemaVersion}` - ] - .encode(record) - .finish() + const partial = ProtobufSchemas[key].fromPartial(record) + const protobuf = ProtobufSchemas[key].encode(partial).finish() return Buffer.concat([blockPrefix, protobuf]) } diff --git a/proto/observation/v4.proto b/proto/observation/v4.proto index 658a8223..ddfc14c3 100644 --- a/proto/observation/v4.proto +++ b/proto/observation/v4.proto @@ -14,7 +14,7 @@ message Observation_4 { optional float lon = 10; repeated google.protobuf.Struct refs = 11; repeated google.protobuf.Struct attachments = 12; - google.protobuf.Struct tags = 13; + optional google.protobuf.Struct tags = 13; message Metadata { optional bool manual_location = 1; From 5668f7389c7d3f77f4ec93c1272113b314e8aae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Tue, 21 Feb 2023 14:25:24 -0300 Subject: [PATCH 05/18] feat: throw error if invalid type_schemaVersion on decoding --- index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/index.js b/index.js index 6dd2dd33..52ac4d06 100644 --- a/index.js +++ b/index.js @@ -176,6 +176,11 @@ export const decode = (buf, { coreId, seq }) => { const version = `${coreId.toString('hex')}/${seq.toString()}` const record = buf.subarray(dataTypeIdSize + schemaVersionSize, buf.length) const key = `${formatSchemaType(type)}_${schemaVersion}` + if (!ProtobufSchemas[key]) { + throw new Error( + `Invalid schemaVersion for ${type} version ${schemaVersion}` + ) + } const protobufObj = ProtobufSchemas[key].decode(record) return protoToJsonSchema(protobufObj, { schemaVersion, type, version }) } From 05278609da60d1610aa49d70edc3ab21301db491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Tue, 21 Feb 2023 14:57:13 -0300 Subject: [PATCH 06/18] feat: add formatSchemaType on utils to get key to schemas --- index.js | 15 +++++++-------- scripts/generate.js | 6 ++---- utils.js | 8 ++++++++ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/index.js b/index.js index 52ac4d06..3656741e 100644 --- a/index.js +++ b/index.js @@ -6,8 +6,7 @@ import * as cenc from 'compact-encoding' import * as JSONSchemas from './dist/schemas.js' import * as ProtobufSchemas from './types/proto/index.js' import schemasPrefix from './schemasPrefix.js' -import { formatSchemaType } from './utils.js' -import { and } from 'ajv/dist/compile/codegen/index.js' +import { formatSchemaType, formatSchemaKey } from './utils.js' const dataTypeIdSize = 6 const schemaVersionSize = 2 @@ -45,7 +44,7 @@ const jsonSchemaToProto = (obj) => { return common }, {}) - const key = `${formatSchemaType(obj.type)}_${obj.schemaVersion}` + const key = formatSchemaKey(obj.type, obj.schemaVersion) // TODO, match for every schema that doesn't inherit common/v1.json if (key === 'Observation_4') { return { @@ -68,7 +67,7 @@ const jsonSchemaToProto = (obj) => { * @returns {import('./types/schema/index').MapeoRecord} */ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { - const key = `${formatSchemaType(type)}_${schemaVersion}` + const key = formatSchemaKey(type, schemaVersion) const obj = { ...protobufObj, schemaVersion, @@ -133,8 +132,8 @@ export const decodeBlockPrefix = (buf) => { * @returns {Boolean} indicating if the object is valid */ export const validate = (obj) => { - const key = `${obj.type.toLowerCase()}_${obj.schemaVersion}` - + const key = formatSchemaKey(obj.type, obj.schemaVersion) + console.log(JSONSchemas) const validatefn = JSONSchemas[key] const isValid = validatefn(obj) if (!isValid) throw new Error(JSON.stringify(validatefn.errors, null, 4)) @@ -146,7 +145,7 @@ export const validate = (obj) => { * @param {import('./types/schema/index').MapeoRecord} obj - Object to be encoded * @returns {Buffer} protobuf encoded buffer with dataTypeIdSize + schemaVersionSize bytes prepended, one for the type of record and the other for the version of the schema */ export const encode = (obj) => { - const key = `${formatSchemaType(obj.type)}_${obj.schemaVersion}` + const key = formatSchemaKey(obj.type, obj.schemaVersion) if (!ProtobufSchemas[key]) { throw new Error( `Invalid schemaVersion for ${obj.type} version ${obj.schemaVersion}` @@ -175,7 +174,7 @@ export const decode = (buf, { coreId, seq }) => { ) const version = `${coreId.toString('hex')}/${seq.toString()}` const record = buf.subarray(dataTypeIdSize + schemaVersionSize, buf.length) - const key = `${formatSchemaType(type)}_${schemaVersion}` + const key = formatSchemaKey(type, schemaVersion) if (!ProtobufSchemas[key]) { throw new Error( `Invalid schemaVersion for ${type} version ${schemaVersion}` diff --git a/scripts/generate.js b/scripts/generate.js index 686452c3..ee10e860 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -11,7 +11,7 @@ import { URL } from 'url' import Ajv from 'ajv' import standaloneCode from 'ajv/dist/standalone/index.js' import glob from 'glob-promise' -import { formatSchemaType } from '../utils.js' +import { formatSchemaKey, formatSchemaType } from '../utils.js' const __dirname = new URL('.', import.meta.url).pathname @@ -28,9 +28,7 @@ const schemas = glob const schemaExports = schemas.reduce((acc, schema) => { const schemaVersion = schema.properties.schemaVersion.enum - const key = `${schema.title.toLowerCase()}_${ - schemaVersion ? schemaVersion : 0 - }` + const key = formatSchemaKey(schema.title, schemaVersion) acc[key] = schema['$id'] return acc }, {}) diff --git a/utils.js b/utils.js index 6b812b0f..07a33135 100644 --- a/utils.js +++ b/utils.js @@ -5,3 +5,11 @@ */ export const formatSchemaType = (str) => str.charAt(0).toUpperCase() + str.slice(1).toLowerCase() + +/** + * Turn a schema type and version into `${type}_${schemaVersion}` + * @param {String} type + * @param {Number} schemaVersion + */ +export const formatSchemaKey = (type, schemaVersion) => + `${formatSchemaType(type)}_${schemaVersion}` From 4bab3a0e405f93ce2d2528412af33039454d6485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 22 Feb 2023 12:13:23 -0300 Subject: [PATCH 07/18] feat: add preset and filter proto and make semi-working example for both --- examples/schema_test.js | 37 ++++++++++++++++++++++++++++++++----- index.js | 23 +++++++++++++++++------ proto/filter/v1.proto | 11 +++++++++++ proto/preset/v1.proto | 11 +++++++++++ schemasPrefix.js | 2 ++ 5 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 proto/filter/v1.proto create mode 100644 proto/preset/v1.proto diff --git a/examples/schema_test.js b/examples/schema_test.js index 9b3c28c7..178ea732 100644 --- a/examples/schema_test.js +++ b/examples/schema_test.js @@ -4,15 +4,42 @@ import Hypercore from 'hypercore' import ram from 'random-access-memory' import { randomBytes } from 'node:crypto' +// FILTER +// const obj = { +// id: randomBytes(32).toString('hex'), +// type: 'filter', +// schemaVersion: 1, +// created_at: new Date().toJSON(), +// filter: ['observation'], +// name: 'john', +// } + +// PRESET +// const obj = { +// id: randomBytes(32).toString('hex'), +// type: 'preset', +// schemaVersion: 1, +// created_at: new Date().toJSON(), +// tags: ['observation'], +// name: 'john', +// } + +// OBSERVATION 4 +// const obj = { +// id: randomBytes(32).toString('hex'), +// type: 'Observation', +// schemaVersion: 4, +// created_at: new Date().toJSON(), +// } + +// OBSERVATION 5 const obj = { id: randomBytes(32).toString('hex'), - type: 'observation', - schemaVersion: 4, + type: 'Observation', + schemaVersion: 5, created_at: new Date().toJSON(), - metadata: { - manual_location: true, - }, } + const record = encode(obj) const core = new Hypercore(ram, { valueEncoding: 'binary' }) diff --git a/index.js b/index.js index 3656741e..a317fbbe 100644 --- a/index.js +++ b/index.js @@ -45,8 +45,8 @@ const jsonSchemaToProto = (obj) => { }, {}) const key = formatSchemaKey(obj.type, obj.schemaVersion) - // TODO, match for every schema that doesn't inherit common/v1.json - if (key === 'Observation_4') { + // this matches for every schema that doesn't inherit common/v1.json + if (key === 'Observation_4' || key === 'Preset_1' || key === 'Filter_1') { return { ...uncommon, ...common, @@ -72,22 +72,33 @@ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { ...protobufObj, schemaVersion, type, - version, } - // TODO, match for every schema that doesn't inherit common/v1.json - if (key === 'Observation_4') { + // Observation_4 and Filter_1 have a lowecase 'type' + if (key === 'Observation_4' || key === 'Filter_1') { return { ...obj, id: obj.id.toString('hex'), type: obj.type.toLowerCase(), + version, + } + } + + // Preset_1 doesn't have a version field and doesn't accept additional fields + if (key === 'Preset_1') { + delete obj['version'] + return { + ...obj, + id: obj.id.toString('hex'), } } + const common = protobufObj.common delete obj.common return { ...obj, ...common, id: common ? common.id.toString('hex') : '', + version, } } @@ -133,7 +144,6 @@ export const decodeBlockPrefix = (buf) => { */ export const validate = (obj) => { const key = formatSchemaKey(obj.type, obj.schemaVersion) - console.log(JSONSchemas) const validatefn = JSONSchemas[key] const isValid = validatefn(obj) if (!isValid) throw new Error(JSON.stringify(validatefn.errors, null, 4)) @@ -151,6 +161,7 @@ export const encode = (obj) => { `Invalid schemaVersion for ${obj.type} version ${obj.schemaVersion}` ) } + const blockPrefix = encodeBlockPrefix({ dataTypeId: schemasPrefix[formatSchemaType(obj.type)].dataTypeId, schemaVersion: obj.schemaVersion, diff --git a/proto/filter/v1.proto b/proto/filter/v1.proto new file mode 100644 index 00000000..3e0366da --- /dev/null +++ b/proto/filter/v1.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package mapeo; + +import "google/protobuf/struct.proto"; + +message Filter_1 { + bytes id = 1; + string created_at = 2; + repeated string filter = 3; + string name = 4; +} diff --git a/proto/preset/v1.proto b/proto/preset/v1.proto new file mode 100644 index 00000000..19f61ce9 --- /dev/null +++ b/proto/preset/v1.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package mapeo; + +import "google/protobuf/struct.proto"; + +message Preset_1 { + bytes id = 1; + string name = 2; + repeated string geometry = 3; + optional google.protobuf.Struct tags = 4; +} diff --git a/schemasPrefix.js b/schemasPrefix.js index 245c8165..7d074f24 100644 --- a/schemasPrefix.js +++ b/schemasPrefix.js @@ -1,4 +1,6 @@ const schemasPrefix = { Observation: { dataTypeId: '71f85084cb94', schemaVersions: [4, 5] }, + Preset: { dataTypeId: '71f85084cb98', schemaVersions: [1] }, + Filter: { dataTypeId: '71f85084cb90', schemaVersions: [1] }, } export default schemasPrefix From 549197e81de2e788426b4bbc80271d0bd60eb0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 22 Feb 2023 12:40:18 -0300 Subject: [PATCH 08/18] feat: build versioning of schemas from filesystem instead of on schemas This is so that schemas that doesn't have schemaVersion can be generated too --- scripts/generate.js | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/scripts/generate.js b/scripts/generate.js index ee10e860..2de30736 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -16,26 +16,31 @@ import { formatSchemaKey, formatSchemaType } from '../utils.js' const __dirname = new URL('.', import.meta.url).pathname /** - * @param {string} path - * @returns {Object} + * @param {string} p + * @returns {{type: String, schemaVersion: string, schema:Object}} */ -const loadJSON = (path) => - JSON.parse(fs.readFileSync(new URL(path, import.meta.url)).toString()) +const loadJSON = (p) => { + const parsedPath = path.parse(p) + return { + type: formatSchemaType(parsedPath.dir.replace('../schema/', '')), + schemaVersion: parsedPath.name.replace('v', ''), + schema: JSON.parse(fs.readFileSync(new URL(p, import.meta.url)).toString()), + } +} const schemas = glob .sync('../schema/*/*.json', { cwd: 'scripts' }) .map(loadJSON) -const schemaExports = schemas.reduce((acc, schema) => { - const schemaVersion = schema.properties.schemaVersion.enum - const key = formatSchemaKey(schema.title, schemaVersion) +const schemaExports = schemas.reduce((acc, { schema, schemaVersion, type }) => { + const key = formatSchemaKey(type, parseInt(schemaVersion)) acc[key] = schema['$id'] return acc }, {}) // compile schemas const ajv = new Ajv({ - schemas: schemas, + schemas: schemas.map(({ schema }) => schema), code: { source: true, esm: true }, formats: { 'date-time': true }, }) @@ -59,14 +64,11 @@ const jsonSchemaType = ` ${schemas .map( /** @param {Object} schema */ - (schema) => { - const version = schema.properties.schemaVersion.enum - const varName = `${schema.title.toLowerCase()}_${version ? version : 0}` + ({ schemaVersion, type }) => { + const varName = `${type.toLowerCase()}_${schemaVersion}` return `import { ${formatSchemaType( - schema.title - )} as ${varName} } from './${schema.title.toLowerCase()}/v${ - version || 1 - }'` + type + )} as ${varName} } from './${type.toLowerCase()}/v${schemaVersion}'` } ) .join('\n')} @@ -79,9 +81,8 @@ schemaVersion: number; export type MapeoRecord = (${schemas .map( /** @param {Object} schema */ - (schema) => { - const version = schema.properties.schemaVersion.enum - return `${schema.title.toLowerCase()}_${version ? version : 0}` + ({ schemaVersion, type }) => { + return `${type.toLowerCase()}_${schemaVersion}` } ) .join(' | ')}) & base From 89965ac134299c8b52e000b00f5add9ff6bde782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 22 Feb 2023 14:00:16 -0300 Subject: [PATCH 09/18] feat: * added optional fields to filter and preset protobufs * added field protobuf and schema * type is optional on Preset_1 so need to be made optional on base type (and deleted before validation) * schemaVersion is optional on Field_1 so need to be made optional on base type (and deleted before validation) * this means utils.js/formatSchema{Key,Type} no takes optional values --- examples/schema_test.js | 27 ++++++--- index.js | 15 ++++- proto/field/v1.proto | 10 ++++ proto/filter/v1.proto | 4 ++ proto/preset/v1.proto | 7 +++ schema/field/v1.json | 129 ++++++++++++++++++++++++++++++++++++++++ schemasPrefix.js | 1 + scripts/generate.js | 43 ++++++++------ utils.js | 4 +- 9 files changed, 208 insertions(+), 32 deletions(-) create mode 100644 proto/field/v1.proto create mode 100644 schema/field/v1.json diff --git a/examples/schema_test.js b/examples/schema_test.js index 178ea732..4d41dad7 100644 --- a/examples/schema_test.js +++ b/examples/schema_test.js @@ -4,7 +4,7 @@ import Hypercore from 'hypercore' import ram from 'random-access-memory' import { randomBytes } from 'node:crypto' -// FILTER +// FILTER_1 // const obj = { // id: randomBytes(32).toString('hex'), // type: 'filter', @@ -14,16 +14,25 @@ import { randomBytes } from 'node:crypto' // name: 'john', // } -// PRESET +// PRESET_1 // const obj = { // id: randomBytes(32).toString('hex'), // type: 'preset', // schemaVersion: 1, // created_at: new Date().toJSON(), -// tags: ['observation'], +// tags: { nature: 'tree' }, +// geometry: ['point'], // name: 'john', // } +// FIELD_1 +const obj = { + id: randomBytes(32).toString('hex'), + type: 'Field', + schemaVersion: 1, + key: 'hi', +} + // OBSERVATION 4 // const obj = { // id: randomBytes(32).toString('hex'), @@ -33,12 +42,12 @@ import { randomBytes } from 'node:crypto' // } // OBSERVATION 5 -const obj = { - id: randomBytes(32).toString('hex'), - type: 'Observation', - schemaVersion: 5, - created_at: new Date().toJSON(), -} +// const obj = { +// id: randomBytes(32).toString('hex'), +// type: 'Observation', +// schemaVersion: 5, +// created_at: new Date().toJSON(), +// } const record = encode(obj) diff --git a/index.js b/index.js index a317fbbe..9af1900c 100644 --- a/index.js +++ b/index.js @@ -46,7 +46,12 @@ const jsonSchemaToProto = (obj) => { const key = formatSchemaKey(obj.type, obj.schemaVersion) // this matches for every schema that doesn't inherit common/v1.json - if (key === 'Observation_4' || key === 'Preset_1' || key === 'Filter_1') { + if ( + key === 'Observation_4' || + key === 'Preset_1' || + key === 'Filter_1' || + key === 'Field_1' + ) { return { ...uncommon, ...common, @@ -84,7 +89,7 @@ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { } // Preset_1 doesn't have a version field and doesn't accept additional fields - if (key === 'Preset_1') { + if (key === 'Preset_1' || key === 'Field_1') { delete obj['version'] return { ...obj, @@ -144,6 +149,12 @@ export const decodeBlockPrefix = (buf) => { */ export const validate = (obj) => { const key = formatSchemaKey(obj.type, obj.schemaVersion) + // Preset_1 doesn't have a type field, so validation won't pass + // but we still need it to now which schema to validate, so we delete it after grabbing the key + if (key === 'Preset_1') delete obj['type'] + // Field_1 doesn't have a schemaVersion field, so validation won't pass + // but we still need it to now which schema to validate, so we delete it after grabbing the key + if (key === 'Field_1') delete obj['schemaVersion'] const validatefn = JSONSchemas[key] const isValid = validatefn(obj) if (!isValid) throw new Error(JSON.stringify(validatefn.errors, null, 4)) diff --git a/proto/field/v1.proto b/proto/field/v1.proto new file mode 100644 index 00000000..1a2bc1f4 --- /dev/null +++ b/proto/field/v1.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; +package mapeo; + +import "google/protobuf/any.proto"; + +message Field_1 { + bytes id = 1; + // keys can be an array of strings or a string + google.protobuf.Any key = 2; +} diff --git a/proto/filter/v1.proto b/proto/filter/v1.proto index 3e0366da..43c20502 100644 --- a/proto/filter/v1.proto +++ b/proto/filter/v1.proto @@ -8,4 +8,8 @@ message Filter_1 { string created_at = 2; repeated string filter = 3; string name = 4; + optional string timestamp = 5; + optional string userId = 6; + optional string deviceId = 7; + repeated string links = 8; } diff --git a/proto/preset/v1.proto b/proto/preset/v1.proto index 19f61ce9..71d5413f 100644 --- a/proto/preset/v1.proto +++ b/proto/preset/v1.proto @@ -8,4 +8,11 @@ message Preset_1 { string name = 2; repeated string geometry = 3; optional google.protobuf.Struct tags = 4; + optional google.protobuf.Struct addTags = 5; + optional google.protobuf.Struct removeTags = 6; + repeated string fields = 7; + repeated string additionalFields = 8; + optional string icon = 9; + repeated string terms = 10; + optional int32 sort = 11; } diff --git a/schema/field/v1.json b/schema/field/v1.json new file mode 100644 index 00000000..adfaf32c --- /dev/null +++ b/schema/field/v1.json @@ -0,0 +1,129 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://mapeo.world/schemas/field.json", + "title": "Field", + "description": "A field defines a form field that will be shown to the user when creating or editing a map entity. Presets define which fields are shown to the user for a particular map entity. The field definition defines whether the field should show as a text box, multiple choice, single-select, etc. It defines what tag-value is set when the field is entered.", + "type": "object", + "properties": { + "id": { + "description": "Unique value that identifies this element", + "type": "string" + }, + "key": { + "description": "They key in a tags object that this field applies to. For nested properties, key can be an array e.g. for tags = `{ foo: { bar: 1 } }` the key is `['foo', 'bar']`", + "oneOf": [{ + "type": "string" + }, { + "type": "array", + "items": { + "type": "string" + } + }] + }, + "type": { + "description": "Type of field - defines how the field is displayed to the user.", + "type": "string", + "meta:enum": { + "text": "Freeform text field", + "localized": "Text field with localization abilities (e.g. name=*, name:es=*, etc.). Currently only supported in Mapeo Desktop territory view.", + "number": "Allows only numbers", + "select_one": "Select one item from a list of pre-defined options", + "select_multiple": "Select any number of items from a list of pre-defined options", + "date": "Select a date", + "datetime": "Select a date and time" + }, + "enum": [ + "text", + "localized", + "number", + "select_one", + "select_multiple", + "date", + "datetime" + ] + }, + "label": { + "description": "Default language label for the form field label", + "type": "string" + }, + "readonly": { + "description": "Field is displayed, but it can't be edited", + "type": "boolean", + "default": false + }, + "appearance": { + "description": "For text fields, display as a single-line or multi-line field", + "type": "string", + "meta:enum": { + "singleline": "Text will be cut-off if more than one line", + "multiline": "Text will wrap to multiple lines within text field" + }, + "enum": ["singleline", "multiline"], + "default": "multiline" + }, + "snake_case": { + "description": "Convert field value into snake_case (replace spaces with underscores and convert to lowercase)", + "type": "boolean", + "default": false + }, + "options": { + "description": "List of options the user can select for single- or multi-select fields", + "type": "array", + "items": { + "anyOf": [{ + "type": "string" + }, { + "type": "boolean" + }, { + "type": "number" + }, { + "type": "null" + }, { + "type": "object", + "properties": { + "label": { + "description": "Label in default language to display to the user for this option", + "type": "string" + }, + "value": { + "description": "Value for tag when this option is selected", + "anyOf": [{ + "type": "string" + }, { + "type": "boolean" + }, { + "type": "number" + }, { + "type": "null" + }] + } + }, + "required": ["value"] + }] + } + }, + "universal": { + "description": "If true, this field will appear in the Add Field list for all presets", + "type": "boolean", + "default": false + }, + "placeholder": { + "description": "Displayed as a placeholder in an empty text or number field before the user begins typing. Use 'helperText' for important information, because the placeholder is not visible after the user has entered data.", + "type": "string" + }, + "helperText": { + "description": "Additional context about the field, e.g. hints about how to answer the question.", + "type": "string" + }, + "min_value": { + "description": "Minimum field value (number, date or datetime fields only). For date or datetime fields, is seconds since unix epoch", + "type": "integer" + }, + "max_value": { + "description": "Maximum field value (number, date or datetime fields only). For date or datetime fields, is seconds since unix epoch", + "type": "integer" + } + }, + "required": ["id", "key", "type"], + "additionalProperties": false +} diff --git a/schemasPrefix.js b/schemasPrefix.js index 7d074f24..1fe306b3 100644 --- a/schemasPrefix.js +++ b/schemasPrefix.js @@ -2,5 +2,6 @@ const schemasPrefix = { Observation: { dataTypeId: '71f85084cb94', schemaVersions: [4, 5] }, Preset: { dataTypeId: '71f85084cb98', schemaVersions: [1] }, Filter: { dataTypeId: '71f85084cb90', schemaVersions: [1] }, + Field: { dataTypeId: '71f85084cb80', schemaVersions: [1] }, } export default schemasPrefix diff --git a/scripts/generate.js b/scripts/generate.js index 2de30736..0e359c8e 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -17,23 +17,25 @@ const __dirname = new URL('.', import.meta.url).pathname /** * @param {string} p - * @returns {{type: String, schemaVersion: string, schema:Object}} + * @returns {{type: String, schemaVersion: Number, schema:Object}} */ -const loadJSON = (p) => { - const parsedPath = path.parse(p) +const loadSchema = (p) => { + const { dir, name } = path.parse(p) return { - type: formatSchemaType(parsedPath.dir.replace('../schema/', '')), - schemaVersion: parsedPath.name.replace('v', ''), + // we get the type of the schema from the directory + type: formatSchemaType(dir.replace('../schema/', '')), + // we get the version from the filename + schemaVersion: parseInt(name.replace('v', '')), schema: JSON.parse(fs.readFileSync(new URL(p, import.meta.url)).toString()), } } const schemas = glob .sync('../schema/*/*.json', { cwd: 'scripts' }) - .map(loadJSON) + .map(loadSchema) const schemaExports = schemas.reduce((acc, { schema, schemaVersion, type }) => { - const key = formatSchemaKey(type, parseInt(schemaVersion)) + const key = formatSchemaKey(type, schemaVersion) acc[key] = schema['$id'] return acc }, {}) @@ -74,8 +76,8 @@ ${schemas .join('\n')} interface base { -type: string; -schemaVersion: number; +type?: string; +schemaVersion?: number; [key:string]: any; } export type MapeoRecord = (${schemas @@ -95,31 +97,34 @@ fs.writeFileSync( // generate index.js for protobuf schemas and index.d.ts const protobufFiles = glob.sync('../types/proto/*/*.ts', { cwd: 'scripts' }) const obj = protobufFiles - .filter((f) => !f.match(/.d/)) + .filter((f) => !f.match(/.d.ts/)) .map((p) => { - const arr = p.split('/') + const { name, dir } = path.parse(p) return { - type: arr[arr.length - 2], - version: arr[arr.length - 1].split('.')[0], + type: dir.replace('../types/proto/', ''), + schemaVersion: name, } }) const linesjs = [] const linesdts = [] const union = obj - .map((t) => `${formatSchemaType(t.type)}_${t.version.replace('v', '')}`) + .map( + ({ type, schemaVersion }) => + `${formatSchemaType(type)}_${schemaVersion.replace('v', '')}` + ) .join(' & ') -obj.forEach((f) => { - const linejs = `export { ${formatSchemaType(f.type)}_${f.version.replace( +obj.forEach(({ type, schemaVersion }) => { + const linejs = `export { ${formatSchemaType(type)}_${schemaVersion.replace( 'v', '' - )} } from './${f.type}/${f.version}.js'` + )} } from './${type}/${schemaVersion}.js'` - const linedts = `import { ${formatSchemaType(f.type)}_${f.version.replace( + const linedts = `import { ${formatSchemaType(type)}_${schemaVersion.replace( 'v', '' - )} } from './${f.type}/${f.version}'` + )} } from './${type}/${schemaVersion}'` linesdts.push(linedts) linesjs.push(linejs) }) diff --git a/utils.js b/utils.js index 07a33135..9ba4ccd6 100644 --- a/utils.js +++ b/utils.js @@ -8,8 +8,8 @@ export const formatSchemaType = (str) => /** * Turn a schema type and version into `${type}_${schemaVersion}` - * @param {String} type - * @param {Number} schemaVersion + * @param {String | undefined} type + * @param {Number | undefined} schemaVersion */ export const formatSchemaKey = (type, schemaVersion) => `${formatSchemaType(type)}_${schemaVersion}` From 434821959583f8787e5018bbbe252eeb2a0469e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 22 Feb 2023 15:24:00 -0300 Subject: [PATCH 10/18] chore: solved minor type error --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 9af1900c..022ffe82 100644 --- a/index.js +++ b/index.js @@ -88,7 +88,7 @@ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { } } - // Preset_1 doesn't have a version field and doesn't accept additional fields + // Preset_1 and Field_1 don't have a version field and doesn't accept additional fields if (key === 'Preset_1' || key === 'Field_1') { delete obj['version'] return { @@ -111,7 +111,7 @@ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { * given a schemaVersion and type, return a buffer with the corresponding data * @param {Object} obj * @param {string} obj.dataTypeId hex encoded string of a 6-byte buffer indicating type - * @param {number} obj.schemaVersion number to indicate version. Gets converted to a padded 4-byte hex string + * @param {number | undefined} obj.schemaVersion number to indicate version. Gets converted to a padded 4-byte hex string * @returns {Buffer} blockPrefix for corresponding schema */ export const encodeBlockPrefix = ({ dataTypeId, schemaVersion }) => { From 17750cf4e9ae29dd0ef67430fe0684764fefc172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Thu, 23 Feb 2023 15:02:57 -0300 Subject: [PATCH 11/18] feat: handle type casing with schemaPrefix Casing on a doc type shouldn't be handled specifically by the code. Nevertheless, some schemas enforce a type to a specific value (type must be i.e. 'observation', which is different from 'Observation'). This means that type "observation" and "Observation" should have different dataTypeIds. That way we know what "type" to fill-in on returning the decoded object to be validated... If we want the same `dataTypeId` for 'observation' and 'Observation' (since we know we're actually talking about roughly the same type of 'document'), we could format the schemasPrefix object differently like: Observation: `{dataTypeId: '924892', versions: {'observation': 4, 'Observation': 5}}` or something less worse... --- index.js | 30 +++++++++--------------------- schemasPrefix.js | 5 +++-- utils.js | 11 +++++++++++ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/index.js b/index.js index 022ffe82..9827d6c2 100644 --- a/index.js +++ b/index.js @@ -6,7 +6,7 @@ import * as cenc from 'compact-encoding' import * as JSONSchemas from './dist/schemas.js' import * as ProtobufSchemas from './types/proto/index.js' import schemasPrefix from './schemasPrefix.js' -import { formatSchemaType, formatSchemaKey } from './utils.js' +import { inheritsFromCommon, formatSchemaKey } from './utils.js' const dataTypeIdSize = 6 const schemaVersionSize = 2 @@ -45,21 +45,16 @@ const jsonSchemaToProto = (obj) => { }, {}) const key = formatSchemaKey(obj.type, obj.schemaVersion) - // this matches for every schema that doesn't inherit common/v1.json - if ( - key === 'Observation_4' || - key === 'Preset_1' || - key === 'Filter_1' || - key === 'Field_1' - ) { + if (inheritsFromCommon(key)) { return { ...uncommon, - ...common, + common, } } + // this matches for every schema that doesn't inherit common/v1.json return { ...uncommon, - common, + ...common, } } @@ -78,15 +73,6 @@ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { schemaVersion, type, } - // Observation_4 and Filter_1 have a lowecase 'type' - if (key === 'Observation_4' || key === 'Filter_1') { - return { - ...obj, - id: obj.id.toString('hex'), - type: obj.type.toLowerCase(), - version, - } - } // Preset_1 and Field_1 don't have a version field and doesn't accept additional fields if (key === 'Preset_1' || key === 'Field_1') { @@ -167,14 +153,16 @@ export const validate = (obj) => { * @returns {Buffer} protobuf encoded buffer with dataTypeIdSize + schemaVersionSize bytes prepended, one for the type of record and the other for the version of the schema */ export const encode = (obj) => { const key = formatSchemaKey(obj.type, obj.schemaVersion) + // some schemas don't have type field so it can be undefined + const type = obj.type || '' if (!ProtobufSchemas[key]) { throw new Error( - `Invalid schemaVersion for ${obj.type} version ${obj.schemaVersion}` + `Invalid schemaVersion for ${type} version ${obj.schemaVersion}` ) } const blockPrefix = encodeBlockPrefix({ - dataTypeId: schemasPrefix[formatSchemaType(obj.type)].dataTypeId, + dataTypeId: schemasPrefix[type].dataTypeId, schemaVersion: obj.schemaVersion, }) const record = jsonSchemaToProto(obj) diff --git a/schemasPrefix.js b/schemasPrefix.js index 1fe306b3..99d2a56c 100644 --- a/schemasPrefix.js +++ b/schemasPrefix.js @@ -1,7 +1,8 @@ const schemasPrefix = { - Observation: { dataTypeId: '71f85084cb94', schemaVersions: [4, 5] }, + Observation: { dataTypeId: '71f85084cb94', schemaVersions: [5] }, + observation: { dataTypeId: '70f85084cb94', schemaVersions: [4] }, Preset: { dataTypeId: '71f85084cb98', schemaVersions: [1] }, - Filter: { dataTypeId: '71f85084cb90', schemaVersions: [1] }, + filter: { dataTypeId: '71f85084cb90', schemaVersions: [1] }, Field: { dataTypeId: '71f85084cb80', schemaVersions: [1] }, } export default schemasPrefix diff --git a/utils.js b/utils.js index 9ba4ccd6..610defa7 100644 --- a/utils.js +++ b/utils.js @@ -13,3 +13,14 @@ export const formatSchemaType = (str) => */ export const formatSchemaKey = (type, schemaVersion) => `${formatSchemaType(type)}_${schemaVersion}` + +/** + * Checks if the type of record inherits from a common one + * @param {String} key - type of doc build from ${type}_${schemaVersion} + * @returns {boolean} + */ +export const inheritsFromCommon = (key) => + key !== 'Observation_4' || + key !== 'Preset_1' || + key !== 'Filter_1' || + key !== 'Field_1' From 88653d8b511662300ed464b476ed372044140397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Thu, 23 Feb 2023 16:27:09 -0300 Subject: [PATCH 12/18] chore: logic error on `inheritsFromCommon` --- index.js | 73 +++++++++++++++++++++----------------------------------- utils.js | 6 ++--- 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/index.js b/index.js index 9827d6c2..cf053f7a 100644 --- a/index.js +++ b/index.js @@ -28,34 +28,20 @@ const jsonSchemaToProto = (obj) => { /** @type {Object} */ const uncommon = Object.keys(obj) .filter((k) => !commonKeys.includes(k)) - .reduce((acc, k) => { - acc[k] = obj[k] - return acc - }, {}) - - const common = commonKeys.reduce((common, field) => { - if (obj[field]) { - if (field === 'id') { - common[field] = Buffer.from(obj[field], 'hex') - } else { - common[field] = obj[field] - } - } - return common - }, {}) + .reduce((uncommon, field) => ({ ...uncommon, [field]: obj[field] }), {}) + + /** @type {Object} */ + const common = commonKeys + .filter((field) => obj[field]) + .reduce((common, field) => ({ ...common, [field]: obj[field] }), {}) + common.id = Buffer.from(obj['id'], 'hex') const key = formatSchemaKey(obj.type, obj.schemaVersion) - if (inheritsFromCommon(key)) { - return { - ...uncommon, - common, - } - } - // this matches for every schema that doesn't inherit common/v1.json - return { - ...uncommon, - ...common, - } + // when we inherit from common, common is actually a field inside the protobuf object, + // so we don't destructure it + return inheritsFromCommon(key) + ? { ...uncommon, common } + : { ...uncommon, ...common } } /** @@ -68,29 +54,20 @@ const jsonSchemaToProto = (obj) => { */ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { const key = formatSchemaKey(type, schemaVersion) - const obj = { - ...protobufObj, - schemaVersion, - type, + /** @type {Object} */ + let obj = { ...protobufObj, schemaVersion, type } + if (obj.common) { + obj = { ...obj, ...obj.common } + delete obj.common } // Preset_1 and Field_1 don't have a version field and doesn't accept additional fields - if (key === 'Preset_1' || key === 'Field_1') { - delete obj['version'] - return { - ...obj, - id: obj.id.toString('hex'), - } + if (key !== 'Preset_1' && key !== 'Field_1') { + obj.version = version } - const common = protobufObj.common - delete obj.common - return { - ...obj, - ...common, - id: common ? common.id.toString('hex') : '', - version, - } + obj.id = obj.id.toString('hex') + return obj } /** @@ -135,12 +112,14 @@ export const decodeBlockPrefix = (buf) => { */ export const validate = (obj) => { const key = formatSchemaKey(obj.type, obj.schemaVersion) + // Preset_1 doesn't have a type field, so validation won't pass // but we still need it to now which schema to validate, so we delete it after grabbing the key if (key === 'Preset_1') delete obj['type'] // Field_1 doesn't have a schemaVersion field, so validation won't pass // but we still need it to now which schema to validate, so we delete it after grabbing the key if (key === 'Field_1') delete obj['schemaVersion'] + const validatefn = JSONSchemas[key] const isValid = validatefn(obj) if (!isValid) throw new Error(JSON.stringify(validatefn.errors, null, 4)) @@ -182,14 +161,16 @@ export const decode = (buf, { coreId, seq }) => { (type, key) => (schemasPrefix[key].dataTypeId === dataTypeId ? key : type), '' ) - const version = `${coreId.toString('hex')}/${seq.toString()}` - const record = buf.subarray(dataTypeIdSize + schemaVersionSize, buf.length) const key = formatSchemaKey(type, schemaVersion) if (!ProtobufSchemas[key]) { throw new Error( `Invalid schemaVersion for ${type} version ${schemaVersion}` ) } + + const version = `${coreId.toString('hex')}/${seq.toString()}` + const record = buf.subarray(dataTypeIdSize + schemaVersionSize, buf.length) + const protobufObj = ProtobufSchemas[key].decode(record) return protoToJsonSchema(protobufObj, { schemaVersion, type, version }) } diff --git a/utils.js b/utils.js index 610defa7..36597f2a 100644 --- a/utils.js +++ b/utils.js @@ -20,7 +20,7 @@ export const formatSchemaKey = (type, schemaVersion) => * @returns {boolean} */ export const inheritsFromCommon = (key) => - key !== 'Observation_4' || - key !== 'Preset_1' || - key !== 'Filter_1' || + key !== 'Observation_4' && + key !== 'Preset_1' && + key !== 'Filter_1' && key !== 'Field_1' From 2193743dbb22117831daea49a3404bdb1ed751f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Tue, 28 Feb 2023 11:58:45 -0300 Subject: [PATCH 13/18] feat: add tests for many doc types. Test present fields and values --- examples/schema_test.js | 33 ++++++++++--------- index.js | 4 +-- test/docs/badDocType.json | 4 +-- test/docs/badSchemaVersion.json | 4 +-- test/docs/good.json | 56 +++++++++++++++++++++++++++------ test/docs/onlyId.json | 2 +- test/index.js | 52 +++++++++++++++++++----------- 7 files changed, 103 insertions(+), 52 deletions(-) diff --git a/examples/schema_test.js b/examples/schema_test.js index 4d41dad7..d090e696 100644 --- a/examples/schema_test.js +++ b/examples/schema_test.js @@ -5,38 +5,37 @@ import ram from 'random-access-memory' import { randomBytes } from 'node:crypto' // FILTER_1 -// const obj = { -// id: randomBytes(32).toString('hex'), -// type: 'filter', -// schemaVersion: 1, -// created_at: new Date().toJSON(), -// filter: ['observation'], -// name: 'john', -// } +const obj = { + id: randomBytes(32).toString('hex'), + type: 'filter', + schemaVersion: 1, + created_at: new Date().toJSON(), + filter: ['observation'], + name: 'john', +} // PRESET_1 // const obj = { // id: randomBytes(32).toString('hex'), -// type: 'preset', +// type: 'Preset', // schemaVersion: 1, -// created_at: new Date().toJSON(), // tags: { nature: 'tree' }, // geometry: ['point'], // name: 'john', // } // FIELD_1 -const obj = { - id: randomBytes(32).toString('hex'), - type: 'Field', - schemaVersion: 1, - key: 'hi', -} +// const obj = { +// id: randomBytes(32).toString('hex'), +// type: 'Field', +// schemaVersion: 1, +// key: 'hi', +// } // OBSERVATION 4 // const obj = { // id: randomBytes(32).toString('hex'), -// type: 'Observation', +// type: 'observation', // schemaVersion: 4, // created_at: new Date().toJSON(), // } diff --git a/index.js b/index.js index cf053f7a..507789d6 100644 --- a/index.js +++ b/index.js @@ -53,7 +53,6 @@ const jsonSchemaToProto = (obj) => { * @returns {import('./types/schema/index').MapeoRecord} */ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { - const key = formatSchemaKey(type, schemaVersion) /** @type {Object} */ let obj = { ...protobufObj, schemaVersion, type } if (obj.common) { @@ -61,7 +60,8 @@ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { delete obj.common } - // Preset_1 and Field_1 don't have a version field and doesn't accept additional fields + // Preset_1 and Field_1 don't have a version field and don't accept additional fields + const key = formatSchemaKey(type, schemaVersion) if (key !== 'Preset_1' && key !== 'Field_1') { obj.version = version } diff --git a/test/docs/badDocType.json b/test/docs/badDocType.json index 7da61d41..6cec7304 100644 --- a/test/docs/badDocType.json +++ b/test/docs/badDocType.json @@ -1,9 +1,9 @@ { - "id": "43cce269106f8da2e41198b414d4a39515b2ca0384955350bbdb87d57a8c055e", + "id": "d436a4bdde139eebb1c5ad18f8d6a31ecbf51f498dca616b24d68d845940651f", "type": "doesnotexist", "schemaVersion": 4, "links": [], - "created_at": "2023-01-02T13:17:59.775Z", + "created_at": "2023-02-28T14:58:07.711Z", "refs": [], "attachments": [], "metadata": { diff --git a/test/docs/badSchemaVersion.json b/test/docs/badSchemaVersion.json index c402969a..95238891 100644 --- a/test/docs/badSchemaVersion.json +++ b/test/docs/badSchemaVersion.json @@ -1,9 +1,9 @@ { - "id": "3e4b3f80171b8651b1cf28bbf27313a3e7d89c51f8feb8896d79b28a7cb26145", + "id": "ef36683c63166a0001fc0bc1ef5f5d9a3420fb65cc5a02b98824ca582530b4d0", "type": "observation", "schemaVersion": null, "links": [], - "created_at": "2023-01-02T13:17:59.775Z", + "created_at": "2023-02-28T14:58:07.711Z", "refs": [], "attachments": [], "metadata": { diff --git a/test/docs/good.json b/test/docs/good.json index 4ce61290..dfdec377 100644 --- a/test/docs/good.json +++ b/test/docs/good.json @@ -1,12 +1,48 @@ { - "id": "0e4e04b9d06413b233dcb5b0bbd03d5a07836f7bafd41f82975217b70e690f10", - "type": "Observation", - "schemaVersion": 5, - "links": [], - "created_at": "2023-01-02T13:17:59.775Z", - "refs": [], - "attachments": [], - "metadata": { - "manual_location": true + "observation_4": { + "id": "295f6861699930714d04edbe31b26e19214940c5be190e12fe2b1c82e454bed7", + "type": "observation", + "schemaVersion": 4, + "links": [], + "created_at": "2023-02-28T14:58:07.711Z", + "refs": [], + "attachments": [], + "metadata": { + "manual_location": true + } + }, + "observation_5": { + "id": "a246a13b1bdddc5d312f5051f7c83cd3a912a0051087b5d13404334c00f03ab3", + "type": "Observation", + "schemaVersion": 5, + "created_at": "2023-02-28T14:58:07.711Z" + }, + "filter": { + "id": "dac6892c0d4259080f32948ecd3a3231a3159141fa6d9d5a68c80cd2671fa895", + "type": "filter", + "schemaVersion": 1, + "created_at": "2023-02-28T14:58:07.711Z", + "filter": [ + "observation" + ], + "name": "john" + }, + "preset": { + "id": "d50c7526e78e938ed9b85b82062897e05f45ec316071f92d5a86ad384ad56d6f", + "type": "Preset", + "schemaVersion": 1, + "tags": { + "nature": "tree" + }, + "geometry": [ + "point" + ], + "name": "john" + }, + "field": { + "id": "da6445be7c2d5b891cf2783131481ca55c3f0bd898cb7744c143d921c42fe05f", + "type": "Field", + "schemaVersion": 1, + "key": "hi" } -} +} \ No newline at end of file diff --git a/test/docs/onlyId.json b/test/docs/onlyId.json index c9616119..5dcf5872 100644 --- a/test/docs/onlyId.json +++ b/test/docs/onlyId.json @@ -1,3 +1,3 @@ { - "id": "6bf891d28d5c2f10e002426101a01df1d077d725d902a3bc3ca98567c5c32061" + "id": "841d7233357a60d3a1950cd07b02c4265825b284182db5eb81a79eaaf805ebdf" } \ No newline at end of file diff --git a/test/index.js b/test/index.js index c8114293..145f7e44 100644 --- a/test/index.js +++ b/test/index.js @@ -42,31 +42,47 @@ test('test encoding of record with wrong schema version', async (t) => { }) test('test encoding of rightfully formated record', async (t) => { - t.plan(1) - t.doesNotThrow(() => { - encode(docs.good) + const goodDocs = Object.keys(docs.good) + t.plan(goodDocs.length) + goodDocs.forEach((k) => { + const doc = docs.good[k] + t.doesNotThrow(() => { + encode(doc) + }) }) }) test('test encoding, decoding of record and comparing the two versions', async (t) => { - const record = decode(encode(docs.good), { coreId: randomBytes(32), seq: 0 }) - const fields = Object.keys(docs.good) - fields.forEach((field) => { - const msg = `checking existence of ${field}` - // check if field exists - record[field] && docs.good[field] ? t.pass(msg) : t.fail(msg) - // if field is not an object, check equality - // since objects as fields mean the possibility of additionalFields in jsonSchemas - if (typeof record[field] !== 'object') { - t.equal(record[field], docs.good[field], `comparing value of ${field}`) - } + const goodDocs = Object.keys(docs.good) + goodDocs.forEach((k) => { + const doc = docs.good[k] + const record = decode(encode(doc), { + coreId: randomBytes(32), + seq: 0, + }) + const fields = Object.keys(doc) + // t.plan(goodDocs.length * fields.length * 2) + fields.forEach((f) => { + const msg = `checking ${f} for ${k}` + record[f] && doc[f] ? t.pass(msg) : t.fail(msg) + + // if field is not an object, check equality + // since objects as fields usually mean the possibility of additionalFields in jsonSchemas + if (typeof record[f] !== 'object') { + t.equal(record[f], doc[f], `comparing value of ${f} for ${k}`) + } + }) }) }) test('test decoding of record without passing core key and index', async (t) => { - t.plan(1) - const record = encode(docs.good) - t.throws(() => { - decode(record) + const goodDocs = Object.keys(docs.good) + t.plan(goodDocs.length) + goodDocs.forEach((key) => { + const doc = docs.good[key] + const record = encode(doc) + t.throws(() => { + decode(record) + }, `testing ${key}`) }) }) From ffd7b2532dae1fa2fcbd12b542ceba530ee16d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 1 Mar 2023 14:13:55 -0300 Subject: [PATCH 14/18] feat: add coreOwnership and device schema --- examples/schema_test.js | 36 ++++++++++++++++++++++++++----- proto/coreOwnership/v1.proto | 16 ++++++++++++++ proto/device/v1.proto | 15 +++++++++++++ schema/coreOwnership/v1.json | 25 ++++++++++++++++++++++ schema/device/v1.json | 36 +++++++++++++++++++++++++++++++ schemasPrefix.js | 2 ++ scripts/generate.js | 9 ++++---- test/docs/badDateFormat.json | 1 - test/docs/badDocType.json | 4 ++-- test/docs/badSchemaVersion.json | 4 ++-- test/docs/good.json | 38 ++++++++++++++++++++++++++------- test/docs/onlyId.json | 2 +- utils.js | 2 +- 13 files changed, 165 insertions(+), 25 deletions(-) create mode 100644 proto/coreOwnership/v1.proto create mode 100644 proto/device/v1.proto create mode 100644 schema/coreOwnership/v1.json create mode 100644 schema/device/v1.json delete mode 100644 test/docs/badDateFormat.json diff --git a/examples/schema_test.js b/examples/schema_test.js index d090e696..54f586ed 100644 --- a/examples/schema_test.js +++ b/examples/schema_test.js @@ -5,15 +5,41 @@ import ram from 'random-access-memory' import { randomBytes } from 'node:crypto' // FILTER_1 +// const obj = { +// id: randomBytes(32).toString('hex'), +// type: 'filter', +// schemaVersion: 1, +// created_at: new Date().toJSON(), +// filter: ['observation'], +// name: 'john', +// } + +// DEVICE const obj = { - id: randomBytes(32).toString('hex'), - type: 'filter', + type: 'Device', schemaVersion: 1, - created_at: new Date().toJSON(), - filter: ['observation'], - name: 'john', + id: randomBytes(32).toString('hex'), + action: 'device:add', + authorId: randomBytes(32).toString('hex'), + projectId: randomBytes(32).toString('hex'), + signature: 'hi', + authorIndex: 10, + deviceIndex: 10, } +// CORE OWNERSHIP +// const obj = { +// type: 'coreOwnership', +// schemaVersion: 1, +// id: randomBytes(32).toString('hex'), +// coreId: randomBytes(32).toString('hex'), +// projectId: randomBytes(32).toString('hex'), +// storeType: 'blob', +// authorIndex: 10, +// deviceIndex: 10, +// action: 'core:owner', +// } + // PRESET_1 // const obj = { // id: randomBytes(32).toString('hex'), diff --git a/proto/coreOwnership/v1.proto b/proto/coreOwnership/v1.proto new file mode 100644 index 00000000..513e51ee --- /dev/null +++ b/proto/coreOwnership/v1.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; +package mapeo; + +import "google/protobuf/struct.proto"; +import "common/v1.proto"; + +message CoreOwnership_1 { + Common_1 common = 1; + string action = 2; + string coreId = 3; + string projectId = 4; + string storeType = 5; + string signature = 6; + int32 authorIndex = 7; + int32 deviceIndex = 8; +} diff --git a/proto/device/v1.proto b/proto/device/v1.proto new file mode 100644 index 00000000..6d9379b6 --- /dev/null +++ b/proto/device/v1.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; +package mapeo; + +import "google/protobuf/struct.proto"; +import "common/v1.proto"; + +message Device_1 { + Common_1 common = 1; + string action = 2; + string authorId = 3; + string projectId = 4; + string signature = 5; + int32 authorIndex = 6; + int32 deviceIndex = 7; +} diff --git a/schema/coreOwnership/v1.json b/schema/coreOwnership/v1.json new file mode 100644 index 00000000..64b2be2f --- /dev/null +++ b/schema/coreOwnership/v1.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://mapeo.world/schemas/coreOwnership/v1.json", + "title": "CoreOwnership", + "type": "object", + "allOf":[{"$ref": "../common/v1.json"}], + "properties": { + "type": { "type": "string", "pattern": "^coreOwnership$" }, + "action": { + "type": "string", + "enum": ["core:owner"] + }, + "schemaVersion": { + "type": "number", + "minimum": 1, + "enum": [1] + }, + "coreId": { "type": "string" }, + "projectId": { "type": "string" }, + "storeType": { "type": "string" }, + "signature": { "type": "string" }, + "authorIndex": { "type": "integer" }, + "deviceIndex": { "type": "integer" } + } +} diff --git a/schema/device/v1.json b/schema/device/v1.json new file mode 100644 index 00000000..0aafe92d --- /dev/null +++ b/schema/device/v1.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://mapeo.world/schemas/device/v1.json", + "title": "Device", + "type": "object", + "allOf":[{"$ref": "../common/v1.json"}], + "properties": { + "type": { + "type": "string", + "pattern": "^Device$" + }, + "action": { + "type": "string", + "enum": [ + "device:add", + "device:remove", + "device:restore" + ] + }, + "authorId": { + "type": "string" + }, + "projectId": { + "type": "string" + }, + "signature": { + "type": "string" + }, + "authorIndex": { + "type": "integer" + }, + "deviceIndex": { + "type": "integer" + } + } +} diff --git a/schemasPrefix.js b/schemasPrefix.js index 99d2a56c..363c252b 100644 --- a/schemasPrefix.js +++ b/schemasPrefix.js @@ -4,5 +4,7 @@ const schemasPrefix = { Preset: { dataTypeId: '71f85084cb98', schemaVersions: [1] }, filter: { dataTypeId: '71f85084cb90', schemaVersions: [1] }, Field: { dataTypeId: '71f85084cb80', schemaVersions: [1] }, + coreOwnership: { dataTypeId: '73f85084cb80', schemaVersions: [1] }, + Device: { dataTypeId: '13f85084cb80', schemaVersions: [1] }, } export default schemasPrefix diff --git a/scripts/generate.js b/scripts/generate.js index 0e359c8e..fdbfa239 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -23,7 +23,7 @@ const loadSchema = (p) => { const { dir, name } = path.parse(p) return { // we get the type of the schema from the directory - type: formatSchemaType(dir.replace('../schema/', '')), + type: dir.replace('../schema/', ''), // we get the version from the filename schemaVersion: parseInt(name.replace('v', '')), schema: JSON.parse(fs.readFileSync(new URL(p, import.meta.url)).toString()), @@ -67,10 +67,10 @@ ${schemas .map( /** @param {Object} schema */ ({ schemaVersion, type }) => { - const varName = `${type.toLowerCase()}_${schemaVersion}` + const varName = `${formatSchemaType(type)}_${schemaVersion}` return `import { ${formatSchemaType( type - )} as ${varName} } from './${type.toLowerCase()}/v${schemaVersion}'` + )} as ${varName} } from './${type}/v${schemaVersion}'` } ) .join('\n')} @@ -78,13 +78,12 @@ ${schemas interface base { type?: string; schemaVersion?: number; -[key:string]: any; } export type MapeoRecord = (${schemas .map( /** @param {Object} schema */ ({ schemaVersion, type }) => { - return `${type.toLowerCase()}_${schemaVersion}` + return `${formatSchemaType(type)}_${schemaVersion}` } ) .join(' | ')}) & base diff --git a/test/docs/badDateFormat.json b/test/docs/badDateFormat.json deleted file mode 100644 index 0967ef42..00000000 --- a/test/docs/badDateFormat.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/test/docs/badDocType.json b/test/docs/badDocType.json index 6cec7304..0ab12907 100644 --- a/test/docs/badDocType.json +++ b/test/docs/badDocType.json @@ -1,9 +1,9 @@ { - "id": "d436a4bdde139eebb1c5ad18f8d6a31ecbf51f498dca616b24d68d845940651f", + "id": "e4b4644ccb71910436371ff766f72340cbeae07f02036433d09e34a984a789a0", "type": "doesnotexist", "schemaVersion": 4, "links": [], - "created_at": "2023-02-28T14:58:07.711Z", + "created_at": "2023-03-01T17:13:04.628Z", "refs": [], "attachments": [], "metadata": { diff --git a/test/docs/badSchemaVersion.json b/test/docs/badSchemaVersion.json index 95238891..e079b062 100644 --- a/test/docs/badSchemaVersion.json +++ b/test/docs/badSchemaVersion.json @@ -1,9 +1,9 @@ { - "id": "ef36683c63166a0001fc0bc1ef5f5d9a3420fb65cc5a02b98824ca582530b4d0", + "id": "e3381660d4bdbc3477f3d2777d442896fa2c9b330373b12a2b460809ad780660", "type": "observation", "schemaVersion": null, "links": [], - "created_at": "2023-02-28T14:58:07.711Z", + "created_at": "2023-03-01T17:13:04.628Z", "refs": [], "attachments": [], "metadata": { diff --git a/test/docs/good.json b/test/docs/good.json index dfdec377..d4bad6b4 100644 --- a/test/docs/good.json +++ b/test/docs/good.json @@ -1,10 +1,10 @@ { "observation_4": { - "id": "295f6861699930714d04edbe31b26e19214940c5be190e12fe2b1c82e454bed7", + "id": "e5f4fc2dcc561a34b91d9d5bdfaebc996d4d50963825fc43df4111026100eae4", "type": "observation", "schemaVersion": 4, "links": [], - "created_at": "2023-02-28T14:58:07.711Z", + "created_at": "2023-03-01T17:13:04.628Z", "refs": [], "attachments": [], "metadata": { @@ -12,23 +12,23 @@ } }, "observation_5": { - "id": "a246a13b1bdddc5d312f5051f7c83cd3a912a0051087b5d13404334c00f03ab3", + "id": "0137f29f892d9160b445af146e90b47858416d83713f9f63f019121b3de4d87b", "type": "Observation", "schemaVersion": 5, - "created_at": "2023-02-28T14:58:07.711Z" + "created_at": "2023-03-01T17:13:04.628Z" }, "filter": { - "id": "dac6892c0d4259080f32948ecd3a3231a3159141fa6d9d5a68c80cd2671fa895", + "id": "4b6c1ddbc6166a95eebe48f50477c151bb033b879628e494a3c8b4ff98a9646f", "type": "filter", "schemaVersion": 1, - "created_at": "2023-02-28T14:58:07.711Z", + "created_at": "2023-03-01T17:13:04.628Z", "filter": [ "observation" ], "name": "john" }, "preset": { - "id": "d50c7526e78e938ed9b85b82062897e05f45ec316071f92d5a86ad384ad56d6f", + "id": "b83340d248ee599a80a70ced2671e14d0909b39c9f6ede7be8e0bb1961c1b756", "type": "Preset", "schemaVersion": 1, "tags": { @@ -40,9 +40,31 @@ "name": "john" }, "field": { - "id": "da6445be7c2d5b891cf2783131481ca55c3f0bd898cb7744c143d921c42fe05f", + "id": "8772606a3b1a2f537d5cd4a84a3024e2f553ae4a161255b78acfc05cadf2b2d9", "type": "Field", "schemaVersion": 1, "key": "hi" + }, + "coreOwnership": { + "type": "coreOwnership", + "schemaVersion": 1, + "id": "d0c87aaf2afa65959d708dd779965b1294e22f326ad05c44714a4b4c131341b6", + "coreId": "c79ebaf961bf8598d6200c99c929bf4212346a1911122090367764990d105313", + "projectId": "1984d0fec810d324a8d600a26462e74a9398927ad18e70735ac1958d1f2f0bea", + "storeType": "blob", + "authorIndex": 10, + "deviceIndex": 10, + "action": "core:owner" + }, + "device": { + "type": "Device", + "schemaVersion": 1, + "id": "637ddc6abcab6bfcad7d79f4388581d962dcce26744b9e0125dd9fa079dd2ede", + "action": "device:add", + "authorId": "8cdea85a935f9eb0fc34266c66c3ec70c183ed7f4124279ff090bd9368b88ce6", + "projectId": "268827d07807accd55f4a2cca624ef2b97f2adbb65d2b5033d51a8b42ed7d469", + "signature": "hi", + "authorIndex": 10, + "deviceIndex": 10 } } \ No newline at end of file diff --git a/test/docs/onlyId.json b/test/docs/onlyId.json index 5dcf5872..a6b9ccc8 100644 --- a/test/docs/onlyId.json +++ b/test/docs/onlyId.json @@ -1,3 +1,3 @@ { - "id": "841d7233357a60d3a1950cd07b02c4265825b284182db5eb81a79eaaf805ebdf" + "id": "6dad7d71c72401ca05011a144f55356ab4612924cd05adc8a2c4ae44e6710bac" } \ No newline at end of file diff --git a/utils.js b/utils.js index 36597f2a..150b78a3 100644 --- a/utils.js +++ b/utils.js @@ -4,7 +4,7 @@ * @returns {String} First letter capitalized, the rest lowercased */ export const formatSchemaType = (str) => - str.charAt(0).toUpperCase() + str.slice(1).toLowerCase() + str.charAt(0).toUpperCase() + str.slice(1) /** * Turn a schema type and version into `${type}_${schemaVersion}` From 760a9feb85a30932044f4dac32c79abec2d460ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 1 Mar 2023 14:24:14 -0300 Subject: [PATCH 15/18] feat: add schema for role --- examples/schema_test.js | 21 +++++++++++++++++---- proto/role/v1.proto | 15 +++++++++++++++ schema/role/v1.json | 40 ++++++++++++++++++++++++++++++++++++++++ schemasPrefix.js | 1 + test/docs/good.json | 39 +++++++++++++++++++++++++-------------- 5 files changed, 98 insertions(+), 18 deletions(-) create mode 100644 proto/role/v1.proto create mode 100644 schema/role/v1.json diff --git a/examples/schema_test.js b/examples/schema_test.js index 54f586ed..280c40e2 100644 --- a/examples/schema_test.js +++ b/examples/schema_test.js @@ -15,13 +15,26 @@ import { randomBytes } from 'node:crypto' // } // DEVICE +// const obj = { +// type: 'Device', +// schemaVersion: 1, +// id: randomBytes(32).toString('hex'), +// action: 'device:add', +// authorId: randomBytes(32).toString('hex'), +// projectId: randomBytes(32).toString('hex'), +// signature: 'hi', +// authorIndex: 10, +// deviceIndex: 10, +// } + +// ROLE const obj = { - type: 'Device', - schemaVersion: 1, id: randomBytes(32).toString('hex'), - action: 'device:add', - authorId: randomBytes(32).toString('hex'), + type: 'Role', + schemaVersion: 1, + role: 'project-creator', projectId: randomBytes(32).toString('hex'), + action: 'role:set', signature: 'hi', authorIndex: 10, deviceIndex: 10, diff --git a/proto/role/v1.proto b/proto/role/v1.proto new file mode 100644 index 00000000..6d58419e --- /dev/null +++ b/proto/role/v1.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; +package mapeo; + +import "google/protobuf/struct.proto"; +import "common/v1.proto"; + +message Role_1 { + Common_1 common = 1; + string role = 2; + string projectId = 3; + string action = 4; + string signature = 5; + int32 authorIndex = 6; + int32 deviceIndex = 7; +} diff --git a/schema/role/v1.json b/schema/role/v1.json new file mode 100644 index 00000000..01786968 --- /dev/null +++ b/schema/role/v1.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://mapeo.world/schemas/role/v1.json", + "title": "Role", + "type": "object", + "allOf":[{"$ref": "../common/v1.json"}], + "properties": { + "type": { + "type": "string", + "pattern": "^Role$" + }, + "role": { + "type": "string", + "enum": [ + "project-creator", + "coordinator", + "member", + "non-member" + ] + }, + "projectId": { + "type": "string" + }, + "action": { + "type": "string", + "enum": [ + "role:set" + ] + }, + "signature": { + "type": "string" + }, + "authorIndex": { + "type": "integer" + }, + "deviceIndex": { + "type": "integer" + } + } +} diff --git a/schemasPrefix.js b/schemasPrefix.js index 363c252b..a36122ba 100644 --- a/schemasPrefix.js +++ b/schemasPrefix.js @@ -6,5 +6,6 @@ const schemasPrefix = { Field: { dataTypeId: '71f85084cb80', schemaVersions: [1] }, coreOwnership: { dataTypeId: '73f85084cb80', schemaVersions: [1] }, Device: { dataTypeId: '13f85084cb80', schemaVersions: [1] }, + Role: { dataTypeId: '13fa5384cb80', schemaVersions: [1] }, } export default schemasPrefix diff --git a/test/docs/good.json b/test/docs/good.json index d4bad6b4..82100dc4 100644 --- a/test/docs/good.json +++ b/test/docs/good.json @@ -1,10 +1,10 @@ { "observation_4": { - "id": "e5f4fc2dcc561a34b91d9d5bdfaebc996d4d50963825fc43df4111026100eae4", + "id": "489319aa2f9c58b359b0312a12fad36b7c39c5492e506a3e2d774da3bb370607", "type": "observation", "schemaVersion": 4, "links": [], - "created_at": "2023-03-01T17:13:04.628Z", + "created_at": "2023-03-01T17:23:21.037Z", "refs": [], "attachments": [], "metadata": { @@ -12,23 +12,23 @@ } }, "observation_5": { - "id": "0137f29f892d9160b445af146e90b47858416d83713f9f63f019121b3de4d87b", + "id": "97b3f1dbfed5f3b4f49780e94bf0f49a645d5fdb029ba558a04915adbd84a5b2", "type": "Observation", "schemaVersion": 5, - "created_at": "2023-03-01T17:13:04.628Z" + "created_at": "2023-03-01T17:23:21.037Z" }, "filter": { - "id": "4b6c1ddbc6166a95eebe48f50477c151bb033b879628e494a3c8b4ff98a9646f", + "id": "f57fbf659e23143a84eae28f7b42a71c59234462c35f63455ba779cc2e005a7f", "type": "filter", "schemaVersion": 1, - "created_at": "2023-03-01T17:13:04.628Z", + "created_at": "2023-03-01T17:23:21.037Z", "filter": [ "observation" ], "name": "john" }, "preset": { - "id": "b83340d248ee599a80a70ced2671e14d0909b39c9f6ede7be8e0bb1961c1b756", + "id": "0f915eb552ae885711a5a5979aa8b80455a29edc44b0590805b0151d2b2fa6e5", "type": "Preset", "schemaVersion": 1, "tags": { @@ -40,7 +40,7 @@ "name": "john" }, "field": { - "id": "8772606a3b1a2f537d5cd4a84a3024e2f553ae4a161255b78acfc05cadf2b2d9", + "id": "e5af32dec58c1950fb7fe5d0083939de0816447f51e130ef8f9de2be77155d85", "type": "Field", "schemaVersion": 1, "key": "hi" @@ -48,9 +48,9 @@ "coreOwnership": { "type": "coreOwnership", "schemaVersion": 1, - "id": "d0c87aaf2afa65959d708dd779965b1294e22f326ad05c44714a4b4c131341b6", - "coreId": "c79ebaf961bf8598d6200c99c929bf4212346a1911122090367764990d105313", - "projectId": "1984d0fec810d324a8d600a26462e74a9398927ad18e70735ac1958d1f2f0bea", + "id": "7384d04d3f9b72fc50b2ea4f348c137f44ea15b7c7321a95411380865354365e", + "coreId": "12aed1dc8c978a0338acf40606de23224c9eb4d748f6da9af1269a7a46fb910a", + "projectId": "5516e90e7c6d72872c414cd1d93c16a033896afe81fa86e2255444e0be10d3a8", "storeType": "blob", "authorIndex": 10, "deviceIndex": 10, @@ -59,10 +59,21 @@ "device": { "type": "Device", "schemaVersion": 1, - "id": "637ddc6abcab6bfcad7d79f4388581d962dcce26744b9e0125dd9fa079dd2ede", + "id": "7b754ac86bfa4bf7ba32e1ada8239b6122da885ddf691c525aa86d767592a646", "action": "device:add", - "authorId": "8cdea85a935f9eb0fc34266c66c3ec70c183ed7f4124279ff090bd9368b88ce6", - "projectId": "268827d07807accd55f4a2cca624ef2b97f2adbb65d2b5033d51a8b42ed7d469", + "authorId": "495e98072bc616a783c243b0461dafb6595d9133a197e894f2ce545617e9da06", + "projectId": "0711767837741ade5eddf6ad49f88773840edb194d6066b9532fc02be4729e5e", + "signature": "hi", + "authorIndex": 10, + "deviceIndex": 10 + }, + "role": { + "id": "ae16a20a443d89d4af57e3588f7fdfa0922955de922ae04292598387ed881195", + "type": "Role", + "schemaVersion": 1, + "role": "project-creator", + "projectId": "f7e70d9d4539ee5f8b53ba71953940e16f279a8faea8ef10b0eba2f6e503cfe3", + "action": "role:set", "signature": "hi", "authorIndex": 10, "deviceIndex": 10 From 604161cadd609b735f896c97c5c1f39a87e6ece0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 1 Mar 2023 14:38:18 -0300 Subject: [PATCH 16/18] feat: add testing for validation of JSONSchema docs --- test/docs/good.json | 41 ++++++++++++++++++++++------------------- test/index.js | 18 +++++++++++++++++- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/test/docs/good.json b/test/docs/good.json index 82100dc4..63bd3480 100644 --- a/test/docs/good.json +++ b/test/docs/good.json @@ -1,10 +1,10 @@ { "observation_4": { - "id": "489319aa2f9c58b359b0312a12fad36b7c39c5492e506a3e2d774da3bb370607", + "id": "ed9cd606c88325ae45a63d1177dd4ce2a70222b65a92e1e4332eb870966c62a7", "type": "observation", "schemaVersion": 4, "links": [], - "created_at": "2023-03-01T17:23:21.037Z", + "created_at": "2023-03-01T17:34:40.539Z", "refs": [], "attachments": [], "metadata": { @@ -12,23 +12,23 @@ } }, "observation_5": { - "id": "97b3f1dbfed5f3b4f49780e94bf0f49a645d5fdb029ba558a04915adbd84a5b2", + "id": "13f8b3b226e906c18e9647e0bfe2cf4cab88b79a53b45368a64ae9db68b5caab", "type": "Observation", "schemaVersion": 5, - "created_at": "2023-03-01T17:23:21.037Z" + "created_at": "2023-03-01T17:34:40.539Z" }, "filter": { - "id": "f57fbf659e23143a84eae28f7b42a71c59234462c35f63455ba779cc2e005a7f", + "id": "3cf15bb3f999ca0dbd806c3b8cce3018a7bb4f677f2fde04a4006b963057e537", "type": "filter", "schemaVersion": 1, - "created_at": "2023-03-01T17:23:21.037Z", + "created_at": "2023-03-01T17:34:40.539Z", "filter": [ "observation" ], "name": "john" }, "preset": { - "id": "0f915eb552ae885711a5a5979aa8b80455a29edc44b0590805b0151d2b2fa6e5", + "id": "6417980b55894dcd98dc22efaa211d7a150b15b49514189ea9476a90a1907302", "type": "Preset", "schemaVersion": 1, "tags": { @@ -40,7 +40,7 @@ "name": "john" }, "field": { - "id": "e5af32dec58c1950fb7fe5d0083939de0816447f51e130ef8f9de2be77155d85", + "id": "9b8b58c4c22e4078ce4e2449ebc099618ea59223fd3b31c12adfabc2eec03ed3", "type": "Field", "schemaVersion": 1, "key": "hi" @@ -48,34 +48,37 @@ "coreOwnership": { "type": "coreOwnership", "schemaVersion": 1, - "id": "7384d04d3f9b72fc50b2ea4f348c137f44ea15b7c7321a95411380865354365e", - "coreId": "12aed1dc8c978a0338acf40606de23224c9eb4d748f6da9af1269a7a46fb910a", - "projectId": "5516e90e7c6d72872c414cd1d93c16a033896afe81fa86e2255444e0be10d3a8", + "id": "8e08069ad265c0b9ba3dec0c2bce064eec909da1a28a7c222cb3cca3ab87f7bb", + "coreId": "01076555a03066704e5f328eef50aec2b2a4ebdda9949dc927fc648a79753cee", + "projectId": "74de52fa0ac6e0b921f57592ba668cc1a343f455cce3d883150ed2ac93b634b1", "storeType": "blob", "authorIndex": 10, "deviceIndex": 10, - "action": "core:owner" + "action": "core:owner", + "created_at": "2023-03-01T17:34:40.539Z" }, "device": { "type": "Device", "schemaVersion": 1, - "id": "7b754ac86bfa4bf7ba32e1ada8239b6122da885ddf691c525aa86d767592a646", + "id": "82efac8f78c7233c6f7bf721f9989b97a8ab53a340d2e788d12f83e2863509a6", "action": "device:add", - "authorId": "495e98072bc616a783c243b0461dafb6595d9133a197e894f2ce545617e9da06", - "projectId": "0711767837741ade5eddf6ad49f88773840edb194d6066b9532fc02be4729e5e", + "authorId": "d2a557903924bebe5b45629b2ae3730b7a59c7bc2e379e86f9f8b879d38808f2", + "projectId": "480974b45f1717975bfef0c000ff46b9835fac2cbae342ad6a985552a24b5feb", "signature": "hi", "authorIndex": 10, - "deviceIndex": 10 + "deviceIndex": 10, + "created_at": "2023-03-01T17:34:40.539Z" }, "role": { - "id": "ae16a20a443d89d4af57e3588f7fdfa0922955de922ae04292598387ed881195", + "id": "5129b0c2ed56315c71eeb24562c20ec729467b5e42bfe15a7effab6089f12b9b", "type": "Role", "schemaVersion": 1, "role": "project-creator", - "projectId": "f7e70d9d4539ee5f8b53ba71953940e16f279a8faea8ef10b0eba2f6e503cfe3", + "projectId": "3b75544dae53346c00aa8a297cab71938f54ce6962148cfdd210f93782fb4a53", "action": "role:set", "signature": "hi", "authorIndex": 10, - "deviceIndex": 10 + "deviceIndex": 10, + "created_at": "2023-03-01T17:34:40.539Z" } } \ No newline at end of file diff --git a/test/index.js b/test/index.js index 145f7e44..d0ca499d 100644 --- a/test/index.js +++ b/test/index.js @@ -1,5 +1,5 @@ import test from 'tape' -import { encode, decode } from '../index.js' +import { encode, decode, validate } from '../index.js' import glob from 'glob-promise' import { readFileSync } from 'node:fs' import { basename } from 'node:path' @@ -52,6 +52,22 @@ test('test encoding of rightfully formated record', async (t) => { }) }) +test('test validation of record', async (t) => { + const goodDocs = Object.keys(docs.good) + t.plan(goodDocs.length) + goodDocs.forEach((k) => { + const doc = docs.good[k] + const record = decode(encode(doc), { + coreId: randomBytes(32), + seq: 0, + }) + t.doesNotThrow(() => { + // field has a type which is different from the rest :| + if (k !== 'field') validate(record) + }, `testing validation of ${k}`) + }) +}) + test('test encoding, decoding of record and comparing the two versions', async (t) => { const goodDocs = Object.keys(docs.good) goodDocs.forEach((k) => { From 670cc9e2befb96a8feb1f5b920d79f3cce6706e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 1 Mar 2023 16:57:23 -0300 Subject: [PATCH 17/18] feat: created_at is saved as Timestamp on protobuf --- examples/schema_test.js | 1 + index.js | 5 ++ proto/common/v1.proto | 4 +- proto/filter/v1.proto | 3 +- proto/observation/v4.proto | 3 +- test/docs.js | 108 ++++++++++++++++++++++++++++++++ test/docs/badDocType.json | 12 ---- test/docs/badSchemaVersion.json | 12 ---- test/docs/good.json | 84 ------------------------- test/docs/onlyId.json | 3 - test/index.js | 22 +------ 11 files changed, 124 insertions(+), 133 deletions(-) create mode 100644 test/docs.js delete mode 100644 test/docs/badDocType.json delete mode 100644 test/docs/badSchemaVersion.json delete mode 100644 test/docs/good.json delete mode 100644 test/docs/onlyId.json diff --git a/examples/schema_test.js b/examples/schema_test.js index 280c40e2..efe34f96 100644 --- a/examples/schema_test.js +++ b/examples/schema_test.js @@ -33,6 +33,7 @@ const obj = { type: 'Role', schemaVersion: 1, role: 'project-creator', + created_at: new Date(), projectId: randomBytes(32).toString('hex'), action: 'role:set', signature: 'hi', diff --git a/index.js b/index.js index 507789d6..f13587e9 100644 --- a/index.js +++ b/index.js @@ -34,7 +34,10 @@ const jsonSchemaToProto = (obj) => { const common = commonKeys .filter((field) => obj[field]) .reduce((common, field) => ({ ...common, [field]: obj[field] }), {}) + common.id = Buffer.from(obj['id'], 'hex') + // turn date represented as string to Date + common.created_at = new Date(common.created_at) const key = formatSchemaKey(obj.type, obj.schemaVersion) // when we inherit from common, common is actually a field inside the protobuf object, @@ -67,6 +70,8 @@ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { } obj.id = obj.id.toString('hex') + // turn date represented as Date to string + if (obj.created_at) obj.created_at = obj.created_at.toJSON() return obj } diff --git a/proto/common/v1.proto b/proto/common/v1.proto index 504b5d07..a0f5dc21 100644 --- a/proto/common/v1.proto +++ b/proto/common/v1.proto @@ -1,8 +1,10 @@ syntax = "proto3"; package mapeo; +import "google/protobuf/timestamp.proto"; + message Common_1 { - string created_at = 1; + google.protobuf.Timestamp created_at = 1; optional string deviceId = 2; // 32-byte random generated number bytes id = 3; diff --git a/proto/filter/v1.proto b/proto/filter/v1.proto index 43c20502..9d0b78ba 100644 --- a/proto/filter/v1.proto +++ b/proto/filter/v1.proto @@ -2,10 +2,11 @@ syntax = "proto3"; package mapeo; import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; message Filter_1 { bytes id = 1; - string created_at = 2; + google.protobuf.Timestamp created_at = 2; repeated string filter = 3; string name = 4; optional string timestamp = 5; diff --git a/proto/observation/v4.proto b/proto/observation/v4.proto index ddfc14c3..c22391be 100644 --- a/proto/observation/v4.proto +++ b/proto/observation/v4.proto @@ -2,10 +2,11 @@ syntax = "proto3"; package mapeo; import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; message Observation_4 { bytes id = 1; - string created_at = 4; + google.protobuf.Timestamp created_at = 4; optional string timestamp = 5; optional string userId = 6; optional string deviceId = 7; diff --git a/test/docs.js b/test/docs.js new file mode 100644 index 00000000..52a94c08 --- /dev/null +++ b/test/docs.js @@ -0,0 +1,108 @@ +import { randomBytes } from 'node:crypto' + +export const docs = { + onlyId: { id: randomBytes(32).toString('hex') }, + badDocType: { + id: randomBytes(32).toString('hex'), + type: 'doesnotexist', + schemaVersion: 4, + links: [], + created_at: new Date().toJSON(), + refs: [], + attachments: [], + metadata: { + manual_location: true, + }, + }, + badSchemaVersion: { + id: randomBytes(32).toString('hex'), + type: 'observation', + schemaVersion: null, + links: [], + created_at: new Date().toJSON(), + refs: [], + attachments: [], + metadata: { + manual_location: true, + }, + }, + good: { + observation_4: { + id: randomBytes(32).toString('hex'), + type: 'observation', + schemaVersion: 4, + links: [], + created_at: new Date().toJSON(), + refs: [], + attachments: [], + metadata: { + manual_location: true, + }, + }, + observation_5: { + id: randomBytes(32).toString('hex'), + type: 'Observation', + schemaVersion: 5, + created_at: new Date().toJSON(), + }, + filter: { + id: randomBytes(32).toString('hex'), + type: 'filter', + schemaVersion: 1, + created_at: new Date().toJSON(), + filter: ['observation'], + name: 'john', + }, + preset: { + id: randomBytes(32).toString('hex'), + type: 'Preset', + schemaVersion: 1, + tags: { nature: 'tree' }, + geometry: ['point'], + name: 'john', + }, + field: { + id: randomBytes(32).toString('hex'), + type: 'Field', + schemaVersion: 1, + key: 'hi', + }, + coreOwnership: { + type: 'coreOwnership', + schemaVersion: 1, + id: randomBytes(32).toString('hex'), + coreId: randomBytes(32).toString('hex'), + projectId: randomBytes(32).toString('hex'), + storeType: 'blob', + authorIndex: 10, + deviceIndex: 10, + action: 'core:owner', + created_at: new Date().toJSON(), + }, + device: { + type: 'Device', + schemaVersion: 1, + id: randomBytes(32).toString('hex'), + action: 'device:add', + authorId: randomBytes(32).toString('hex'), + projectId: randomBytes(32).toString('hex'), + signature: 'hi', + authorIndex: 10, + deviceIndex: 10, + created_at: new Date().toJSON(), + }, + role: { + id: randomBytes(32).toString('hex'), + type: 'Role', + schemaVersion: 1, + role: 'project-creator', + projectId: randomBytes(32).toString('hex'), + action: 'role:set', + signature: 'hi', + authorIndex: 10, + deviceIndex: 10, + created_at: new Date().toJSON(), + }, + }, +} +// Object.keys(docs).forEach(save) diff --git a/test/docs/badDocType.json b/test/docs/badDocType.json deleted file mode 100644 index 0ab12907..00000000 --- a/test/docs/badDocType.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "e4b4644ccb71910436371ff766f72340cbeae07f02036433d09e34a984a789a0", - "type": "doesnotexist", - "schemaVersion": 4, - "links": [], - "created_at": "2023-03-01T17:13:04.628Z", - "refs": [], - "attachments": [], - "metadata": { - "manual_location": true - } -} \ No newline at end of file diff --git a/test/docs/badSchemaVersion.json b/test/docs/badSchemaVersion.json deleted file mode 100644 index e079b062..00000000 --- a/test/docs/badSchemaVersion.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "e3381660d4bdbc3477f3d2777d442896fa2c9b330373b12a2b460809ad780660", - "type": "observation", - "schemaVersion": null, - "links": [], - "created_at": "2023-03-01T17:13:04.628Z", - "refs": [], - "attachments": [], - "metadata": { - "manual_location": true - } -} \ No newline at end of file diff --git a/test/docs/good.json b/test/docs/good.json deleted file mode 100644 index 63bd3480..00000000 --- a/test/docs/good.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "observation_4": { - "id": "ed9cd606c88325ae45a63d1177dd4ce2a70222b65a92e1e4332eb870966c62a7", - "type": "observation", - "schemaVersion": 4, - "links": [], - "created_at": "2023-03-01T17:34:40.539Z", - "refs": [], - "attachments": [], - "metadata": { - "manual_location": true - } - }, - "observation_5": { - "id": "13f8b3b226e906c18e9647e0bfe2cf4cab88b79a53b45368a64ae9db68b5caab", - "type": "Observation", - "schemaVersion": 5, - "created_at": "2023-03-01T17:34:40.539Z" - }, - "filter": { - "id": "3cf15bb3f999ca0dbd806c3b8cce3018a7bb4f677f2fde04a4006b963057e537", - "type": "filter", - "schemaVersion": 1, - "created_at": "2023-03-01T17:34:40.539Z", - "filter": [ - "observation" - ], - "name": "john" - }, - "preset": { - "id": "6417980b55894dcd98dc22efaa211d7a150b15b49514189ea9476a90a1907302", - "type": "Preset", - "schemaVersion": 1, - "tags": { - "nature": "tree" - }, - "geometry": [ - "point" - ], - "name": "john" - }, - "field": { - "id": "9b8b58c4c22e4078ce4e2449ebc099618ea59223fd3b31c12adfabc2eec03ed3", - "type": "Field", - "schemaVersion": 1, - "key": "hi" - }, - "coreOwnership": { - "type": "coreOwnership", - "schemaVersion": 1, - "id": "8e08069ad265c0b9ba3dec0c2bce064eec909da1a28a7c222cb3cca3ab87f7bb", - "coreId": "01076555a03066704e5f328eef50aec2b2a4ebdda9949dc927fc648a79753cee", - "projectId": "74de52fa0ac6e0b921f57592ba668cc1a343f455cce3d883150ed2ac93b634b1", - "storeType": "blob", - "authorIndex": 10, - "deviceIndex": 10, - "action": "core:owner", - "created_at": "2023-03-01T17:34:40.539Z" - }, - "device": { - "type": "Device", - "schemaVersion": 1, - "id": "82efac8f78c7233c6f7bf721f9989b97a8ab53a340d2e788d12f83e2863509a6", - "action": "device:add", - "authorId": "d2a557903924bebe5b45629b2ae3730b7a59c7bc2e379e86f9f8b879d38808f2", - "projectId": "480974b45f1717975bfef0c000ff46b9835fac2cbae342ad6a985552a24b5feb", - "signature": "hi", - "authorIndex": 10, - "deviceIndex": 10, - "created_at": "2023-03-01T17:34:40.539Z" - }, - "role": { - "id": "5129b0c2ed56315c71eeb24562c20ec729467b5e42bfe15a7effab6089f12b9b", - "type": "Role", - "schemaVersion": 1, - "role": "project-creator", - "projectId": "3b75544dae53346c00aa8a297cab71938f54ce6962148cfdd210f93782fb4a53", - "action": "role:set", - "signature": "hi", - "authorIndex": 10, - "deviceIndex": 10, - "created_at": "2023-03-01T17:34:40.539Z" - } -} \ No newline at end of file diff --git a/test/docs/onlyId.json b/test/docs/onlyId.json deleted file mode 100644 index a6b9ccc8..00000000 --- a/test/docs/onlyId.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "id": "6dad7d71c72401ca05011a144f55356ab4612924cd05adc8a2c4ae44e6710bac" -} \ No newline at end of file diff --git a/test/index.js b/test/index.js index d0ca499d..0f431545 100644 --- a/test/index.js +++ b/test/index.js @@ -1,24 +1,7 @@ import test from 'tape' import { encode, decode, validate } from '../index.js' -import glob from 'glob-promise' -import { readFileSync } from 'node:fs' -import { basename } from 'node:path' import { randomBytes } from 'node:crypto' - -const loadJSON = (path) => { - return { - name: basename(path).replace('.json', ''), - doc: JSON.parse(readFileSync(new URL(path, import.meta.url)).toString()), - } -} - -const docs = glob - .sync('./docs/*.json', { cwd: 'test' }) - .map(loadJSON) - .reduce((acc, val) => { - acc[val.name] = val.doc - return acc - }, {}) +import { docs } from './docs.js' test('test encoding of record with missing fields', async (t) => { t.plan(1) @@ -48,7 +31,7 @@ test('test encoding of rightfully formated record', async (t) => { const doc = docs.good[k] t.doesNotThrow(() => { encode(doc) - }) + }, `testing ${k}`) }) }) @@ -79,6 +62,7 @@ test('test encoding, decoding of record and comparing the two versions', async ( const fields = Object.keys(doc) // t.plan(goodDocs.length * fields.length * 2) fields.forEach((f) => { + console.log(typeof record[f], typeof doc[f]) const msg = `checking ${f} for ${k}` record[f] && doc[f] ? t.pass(msg) : t.fail(msg) From 4cdcb4add3dba8cfcd825f12d4eec7e3546c7bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Ciccola?= Date: Wed, 1 Mar 2023 17:09:03 -0300 Subject: [PATCH 18/18] feat: timestamp is saved as Timestamp on protobuf --- index.js | 2 ++ proto/common/v1.proto | 2 +- proto/filter/v1.proto | 2 +- proto/observation/v4.proto | 2 +- test/docs.js | 6 ++++++ 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index f13587e9..008c00d8 100644 --- a/index.js +++ b/index.js @@ -38,6 +38,7 @@ const jsonSchemaToProto = (obj) => { common.id = Buffer.from(obj['id'], 'hex') // turn date represented as string to Date common.created_at = new Date(common.created_at) + common.timestamp = new Date(common.timestamp) const key = formatSchemaKey(obj.type, obj.schemaVersion) // when we inherit from common, common is actually a field inside the protobuf object, @@ -72,6 +73,7 @@ const protoToJsonSchema = (protobufObj, { schemaVersion, type, version }) => { obj.id = obj.id.toString('hex') // turn date represented as Date to string if (obj.created_at) obj.created_at = obj.created_at.toJSON() + if (obj.timestamp) obj.timestamp = obj.timestamp.toJSON() return obj } diff --git a/proto/common/v1.proto b/proto/common/v1.proto index a0f5dc21..ac72ca4c 100644 --- a/proto/common/v1.proto +++ b/proto/common/v1.proto @@ -9,7 +9,7 @@ message Common_1 { // 32-byte random generated number bytes id = 3; repeated string links = 4; - optional string timestamp = 5; + optional google.protobuf.Timestamp timestamp = 5; optional string userId = 6; } /* ignored fields and differences from common.json jsonSchema diff --git a/proto/filter/v1.proto b/proto/filter/v1.proto index 9d0b78ba..b4c7d615 100644 --- a/proto/filter/v1.proto +++ b/proto/filter/v1.proto @@ -9,7 +9,7 @@ message Filter_1 { google.protobuf.Timestamp created_at = 2; repeated string filter = 3; string name = 4; - optional string timestamp = 5; + optional google.protobuf.Timestamp timestamp = 5; optional string userId = 6; optional string deviceId = 7; repeated string links = 8; diff --git a/proto/observation/v4.proto b/proto/observation/v4.proto index c22391be..eca459a6 100644 --- a/proto/observation/v4.proto +++ b/proto/observation/v4.proto @@ -7,7 +7,7 @@ import "google/protobuf/timestamp.proto"; message Observation_4 { bytes id = 1; google.protobuf.Timestamp created_at = 4; - optional string timestamp = 5; + optional google.protobuf.Timestamp timestamp = 5; optional string userId = 6; optional string deviceId = 7; repeated string links = 8; diff --git a/test/docs.js b/test/docs.js index 52a94c08..c1c0d086 100644 --- a/test/docs.js +++ b/test/docs.js @@ -33,6 +33,7 @@ export const docs = { schemaVersion: 4, links: [], created_at: new Date().toJSON(), + timestamp: new Date().toJSON(), refs: [], attachments: [], metadata: { @@ -44,9 +45,11 @@ export const docs = { type: 'Observation', schemaVersion: 5, created_at: new Date().toJSON(), + timestamp: new Date().toJSON(), }, filter: { id: randomBytes(32).toString('hex'), + timestamp: new Date().toJSON(), type: 'filter', schemaVersion: 1, created_at: new Date().toJSON(), @@ -78,6 +81,7 @@ export const docs = { deviceIndex: 10, action: 'core:owner', created_at: new Date().toJSON(), + timestamp: new Date().toJSON(), }, device: { type: 'Device', @@ -90,6 +94,7 @@ export const docs = { authorIndex: 10, deviceIndex: 10, created_at: new Date().toJSON(), + timestamp: new Date().toJSON(), }, role: { id: randomBytes(32).toString('hex'), @@ -102,6 +107,7 @@ export const docs = { authorIndex: 10, deviceIndex: 10, created_at: new Date().toJSON(), + timestamp: new Date().toJSON(), }, }, }