Create function to traverse the object for encoding - Closes #5217 & #5228 & #5235

elements/lisk-codec/benchmark/encode.js

@@ -0,0 +1,87 @@
+/*
+ * Copyright © 2020 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+// writeBoolean x 3,543,238 ops/sec ±1.59% (89 runs sampled)
+
+const { Suite } = require('benchmark');
+const {
+    codec,
+} = require('../dist-node/codec');
+
+const suite = new Suite();
+
+const testSchema = {
+	$id: 'testSchema',
+	type: 'object',
+	properties: {
+		b: { fieldNumber: 2, dataType: 'string' },
+		a: { fieldNumber: 1, dataType: 'string' },
+		d: { fieldNumber: 4, dataType: 'bytes' },
+		e: { fieldNumber: 5, dataType: 'uint32' },
+		c: {
+			type: 'object',
+			fieldNumber: 3,
+			properties: {
+				cc: { fieldNumber: 3, dataType: 'string' },
+				ca: { fieldNumber: 1, dataType: 'string' },
+				cb: {
+					type: 'object',
+					fieldNumber: 2,
+					properties: {
+						cbb: { fieldNumber: 2, dataType: 'string' },
+						cba: { fieldNumber: 1, dataType: 'string' },
+						cbc: {
+							type: 'object',
+							fieldNumber: 3,
+							properties: {
+								cbcb: { fieldNumber: 3, dataType: 'string' },
+								cbca: { fieldNumber: 2, dataType: 'string' },
+							},
+						},
+						cbd: { fieldNumber: 4, dataType: 'string' },
+					},
+				},
+			},
+		},
+	},
+};
+
+const obj = {
+	b: '1371893719313718937193137189371931371893719313718937193',
+	a: '1371893719313718937193137189371931371893719313718937193',
+	d: Buffer.from('1371893719313718937193137189371931371893719313718937193'),
+	e: 10000,
+	c: {
+		cc: '1371893719313718937193137189371931371893719313718937193',
+		ca: '1371893719313718937193137189371931371893719313718937193',
+		cb: {
+			cbb: '1371893719313718937193137189371931371893719313718937193',
+			cba: '1371893719313718937193137189371931371893719313718937193',
+			cbc: {
+				cbcb: '1371893719313718937193137189371931371893719313718937193',
+				cbca: '1371893719313718937193137189371931371893719313718937193',
+			},
+			cbd: '1371893719313718937193137189371931371893719313718937193',
+		},
+	},
+};
+
+
+suite
+    .add('encode', () => {
+			codec.encode(testSchema, obj);
+    })
+    .on('cycle', function (event) {
+        console.log(String(event.target));
+    })
+    .run({ async: true });

elements/lisk-codec/fixtures

@@ -0,0 +1 @@
+../../protocol-specs/generator_outputs/lisk_codec

elements/lisk-codec/src/codec.ts

@@ -12,19 +12,153 @@
  * Removal or modification of this copyright notice is prohibited.
  */
 
+import { generateKey } from './utils';
+
+import {
+	CompiledSchema,
+	CompiledSchemas,
+	CompiledSchemasArray,
+	GenericObject,
+	Schema,
+	SchemaProps,
+} from './types';
+
+import { writeObject } from './collection';
+
 export class Codec {
-	// eslint-disable-next-line
-	public addSchema(_schema: object): void {}
+	private readonly _compileSchemas: CompiledSchemas = {};
 
-	// eslint-disable-next-line
-	public encode(_schema: object, _message: any): Buffer {
-		return Buffer.alloc(0);
+	public addSchema(schema: Schema): void {
+		const schemaName = schema.$id;
+		this._compileSchemas[schemaName] = this.compileSchema(schema, [], []);
+	}
+
+	public encode(schema: Schema, message: GenericObject): Buffer {
+		if (this._compileSchemas[schema.$id] === undefined) {
+			this.addSchema(schema);
+		}
+		const compiledSchema = this._compileSchemas[schema.$id];
+		const res = writeObject(compiledSchema, message, []);
+		return Buffer.concat(res[0]);
 	}
 
 	// eslint-disable-next-line
 	public decode<T>(_schema: object, _message: Buffer): T {
 		return {} as T;
 	}
+
+	private compileSchema(
+		schema: Schema | SchemaProps,
+		compiledSchema: CompiledSchemasArray,
+		dataPath: string[],
+	): CompiledSchemasArray {
+		if (schema.type === 'object') {
+			const { properties } = schema;
+			if (properties === undefined) {
+				throw new Error('Invalid schema. Missing "properties" property');
+			}
+			const currentDepthSchema = Object.entries(properties).sort(
+				(a, b) => a[1].fieldNumber - b[1].fieldNumber,
+			);
+
+			// eslint-disable-next-line @typescript-eslint/prefer-for-of
+			for (let i = 0; i < currentDepthSchema.length; i += 1) {
+				const [schemaPropertyName, schemaPropertyValue] = currentDepthSchema[i];
+				if (schemaPropertyValue.type === 'object') {
+					// Object recursive case
+					dataPath.push(schemaPropertyName);
+					const nestedSchema = [
+						{
+							propertyName: schemaPropertyName,
+							schemaProp: {
+								type: schemaPropertyValue.type,
+								fieldNumber: schemaPropertyValue.fieldNumber,
+							},
+							dataPath: [...dataPath],
+							binaryKey: generateKey(schemaPropertyValue),
+						},
+					];
+					const res = this.compileSchema(
+						schemaPropertyValue,
+						nestedSchema,
+						dataPath,
+					);
+					compiledSchema.push(res as CompiledSchema[]);
+					dataPath.pop();
+				} else if (schemaPropertyValue.type === 'array') {
+					// Array recursive case
+					if (schemaPropertyValue.items === undefined) {
+						throw new Error(
+							'Invalid schema. Missing "items" property for Array schema',
+						);
+					}
+					dataPath.push(schemaPropertyName);
+					if (schemaPropertyValue.items.type === 'object') {
+						const nestedSchema = [
+							{
+								propertyName: schemaPropertyName,
+								schemaProp: {
+									type: 'object',
+									fieldNumber: schemaPropertyValue.fieldNumber,
+								},
+								dataPath: [...dataPath],
+								binaryKey: generateKey(schemaPropertyValue),
+							},
+						];
+						const res = this.compileSchema(
+							schemaPropertyValue.items,
+							nestedSchema,
+							dataPath,
+						);
+						compiledSchema.push([
+							{
+								propertyName: schemaPropertyName,
+								schemaProp: {
+									type: schemaPropertyValue.type,
+									fieldNumber: schemaPropertyValue.fieldNumber,
+								},
+								dataPath: [...dataPath],
+								binaryKey: generateKey(schemaPropertyValue),
+							},
+							(res as unknown) as CompiledSchema,
+						]);
+						dataPath.pop();
+					} else {
+						compiledSchema.push([
+							{
+								propertyName: schemaPropertyName,
+								schemaProp: {
+									type: schemaPropertyValue.type,
+									fieldNumber: schemaPropertyValue.fieldNumber,
+								},
+								dataPath: [...dataPath],
+								binaryKey: generateKey(schemaPropertyValue),
+							},
+							{
+								propertyName: schemaPropertyName,
+								schemaProp: {
+									dataType: schemaPropertyValue.items.dataType,
+									fieldNumber: schemaPropertyValue.fieldNumber,
+								},
+								dataPath: [...dataPath],
+								binaryKey: generateKey(schemaPropertyValue),
+							},
+						]);
+						dataPath.pop();
+					}
+				} else {
+					// Base case
+					compiledSchema.push({
+						propertyName: schemaPropertyName,
+						schemaProp: schemaPropertyValue,
+						dataPath: [...dataPath],
+						binaryKey: generateKey(schemaPropertyValue),
+					});
+				}
+			}
+		}
+		return compiledSchema;
+	}
 }
 
 export const codec = new Codec();

elements/lisk-codec/src/collection.ts

@@ -0,0 +1,92 @@
+/* eslint-disable no-param-reassign */
+/*
+ * Copyright © 2020 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+import { findObjectByPath, generateKey } from './utils';
+
+import { GenericObject, CompiledSchemasArray } from './types';
+
+import { writeSInt32, writeSInt64, writeUInt32, writeUInt64 } from './varint';
+import { writeString } from './string';
+import { writeBytes } from './bytes';
+import { writeBoolean } from './boolean';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const _writers: { readonly [key: string]: (value: any) => Buffer } = {
+	uint32: writeUInt32,
+	sint32: writeSInt32,
+	uint64: writeUInt64,
+	sint64: writeSInt64,
+	string: writeString,
+	bytes: writeBytes,
+	boolean: writeBoolean,
+};
+
+export const writeObject = (
+	compiledSchema: CompiledSchemasArray,
+	message: GenericObject,
+	chunks: Buffer[],
+): [Buffer[], number] => {
+	let simpleObjectSize = 0;
+	// eslint-disable-next-line @typescript-eslint/prefer-for-of
+	for (let i = 0; i < compiledSchema.length; i += 1) {
+		const property = compiledSchema[i];
+		if (Array.isArray(property)) {
+			const headerProp = property[0];
+			const nestedObject = findObjectByPath(message, [headerProp.propertyName]);
+			// Write the key for container object
+			const key = generateKey(headerProp.schemaProp);
+			chunks.push(key);
+			const [encodedValues, totalWrittenSize] = writeObject(
+				property,
+				nestedObject as GenericObject,
+				[],
+			);
+			// Add nested object size to total size
+			chunks.push(_writers.uint32(totalWrittenSize));
+			// eslint-disable-next-line @typescript-eslint/prefer-for-of
+			for (let e = 0; e < encodedValues.length; e += 1) {
+				chunks.push(encodedValues[e]);
+			}
+		} else {
+			// This is the header object so it does not need to be written
+			if (property.schemaProp.type === 'object') {
+				// eslint-disable-next-line no-continue
+				continue;
+			}
+			const value = message[property.propertyName];
+			// Missing properties are not encoded as per LIP-0027
+			if (value === undefined) {
+				// eslint-disable-next-line no-continue
+				continue;
+			}
+
+			const {
+				schemaProp: { dataType },
+				binaryKey,
+			} = property;
+			if (dataType === undefined) {
+				throw new Error(
+					'Compiled Schema is corrutped as "dataType" can not be undefined.',
+				);
+			}
+
+			const binaryValue = _writers[dataType](value);
+
+			chunks.push(binaryKey);
+			chunks.push(binaryValue);
+			simpleObjectSize += binaryKey.length + binaryValue.length;
+		}
+	}
+	return [chunks, simpleObjectSize];
+};

elements/lisk-codec/src/types.ts

@@ -0,0 +1,35 @@
+export interface GenericObject {
+	[key: string]: GenericObject | string | number | Buffer;
+}
+export interface SchemaPair {
+	readonly [key: string]: SchemaProps;
+}
+export interface Schema {
+	readonly $id: string;
+	readonly type: string;
+	properties: SchemaPair;
+}
+export interface SchemaProps {
+	readonly fieldNumber: number;
+	readonly type?: string;
+	readonly dataType?: string;
+	readonly properties?: SchemaPair;
+	readonly items?: SchemaProps;
+}
+export interface SchemaScalarType {
+	readonly dataType?: string;
+	readonly type?: string;
+}
+
+export interface CompiledSchema {
+	schemaProp: SchemaProps;
+	propertyName: string;
+	binaryKey: Buffer;
+	dataPath: string[];
+}
+
+export type CompiledSchemasArray = Array<CompiledSchema | CompiledSchema[]>;
+
+export interface CompiledSchemas {
+	[key: string]: CompiledSchemasArray;
+}