feat(jsbattle): add cli to dump and restore DB

packages/jsbattle-server/.eslintrc.js

@@ -121,7 +121,7 @@ module.exports = {
         "no-alert": "error",
         "no-array-constructor": "error",
         "no-async-promise-executor": "off",
-        "no-await-in-loop": "error",
+        "no-await-in-loop": "off",
         "no-bitwise": "error",
         "no-buffer-constructor": "error",
         "no-caller": "error",

packages/jsbattle-server/app/Node.js

@@ -6,6 +6,7 @@ require('dotenv').config();
 
 const GATEWAY = 'gateway';
 const WORKER = 'worker';
+const CLI = 'cli';
 
 class Node {
 
@@ -32,7 +33,6 @@ class Node {
       };
     }
 
-
     return new Promise((resolve) => {
       this.broker = new ServiceBroker(
         {
@@ -82,6 +82,17 @@ class Node {
             'node',
           ];
           break;
+        case CLI:
+          serviceList = [
+            'cli',
+            'battleStore',
+            'challenges',
+            'league',
+            'scriptStore',
+            'userStore',
+            'ubdValidator',
+          ];
+          break;
         default:
           throw Error('unknown node type: ' + this.type);
 

packages/jsbattle-server/app/runner-cli.js

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+
+const Node = require('./Node.js');
+
+(async () => {
+  let gateway = new Node('cli');
+  let config = {
+    "loglevel": "warn",
+    "logger": {
+        "type": "Console",
+        "options": {
+            colors: true,
+            moduleColors: true,
+            formatter: "short",
+            autoPadding: true
+        }
+    }
+  };
+  await gateway.init(config);
+  await gateway.start();
+  console.log(await gateway.broker.call('cli.dumpDb', {dumpPath: '../../../tmp/dump/v3'}))
+  console.log(await gateway.broker.call('cli.restoreDb', {dumpPath: '../../../tmp/dump/v3'}))
+  console.log(await gateway.broker.call('cli.dumpDb', {dumpPath: '../../../tmp/dump/v3'}))
+  await gateway.stop();
+})()
+

packages/jsbattle-server/app/services/cli/actions/dumpDb.js

@@ -0,0 +1,49 @@
+const { ValidationError } = require("moleculer").Errors;
+const path = require('path');
+const fs = require('fs').promises;
+
+const dataServices = [
+  'battleStore',
+  'challenges',
+  'league',
+  'scriptStore',
+  'userStore'
+]
+
+module.exports = async function(ctx) {
+  const {params} = ctx;
+  if(!params.dumpPath) {
+    throw new ValidationError('dumpPath parameter is required', 400);
+  }
+  const dest = path.resolve(params.dumpPath);
+
+  const stats = {};
+
+  for(let service of dataServices) {
+    this.logger.info(`Dumping data of '${service}' service`);
+
+    let serviceDest = path.join(dest, service);
+    this.logger.debug(`Creating '${service}' service dump location at ${serviceDest}`);
+    await fs.mkdir(serviceDest, { recursive: true });
+
+    this.logger.debug(`reading entities of '${service}'`);
+    const entities = await ctx.call(service + '.find');
+    this.logger.debug(`${entities.length} entities of '${service}' found`);
+    stats[service] = entities.length;
+
+    for(let entity of entities) {
+      entity._id = entity.id;
+      delete entity.id;
+      let entityDest = path.join(serviceDest, entity._id + ".json");
+
+      this.logger.trace(`Dumping '${service}' entity to ${entityDest}`);
+      await fs.writeFile(entityDest, JSON.stringify(entity));
+    }
+    this.logger.info(`${entities.length} entities of '${service}' dumped`);
+  }
+
+  return {
+    entities: stats,
+    dumpPath: dest
+  };
+}

packages/jsbattle-server/app/services/cli/actions/restoreDb.js

@@ -0,0 +1,56 @@
+const { ValidationError } = require("moleculer").Errors;
+const path = require('path');
+const fs = require('fs').promises;
+
+module.exports = async function(ctx) {
+  const {params} = ctx;
+  if(!params.dumpPath) {
+    throw new ValidationError('dumpPath parameter is required', 400);
+  }
+  const dumpPath = path.resolve(params.dumpPath);
+  const stats = {};
+  let errors = 0;
+
+  const dirs = await fs.readdir(dumpPath);
+  const services = [];
+  for(let dir of dirs) {
+    let stats = await fs.lstat(path.join(dumpPath, dir));
+    if(stats.isDirectory()) {
+      services.push({
+        name: dir,
+        path: path.join(dumpPath, dir)
+      })
+    }
+  }
+  for(let service of services) {
+    this.logger.info(`Restoring '${service.name}' from ${service.path}`);
+
+    let files = await fs.readdir(service.path);
+    this.logger.debug(`${files.length} entities of ${service.path} service found`);
+    stats[service.name] = files.length;
+
+    const allEntities = await ctx.call(service.name + '.find', {fields: ['id']});
+
+    Promise.all(allEntities.map((item) => ctx.call(service.name + '.remove', {id: item.id})));
+
+    for(let file of files) {
+      let data = await fs.readFile(path.join(service.path, file), 'utf8');
+      try {
+        let entity = JSON.parse(data);
+        await await ctx.call(service.name + '.create', entity);
+      } catch(err) {
+        this.logger.warn(`unable to restore ${path.join(service.path, file)}`);
+        this.logger.error(err);
+        errors = errors + 1;
+      }
+
+    }
+
+  }
+
+  return {
+    entities: stats,
+    dumpPath,
+    errors
+  };
+}

packages/jsbattle-server/app/services/cli/index.js

@@ -0,0 +1,7 @@
+module.exports = () => ({
+  name: "cli",
+  actions: {
+    "dumpDb": require('./actions/dumpDb.js'),
+    "restoreDb": require('./actions/restoreDb.js'),
+  }
+});

packages/jsbattle-server/package.json

@@ -17,7 +17,7 @@
     "test": "npm run test:scoring && npm run test:app",
     "lint": "eslint app/",
     "test:scoring": "node ./tools/scoringBenchmark/run.js",
-    "test:app": "rimraf test/logs && jest --env node ..."
+    "test:app": "rimraf test/logs && rimraf test/tmp && jest --env node ..."
   },
   "devDependencies": {
     "axios": "^0.23.0",

packages/jsbattle-server/test/unit/lib/dbAdapters.spec.js

@@ -78,7 +78,7 @@ for(let adapterName of adapterList) {
       expect(newItem.blob).toHaveProperty('bar', 'ABC');
 
       const retrievedItem = await broker.call('testDbService.get', {id: newItem._id});
-      console.log(retrievedItem)
+
       expect(retrievedItem).toHaveProperty('_id');
       expect(retrievedItem).toHaveProperty('foo', 'bar8732');
       expect(retrievedItem).toHaveProperty('yes', true);

packages/jsbattle-server/test/unit/services/Cli.spec.js

@@ -0,0 +1,76 @@
+"use strict";
+
+const serviceConfig = require('../../../app/lib/serviceConfig.js');
+const { ServiceBroker } = require("moleculer");
+const { ValidationError } = require("moleculer").Errors;
+const { MoleculerClientError } = require("moleculer").Errors;
+const path = require('path');
+
+const dataServices = [
+  'battleStore',
+  'challenges',
+  'league',
+  'scriptStore',
+  'userStore'
+]
+
+describe("Test 'CLI' service", () => {
+	let broker;
+	let createMock;
+	let removeMock;
+
+	beforeEach(async () => {
+		broker = new ServiceBroker(require('../../utils/getLoggerSettings.js')(path.resolve(__dirname, '..', '..'), __filename, expect.getState()));
+		const schemaBuilder = require(__dirname + "../../../../app/services/cli/index.js");
+		await broker.createService(schemaBuilder(serviceConfig.data));
+		await broker.start()
+
+		createMock = jest.fn();
+		removeMock = jest.fn();
+
+		for(let service of dataServices) {
+			broker.createService({
+				name: service,
+				actions: {
+					find: () => [
+						{
+							id: 'ID_' + service + "_1",
+							foo: 'bar-' + service + '-1'
+						},
+						{
+							id: 'ID_' + service + "_2",
+							foo: 'bar-' + service + '-2'
+						}
+					],
+					remove: removeMock,
+					create: createMock
+				}
+			})
+		}
+
+	});
+	afterEach(async () => await broker.stop());
+
+	it.only('should dump and restore', async () => {
+		const dumpPath = path.resolve(__dirname, '..', '..', 'tmp', 'dump_' + Math.round(Math.random()*0xffffffff)).toString(16);
+		const response1 = await broker.call('cli.dumpDb', { dumpPath });
+		expect(response1).toHaveProperty('dumpPath', dumpPath);
+		expect(response1).toHaveProperty('entities');
+
+		const response2 = await broker.call('cli.restoreDb', { dumpPath });
+		expect(response2).toHaveProperty('dumpPath', dumpPath)
+		expect(response1).toHaveProperty('entities');
+		expect(response2).toHaveProperty('errors', 0);
+
+		expect(createMock.mock.calls.length).toBe(10)
+		expect(removeMock.mock.calls.length).toBe(10)
+
+		expect(createMock.mock.calls.map(c => c[0].params._id)).toContain('ID_battleStore_1')
+		expect(createMock.mock.calls.map(c => c[0].params._id)).toContain('ID_battleStore_2')
+		expect(createMock.mock.calls.map(c => c[0].params._id)).toContain('ID_challenges_1')
+		expect(createMock.mock.calls.map(c => c[0].params._id)).toContain('ID_league_2')
+		expect(createMock.mock.calls.map(c => c[0].params._id)).toContain('ID_scriptStore_1')
+		expect(createMock.mock.calls.map(c => c[0].params._id)).toContain('ID_userStore_2')
+	});
+
+});

packages/jsbattle/src/jsbattle.js

@@ -80,6 +80,62 @@ yargs
       .catch(console.error);
     }
   )
+  .command(
+    'dump [dumpPath]',
+    'Dump JsBattle DB to files',
+    (yargs) => {
+      return yargs.positional('dumpPath', {
+        describe: 'path to directory where DB dump will be stored',
+        default: './jsbattle-dump'
+      })
+    },
+    (argv) => {
+      let config = {};
+      if(argv.config) {
+        config = require(path.resolve(argv.config));
+      }
+
+      // override config by CLI arguments
+      if(argv.loglevel) {
+        config.loglevel = argv.loglevel
+      }
+
+      let cli = new Node('cli');
+      cli.init(config)
+        .then(() => cli.start())
+        .then(() => cli.broker.call('cli.dumpDb', {dumpPath: argv.dumpPath}))
+        .then(() => cli.stop())
+        .catch(console.error);
+    }
+  )
+  .command(
+    'restore [dumpPath]',
+    'Restore JsBattle DB from dump files',
+    (yargs) => {
+      return yargs.positional('dumpPath', {
+        describe: 'path to directory where DB dump will be read',
+        default: './jsbattle-dump'
+      })
+    },
+    (argv) => {
+      let config = {};
+      if(argv.config) {
+        config = require(path.resolve(argv.config));
+      }
+
+      // override config by CLI arguments
+      if(argv.loglevel) {
+        config.loglevel = argv.loglevel
+      }
+
+      let cli = new Node('cli');
+      cli.init(config)
+        .then(() => cli.start())
+        .then(() => cli.broker.call('cli.restoreDb', {dumpPath: argv.dumpPath}))
+        .then(() => cli.stop())
+        .catch(console.error);
+    }
+  )
   .command("*", "", (argv) => {
     console.log("Nothing happened :( Run 'jsbattle.js --help' for more info\n");
   })