Improve Core API

cli/.ci/script.sh

@@ -2,12 +2,8 @@
 
 IMAGE_NAME="cli-test"
 
-cd ./cli
-
 echo "Building test image..."
-docker build -t $IMAGE_NAME:$CI_BUILD_NUMBER .
+docker build -t $IMAGE_NAME:$CI_BUILD_NUMBER -f ./cli/Dockerfile .
 
 echo "Running tests with coverage reporting..."
 docker run --rm --network="host" $IMAGE_NAME:$CI_BUILD_NUMBER
-
-cd ..

cli/Dockerfile

@@ -1,18 +1,54 @@
-FROM node:10.10.0-alpine
+# Run with context of the upper level directory
+
+#
+# Stage 1 
+#
+
+FROM node:10-alpine AS builder
+LABEL Maintainer="Pawel Kosiec <pawel@kosiec.net>"
+
+ENV CORE_DIR=./core
+WORKDIR /app
+
+# Copy sources
+COPY $CORE_DIR/package.json $CORE_DIR/package-lock.json $CORE_DIR/tsconfig.json /app/
+COPY $CORE_DIR/src /app/src/
+
+# Install dependencies
+RUN npm install --no-optional
+
+# Build app
+RUN npm run build
+
+# Remove sources
+RUN rm -rf /app/src/
+
+#
+# Stage 2 
+#
+
+FROM node:10-alpine
 
 WORKDIR /app
 
+ENV CLI_DIR=./cli
+
 # Install dependencies
-COPY package.json package-lock.json /app/
-RUN npm i
+COPY $CLI_DIR/package.json $CLI_DIR/package-lock.json /app/
+RUN npm i --no-optional
+
+# Copy core
+COPY --from=builder /app/ /app/node_modules/mongo-seeding/
 
 # Copy app sources
-COPY tsconfig.json /app/
-COPY src/ /app/src/
-COPY test/ /app/test/
+COPY $CLI_DIR/tsconfig.json /app/
+COPY $CLI_DIR/src/ /app/src/
+COPY $CLI_DIR/bin/ /app/bin/
+COPY $CLI_DIR/test/ /app/test/
 
 # Build app
 RUN npm run build
+
 # Remove built app
 RUN npm run cleanup
 

cli/README.md

@@ -59,10 +59,10 @@ You can use the following parameters while using `seed` tool:
 | `--db-name {DB_NAME}` | `database` | Name of the database |
 | `--db-username {DB_USERNAME}` | *`undefined`*  | Username for connecting with database that requires authentication |
 | `--db-password {DB_PASSWORD}` | *`undefined`*  | Password for connecting with database that requires authentication |
-| `--drop-database` | `false` | Dropping entire database before data import |
-| `--drop-collection` | `false` | Dropping every collection that is being imported |
-| `--replace-id` | `false` | Replacing `id` property with `_id` for every document during data import |
-| `--reconnect-timeout` | `10` (seconds) | Maximum time of waiting for successful MongoDB connection|
+| `--drop-database` | `false` | Drops entire database before data import |
+| `--drop-collections` | `false` | Drops every collection which is being imported |
+| `--replace-id` | `false` | Replaces `id` property with `_id` for every document during data import |
+| `--reconnect-timeout` | `10000` | Maximum time in milliseconds of waiting for successful MongoDB connection |
 | `--help` or `-h` | n/a | Help
 
 ## Environmental variables
@@ -77,7 +77,7 @@ You can use the following environmental variables while using `seed` tool:
 | DB_NAME | `database` | Name of the database |
 | DB_USERNAME | *`undefined`* | Username for connecting with database that requires authentication |
 | DB_PASSWORD | *`undefined`* | Password for connecting with database that requires authentication |
-| DROP_DATABASE | `false` | Dropping entire database before data import |
-| DROP_COLLECTION | `false` | Dropping every collection that is being imported |
-| REPLACE_ID | `false` | Replacing `id` property with `_id` for every document during import; useful for ORMs | 
-| RECONNECT_TIMEOUT | `10` | Maximum time, in which app should keep trying connecting to database |
+| DROP_DATABASE | `false` | Drops entire database before data import |
+| DROP_COLLECTIONS | `false` | Drops every collection which is being imported |
+| REPLACE_ID | `false` | Replaces `id` property with `_id` for every document during import; useful for ORMs | 
+| RECONNECT_TIMEOUT | `10000` | Maximum time in milliseconds of waiting for successful MongoDB connection |

cli/bin/seed.js

@@ -2,4 +2,6 @@
 'use strict';
 
 require('ts-node').register();
-require('../dist/index').run();
+
+const { cliSeeder } = require('../dist/index');
+cliSeeder.run();

cli/package.json

@@ -54,7 +54,7 @@
     "build": "npm run cleanup; tsc",
     "cleanup": "rm -rf ./dist",
     "preversion": "npm run build",
-    "test": "jest",
+    "test": "jest -i",
     "test:unit": "jest unit",
     "test:integration": "jest integration",
     "test:watch": "jest --watch --coverage"

cli/src/help.ts

@@ -12,7 +12,9 @@ const helpSections = [
     header: 'Example usage',
     content: [
       '$ {bold seed}',
+      '$ {bold seed ./seed-data/}',
       '$ {bold seed -u `mongodb://127.0.0.1:27017/mydbname` --drop-database --replace-id}',
+      '$ {bold seed -u `mongodb://127.0.0.1:27017/mydbname` --replace-id /absolute/path/}',
     ],
   },
   {

cli/src/index.ts

@@ -2,7 +2,8 @@
 process.env.DEBUG = 'mongo-seeding';
 
 import * as commandLineArgs from 'command-line-args';
-import { seedDatabase } from 'mongo-seeding';
+import { resolve } from 'path';
+import { Seeder, SeederCollectionReadingOptions } from 'mongo-seeding';
 import {
   cliOptions,
   validateOptions,
@@ -11,34 +12,70 @@ import {
 import { showHelp, shouldShowHelp } from './help';
 import { CommandLineArguments } from './types';
 
-export const run = async () => {
-  let options: CommandLineArguments;
+class CliSeeder {
+  run = async () => {
+    let options: CommandLineArguments;
+
+    try {
+      options = commandLineArgs(cliOptions) as CommandLineArguments;
+    } catch (err) {
+      this.printError(err);
+      return;
+    }
+
+    if (shouldShowHelp(options)) {
+      showHelp();
+      return;
+    }
+
+    try {
+      validateOptions(options);
+    } catch (err) {
+      this.printError(err);
+      return;
+    }
+
+    const config = createConfigFromOptions(options);
+    const seeder = new Seeder(config);
+
+    const collectionsPath = options.data ? options.data : './';
+    const collectionReadingConfig = this.getCollectionReadingConfig(options);
+
+    try {
+      const collections = seeder.readCollectionsFromPath(
+        resolve(collectionsPath),
+        collectionReadingConfig,
+      );
+
+      await seeder.import(collections);
+    } catch (err) {
+      this.printError(err);
+    }
 
-  try {
-    options = commandLineArgs(cliOptions) as CommandLineArguments;
-  } catch (err) {
-    printError(err);
     process.exit(0);
-    return;
-  }
+  };
 
-  if (shouldShowHelp(options)) {
-    showHelp();
-    return;
-  }
+  private getCollectionReadingConfig = (
+    options: CommandLineArguments,
+  ): SeederCollectionReadingOptions => {
+    const transformers = [];
+    const replaceIdWithUnderscoreId =
+      options['replace-id'] || process.env.REPLACE_ID === 'true';
 
-  const config = createConfigFromOptions(options);
+    if (replaceIdWithUnderscoreId) {
+      transformers.push(Seeder.Transformers.replaceDocumentIdWithUnderscoreId);
+    }
 
-  try {
-    validateOptions(options);
-    await seedDatabase(config);
-  } catch (err) {
-    printError(err);
-  }
+    return {
+      extensions: ['ts', 'js', 'json'],
+      transformers,
+    };
+  };
 
-  process.exit(0);
-};
+  private printError = (err: Error) => {
+    console.error(`Error ${err.name}: ${err.message}`);
+    process.exit(0);
+  };
+}
 
-const printError = (err: Error) => {
-  console.error(`Error ${err.name}: ${err.message}`);
-};
+export const cliSeeder = new CliSeeder();

cli/src/options.ts

@@ -1,8 +1,8 @@
-import { resolve } from 'path';
 import * as extend from 'extend';
-import { AppConfig, DeepPartial } from 'mongo-seeding/dist/common';
+import { DeepPartial } from 'mongo-seeding/dist/common';
 import { throwOnNegativeNumber } from './validators';
 import { CommandLineOption, CommandLineArguments } from './types';
+import { SeederConfig } from 'mongo-seeding';
 
 export const cliOptions: CommandLineOption[] = [
   {
@@ -50,7 +50,7 @@ export const cliOptions: CommandLineOption[] = [
     name: 'db-uri',
     alias: 'u',
     description:
-      'If defined, the URI will be used for establishing connection to database, ignoring values defined via other `db-*` parameters, i.e. `db-name`, `db-host`, etc.; Default: {bold undefined}',
+      'If defined, the URI will be used for establishing connection to database, ignoring values defined via other `db-*` parameters, e.g. `db-name`, `db-host`, etc.; Default: {bold undefined}',
     type: String,
   },
   {
@@ -65,13 +65,14 @@ export const cliOptions: CommandLineOption[] = [
     type: Boolean,
   },
   {
-    name: 'drop-collection',
-    description: 'Drops collection before importing it',
+    name: 'drop-collections',
+    description: 'Drops every collection that is being imported',
     type: Boolean,
   },
   {
     name: 'replace-id',
-    description: 'Replaces `id` property with `_id` for every object to import',
+    description:
+      'Replaces `id` property with `_id` for every document before import',
     type: Boolean,
   },
   {
@@ -89,7 +90,7 @@ export const validateOptions = (options: CommandLineArguments) => {
 
 export const createConfigFromOptions = (
   cmdArgs: CommandLineArguments,
-): DeepPartial<AppConfig> => {
+): DeepPartial<SeederConfig> => {
   const commandLineConfig = populateCommandLineOptions(cmdArgs);
   const envConfig = populateEnvOptions();
   const config = {};
@@ -98,45 +99,53 @@ export const createConfigFromOptions = (
 
 function populateCommandLineOptions(
   options: CommandLineArguments,
-): DeepPartial<AppConfig> {
+): DeepPartial<SeederConfig> {
   return {
-    database: {
-      protocol: options['db-protocol'],
-      host: options['db-host'],
-      port: options['db-port'],
-      name: options['db-name'],
-      username: options['db-username'],
-      password: options['db-password'],
-    },
-    databaseConnectionUri: options['db-uri'],
-    inputPath: options.data ? resolve(options.data) : resolve('./'),
+    database: options['db-uri']
+      ? options['db-uri']
+      : convertEmptyObjectToUndefined({
+          protocol: options['db-protocol'],
+          host: options['db-host'],
+          port: options['db-port'],
+          name: options['db-name'],
+          username: options['db-username'],
+          password: options['db-password'],
+        }),
+    databaseReconnectTimeout: options['reconnect-timeout'],
     dropDatabase: options['drop-database'],
-    dropCollection: options['drop-collection'],
-    replaceIdWithUnderscoreId: options['replace-id'],
-    reconnectTimeoutInSeconds: options['reconnect-timeout'],
+    dropCollections: options['drop-collections'],
   };
 }
 
-function populateEnvOptions(): DeepPartial<AppConfig> {
+function populateEnvOptions(): DeepPartial<SeederConfig> {
   const env = process.env;
-  const envOptions: DeepPartial<AppConfig> = {
-    database: {
-      protocol: env.DB_PROTOCOL ? String(env.DB_PROTOCOL) : undefined,
-      host: env.DB_HOST ? String(env.DB_HOST) : undefined,
-      port: env.DB_PORT ? Number(env.DB_PORT) : undefined,
-      name: env.DB_NAME ? String(env.DB_NAME) : undefined,
-      username: env.DB_USERNAME ? String(env.DB_USERNAME) : undefined,
-      password: env.DB_PASSWORD ? String(env.DB_PASSWORD) : undefined,
-    },
-    databaseConnectionUri: env.DB_URI ? String(env.DB_URI) : undefined,
-    dropDatabase: env.DROP_DATABASE === 'true',
-    dropCollection: env.DROP_COLLECTION === 'true',
-    replaceIdWithUnderscoreId: env.REPLACE_ID === 'true',
-    supportedExtensions: ['ts', 'js', 'json'],
-    reconnectTimeoutInSeconds: env.RECONNECT_TIMEOUT
+  const envOptions: DeepPartial<SeederConfig> = {
+    database: env.DB_URI
+      ? String(env.DB_URI)
+      : convertEmptyObjectToUndefined({
+          protocol: env.DB_PROTOCOL ? String(env.DB_PROTOCOL) : undefined,
+          host: env.DB_HOST ? String(env.DB_HOST) : undefined,
+          port: env.DB_PORT ? Number(env.DB_PORT) : undefined,
+          name: env.DB_NAME ? String(env.DB_NAME) : undefined,
+          username: env.DB_USERNAME ? String(env.DB_USERNAME) : undefined,
+          password: env.DB_PASSWORD ? String(env.DB_PASSWORD) : undefined,
+        }),
+    databaseReconnectTimeout: env.RECONNECT_TIMEOUT
       ? Number(env.RECONNECT_TIMEOUT)
       : undefined,
+    dropDatabase: env.DROP_DATABASE === 'true',
+    dropCollections: env.DROP_COLLECTIONS === 'true',
   };
 
   return envOptions;
 }
+
+export function convertEmptyObjectToUndefined(obj: any): object | undefined {
+  for (const key in obj) {
+    if (obj.hasOwnProperty(key) && typeof obj[key] !== 'undefined') {
+      return obj;
+    }
+  }
+
+  return undefined;
+}

cli/src/types.ts

@@ -10,7 +10,7 @@ export interface CommandLineArguments {
   data?: string;
   [key: string]: string | number | boolean | undefined;
   'drop-database'?: boolean;
-  'drop-collection'?: boolean;
+  'drop-collections'?: boolean;
   'replace-id'?: boolean;
   'db-protocol'?: string;
   'db-host'?: string;