Merge pull request #391 from MatrixAI/feature-pkg_integration_tests

Packaged executable integration tests - initial integration tests with docker

.env.example

@@ -31,3 +31,9 @@ AWS_SECRET_ACCESS_KEY=
 
 # Authenticate to GitHub with `gh`
 # GITHUB_TOKEN=
+
+# To allow testing different executables in the bin tests
+# Both PK_TEST_COMMAND and PK_TEST_PLATFORM must be set at the same time
+# PK_TEST_COMMAND= #Specify the shell command we want to test against
+# PK_TEST_PLATFORM=docker #Overrides the auto set `testPlatform` variable used for enabling platform specific tests
+# PK_TEST_TMPDIR= #Sets the `global.tmpDir` variable to allow overriding the temp directory used for tests

.gitlab-ci.yml

@@ -330,15 +330,20 @@ integration:docker:
     - integration:builds
     - job: integration:deployment
       optional: true
-  image: docker:20.10.11
   services:
-    - docker:20.10.11-dind
+    - docker:20.10.16-dind
   variables:
     DOCKER_TLS_CERTDIR: "/certs"
+    FF_NETWORK_PER_BUILD: "true"
   script:
     - docker info
-    - image="$(docker load --input ./builds/*docker* | cut -d' ' -f3)"
-    - docker run "$image"
+    - >
+      nix-shell --run $'
+        PK_TEST_COMMAND="docker run \${DOCKER_OPTIONS} $(docker load --input ./builds/*docker* | cut -d\' \' -f3) polykey" \
+        PK_TEST_PLATFORM=docker \
+        PK_TEST_TMPDIR=/builds/$CI_PROJECT_PATH/tmp \
+        exec npm run test -- tests/bin
+      '
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/

jest.config.js

@@ -15,6 +15,7 @@ moduleNameMapper['^jose/(.*)$'] = "<rootDir>/node_modules/jose/dist/node/cjs/$1"
 
 // Global variables that are shared across the jest worker pool
 // These variables must be static and serializable
+if ((process.env.PK_TEST_PLATFORM != null) !== (process.env.PK_TEST_COMMAND != null)) throw Error('Both PK_TEST_PLATFORM and PK_TEST_COMMAND must be set together.')
 const globals = {
   // Absolute directory to the project root
   projectDir: __dirname,
@@ -31,7 +32,8 @@ const globals = {
   // Timeouts rely on setTimeout which takes 32 bit numbers
   maxTimeout: Math.pow(2, 31) - 1,
   testCmd: process.env.PK_TEST_COMMAND,
-  testPlatform: process.env.PK_TEST_COMMAND_DOCKER,
+  testPlatform: process.env.PK_TEST_PLATFORM,
+  tmpDir: process.env.PK_TEST_TMPDIR ?? os.tmpdir(),
 };
 
 // The `globalSetup` and `globalTeardown` cannot access the `globals`

src/PolykeyAgent.ts

@@ -2,7 +2,7 @@ import type { FileSystem } from './types';
 import type { PolykeyWorkerManagerInterface } from './workers/types';
 import type { ConnectionData, Host, Port } from './network/types';
 import type { SeedNodes } from './nodes/types';
-import type { KeyManagerChangeData } from './keys/types';
+import type { KeyManagerChangeData, PrivateKeyPem } from './keys/types';
 import path from 'path';
 import process from 'process';
 import Logger from '@matrixai/logger';
@@ -108,6 +108,7 @@ class PolykeyAgent {
       rootCertDuration?: number;
       dbKeyBits?: number;
       recoveryCode?: string;
+      privateKeyPemOverride?: PrivateKeyPem;
     };
     proxyConfig?: {
       authToken?: string;

src/bin/agent/CommandStart.ts

@@ -37,6 +37,7 @@ class CommandStart extends CommandPolykey {
     this.addOption(binOptions.backgroundOutFile);
     this.addOption(binOptions.backgroundErrFile);
     this.addOption(binOptions.fresh);
+    this.addOption(binOptions.rootKeyFile);
     this.action(async (options) => {
       options.clientHost =
         options.clientHost ?? config.defaults.networkConfig.clientHost;
@@ -88,12 +89,16 @@ class CommandStart extends CommandPolykey {
       const [seedNodes, defaults] = options.seedNodes;
       let seedNodes_ = seedNodes;
       if (defaults) seedNodes_ = { ...options.network, ...seedNodes };
+      const privateKeyPem = await binProcessors.processRootKey(
+        options.rootKeyFile,
+      );
       const agentConfig = {
         password,
         nodePath: options.nodePath,
         keysConfig: {
           rootKeyPairBits: options.rootKeyPairBits,
           recoveryCode: recoveryCodeIn,
+          privateKeyPemOverride: privateKeyPem,
         },
         proxyConfig: {
           connConnectTime: options.connectionTimeout,

src/bin/bootstrap/CommandBootstrap.ts

@@ -11,6 +11,7 @@ class CommandBootstrap extends CommandPolykey {
     this.addOption(binOptions.recoveryCodeFile);
     this.addOption(binOptions.rootKeyPairBits);
     this.addOption(binOptions.fresh);
+    this.addOption(binOptions.rootKeyFile);
     this.action(async (options) => {
       const bootstrapUtils = await import('../../bootstrap/utils');
       const password = await binProcessors.processNewPassword(
@@ -21,19 +22,23 @@ class CommandBootstrap extends CommandPolykey {
         options.recoveryCodeFile,
         this.fs,
       );
+      const privateKeyPem = await binProcessors.processRootKey(
+        options.rootKeyFile,
+      );
       const recoveryCodeOut = await bootstrapUtils.bootstrapState({
         password,
         nodePath: options.nodePath,
         keysConfig: {
           rootKeyPairBits: options.rootKeyPairBits,
           recoveryCode: recoveryCodeIn,
+          privateKeyPemOverride: privateKeyPem,
         },
         fresh: options.fresh,
         fs: this.fs,
         logger: this.logger,
       });
       this.logger.info(`Bootstrapped ${options.nodePath}`);
-      process.stdout.write(recoveryCodeOut + '\n');
+      if (recoveryCodeOut != null) process.stdout.write(recoveryCodeOut + '\n');
     });
   }
 }

src/bin/errors.ts

@@ -29,6 +29,11 @@ class ErrorCLIRecoveryCodeFileRead<T> extends ErrorCLI<T> {
   exitCode = sysexits.NOINPUT;
 }
 
+class ErrorCLIPrivateKeyFileRead<T> extends ErrorCLI<T> {
+  static description = 'Failed to read private key Pem file';
+  exitCode = sysexits.NOINPUT;
+}
+
 class ErrorCLIFileRead<T> extends ErrorCLI<T> {
   static description = 'Failed to read file';
   exitCode = sysexits.NOINPUT;
@@ -61,6 +66,7 @@ export {
   ErrorCLIPasswordMissing,
   ErrorCLIPasswordFileRead,
   ErrorCLIRecoveryCodeFileRead,
+  ErrorCLIPrivateKeyFileRead,
   ErrorCLIFileRead,
   ErrorCLIPolykeyAgentStatus,
   ErrorCLIPolykeyAgentProcess,

src/bin/utils/options.ts

@@ -163,6 +163,11 @@ const noPing = new commander.Option('--no-ping', 'Skip ping step').default(
   true,
 );
 
+const rootKeyFile = new commander.Option(
+  '--root-key-file <rootKeyFile>',
+  'Override key generation with a private key Pem from a file.',
+);
+
 export {
   nodePath,
   format,
@@ -187,4 +192,5 @@ export {
   pullVault,
   forceNodeAdd,
   noPing,
+  rootKeyFile,
 };

src/bin/utils/processors.ts

@@ -1,5 +1,5 @@
 import type { FileSystem } from '../../types';
-import type { RecoveryCode } from '../../keys/types';
+import type { RecoveryCode, PrivateKeyPem } from '../../keys/types';
 import type { NodeId } from '../../nodes/types';
 import type { Host, Port } from '../../network/types';
 import type {
@@ -403,6 +403,29 @@ async function processAuthentication(
   return meta;
 }
 
+async function processRootKey(
+  privateKeyFile: string | undefined,
+  fs: FileSystem = require('fs'),
+): Promise<PrivateKeyPem | undefined> {
+  if (privateKeyFile != null) {
+    try {
+      return (await fs.promises.readFile(privateKeyFile, 'utf-8')).trim();
+    } catch (e) {
+      throw new binErrors.ErrorCLIPrivateKeyFileRead(e.message, {
+        data: {
+          errno: e.errno,
+          syscall: e.syscall,
+          code: e.code,
+          path: e.path,
+        },
+        cause: e,
+      });
+    }
+  } else if (typeof process.env['PK_ROOT_KEY'] === 'string') {
+    return process.env['PK_ROOT_KEY'];
+  }
+}
+
 export {
   promptPassword,
   promptNewPassword,
@@ -412,4 +435,5 @@ export {
   processClientOptions,
   processClientStatus,
   processAuthentication,
+  processRootKey,
 };

src/bootstrap/utils.ts

@@ -1,5 +1,5 @@
 import type { FileSystem } from '../types';
-import type { RecoveryCode } from '../keys/types';
+import type { RecoveryCode, PrivateKeyPem } from '../keys/types';
 import path from 'path';
 import Logger from '@matrixai/logger';
 import { DB } from '@matrixai/db';
@@ -40,11 +40,12 @@ async function bootstrapState({
     rootCertDuration?: number;
     dbKeyBits?: number;
     recoveryCode?: RecoveryCode;
+    privateKeyPemOverride?: PrivateKeyPem;
   };
   fresh?: boolean;
   fs?: FileSystem;
   logger?: Logger;
-}): Promise<RecoveryCode> {
+}): Promise<RecoveryCode | undefined> {
   const umask = 0o077;
   logger.info(`Setting umask to ${umask.toString(8).padStart(3, '0')}`);
   process.umask(umask);

src/discovery/Discovery.ts

@@ -30,6 +30,7 @@ import * as resources from '@matrixai/resources';
 import * as discoveryUtils from './utils';
 import * as discoveryErrors from './errors';
 import * as nodesErrors from '../nodes/errors';
+import * as networkErrors from '../network/errors';
 import * as gestaltsUtils from '../gestalts/utils';
 import * as claimsUtils from '../claims/utils';
 import * as nodesUtils from '../nodes/utils';
@@ -371,7 +372,8 @@ class Discovery {
                 } catch (e) {
                   if (
                     e instanceof nodesErrors.ErrorNodeConnectionDestroyed ||
-                    e instanceof nodesErrors.ErrorNodeConnectionTimeout
+                    e instanceof nodesErrors.ErrorNodeConnectionTimeout ||
+                    e instanceof networkErrors.ErrorConnectionNotRunning
                   ) {
                     if (!this.visitedVertices.has(linkedVertexGK)) {
                       await this.pushKeyToDiscoveryQueue(linkedVertexGK);

src/keys/KeyManager.ts

@@ -6,6 +6,7 @@ import type {
   CertificatePemChain,
   RecoveryCode,
   KeyManagerChangeData,
+  PrivateKeyPem,
 } from './types';
 import type { FileSystem } from '../types';
 import type { NodeId } from '../nodes/types';
@@ -40,6 +41,7 @@ class KeyManager {
     fs = require('fs'),
     logger = new Logger(this.name),
     recoveryCode,
+    privateKeyPemOverride,
     fresh = false,
   }: {
     keysPath: string;
@@ -51,6 +53,7 @@ class KeyManager {
     fs?: FileSystem;
     logger?: Logger;
     recoveryCode?: RecoveryCode;
+    privateKeyPemOverride?: PrivateKeyPem;
     fresh?: boolean;
   }): Promise<KeyManager> {
     logger.info(`Creating ${this.name}`);
@@ -67,6 +70,7 @@ class KeyManager {
     await keyManager.start({
       password,
       recoveryCode,
+      privateKeyPemOverride,
       fresh,
     });
     logger.info(`Created ${this.name}`);
@@ -134,10 +138,12 @@ class KeyManager {
   public async start({
     password,
     recoveryCode,
+    privateKeyPemOverride,
     fresh = false,
   }: {
     password: string;
     recoveryCode?: RecoveryCode;
+    privateKeyPemOverride?: PrivateKeyPem;
     fresh?: boolean;
   }): Promise<void> {
     this.logger.info(`Starting ${this.constructor.name}`);
@@ -160,6 +166,7 @@ class KeyManager {
       password,
       this.rootKeyPairBits,
       recoveryCode,
+      privateKeyPemOverride,
     );
     const rootCert = await this.setupRootCert(
       rootKeyPair,
@@ -561,7 +568,7 @@ class KeyManager {
     bits: number,
     recoveryCode?: RecoveryCode,
   ): Promise<KeyPair> {
-    let keyPair;
+    let keyPair: KeyPair;
     if (this.workerManager) {
       keyPair = await this.workerManager.call(async (w) => {
         let keyPair;
@@ -588,10 +595,20 @@ class KeyManager {
     return keyPair;
   }
 
+  /**
+   * Generates and writes the encrypted keypair to a the root key file.
+   * If privateKeyPemOverride is provided then key generation is skipped in favor of the provided key.
+   * If state already exists the privateKeyPemOverride is ignored.
+   * @param password
+   * @param bits - Bit-width of the generated key.
+   * @param recoveryCode - Code to generate the key from.
+   * @param privateKeyPemOverride - Override generation with a provided private key.
+   */
   protected async setupRootKeyPair(
     password: string,
     bits: number = 4096,
     recoveryCode: RecoveryCode | undefined,
+    privateKeyPemOverride: PrivateKeyPem | undefined,
   ): Promise<[KeyPair, RecoveryCode | undefined]> {
     let rootKeyPair: KeyPair;
     let recoveryCodeNew: RecoveryCode | undefined;
@@ -610,6 +627,14 @@ class KeyManager {
       }
       return [rootKeyPair, undefined];
     } else {
+      if (privateKeyPemOverride != null) {
+        this.logger.info('Using provided root key pair');
+        const privateKey = keysUtils.privateKeyFromPem(privateKeyPemOverride);
+        const publicKey = keysUtils.publicKeyFromPrivateKey(privateKey);
+        rootKeyPair = { privateKey, publicKey };
+        await this.writeRootKeyPair(rootKeyPair, password);
+        return [rootKeyPair, undefined];
+      }
       this.logger.info('Generating root key pair');
       if (recoveryCode != null) {
         // Deterministic key pair generation from recovery code

src/validation/utils.ts

@@ -12,12 +12,14 @@ import type { GestaltAction, GestaltId } from '../gestalts/types';
 import type { VaultAction, VaultId } from '../vaults/types';
 import type { Host, Hostname, Port } from '../network/types';
 import type { ClaimId } from '../claims/types';
+import type { PrivateKey } from '../keys/types';
 import * as validationErrors from './errors';
 import * as nodesUtils from '../nodes/utils';
 import * as gestaltsUtils from '../gestalts/utils';
 import * as vaultsUtils from '../vaults/utils';
 import * as networkUtils from '../network/utils';
 import * as claimsUtils from '../claims/utils';
+import * as keysUtils from '../keys/utils';
 import config from '../config';
 
 function parseInteger(data: any): number {
@@ -259,6 +261,21 @@ function parseSeedNodes(data: any): [SeedNodes, boolean] {
   return [seedNodes, defaults];
 }
 
+function parsePrivateKeyPem(data: any): PrivateKey {
+  if (typeof data !== 'string') {
+    throw new validationErrors.ErrorParse('Private key Pem must be a string');
+  }
+  let privateKey: PrivateKey;
+  try {
+    privateKey = keysUtils.privateKeyFromPem(data);
+  } catch (e) {
+    throw new validationErrors.ErrorParse(
+      'Must provide a valid private key Pem',
+    );
+  }
+  return privateKey;
+}
+
 export {
   parseInteger,
   parseNumber,
@@ -276,4 +293,5 @@ export {
   parsePort,
   parseNetwork,
   parseSeedNodes,
+  parsePrivateKeyPem,
 };