Google Cloud (GCP/GCE) driver (#54)

* google cloud driver - wip - temporarily switch ssh2 version due to mscdex/ssh2#989 * some fixes * more fixes - rsync permissions - wip - gce calls retry - async trace - remove redundant lines * even more fixes - run rsync with sudo to prevent permission errors from container-generated files - add wait flag to machine deletion - convert gce instance name to be dynamic to prevent collisions with machines being deleted - cap instance name length to google provided max * add google cloud storage profile storage * cosmetics, remove extractFirst * update docs * update package version. set ssh2 to a fixed revision * updated generated readme --------- Co-authored-by: Yshay Yaacobi <yshayy@gmail.com>
livecycle · Apr 23, 2023 · 8bfc9ae · 8bfc9ae
1 parent de0b7b8
commit 8bfc9ae
Show file tree

Hide file tree

Showing 38 changed files with 1,474 additions and 306 deletions.
diff --git a/README.md b/README.md
@@ -22,7 +22,7 @@
 ![Terminal GIF](./terminal.gif)
 
 Preevy is a powerful CLI tool designed to simplify the process of creating ephemeral preview environments.
-Using Preevy, you can easily provision any Docker-Compose application on AWS using affordable [Lightsail](https://aws.amazon.com/free/compute/lightsail) VMs (support for GCP and more cloud is on the way).
+Using Preevy, you can easily provision any Docker-Compose application on AWS using affordable [Lightsail](https://aws.amazon.com/free/compute/lightsail) VMs or simple Google cloud VMS (support more cloud providers is on the way).
 
 ## Documentation
 
@@ -52,14 +52,16 @@ You can read more about the story and philosophy behind Preevy [here](https://pr
 
 To start using the Preevy CLI you will need:
 
-- A local AWS configuration (you can get it by using `aws login` or `aws configure`)
+- A local Cloud provider configuration context:
+  - In AWS, it could be by using `aws login` or `aws configure`
+  - In GCP, it could be by using `gcloud auth application-default login`
 - A Docker-Compose application (examples can be found [here](https://github.com/docker/awesome-compose))
 
 Running Preevy:
 
 1. Install the CLI using npm:`npm install -g preevy` , or use it directly using: `npx preevy <command>`
 2. Set up a profile by using: `preevy init`
-3. Use `up` command to provision a new VM (Lightsail) with your application: `preevy up`
+3. Use `up` command to provision a new VM with your application: `preevy up`
 4. Access and share your new preview environment by using the `*.livecycle.run` outputted by the CLI.
 5. Code changed? Re-run `up` to quickly sync the preview environment with your changes on the existing VM.
 6. Destroy the environment by using: `preevy down`.

diff --git a/packages/cli/README.md b/packages/cli/README.md
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "preevy",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "description": "Quickly deploy preview environments to the cloud!",
   "author": "Livecycle",
   "bin": {
@@ -22,18 +22,21 @@
     "@aws-sdk/client-s3": "^3.271.0",
     "@aws-sdk/client-sts": "^3.289.0",
     "@aws-sdk/util-waiter": "^3.271.0",
+    "@google-cloud/compute": "^3.9.1",
+    "@google-cloud/storage": "^6.9.5",
     "@oclif/core": "^2",
     "@oclif/plugin-help": "^5",
     "@oclif/plugin-plugins": "^2.3.0",
-    "@preevy/common": "0.0.21",
-    "@preevy/compose-tunnel-agent": "0.0.21",
+    "@preevy/common": "0.0.22",
+    "@preevy/compose-tunnel-agent": "0.0.22",
     "@types/node-fetch": "^2.6.3",
     "@types/ssh2": "^1.11.8",
     "chalk": "^4.1.2",
     "cli-color": "^2.0.3",
     "commondir": "^1.0.1",
     "fast-safe-stringify": "^2.1.1",
     "glob": "^9.2.1",
+    "google-gax": "^3.6.0",
     "inquirer": "^8.0.0",
     "is-stream": "^2.0.1",
     "iter-tools-es": "^7.5.1",
@@ -48,7 +51,7 @@
     "rimraf": "^4.4.0",
     "shell-escape": "^0.2.0",
     "source-map-support": "^0.5.21",
-    "ssh2": "^1.11.0",
+    "ssh2": "github:mscdex/ssh2#64d45fd0b36642922bc3ec8f51de6bfc98c37022",
     "tar": "^6.1.13",
     "yaml": "^2.2.1"
   },

diff --git a/packages/cli/src/commands/down/index.ts b/packages/cli/src/commands/down/index.ts
@@ -4,6 +4,7 @@ import { localComposeClient } from '../../lib/compose/client'
 import { composeFlags } from '../../lib/compose/flags'
 import { envIdFlags, findAmbientEnvId, findAmbientProjectName } from '../../lib/env-id'
 import { Logger } from '../../log'
+import { withSpinner } from '../../lib/spinner'
 
 const findEnvId = async (log: Logger, { project, file }: { project?: string; file: string[] }) => {
   const projectName = project || await findAmbientProjectName(localComposeClient(file))
@@ -22,6 +23,10 @@ export default class Down extends DriverCommand<typeof Down> {
       description: 'Do not error if the environment is not found',
       default: false,
     }),
+    wait: Flags.boolean({
+      description: 'Wait for resource deletion to complete. If false (the default), the deletion will be started but not waited for',
+      default: false,
+    }),
   }
 
   static args = {
@@ -41,12 +46,14 @@ export default class Down extends DriverCommand<typeof Down> {
 
     if (!machine) {
       if (!flags.force) {
-        throw new Error(`No machine found for envId ${envId}`)
+        throw new Error(`No machine found for environment ${envId}`)
       }
       return undefined
     }
 
-    await driver.removeMachine(machine.providerId)
+    await withSpinner(async () => {
+      await driver.removeMachine(machine.providerId, flags.wait)
+    }, { opPrefix: `Deleting ${driver.friendlyName} machine ${machine.providerId} for environment ${envId}` })
 
     if (flags.json) {
       return envId

diff --git a/packages/cli/src/commands/init/index.ts b/packages/cli/src/commands/init/index.ts
@@ -1,10 +1,12 @@
 import { Flags, Args, ux } from '@oclif/core'
-import chalk from 'chalk'
 import inquirer from 'inquirer'
-import { pickBy } from 'lodash'
 import BaseCommand from '../../base-command'
-import { DriverFlagName, DriverName, machineDrivers } from '../../lib/machine'
-import { suggestDefaultUrl } from '../../lib/store/fs/s3'
+import { DriverName, machineDrivers } from '../../lib/machine'
+import { defaultBucketName as s3DefaultBucketName } from '../../lib/store/fs/s3'
+import { defaultBucketName as gsDefaultBucketName } from '../../lib/store/fs/google-cloud-storage'
+import { defaultProjectId } from '../../lib/machine/drivers/gce/client'
+import { REGIONS } from '../../lib/machine/drivers/lightsail/client'
+import { ambientAccountId } from '../../lib/aws-utils/account-id'
 
 export default class Init extends BaseCommand {
   static description = 'Initialize or import a new profile'
@@ -57,57 +59,74 @@ export default class Init extends BaseCommand {
             type: 'list',
             name: 'driver',
             message: 'Which cloud provider do you want to use?',
-            choices: [{
-              value: 'lightsail',
-              name: 'AWS Lightsail',
-            }],
+            choices: [
+              { value: 'lightsail', name: 'AWS Lightsail' },
+              { value: 'gce', name: 'Google Compute Engine' },
+            ],
           }])
 
       const driverStatic = machineDrivers[driver]
 
-      type DFS = (typeof driverStatic)['flags']
+      const driverAnswers = await inquirer.prompt<Record<string, unknown>>(await driverStatic.questions())
+      const driverFlags = await driverStatic.flagsFromAnswers(driverAnswers) as Record<string, unknown>
 
-      const requiredFlags = pickBy(
-        machineDrivers[driver].flags,
-        flag => (flag as { required: boolean }).required,
-      ) as DFS
+      const { locationType } = await inquirer.prompt<{ locationType: string }>([
+        {
+          type: 'list',
+          name: 'locationType',
+          message: 'Where do you want to store the profile?',
+          default: 'local',
+          choices: [
+            { value: 'local', name: 'local file' },
+            { value: 's3', name: 'AWS S3' },
+            { value: 'gs', name: 'Google Cloud Storage' },
+          ],
+        },
+      ])
 
-      const questions = Object.entries(requiredFlags).map(([key, flag]) => ({
-        type: 'input',
-        name: key,
-        message: flag.description,
-        default: ('flagHint' in driverStatic) ? driverStatic.flagHint(key as DriverFlagName<DriverName, 'flags'>) : '',
-      }))
-
-      const driverFlags = await inquirer.prompt<Record<string, string>>(questions)
-      const { locationType } = await inquirer.prompt<{
-          locationType: string
-        }>([
-          {
-            type: 'list',
-            name: 'locationType',
-            message: 'Where do you want to store the profile?',
-            default: 'local',
-            choices: [{
-              value: 's3',
-              name: 's3',
-            }, {
-              value: 'local',
-              name: 'local',
-            }],
-          }])
       let location: string
       if (locationType === 's3') {
-        const { s3Url } = await inquirer.prompt<{
-              s3Url: string
-            }>([{
-              type: 'input',
-              name: 's3Url',
-              message: `What is the S3 URL? ${chalk.reset.italic(`(format: s3://${chalk.yellowBright('[bucket]')}?region=${chalk.yellowBright('[region]')})`)}`,
-              default: await suggestDefaultUrl(profileAlias), // might worth generating profile id?
-            }])
+        const { region, bucket } = await inquirer.prompt<{ region: string; bucket: string }>([
+          {
+            type: 'list',
+            name: 'region',
+            message: 'S3 bucket region',
+            choices: REGIONS,
+            default: driver === 'lightsail' ? driverFlags.region as string : 'us-east-1',
+          },
+          {
+            type: 'input',
+            name: 'bucket',
+            message: 'Bucket name',
+            default: async (
+              answers: Record<string, unknown>
+            ) => {
+              const accountId = await ambientAccountId(answers.region as string)
+              return accountId ? s3DefaultBucketName({ profileAlias, accountId }) : undefined
+            },
+          },
+        ])
+
+        location = `s3://${bucket}?region=${region}`
+      } else if (locationType === 'gs') {
+        const { project, bucket } = await inquirer.prompt<{ project: string; bucket: string }>([
+          {
+            type: 'input',
+            name: 'project',
+            message: 'Google Cloud project',
+            default: driver === 'gce' ? driverFlags['project-id'] : defaultProjectId(),
+          },
+          {
+            type: 'input',
+            name: 'bucket',
+            message: 'Bucket name',
+            default: (
+              answers: Record<string, unknown>,
+            ) => gsDefaultBucketName({ profileAlias, project: answers.project as string }),
+          },
+        ])
 
-        location = s3Url
+        location = `gs://${bucket}?project=${project}`
       } else {
         location = `local://${profileAlias}`
       }

diff --git a/packages/cli/src/commands/profile/create.ts b/packages/cli/src/commands/profile/create.ts
@@ -29,7 +29,11 @@ export default class CreateProfile extends DriverCommand<typeof CreateProfile> {
     const alias = this.args.name
     const driver = this.flags.driver as DriverName
 
-    const driverFlags = mapKeys(pickBy(this.flags, (v, k) => k.startsWith(`${driver}-`)), (v, k) => k.substring(`${driver}-`.length))
+    const driverPrefix = `${driver}-`
+    const driverFlags = mapKeys(
+      pickBy(this.flags, (v, k) => k.startsWith(driverPrefix)),
+      (_v, k) => k.substring(driverPrefix.length),
+    )
 
     await this.profileConfig.create(alias, this.args.url, { driver }, async pStore => {
       await pStore.setDefaultFlags(

diff --git a/packages/cli/src/commands/purge.ts b/packages/cli/src/commands/purge.ts
@@ -57,6 +57,10 @@ export default class Purge extends DriverCommand<typeof Purge> {
       description: 'Do not ask for confirmation',
       default: false,
     }),
+    wait: Flags.boolean({
+      description: 'Wait for resource deletion to complete. If false (the default), the deletion will be started but not waited for',
+      default: false,
+    }),
   }
 
   static enableJsonFlag = true
@@ -92,7 +96,7 @@ export default class Purge extends DriverCommand<typeof Purge> {
     }
 
     await Promise.all([
-      ...machines.map(m => driver.removeMachine(m.providerId)),
+      ...machines.map(m => driver.removeMachine(m.providerId, flags.wait)),
       ...snapshots.map(s => driver.removeSnapshot(s.providerId)),
     ])
 

diff --git a/packages/cli/src/commands/up/index.ts b/packages/cli/src/commands/up/index.ts
@@ -64,7 +64,7 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
     const keyStore = sshKeysStore(this.store)
     let keyPair = await keyStore.getKey(keyAlias)
     if (!keyPair) {
-      this.logger.info(`keypair ${keyAlias} not found, creating new one`)
+      this.logger.info(`key pair ${keyAlias} not found, creating a new key pair`)
       keyPair = await driver.createKeyPair()
       await keyStore.addKey(keyPair)
       this.logger.info(`keypair ${keyAlias} created`)

diff --git a/packages/cli/src/driver-command.ts b/packages/cli/src/driver-command.ts
@@ -15,8 +15,8 @@ abstract class DriverCommand<T extends typeof Command> extends ProfileCommand<T>
     driver: Flags.custom<DriverName>({
       description: 'Machine driver to use',
       char: 'd',
-      default: 'lightsail' as const,
       options: Object.keys(machineDrivers),
+      required: false,
     })(),
     ...flagsForAllDrivers,
   }
@@ -42,8 +42,7 @@ abstract class DriverCommand<T extends typeof Command> extends ProfileCommand<T>
     if (this.#driver) {
       return this.#driver
     }
-    const { profile } = this
-    const driverName = this.flags.driver as DriverName
+    const { profile, driverName } = this
     const driverFlags = {
       ...await profileStore(this.store).defaultFlags(driverName),
       ...removeDriverPrefix<DriverFlags<DriverName, 'flags'>>(driverName, this.flags),

diff --git a/packages/cli/src/hooks/init/async-trace.ts b/packages/cli/src/hooks/init/async-trace.ts
@@ -16,7 +16,7 @@ const hook: Hook<'init'> = async () => {
     init(id: number) {
       const trace = {}
       Error.captureStackTrace(trace)
-      traces.set(id, (trace as { stack: string }).stack) // .replace(/(^.+$\n){4}/m, '\n'))
+      traces.set(id, (trace as { stack: string }).stack.replace(/(^.+$\n){6}/m, '\n'))
     },
     destroy(id) {
       traces.delete(id)

diff --git a/packages/cli/src/lib/aws-utils/account-id.ts b/packages/cli/src/lib/aws-utils/account-id.ts
@@ -0,0 +1,7 @@
+import { STS } from '@aws-sdk/client-sts'
+
+export const ambientAccountId = async (region: string) => {
+  const sts = new STS({ region })
+  const { Account: account } = await sts.getCallerIdentity({})
+  return account
+}
diff --git a/packages/cli/src/lib/commands/up/index.ts b/packages/cli/src/lib/commands/up/index.ts
@@ -136,7 +136,7 @@ const up = async ({
   const withDockerSocket = wrapWithDockerSocket({ sshClient, log })
 
   try {
-    await sshClient.execCommand(`sudo mkdir -p "${remoteDir}" && sudo chown $USER:docker "${remoteDir}"`)
+    await sshClient.execCommand(`sudo mkdir -p "${remoteDir}" && sudo chown $USER "${remoteDir}"`)
 
     log.debug('Files to copy', filesToCopy)
 

diff --git a/packages/cli/src/lib/commands/up/machine.ts b/packages/cli/src/lib/commands/up/machine.ts
@@ -1,7 +1,7 @@
 import { EOL } from 'os'
 import retry from 'p-retry'
 import { Logger } from '../../../log'
-import { MachineDriver, scripts } from '../../machine'
+import { MachineDriver } from '../../machine'
 import { connectSshClient } from '../../ssh/client'
 import { SSHKeyConfig } from '../../ssh/keypair'
 import { withSpinner } from '../../spinner'
@@ -38,7 +38,7 @@ const ensureMachine = async ({
   return withSpinner(async spinner => {
     if (recreating) {
       spinner.text = 'Deleting machine'
-      await machineDriver.removeMachine(existingMachine.providerId)
+      await machineDriver.removeMachine(existingMachine.providerId, false)
     }
     spinner.text = 'Checking for existing snapshot'
     const machineCreation = await machineCreationDriver.createMachine({ envId, keyConfig: sshKey })
@@ -99,7 +99,7 @@ export const ensureCustomizedMachine = async ({
   try {
     await withSpinner(async () => {
       log.debug('Executing machine scripts')
-      for (const script of scripts.CUSTOMIZE_BARE_MACHINE) {
+      for (const script of machineDriver.customizationScripts ?? []) {
         // eslint-disable-next-line no-await-in-loop
         await sshClient.execScript(script)
       }

diff --git a/packages/cli/src/lib/compose/model.ts b/packages/cli/src/lib/compose/model.ts
@@ -115,20 +115,19 @@ export const fixModelForRemote = async (
           return volume
         }
 
+        const remote = remoteVolumePath(volume)
         const stats = await statOrUndefined(volume.source) as fs.Stats | undefined
 
-        if (!stats) {
+        if (stats) {
+          if (!stats.isDirectory() && !stats.isFile() && !stats.isSymbolicLink()) {
+            return volume
+          }
+
           // ignore non-existing files like docker and compose do,
           //  they will be created as directories in the container
-          return volume
+          filesToCopy.push({ local: { path: volume.source, stats }, remote })
         }
 
-        if (!stats.isDirectory() && !stats.isFile() && !stats.isSymbolicLink()) {
-          return volume
-        }
-
-        const remote = remoteVolumePath(volume)
-        filesToCopy.push({ local: { path: volume.source, stats }, remote })
         return { ...volume, source: path.join(remoteDir, remote) }
       }, service.volumes)),
     })

diff --git a/packages/cli/src/lib/machine/driver/driver.ts b/packages/cli/src/lib/machine/driver/driver.ts
@@ -17,6 +17,7 @@ export type SpecDiffItem = {
 }
 
 export type MachineDriver = {
+  customizationScripts?: string[]
   friendlyName: string
 
   getMachine: (args: { envId: string }) => Promise<Machine | undefined>
@@ -28,7 +29,7 @@ export type MachineDriver = {
   listMachines: () => AsyncIterableIterator<Machine & { envId: string }>
   listSnapshots: () => AsyncIterableIterator<{ providerId: string }>
 
-  removeMachine: (driverMachineId: string) => Promise<void>
+  removeMachine: (driverMachineId: string, wait: boolean) => Promise<void>
   removeSnapshot: (providerId: string) => Promise<void>
   removeKeyPair: (alias: string) => Promise<void>
 }