diff --git a/.gitignore b/.gitignore index 1045197a..5e8238af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store node_modules .eslintcache +.idea .yarn/* !.yarn/patches diff --git a/packages/events-example/package.json b/packages/events-example/package.json index 80816f96..eda641ca 100644 --- a/packages/events-example/package.json +++ b/packages/events-example/package.json @@ -24,7 +24,7 @@ "eslint": "^8.21.0", "eslint-config-oclif": "^4", "eslint-config-oclif-typescript": "^1.0.2", - "globby": "^13", + "globby": "^11", "oclif": "^3", "shx": "^0.3.3", "ts-node": "^10.2.1", diff --git a/packages/events-example/src/commands/reconcile/ReconciliationPlan.ts b/packages/events-example/src/commands/reconcile/ReconciliationPlan.ts new file mode 100644 index 00000000..a7de82da --- /dev/null +++ b/packages/events-example/src/commands/reconcile/ReconciliationPlan.ts @@ -0,0 +1,22 @@ +import { Instance } from '@theniledev/js'; +import { StackSummary } from '@pulumi/pulumi/automation'; + +export class ReconciliationPlan { + + readonly creationSpecs!: Instance[]; + readonly destructionIds!: string[]; + + get creationIds(): string[] { + return this.creationSpecs.map((i: Instance) => i.id); + } + + constructor( + instances: { [key: string]: Instance; }, + stacks: { [key: string]: StackSummary; } + ) { + this.destructionIds = Object.keys(stacks) + .filter((id: string) => id !== null && id !== undefined && !instances[id]); + this.creationSpecs = Object.values(instances) + .filter((i: Instance) => i.id !== null && i.id !== undefined && !stacks[i.id]); + } +} diff --git a/packages/events-example/src/commands/reconcile/index.ts b/packages/events-example/src/commands/reconcile/index.ts index e2b73b95..472d1c4f 100644 --- a/packages/events-example/src/commands/reconcile/index.ts +++ b/packages/events-example/src/commands/reconcile/index.ts @@ -1,16 +1,9 @@ -import { CliUx, Command, Flags } from '@oclif/core'; +import { Command, Flags } from '@oclif/core'; import Nile, { Instance, NileApi } from '@theniledev/js'; -import { - DestroyResult, - InlineProgramArgs, - LocalWorkspace, - StackSummary, - Stack, - UpResult, - PulumiFn, -} from '@pulumi/pulumi/automation'; import { pulumiProgram } from '../../pulumiS3'; +import PulumiAwsDeployment from '../../deployments/PulumiAwsDeployment'; +import { ReconciliationPlan } from './ReconciliationPlan'; export default class Reconcile extends Command { static enableJsonFlag = true; @@ -18,88 +11,75 @@ export default class Reconcile extends Command { static flags = { basePath: Flags.string({ - description: 'basePath', + description: 'root URL for the Nile API', default: 'http://localhost:8080', }), workspace: Flags.string({ - description: 'workspace', - default: 'tryhard', + description: 'your Nile workspace name', + default: 'dev', }), email: Flags.string({ - description: 'email', - default: 'trying@demo.com', + description: 'developer email address', + default: 'developer@demo.com', }), password: Flags.string({ - description: 'password', - default: 'trying', + description: 'developer password', + default: 'very_secret', }), - organization: Flags.string({ description: 'organization' }), - entity: Flags.string({ description: 'entity' }), - status: Flags.boolean({ char: 's', description: 'status', default: false }), + organization: Flags.string({ description: 'an organization in your Nile workspace' }), + entity: Flags.string({ description: 'an entity type in your Nile workspace' }), + status: Flags.boolean({ char: 's', description: 'check current status of your control and data planes', default: false }), + region: Flags.string({ description: 'AWS region', default: 'us-west-2'}), }; - localWorkspace!: LocalWorkspace; + deployment!: PulumiAwsDeployment; nile!: NileApi; - async waitOnStack(stack: Stack): Promise { - let stackInfo; - do { - stackInfo = await stack.info(); - this.debug(stackInfo); - } while (stackInfo != undefined && stackInfo?.result !== 'succeeded'); - } + async run(): Promise { + const { flags } = await this.parse(Reconcile); - async getStack(stackName: string, program: PulumiFn): Promise { - const args: InlineProgramArgs = { - stackName, - projectName: 'tryhard', - program, - }; - const stack = await LocalWorkspace.createOrSelectStack(args); - await stack.setConfig('aws:region', { value: 'us-west-2' }); - return stack; - } + // nile setup + await this.connectNile(flags); - async createStack(instance: Instance): Promise { - const stack = await this.getStack(instance.id, pulumiProgram(instance)); - await this.waitOnStack(stack); - try { - CliUx.ux.action.start(`Creating a stack id=${instance.id}`); - return await stack.up({ onOutput: console.log }); - } finally { - CliUx.ux.action.stop(); - } - } + // pulumi setup + this.deployment = await PulumiAwsDeployment.create("nile-examples", pulumiProgram, { region: flags.region }); - async destroyStack(id: string): Promise { - const stack = await this.getStack(id, pulumiProgram({})); - await this.waitOnStack(stack); - try { - CliUx.ux.action.start(`Destroying a stack id=${id}`); - return await stack.destroy({ onOutput: console.log }); - } finally { - CliUx.ux.action.stop(); + // load our data + const stacks = await this.deployment.loadPulumiStacks(); + const instances = await this.loadNileInstances( + flags.organization, + flags.entity + ); + const plan = new ReconciliationPlan(instances, stacks); + + if (flags.status) { + this.log('Status check only.'); + this.log(`Pending destruction: ${ plan.destructionIds } (${ plan.destructionIds.length})`); + this.log(`Pending creation: ${ plan.creationIds } (${ plan.creationIds.length })`) + return { stacks, instances }; } + + await this.synchronizeDataPlane(plan); + await this.listenForNileEvents(flags.entity, this.findLastSeq(Object.values(instances))); } - async loadPulumiStacks(): Promise<{ [key: string]: StackSummary }> { - const stacks = await ( - await this.localWorkspace.listStacks() - ).reduce(async (accP, stack) => { - const acc = await accP; - const fullStack = await this.getStack(stack.name, pulumiProgram({})); - const info = await fullStack.info(); - if (info?.kind != 'destroy') { - acc[stack.name] = stack; - this.debug('adding stack', stack); - } - return acc; - }, Promise.resolve({} as { [key: string]: StackSummary })); - this.debug(stacks); - return stacks; + async connectNile(parsedFlags: {[name: string]: any}) { + this.nile = Nile({ basePath: parsedFlags.basePath, workspace: parsedFlags.workspace }); + const token = await this.nile.developers + .loginDeveloper({ + loginInfo: { + email: parsedFlags.email, + password: parsedFlags.password, + }, + }) + .catch((error: unknown) => { + // eslint-disable-next-line no-console + console.error("Nile authentication failed", error); + }); + this.nile.authToken = token?.token; } - async loadInstances( + async loadNileInstances( organization: string, entity: string ): Promise<{ [key: string]: Instance }> { @@ -108,83 +88,53 @@ export default class Reconcile extends Command { org: organization, type: entity, }) - ).reduce((acc, instance: Instance) => { - if (instance) { - acc[instance.id] = instance; - } - + ) + .filter((value: Instance) => value !== null && value !== undefined) + .reduce((acc, instance: Instance) => { + acc[instance.id] = instance; return acc; }, {} as { [key: string]: Instance }); - this.debug(instances); + this.debug("Nile Instances", instances); return instances; } - async run(): Promise { - const { flags } = await this.parse(Reconcile); - - // pulumi setup - this.localWorkspace = await LocalWorkspace.create({ - projectSettings: { name: flags.workspace, runtime: 'nodejs' }, - }); - this.localWorkspace.installPlugin('aws', 'v4.0.0'); - - // nile setup - this.nile = Nile({ basePath: flags.basePath, workspace: flags.workspace }); - const token = await this.nile.developers - .loginDeveloper({ - loginInfo: { - email: flags.email, - password: flags.password, - }, - }) - .catch((error: unknown) => { - // eslint-disable-next-line no-console - console.error(error); - }); - this.nile.authToken = token?.token; - - // load our data - const stacks = await this.loadPulumiStacks(); - const instances = await this.loadInstances( - flags.organization, - flags.entity - ); - let seq = 0; - for (const instance of Object.values(instances)) { - if (seq == null || (instance?.seq || seq) > seq) { - seq = instance?.seq || seq; - } - } + private findLastSeq(instances: Instance[]): number { + return instances + .map((value: Instance) => value?.seq || 0) + .reduce((prev: number, curr: number) => { + return Math.max(prev, curr || 0); + }, 0); + } - if (flags.status) { - return { stacks, instances }; - } + private async synchronizeDataPlane(plan: ReconciliationPlan) { + this.debug('Synchronizing data and control planes...'); + this.debug(plan); - // destroy any stacks that shouldnt exist - for (const id of Object.keys(stacks)) { - if (!instances[id]) { - this.destroyStack(id); - } + // destroy any stacks that should not exist + for (const id of plan.destructionIds) { + await this.deployment.destroyStack(id); } // create any stacks that should exist - for (const id of Object.keys(instances)) { - if (!stacks[id]) { - this.createStack(instances[id]); - } + for (const spec of plan.creationSpecs) { + await this.deployment.createStack(spec) } + } + private async listenForNileEvents(entityType: string, fromSeq: number) { + this.log(`Listening for events for ${ entityType } entities from sequence #${ fromSeq }`); await new Promise(() => { - this.nile.events.on({ type: flags.entity, seq }, async (e) => { + this.nile.events.on({ type: entityType, seq: fromSeq }, async (e) => { this.log(JSON.stringify(e, null, 2)); if (e.after) { const out = await (e.after.deleted - ? this.destroyStack(e.after.id) - : this.createStack(e.after)); + ? this.deployment.destroyStack(e.after.id) + : this.deployment.createStack(e.after)); - this.debug(out); + this.debug("Event Received", out); } }); }); } + } diff --git a/packages/events-example/src/deployments/AWSConfig.ts b/packages/events-example/src/deployments/AWSConfig.ts new file mode 100644 index 00000000..cc92417c --- /dev/null +++ b/packages/events-example/src/deployments/AWSConfig.ts @@ -0,0 +1,3 @@ +export default interface AWSConfig { + region: string; +} diff --git a/packages/events-example/src/deployments/PulumiAwsDeployment.ts b/packages/events-example/src/deployments/PulumiAwsDeployment.ts new file mode 100644 index 00000000..f7ea48b4 --- /dev/null +++ b/packages/events-example/src/deployments/PulumiAwsDeployment.ts @@ -0,0 +1,95 @@ +import { CliUx } from "@oclif/core"; +import { DestroyResult, InlineProgramArgs, LocalWorkspace, PulumiFn, Stack, StackSummary, UpResult } from "@pulumi/pulumi/automation"; +import { Instance } from "@theniledev/js"; +import AWSConfig from "./AWSConfig"; + +export interface PulumiFnGen { + (staticContent: any): PulumiFn + } + +export default class PulumiAwsDeployment { + + projectName!: string; + private localWorkspace!: LocalWorkspace; + private pulumiProgram: PulumiFnGen; + private awsConfig: AWSConfig; + + static async create( + projectName: string, + pulumiProgram: PulumiFnGen, + awsConfig: AWSConfig, + ): Promise { + const ws = await LocalWorkspace.create({ + projectSettings: { name: projectName, runtime: 'nodejs' }, + }); + ws.installPlugin('aws', 'v4.0.0'); + return new PulumiAwsDeployment(projectName, ws, pulumiProgram, awsConfig) + } + + constructor( + projectName: string, + localWorkspace: LocalWorkspace, + pulumiProgram: PulumiFnGen, + awsConfig: AWSConfig, + ) { + this.projectName = projectName; + this.localWorkspace = localWorkspace; + this.pulumiProgram = pulumiProgram; + this.awsConfig = awsConfig; + } + + async loadPulumiStacks(): Promise<{ [key: string]: StackSummary }> { + const stacks = await ( + await this.localWorkspace.listStacks() + ).reduce(async (accP, stack) => { + const acc = await accP; + const fullStack = await this.getStack(stack.name, this.pulumiProgram({})); + const info = await fullStack.info(); + if (info?.kind != 'destroy') { + acc[stack.name] = stack; + } + return acc; + }, Promise.resolve({} as { [key: string]: StackSummary })); + return stacks; + } + + async waitOnStack(stack: Stack): Promise { + let stackInfo; + do { + stackInfo = await stack.info(); + } while (stackInfo != undefined && stackInfo?.result !== 'succeeded'); + } + + async getStack(stackName: string, program: PulumiFn): Promise { + const args: InlineProgramArgs = { + stackName, + projectName: 'tryhard', + program, + }; + const stack = await LocalWorkspace.createOrSelectStack(args); + await stack.setConfig('aws:region', { value: this.awsConfig.region }); + return stack; + } + + async createStack(instance: Instance): Promise { + const stack = await this.getStack(instance.id, this.pulumiProgram(instance)); + await this.waitOnStack(stack); + try { + CliUx.ux.action.start(`Creating a stack id=${instance.id}`); + return await stack.up({ onOutput: console.log }); + } finally { + CliUx.ux.action.stop(); + } + } + + async destroyStack(id: string): Promise { + const stack = await this.getStack(id, this.pulumiProgram({})); + await this.waitOnStack(stack); + try { + CliUx.ux.action.start(`Destroying a stack id=${id}`); + return await stack.destroy({ onOutput: console.log }); + } finally { + CliUx.ux.action.stop(); + } + } +} diff --git a/yarn.lock b/yarn.lock index 879cad62..414b1182 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9746,7 +9746,7 @@ fast-glob@^2.2.6: merge2 "^1.2.3" micromatch "^3.1.10" -fast-glob@^3.0.3, fast-glob@^3.2.11, fast-glob@^3.2.9: +fast-glob@^3.0.3, fast-glob@^3.2.9: version "3.2.11" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.11.tgz#a1172ad95ceb8a16e20caa5c5e56480e5129c1d9" integrity sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew== @@ -10629,7 +10629,7 @@ globby@^10.0.1: merge2 "^1.2.3" slash "^3.0.0" -globby@^11.0.1, globby@^11.0.2, globby@^11.0.3, globby@^11.1.0: +globby@^11, globby@^11.0.1, globby@^11.0.2, globby@^11.0.3, globby@^11.1.0: version "11.1.0" resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== @@ -10641,17 +10641,6 @@ globby@^11.0.1, globby@^11.0.2, globby@^11.0.3, globby@^11.1.0: merge2 "^1.4.1" slash "^3.0.0" -globby@^13: - version "13.1.2" - resolved "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz" - integrity sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ== - dependencies: - dir-glob "^3.0.1" - fast-glob "^3.2.11" - ignore "^5.2.0" - merge2 "^1.4.1" - slash "^4.0.0" - globby@^9.2.0: version "9.2.0" resolved "https://registry.npmjs.org/globby/-/globby-9.2.0.tgz" @@ -16943,11 +16932,6 @@ slash@^3.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -slash@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz" - integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew== - slice-ansi@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz"