feat(python): idiomatic capitalization for structs (#586)

This change addresses 4 issues:

- Structs now use idiomatic (snake_case) capitalization of fields
  (instead of JSII-inherited camelCase).
- IDE support -- replace TypedDict usage with regular classes. This
  makes it so that we can't use dicts anymore, but mypy support in
  IDEs wasn't great and by using POPOs (Plain Old Python Objects)
  IDEs get their support back.
- Structs in a variadic argument use to be incorrectly lifted to
  keyword arguments, this no longer happens.
- Stop emitting "Stable" stabilities in docstrings, "Stable" is implied.

In order to make this change, I've had to make `jsii-pacmak` depend on
`jsii-reflect`. This is the proper layering of libraries anyway, since
`jsii-reflect` adds much-needed--and otherwise
duplicated--interpretation of the spec.

Complete refactoring of `jsii-pacmak` to build on `jsii-reflect` is too
much work right now, however, so I've added "escape hatching" where
generators can take advantage of the power of jsii-reflect if they want
to, but most of the code still works on the raw spec level.

Added a refactoring where we load the assembly once and reuse the same
instance for all generators, instead of loading the assembly for every
generator. Assembly-loading, especially with a lot of dependencies, takes
a non-negligible amount of time, so this has the side effect of making
the packaging step faster (shaves off 100 packages * 3 targets * a
couple of seconds).

Fixes #537 
Fixes #577 
Fixes #578 
Fixes #588

packages/codemaker/lib/case-utils.ts

@@ -1,5 +1,11 @@
 import { default as camelcase } from 'camelcase';
-import * as decamelize from 'decamelize';
+import decamelize = require('decamelize');
+
+const COMMON_ABBREVIATIONS = [
+    'KiB',
+    'MiB',
+    'GiB',
+];
 
 export function toCamelCase(...args: string[]) {
     return camelcase(args);
@@ -10,5 +16,17 @@ export function toPascalCase(...args: string[]) {
 }
 
 export function toSnakeCase(s: string, sep = '_') {
+    // Save common abbrevations
+    s = s.replace(ABBREV_RE, (_, before, abbr, after) => before + ucfirst(abbr.toLowerCase()) + after);
     return decamelize(s, sep);
 }
+
+function regexQuote(s: string) {
+    return s.replace(/[.?*+^$[\]\\(){}|-]/g, "\\$&");
+}
+
+const ABBREV_RE = new RegExp('(^|[^A-Z])(' + COMMON_ABBREVIATIONS.map(regexQuote).join('|') + ')($|[^a-z])', 'g');
+
+function ucfirst(s: string) {
+    return s.substr(0, 1).toUpperCase() + s.substr(1).toLowerCase();
+}

packages/codemaker/test/test.case-utils.ts

@@ -2,15 +2,32 @@ import nodeunit = require('nodeunit');
 import caseUtils = require('../lib/case-utils');
 
 export = nodeunit.testCase({
-  toCamelCase(test: nodeunit.Test) {
+  'toCamelCase'(test: nodeunit.Test) {
     test.equal(caseUtils.toCamelCase('EXAMPLE_VALUE'), 'exampleValue');
     test.equal(caseUtils.toCamelCase('example', 'value'), 'exampleValue');
     test.done();
   },
 
-  toPascalCase(test: nodeunit.Test) {
+  'toPascalCase'(test: nodeunit.Test) {
     test.equal(caseUtils.toPascalCase('EXAMPLE_VALUE'), 'ExampleValue');
     test.equal(caseUtils.toPascalCase('example', 'value'), 'ExampleValue');
+    test.done();
+  },
+
+  'toSnakeCase'(test: nodeunit.Test) {
+    test.equal(caseUtils.toSnakeCase('EXAMPLE_VALUE'), 'example_value');
+    test.equal(caseUtils.toSnakeCase('exampleValue'), 'example_value');
+    test.equal(caseUtils.toSnakeCase('ExampleValue'), 'example_value');
+    test.equal(caseUtils.toSnakeCase('EPSConduit'), 'eps_conduit');
+    test.equal(caseUtils.toSnakeCase('SomeEBSVolume'), 'some_ebs_volume');
+    test.done();
+  },
+
+  'reserved word snake-casing'(test: nodeunit.Test) {
+    test.equal(caseUtils.toSnakeCase('SizeMiB'), 'size_mib');
+    test.equal(caseUtils.toSnakeCase('SizeMiBiBytes'), 'size_mi_bi_bytes');
+    test.equal(caseUtils.toSnakeCase('MiBSize'), 'mib_size');
+
     test.done();
   }
 });

packages/jsii-calc/lib/compliance.ts

@@ -1741,3 +1741,46 @@ export class DataRenderer {
         return JSON.stringify(map, null, 2);
     }
 }
+
+export interface TopLevelStruct {
+    /**
+     * This is a required field
+     */
+    readonly required: string;
+
+    /**
+     * You don't have to pass this
+     */
+    readonly optional?: string;
+
+    /**
+     * A union to really stress test our serialization
+     */
+    readonly secondLevel: SecondLevelStruct | number;
+}
+
+export interface SecondLevelStruct {
+    /**
+     * It's long and required
+     */
+    readonly deeperRequiredProp: string;
+
+    /**
+     * It's long, but you'll almost never pass it.
+     */
+    readonly deeperOptionalProp?: string;
+}
+
+export class StructPassing {
+    public static roundTrip(_positional: number, input: TopLevelStruct): TopLevelStruct {
+        return {
+            required: input.required,
+            optional: input.optional,
+            secondLevel: input.secondLevel,
+        };
+    }
+
+    public static howManyVarArgsDidIPass(_positional: number, inputs: TopLevelStruct[]): number {
+        return inputs.length;
+    }
+}

packages/jsii-calc/test/assembly.jsii

@@ -7150,6 +7150,55 @@
       ],
       "name": "RuntimeTypeChecking"
     },
+    "jsii-calc.SecondLevelStruct": {
+      "assembly": "jsii-calc",
+      "datatype": true,
+      "docs": {
+        "stability": "experimental"
+      },
+      "fqn": "jsii-calc.SecondLevelStruct",
+      "kind": "interface",
+      "locationInModule": {
+        "filename": "lib/compliance.ts",
+        "line": 1762
+      },
+      "name": "SecondLevelStruct",
+      "properties": [
+        {
+          "abstract": true,
+          "docs": {
+            "stability": "experimental",
+            "summary": "It's long and required."
+          },
+          "immutable": true,
+          "locationInModule": {
+            "filename": "lib/compliance.ts",
+            "line": 1766
+          },
+          "name": "deeperRequiredProp",
+          "type": {
+            "primitive": "string"
+          }
+        },
+        {
+          "abstract": true,
+          "docs": {
+            "stability": "experimental",
+            "summary": "It's long, but you'll almost never pass it."
+          },
+          "immutable": true,
+          "locationInModule": {
+            "filename": "lib/compliance.ts",
+            "line": 1771
+          },
+          "name": "deeperOptionalProp",
+          "optional": true,
+          "type": {
+            "primitive": "string"
+          }
+        }
+      ]
+    },
     "jsii-calc.SingleInstanceTwoTypes": {
       "assembly": "jsii-calc",
       "docs": {
@@ -7756,6 +7805,87 @@
         }
       ]
     },
+    "jsii-calc.StructPassing": {
+      "assembly": "jsii-calc",
+      "docs": {
+        "stability": "experimental"
+      },
+      "fqn": "jsii-calc.StructPassing",
+      "initializer": {},
+      "kind": "class",
+      "locationInModule": {
+        "filename": "lib/compliance.ts",
+        "line": 1774
+      },
+      "methods": [
+        {
+          "docs": {
+            "stability": "experimental"
+          },
+          "locationInModule": {
+            "filename": "lib/compliance.ts",
+            "line": 1783
+          },
+          "name": "howManyVarArgsDidIPass",
+          "parameters": [
+            {
+              "name": "_positional",
+              "type": {
+                "primitive": "number"
+              }
+            },
+            {
+              "name": "inputs",
+              "type": {
+                "collection": {
+                  "elementtype": {
+                    "fqn": "jsii-calc.TopLevelStruct"
+                  },
+                  "kind": "array"
+                }
+              }
+            }
+          ],
+          "returns": {
+            "type": {
+              "primitive": "number"
+            }
+          },
+          "static": true
+        },
+        {
+          "docs": {
+            "stability": "experimental"
+          },
+          "locationInModule": {
+            "filename": "lib/compliance.ts",
+            "line": 1775
+          },
+          "name": "roundTrip",
+          "parameters": [
+            {
+              "name": "_positional",
+              "type": {
+                "primitive": "number"
+              }
+            },
+            {
+              "name": "input",
+              "type": {
+                "fqn": "jsii-calc.TopLevelStruct"
+              }
+            }
+          ],
+          "returns": {
+            "type": {
+              "fqn": "jsii-calc.TopLevelStruct"
+            }
+          },
+          "static": true
+        }
+      ],
+      "name": "StructPassing"
+    },
     "jsii-calc.Sum": {
       "assembly": "jsii-calc",
       "base": "jsii-calc.composition.CompositeOperation",
@@ -8104,6 +8234,80 @@
       ],
       "name": "Thrower"
     },
+    "jsii-calc.TopLevelStruct": {
+      "assembly": "jsii-calc",
+      "datatype": true,
+      "docs": {
+        "stability": "experimental"
+      },
+      "fqn": "jsii-calc.TopLevelStruct",
+      "kind": "interface",
+      "locationInModule": {
+        "filename": "lib/compliance.ts",
+        "line": 1745
+      },
+      "name": "TopLevelStruct",
+      "properties": [
+        {
+          "abstract": true,
+          "docs": {
+            "stability": "experimental",
+            "summary": "This is a required field."
+          },
+          "immutable": true,
+          "locationInModule": {
+            "filename": "lib/compliance.ts",
+            "line": 1749
+          },
+          "name": "required",
+          "type": {
+            "primitive": "string"
+          }
+        },
+        {
+          "abstract": true,
+          "docs": {
+            "stability": "experimental",
+            "summary": "A union to really stress test our serialization."
+          },
+          "immutable": true,
+          "locationInModule": {
+            "filename": "lib/compliance.ts",
+            "line": 1759
+          },
+          "name": "secondLevel",
+          "type": {
+            "union": {
+              "types": [
+                {
+                  "primitive": "number"
+                },
+                {
+                  "fqn": "jsii-calc.SecondLevelStruct"
+                }
+              ]
+            }
+          }
+        },
+        {
+          "abstract": true,
+          "docs": {
+            "stability": "experimental",
+            "summary": "You don't have to pass this."
+          },
+          "immutable": true,
+          "locationInModule": {
+            "filename": "lib/compliance.ts",
+            "line": 1754
+          },
+          "name": "optional",
+          "optional": true,
+          "type": {
+            "primitive": "string"
+          }
+        }
+      ]
+    },
     "jsii-calc.UnaryOperation": {
       "abstract": true,
       "assembly": "jsii-calc",
@@ -8851,5 +9055,5 @@
     }
   },
   "version": "0.13.4",
-  "fingerprint": "T+Xf0Qtu6HVsykkz6vGH8OhMtRQbt7Jzq4O7Rsv5SeM="
+  "fingerprint": "7VlL0zBte+Qeew8qdhvaeS0ZtQtbbT8m3wjT7mDtAoE="
 }

packages/jsii-pacmak/bin/jsii-pacmak.ts

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 import fs = require('fs-extra');
+import reflect = require('jsii-reflect');
 import spec = require('jsii-spec');
 import os = require('os');
 import path = require('path');
@@ -136,6 +137,10 @@ import { VERSION_DESC } from '../lib/version';
             const tarball = await timers.recordAsync('npm pack', () => {
                 return npmPack(packageDir, tmpdir);
             });
+
+            const ts = new reflect.TypeSystem();
+            const assembly = await ts.loadModule(packageDir);
+
             for (const targetName of targets) {
                 // if we are targeting a single language, output to outdir, otherwise outdir/<target>
                 const targetOutputDir = (targets.length > 1 || forceSubdirectory)
@@ -144,7 +149,7 @@ import { VERSION_DESC } from '../lib/version';
                 logging.debug(`Building ${pkg.name}/${targetName}: ${targetOutputDir}`);
 
                 await timers.recordAsync(targetName.toString(), () =>
-                    generateTarget(packageDir, targetName.toString(), targetOutputDir, tarball)
+                    generateTarget(assembly, packageDir, targetName.toString(), targetOutputDir, tarball)
                 );
             }
         } finally {
@@ -159,7 +164,7 @@ import { VERSION_DESC } from '../lib/version';
         logging.info(`Packaged. ${timers.display()}`);
     }
 
-    async function generateTarget(packageDir: string, targetName: string, targetOutputDir: string, tarball: string) {
+    async function generateTarget(assembly: reflect.Assembly, packageDir: string, targetName: string, targetOutputDir: string, tarball: string) {
         // ``argv.target`` is guaranteed valid by ``yargs`` through the ``choices`` directive.
         const targetConstructor = targetConstructors[targetName];
         if (!targetConstructor) {
@@ -169,6 +174,7 @@ import { VERSION_DESC } from '../lib/version';
         const target = new targetConstructor({
             targetName,
             packageDir,
+            assembly,
             fingerprint: argv.fingerprint,
             force: argv.force,
             arguments: argv