feat: simplify jsonschema ref names in postgen

Signed-off-by: Grant Timmerman <timmerman+devrel@google.com>
googleapis · Nov 4, 2020 · 63da610 · 63da610
1 parent 09da596
commit 63da610
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 36 deletions.
diff --git a/jsonschema/google/events/cloud/cloudbuild/v1/BuildEventData.json b/jsonschema/google/events/cloud/cloudbuild/v1/BuildEventData.json
@@ -50,13 +50,13 @@
     "source": {
       "properties": {
         "storageSource": {
-          "$ref": "#/definitions/googleEventsCloudCloudbuildV1StorageSource",
+          "$ref": "#/definitions/StorageSource",
           "additionalProperties": true,
           "type": "object",
           "description": "If provided, get the source from this location in Google Cloud Storage."
         },
         "repoSource": {
-          "$ref": "#/definitions/googleEventsCloudCloudbuildV1RepoSource",
+          "$ref": "#/definitions/RepoSource",
           "additionalProperties": true,
           "type": "object",
           "description": "If provided, get the source from this location in a Cloud Source\n Repository."
@@ -115,19 +115,19 @@
           },
           "volumes": {
             "items": {
-              "$ref": "#/definitions/googleEventsCloudCloudbuildV1Volume"
+              "$ref": "#/definitions/Volume"
             },
             "type": "array",
             "description": "List of volumes to mount into the build step.\n\n Each volume is created as an empty volume prior to execution of the\n build step. Upon completion of the build, volumes and their contents are\n discarded.\n\n Using a named volume in only one step is not valid as it is indicative\n of a build request with an incorrect configuration."
           },
           "timing": {
-            "$ref": "#/definitions/googleEventsCloudCloudbuildV1TimeSpan",
+            "$ref": "#/definitions/TimeSpan",
             "additionalProperties": true,
             "type": "object",
             "description": "Stores timing information for executing this build step."
           },
           "pullTiming": {
-            "$ref": "#/definitions/googleEventsCloudCloudbuildV1TimeSpan",
+            "$ref": "#/definitions/TimeSpan",
             "additionalProperties": true,
             "type": "object",
             "description": "Stores timing information for pulling this build step's\n builder image only."
@@ -187,7 +187,7 @@
                 "description": "Docker Registry 2.0 digest."
               },
               "pushTiming": {
-                "$ref": "#/definitions/googleEventsCloudCloudbuildV1TimeSpan",
+                "$ref": "#/definitions/TimeSpan",
                 "additionalProperties": true,
                 "type": "object",
                 "description": "Stores timing information for pushing the specified image."
@@ -230,7 +230,7 @@
           "description": "List of build step outputs, produced by builder images, in the order\n corresponding to build step indices.\n\n [Cloud Builders](https://cloud.google.com/cloud-build/docs/cloud-builders)\n can produce this output by writing to `$BUILDER_OUTPUT/output`.\n Only the first 4KB of data is stored."
         },
         "artifactTiming": {
-          "$ref": "#/definitions/googleEventsCloudCloudbuildV1TimeSpan",
+          "$ref": "#/definitions/TimeSpan",
           "additionalProperties": true,
           "type": "object",
           "description": "Time to push all non-container artifacts."
@@ -329,7 +329,7 @@
               "description": "Path globs used to match files in the build's workspace."
             },
             "timing": {
-              "$ref": "#/definitions/googleEventsCloudCloudbuildV1TimeSpan",
+              "$ref": "#/definitions/TimeSpan",
               "additionalProperties": true,
               "type": "object",
               "description": "Stores timing information for pushing all artifact objects."
@@ -351,13 +351,13 @@
     "sourceProvenance": {
       "properties": {
         "resolvedStorageSource": {
-          "$ref": "#/definitions/googleEventsCloudCloudbuildV1StorageSource",
+          "$ref": "#/definitions/StorageSource",
           "additionalProperties": true,
           "type": "object",
           "description": "A copy of the build's `source.storage_source`, if exists, with any\n generations resolved."
         },
         "resolvedRepoSource": {
-          "$ref": "#/definitions/googleEventsCloudCloudbuildV1RepoSource",
+          "$ref": "#/definitions/RepoSource",
           "additionalProperties": true,
           "type": "object",
           "description": "A copy of the build's `source.repo_source`, if exists, with any\n revisions resolved."
@@ -553,7 +553,7 @@
         },
         "volumes": {
           "items": {
-            "$ref": "#/definitions/googleEventsCloudCloudbuildV1Volume"
+            "$ref": "#/definitions/Volume"
           },
           "type": "array",
           "description": "Global list of volumes to mount for ALL build steps\n\n Each volume is created as an empty volume prior to starting the build\n process. Upon completion of the build, volumes and their contents are\n discarded. Global volume names and paths cannot conflict with the volumes\n defined a build step.\n\n Using a global volume in a build with only one step is not valid as\n it is indicative of a build request with an incorrect configuration."
@@ -605,7 +605,7 @@
     },
     "timing": {
       "additionalProperties": {
-        "$ref": "#/definitions/googleEventsCloudCloudbuildV1TimeSpan",
+        "$ref": "#/definitions/TimeSpan",
         "additionalProperties": true,
         "type": "object"
       },
@@ -617,7 +617,7 @@
   "type": "object",
   "description": "Build event data\n Common build format for Google Cloud Platform API operations.\n Copied from\n https://github.com/googleapis/googleapis/blob/master/google/devtools/cloudbuild/v1/cloudbuild.proto.",
   "definitions": {
-    "googleEventsCloudCloudbuildV1RepoSource": {
+    "RepoSource": {
       "properties": {
         "projectId": {
           "type": "string",
@@ -660,7 +660,7 @@
       "description": "Location of the source in a Google Cloud Source Repository.",
       "id": "google.events.cloud.cloudbuild.v1.RepoSource"
     },
-    "googleEventsCloudCloudbuildV1StorageSource": {
+    "StorageSource": {
       "properties": {
         "bucket": {
           "type": "string",
@@ -687,7 +687,7 @@
       "description": "Location of the source in an archive file in Google Cloud Storage.",
       "id": "google.events.cloud.cloudbuild.v1.StorageSource"
     },
-    "googleEventsCloudCloudbuildV1TimeSpan": {
+    "TimeSpan": {
       "properties": {
         "startTime": {
           "type": "string",
@@ -705,7 +705,7 @@
       "description": "Start and end times for a build execution phase.",
       "id": "google.events.cloud.cloudbuild.v1.TimeSpan"
     },
-    "googleEventsCloudCloudbuildV1Volume": {
+    "Volume": {
       "properties": {
         "name": {
           "type": "string",

diff --git a/jsonschema/google/events/cloud/firestore/v1/DocumentEventData.json b/jsonschema/google/events/cloud/firestore/v1/DocumentEventData.json
@@ -5,13 +5,13 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "properties": {
     "value": {
-      "$ref": "#/definitions/googleEventsCloudFirestoreV1Document",
+      "$ref": "#/definitions/Document",
       "additionalProperties": true,
       "type": "object",
       "description": "A Document object containing a post-operation document snapshot.\n This is not populated for delete events. (TODO: check this!)"
     },
     "oldValue": {
-      "$ref": "#/definitions/googleEventsCloudFirestoreV1Document",
+      "$ref": "#/definitions/Document",
       "additionalProperties": true,
       "type": "object",
       "description": "A Document object containing a pre-operation document snapshot.\n This is only populated for update and delete events."
@@ -35,15 +35,15 @@
   "type": "object",
   "description": "The data within all Firestore document events.",
   "definitions": {
-    "googleEventsCloudFirestoreV1Document": {
+    "Document": {
       "properties": {
         "name": {
           "type": "string",
           "description": "The resource name of the document, for example\n `projects/{project_id}/databases/{database_id}/documents/{document_path}`."
         },
         "fields": {
           "additionalProperties": {
-            "$ref": "#/definitions/googleEventsCloudFirestoreV1Value",
+            "$ref": "#/definitions/Value",
             "additionalProperties": true,
             "type": "object"
           },
@@ -66,7 +66,7 @@
       "description": "A Firestore document.",
       "id": "google.events.cloud.firestore.v1.Document"
     },
-    "googleEventsCloudFirestoreV1Value": {
+    "Value": {
       "properties": {
         "nullValue": {
           "oneOf": [
@@ -134,7 +134,7 @@
           "properties": {
             "values": {
               "items": {
-                "$ref": "#/definitions/googleEventsCloudFirestoreV1Value"
+                "$ref": "#/definitions/Value"
               },
               "type": "array",
               "description": "Values in the array."
@@ -148,7 +148,7 @@
           "properties": {
             "fields": {
               "additionalProperties": {
-                "$ref": "#/definitions/googleEventsCloudFirestoreV1Value",
+                "$ref": "#/definitions/Value",
                 "additionalProperties": true,
                 "type": "object"
               },

diff --git a/jsonschema/google/events/firebase/analytics/v1/AnalyticsLogData.json b/jsonschema/google/events/firebase/analytics/v1/AnalyticsLogData.json
@@ -25,7 +25,7 @@
           "additionalProperties": {
             "properties": {
               "value": {
-                "$ref": "#/definitions/googleEventsFirebaseAnalyticsV1AnalyticsValue",
+                "$ref": "#/definitions/AnalyticsValue",
                 "additionalProperties": true,
                 "type": "object",
                 "description": "Last set value of user property."
@@ -227,7 +227,7 @@
           },
           "params": {
             "additionalProperties": {
-              "$ref": "#/definitions/googleEventsFirebaseAnalyticsV1AnalyticsValue",
+              "$ref": "#/definitions/AnalyticsValue",
               "additionalProperties": true,
               "type": "object"
             },
@@ -273,7 +273,7 @@
   "type": "object",
   "description": "The data within Firebase Analytics log events.",
   "definitions": {
-    "googleEventsFirebaseAnalyticsV1AnalyticsValue": {
+    "AnalyticsValue": {
       "properties": {
         "stringValue": {
           "type": "string"

diff --git a/tools/proto2jsonschema/gen.sh b/tools/proto2jsonschema/gen.sh
@@ -42,7 +42,7 @@ cd ..
 
 echo "- Setting up protoc plugin (chrusty/protoc-gen-jsonschema)"
 # Pin chrusty tool to specific version: https://github.com/chrusty/protoc-gen-jsonschema/tags
-GO111MODULE=on go get -v github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@0.9.4
+GO111MODULE=on go get -v github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@0.9.5
 go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema
 
 echo "- Converting protos to JSON Schemas"
@@ -70,11 +70,11 @@ for proto in $DATA_PROTOS; do
   mkdir -p $OUT_PROTO_DIR
 
   # Run protoc
-  # - Option: proto_and_json_fieldnames – use JSON field names (camelCase)
+  # - Option: json_fieldnames – use JSON field names (camelCase)
   # - Input: proto/ – our CloudEvent protos
   # - Input: googlapis/ – common googleapis protos
   $PROTOC \
-  --jsonschema_out=proto_and_json_fieldnames:$OUT_PROTO_DIR \
+  --jsonschema_out=json_fieldnames:$OUT_PROTO_DIR \
   --proto_path=proto/ \
   --proto_path=third_party/googleapis \
   "$proto"
@@ -90,7 +90,8 @@ find $OUT_DIR -type f ! -name "*Data.json" -exec rm {} \;
 echo "- Done with gen."
 
 # Postgen
-echo "- Postgen"
+echo "- Postgen: Start"
 cd $(dirname $0)
 npm i
 node postgen.js
+echo "- Postgen: End"
diff --git a/tools/proto2jsonschema/postgen.js b/tools/proto2jsonschema/postgen.js
@@ -47,10 +47,6 @@ console.log(`Fixing paths in dir: ${ROOT}`);
           // Delete these keys from object
           delete obj[prop];
         } else if (isRef) {
-          const ucfirst = (w) => w.charAt(0).toUpperCase() + w.slice(1);
-          const lcfirst = (w) => w.charAt(0).toLowerCase() + w.slice(1);
-
-          // Fix the $ref prop
           const uri = lcfirst(obj[prop].split('.').map(ucfirst).join(''));
           obj[prop] = `#/definitions/${uri}`;
         } else if (typeof obj[prop] === 'object') {
@@ -61,9 +57,54 @@ console.log(`Fixing paths in dir: ${ROOT}`);
     };
     cleanSchema(resultJSON, true);
 
-    // Write back file with formatting
-    fs.writeFileSync(filePath, JSON.stringify(resultJSON, null, 2));
+    /**
+     * Simplify the $ref tags. Used in some languagues like Java.
+     *
+     * This couldn't be done in the previous step, because the $ref was simply wrong in that step.
+     * Now the $refs are correct (but long).
+     *
+     * We need to change the $ref name and definition.
+     *
+     * We do so by:
+     * - traversing the json, getting all refs
+     * - getting the simplified replacement $ref without the prefix
+     * - replaceAll ref ids (fixes "definition" and corresponding "$ref" tags)
+     */
+    const allRefs = [];
+    const traverseObjAndGatherRefs = (obj) => {
+      for (const prop in obj) {
+        if (prop === "$ref") {
+          allRefs.push(obj[prop]);
+        } else if (typeof obj[prop] === 'object') {
+          // Recursive case
+          traverseObjAndGatherRefs(obj[prop]);
+        }
+      }
+    };
+    traverseObjAndGatherRefs(resultJSON);
+
+    // Replace all complex $refs
+    const refReplacementList = []; // a list of [before, after] strings
+    const typePrefix = lcfirst(getCloudEventType(filePath).split('.').map(ucfirst).join(''));
+    const unnecessaryTypePrefix = `#/definitions/${typePrefix}`;
+    // Go through all $refs
+    const uniqueRefs = [...new Set(allRefs)];
+    uniqueRefs.forEach((ref) => {
+      const replacementType = ref.substring(unnecessaryTypePrefix.length);
+      refReplacementList.push([`${typePrefix}${replacementType}`, replacementType]);
+    });
+
+    // Format JSON
+    let jsonString = JSON.stringify(resultJSON, null, 2);
+    // Replace all $refs and definitions with simpler $refs
+    refReplacementList.forEach(([before, after]) => {
+      jsonString = jsonString.split(before).join(after);
+    });
+
+    // Write back JSON Schema
+    fs.writeFileSync(filePath, jsonString);
   });
+  console.log(`Fixed ${filePaths.length} schemas!`);
 })();
 
 /**
@@ -90,3 +131,6 @@ function getCloudEventType(filepath) {
   const removeSuffix = removePrefix.substring(0, removePrefix.lastIndexOf("/"));
   return removeSuffix.replace(/\//g, '.');
 }
+
+const ucfirst = (w) => w.charAt(0).toUpperCase() + w.slice(1);
+const lcfirst = (w) => w.charAt(0).toLowerCase() + w.slice(1);