Merge pull request #294 from JupiterOne/293-dedup-schema

Fixes #293 - "toMatchGraphObjectSchema" generates an invalid JSON

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/cli",
-  "version": "2.11.0",
+  "version": "2.11.1",
   "description": "The JupiterOne cli",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -22,8 +22,8 @@
     "prepack": "yarn build:dist"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-core": "^2.11.0",
-    "@jupiterone/integration-sdk-runtime": "^2.11.0",
+    "@jupiterone/integration-sdk-core": "^2.11.1",
+    "@jupiterone/integration-sdk-runtime": "^2.11.1",
     "@lifeomic/attempt": "^3.0.0",
     "commander": "^5.0.0",
     "globby": "^11.0.1",

packages/integration-sdk-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-cli",
-  "version": "2.11.0",
+  "version": "2.11.1",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -22,15 +22,15 @@
     "prepack": "yarn build:dist"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-runtime": "^2.11.0",
+    "@jupiterone/integration-sdk-runtime": "^2.11.1",
     "commander": "^5.0.0",
     "globby": "^11.0.0",
     "lodash": "^4.17.19",
     "upath": "^1.2.0",
     "vis": "^4.21.0-EOL"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^2.11.0",
+    "@jupiterone/integration-sdk-private-test-utils": "^2.11.1",
     "@pollyjs/adapter-node-http": "^4.0.4",
     "@pollyjs/core": "^4.0.4",
     "@pollyjs/persister-fs": "^4.0.4",

packages/integration-sdk-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-core",
-  "version": "2.11.0",
+  "version": "2.11.1",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",

packages/integration-sdk-dev-tools/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-dev-tools",
-  "version": "2.11.0",
+  "version": "2.11.1",
   "description": "A collection of developer tools that will assist with building integrations.",
   "repository": "git@github.com:JupiterOne/sdk.git",
   "author": "JupiterOne <dev@jupiterone.io>",
@@ -15,8 +15,8 @@
     "access": "public"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-cli": "^2.11.0",
-    "@jupiterone/integration-sdk-testing": "^2.11.0",
+    "@jupiterone/integration-sdk-cli": "^2.11.1",
+    "@jupiterone/integration-sdk-testing": "^2.11.1",
     "@types/jest": "^25.2.3",
     "@types/node": "^14.0.5",
     "@typescript-eslint/eslint-plugin": "^3.8.0",

packages/integration-sdk-private-test-utils/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@jupiterone/integration-sdk-private-test-utils",
   "private": true,
-  "version": "2.11.0",
+  "version": "2.11.1",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,7 +12,7 @@
     "node": "10.x || 12.x || 14.x"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-core": "^2.11.0",
+    "@jupiterone/integration-sdk-core": "^2.11.1",
     "lodash": "^4.17.15",
     "uuid": "^7.0.3"
   },

packages/integration-sdk-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-runtime",
-  "version": "2.11.0",
+  "version": "2.11.1",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,7 +23,7 @@
     "prepack": "yarn build:dist"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-core": "^2.11.0",
+    "@jupiterone/integration-sdk-core": "^2.11.1",
     "@lifeomic/alpha": "^1.1.3",
     "async-sema": "^3.1.0",
     "axios": "^0.19.2",
@@ -42,7 +42,7 @@
     "uuid": "^7.0.3"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^2.11.0",
+    "@jupiterone/integration-sdk-private-test-utils": "^2.11.1",
     "@types/uuid": "^7.0.2",
     "get-port": "^5.1.1",
     "memfs": "^3.2.0",

packages/integration-sdk-testing/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-testing",
-  "version": "2.11.0",
+  "version": "2.11.1",
   "description": "Testing utilities for JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,8 +23,8 @@
     "prepack": "yarn build:dist"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-core": "^2.11.0",
-    "@jupiterone/integration-sdk-runtime": "^2.11.0",
+    "@jupiterone/integration-sdk-core": "^2.11.1",
+    "@jupiterone/integration-sdk-runtime": "^2.11.1",
     "@pollyjs/adapter-node-http": "^4.0.4",
     "@pollyjs/core": "^4.0.4",
     "@pollyjs/persister-fs": "^4.0.4",
@@ -36,7 +36,7 @@
     "lodash": "^4.17.15"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^2.11.0",
+    "@jupiterone/integration-sdk-private-test-utils": "^2.11.1",
     "@types/lodash": "^4.14.149",
     "get-port": "^5.1.1",
     "memfs": "^3.2.0"

packages/integration-sdk-testing/src/__tests__/jest.test.ts

@@ -100,6 +100,120 @@ describe('#toMatchGraphObjectSchema', () => {
     expect(result.message()).toEqual('Success!');
   });
 
+  test('should dedup "required" and "type" properties', () => {
+    const googleComputeDisk = {
+      id: '6905138577447433802',
+      name: 'testvm',
+      status: 'READY',
+      _class: ['DataStore', 'Disk'],
+      _type: 'google_compute_disk',
+      _key:
+        'https://www.googleapis.com/compute/v1/projects/j1-gc-integration-dev/zones/us-central1-a/disks/testvm',
+      displayName: 'testvm',
+      createdOn: 1597245605930,
+      zone: 'us-central1-a',
+      sizeGB: '10',
+      active: true,
+      sourceImage:
+        'https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-9-stretch-v20200805',
+      sourceImageId: '6709658075886210235',
+      type: 'pd-standard',
+      licenses: [
+        'https://www.googleapis.com/compute/v1/projects/debian-cloud/global/licenses/debian-9-stretch',
+      ],
+      guestOsFeatures: ['VIRTIO_SCSI_MULTIQUEUE'],
+      lastAttachTimestamp: 1597245605930,
+      labelFingerprint: '42WmSpB8rSM=',
+      licenseCodes: ['1000205'],
+      physicalBlockSizeBytes: '4096',
+      kind: 'compute#disk',
+      encrypted: true,
+      classification: null,
+      _rawData: [
+        {
+          name: 'default',
+          rawData: {
+            id: '6905138577447433802',
+            creationTimestamp: '2020-08-12T08:20:05.930-07:00',
+            name: 'testvm',
+            sizeGb: '10',
+            zone:
+              'https://www.googleapis.com/compute/v1/projects/j1-gc-integration-dev/zones/us-central1-a',
+            status: 'READY',
+            selfLink:
+              'https://www.googleapis.com/compute/v1/projects/j1-gc-integration-dev/zones/us-central1-a/disks/testvm',
+            sourceImage:
+              'https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-9-stretch-v20200805',
+            sourceImageId: '6709658075886210235',
+            type:
+              'https://www.googleapis.com/compute/v1/projects/j1-gc-integration-dev/zones/us-central1-a/diskTypes/pd-standard',
+            licenses: [
+              'https://www.googleapis.com/compute/v1/projects/debian-cloud/global/licenses/debian-9-stretch',
+            ],
+            guestOsFeatures: [
+              {
+                type: 'VIRTIO_SCSI_MULTIQUEUE',
+              },
+            ],
+            lastAttachTimestamp: '2020-08-12T08:20:05.930-07:00',
+            users: [
+              'https://www.googleapis.com/compute/v1/projects/j1-gc-integration-dev/zones/us-central1-a/instances/testvm',
+            ],
+            labelFingerprint: '42WmSpB8rSM=',
+            licenseCodes: ['1000205'],
+            physicalBlockSizeBytes: '4096',
+            kind: 'compute#disk',
+            classification: null,
+          },
+        },
+      ],
+    };
+
+    const result = toMatchGraphObjectSchema(googleComputeDisk, {
+      _class: ['DataStore', 'Disk'],
+      schema: {
+        additionalProperties: false,
+        properties: {
+          _type: { const: 'google_compute_disk' },
+          _rawData: {
+            type: 'array',
+            items: { type: 'object' },
+          },
+          description: { type: 'string' },
+          zone: { type: 'string' },
+          sizeGB: { type: 'string' },
+          status: { type: 'string' },
+          sourceImage: { type: 'string' },
+          sourceImageId: { type: 'string' },
+          type: { type: 'string' },
+          licenses: {
+            type: 'array',
+            items: { type: 'string' },
+          },
+          guestOsFeatures: {
+            type: 'array',
+            items: { type: 'string' },
+          },
+          lastAttachTimestamp: { type: 'number' },
+          labelFingerprint: { type: 'string' },
+          licenseCodes: {
+            type: 'array',
+            items: { type: 'string' },
+          },
+          physicalBlockSizeBytes: { type: 'string' },
+          kind: { type: 'string' },
+          encrypted: true,
+        },
+      },
+    });
+
+    expect(result.message()).toEqual('Success!');
+    expect(result).toEqual({
+      message: expect.any(Function),
+      pass: true,
+    });
+  });
+
   test('should match array of custom entities using schema', () => {
     const result = toMatchGraphObjectSchema(
       [generateCollectedEntity(), generateCollectedEntity()],

packages/integration-sdk-testing/src/jest.ts

@@ -63,6 +63,49 @@ function collectSchemasFromRef(
   return schemas;
 }
 
+function dedupSchemaRequiredPropertySchema(
+  schema: GraphObjectSchema,
+): GraphObjectSchema {
+  if (!schema.required) {
+    return schema;
+  }
+
+  const requiredSet = new Set(schema.required);
+
+  return {
+    ...schema,
+    required: Array.from(requiredSet),
+  };
+}
+
+function dedupSchemaPropertyTypes(
+  schema: GraphObjectSchema,
+): GraphObjectSchema {
+  if (!schema.properties) {
+    return schema;
+  }
+
+  const newProperties: Record<string, any> = {};
+
+  for (const propertyName in schema.properties) {
+    const property = schema.properties[propertyName];
+
+    if (Array.isArray(property.type)) {
+      newProperties[propertyName] = {
+        ...property,
+        type: Array.from(new Set(property.type)),
+      };
+    } else {
+      newProperties[propertyName] = property;
+    }
+  }
+
+  return {
+    ...schema,
+    properties: newProperties,
+  };
+}
+
 function generateEntitySchemaFromDataModelSchemas(
   schemas: GraphObjectSchema[],
 ) {
@@ -105,7 +148,9 @@ function generateEntitySchemaFromDataModelSchemas(
     newSchemas.push(schema);
   }
 
-  return deepmerge.all(newSchemas);
+  return dedupSchemaRequiredPropertySchema(
+    dedupSchemaPropertyTypes(deepmerge.all(newSchemas)),
+  );
 }
 
 function graphObjectClassToSchemaRef(_class: string) {
@@ -161,20 +206,20 @@ export function toMatchGraphObjectSchema<T extends Entity>(
     }
   }
 
-  const newEntitySchema = generateEntitySchemaFromDataModelSchemas([
-    // Merging should have the highest-level schemas at the end of the array
-    // so that they can override the parent classes
-    ...schemas.reverse(),
-    {
-      ...schema,
-      properties: {
-        ...schema.properties,
-        _class: {
-          const: _class,
-        },
-      },
-    },
-  ]);
+  const newEntitySchema: GraphObjectSchema = generateEntitySchemaFromDataModelSchemas(
+    [
+      // Merging should have the highest-level schemas at the end of the array
+      // so that they can override the parent classes
+      ...schemas.reverse(),
+      schema,
+    ],
+  );
+
+  if (newEntitySchema.properties) {
+    newEntitySchema.properties._class = {
+      const: _class,
+    };
+  }
 
   received = Array.isArray(received) ? received : [received];
 

packages/integration-sdk/CHANGELOG.md

@@ -9,6 +9,14 @@ and this project adheres to
 
 ## Unreleased
 
+## 2.11.1 - 2020-08-21
+
+### Fixed
+
+- [#293](https://github.com/JupiterOne/sdk/issues/293)
+  `toMatchGraphObjectSchema` generates an invalid JSON schema when using
+  `_class` with duplicate `required` or `types` properties.
+
 ## 2.11.0 - 2020-08-20
 
 ### Changed