fix(integ-runner): asset hashes of nested stack templates are not ign…

…ored
aws · Nov 15, 2022 · f8bb688 · f8bb688
1 parent e31a2fa
commit f8bb688
Show file tree

Hide file tree

Showing 25 changed files with 935 additions and 8 deletions.
diff --git a/packages/@aws-cdk/integ-runner/lib/runner/snapshot-test-runner.ts b/packages/@aws-cdk/integ-runner/lib/runner/snapshot-test-runner.ts
@@ -198,13 +198,16 @@ export class IntegSnapshotRunner extends IntegRunner {
           });
           continue;
         } else {
+          const config = {
+            diffAssets: this.actualTestSuite.getOptionsForStack(stackId)?.diffAssets,
+          };
           let actualTemplate = actual[stackId].templates[templateId];
           let expectedTemplate = expected[stackId].templates[templateId];
 
           // if we are not verifying asset hashes then remove the specific
           // asset hashes from the templates so they are not part of the diff
           // comparison
-          if (!this.actualTestSuite.getOptionsForStack(stackId)?.diffAssets) {
+          if (!config.diffAssets) {
             actualTemplate = this.canonicalizeTemplate(actualTemplate, actual[stackId].assets);
             expectedTemplate = this.canonicalizeTemplate(expectedTemplate, expected[stackId].assets);
           }
@@ -250,6 +253,7 @@ export class IntegSnapshotRunner extends IntegRunner {
               message: writable.data,
               stackName: templateId,
               testName: this.testName,
+              config,
             });
           }
         }

diff --git a/packages/@aws-cdk/integ-runner/lib/workers/common.ts b/packages/@aws-cdk/integ-runner/lib/workers/common.ts
@@ -245,6 +245,11 @@ export interface Diagnostic {
    * Additional messages to print
    */
   readonly additionalMessages?: string[];
+
+  /**
+   * Relevant config options that were used for the integ test
+   */
+  readonly config?: Record<string, any>;
 }
 
 export function printSummary(total: number, failed: number): void {

diff --git a/packages/@aws-cdk/integ-runner/test/runner/snapshot-test-runner.test.ts b/packages/@aws-cdk/integ-runner/test/runner/snapshot-test-runner.test.ts
@@ -65,6 +65,7 @@ describe('IntegTest runSnapshotTests', () => {
         reason: DiagnosticReason.SNAPSHOT_FAILED,
         testName: 'xxxxx.test-with-snapshot',
         message: expect.stringContaining('foobar'),
+        config: { diffAssets: true },
       }),
     ]));
     expect(results.destructiveChanges).not.toEqual([{
@@ -96,6 +97,13 @@ describe('IntegTest runSnapshotTests', () => {
       reason: DiagnosticReason.SNAPSHOT_FAILED,
       testName: 'xxxxx.test-with-new-assets',
       message: expect.stringContaining('S3Key'),
+      config: { diffAssets: true },
+    }),
+    expect.objectContaining({
+      reason: DiagnosticReason.SNAPSHOT_FAILED,
+      testName: 'xxxxx.test-with-new-assets',
+      message: expect.stringContaining('TemplateURL'),
+      config: { diffAssets: true },
     })]));
   });
 
@@ -110,6 +118,21 @@ describe('IntegTest runSnapshotTests', () => {
         testName: 'xxxxx.test-with-nested-stack',
         stackName: expect.stringContaining('teststacknested'),
         message: expect.stringContaining('AWS::SNS::Topic'),
+        config: { diffAssets: false },
+      })]));
+    });
+
+    test('it will diff assets for nested stacks', () => {
+      // WHEN
+      const results = cdkMock.snapshotTest('xxxxx.test-with-nested-stack.js', 'xxxxx.test-with-asset-in-nested-stack.js.snapshot');
+
+      // THEN
+      expect(results.diagnostics).toEqual(expect.arrayContaining([expect.objectContaining({
+        reason: DiagnosticReason.SNAPSHOT_FAILED,
+        testName: 'xxxxx.test-with-nested-stack',
+        stackName: expect.stringContaining('teststacknested'),
+        message: expect.stringContaining('S3Key'),
+        config: { diffAssets: true },
       })]));
     });
   });
@@ -124,6 +147,7 @@ describe('IntegTest runSnapshotTests', () => {
         reason: DiagnosticReason.SNAPSHOT_FAILED,
         testName: 'xxxxx.test-with-snapshot-assets',
         message: expect.stringContaining('Parameters'),
+        config: { diffAssets: true },
       })]));
     });
 

diff --git a/...est-with-new-assets-diff.js.snapshot/SomeNestedStackResourceA7AEBA6B.nested.template.json b/...est-with-new-assets-diff.js.snapshot/SomeNestedStackResourceA7AEBA6B.nested.template.json
@@ -0,0 +1 @@
+{}
diff --git a/...est-data/cdk-integ.out.xxxxx.test-with-new-assets-diff.js.snapshot/test-stack.assets.json b/...est-data/cdk-integ.out.xxxxx.test-with-new-assets-diff.js.snapshot/test-stack.assets.json
@@ -26,6 +26,19 @@
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }
+    },
+    "DIFFERENT_NESTED_STACK_HASH": {
+      "source": {
+        "path": "SomeNestedStackResourceA7AEBA6B.nested.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "DIFFERENT_NESTED_STACK_HASH.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
     }
   },
   "dockerImages": {}

diff --git a/...t-data/cdk-integ.out.xxxxx.test-with-new-assets-diff.js.snapshot/test-stack.template.json b/...t-data/cdk-integ.out.xxxxx.test-with-new-assets-diff.js.snapshot/test-stack.template.json
@@ -38,17 +38,18 @@
           "S3Key": "be270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92825.zip"
         },
         "Role": {
-          "Fn::GetAtt": [
-            "MyFunction1ServiceRole9852B06B",
-            "Arn"
-          ]
+          "Fn::GetAtt": ["MyFunction1ServiceRole9852B06B", "Arn"]
         },
         "Handler": "index.handler",
         "Runtime": "nodejs14.x"
       },
-      "DependsOn": [
-        "MyFunction1ServiceRole9852B06B"
-      ]
+      "DependsOn": ["MyFunction1ServiceRole9852B06B"]
+    },
+    "SomeNestedStackResourceA7AEBA6B": {
+      "Type": "AWS::CloudFormation::Stack",
+      "Properties": {
+        "TemplateURL": "https://s3.unknown-region.amazonaws.com/cdk-hnb659fds-assets-unknown-account-unknown-region/DIFFERENT_NESTED_STACK_HASH.json"
+      }
     }
   }
 }
diff --git a/...xxx.test-with-new-assets.js.snapshot/SomeNestedStackResourceA7AEBA6B.nested.template.json b/...xxx.test-with-new-assets.js.snapshot/SomeNestedStackResourceA7AEBA6B.nested.template.json
@@ -0,0 +1 @@
+{}
diff --git a/...est/test-data/cdk-integ.out.xxxxx.test-with-new-assets.js.snapshot/test-stack.assets.json b/...est/test-data/cdk-integ.out.xxxxx.test-with-new-assets.js.snapshot/test-stack.assets.json
@@ -26,6 +26,19 @@
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }
+    },
+    "6f980d09d47a00a7d5001feeb3994d1909bc5294e309d69bd4fbc815622f6fa9": {
+      "source": {
+        "path": "SomeNestedStackResourceA7AEBA6B.nested.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "6f980d09d47a00a7d5001feeb3994d1909bc5294e309d69bd4fbc815622f6fa9.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
     }
   },
   "dockerImages": {}

diff --git a/...t/test-data/cdk-integ.out.xxxxx.test-with-new-assets.js.snapshot/test-stack.template.json b/...t/test-data/cdk-integ.out.xxxxx.test-with-new-assets.js.snapshot/test-stack.template.json
@@ -49,6 +49,12 @@
       "DependsOn": [
         "MyFunction1ServiceRole9852B06B"
       ]
+    },
+    "SomeNestedStackResourceA7AEBA6B": {
+      "Type": "AWS::CloudFormation::Stack",
+      "Properties": {
+        "TemplateURL": "https://s3.unknown-region.amazonaws.com/cdk-hnb659fds-assets-unknown-account-unknown-region/6f980d09d47a00a7d5001feeb3994d1909bc5294e309d69bd4fbc815622f6fa9.json"
+      }
     }
   }
 }
diff --git a/...n-nested-stack.js.snapshot/TestWithNestedStackDefaultTestDeployAssert9F80D3B6.assets.json b/...n-nested-stack.js.snapshot/TestWithNestedStackDefaultTestDeployAssert9F80D3B6.assets.json
@@ -0,0 +1,19 @@
+{
+  "version": "21.0.0",
+  "files": {
+    "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
+      "source": {
+        "path": "TestWithNestedStackDefaultTestDeployAssert9F80D3B6.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    }
+  },
+  "dockerImages": {}
+}
diff --git a/...nested-stack.js.snapshot/TestWithNestedStackDefaultTestDeployAssert9F80D3B6.template.json b/...nested-stack.js.snapshot/TestWithNestedStackDefaultTestDeployAssert9F80D3B6.template.json
@@ -0,0 +1,36 @@
+{
+ "Parameters": {
+  "BootstrapVersion": {
+   "Type": "AWS::SSM::Parameter::Value<String>",
+   "Default": "/cdk-bootstrap/hnb659fds/version",
+   "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
+  }
+ },
+ "Rules": {
+  "CheckBootstrapVersion": {
+   "Assertions": [
+    {
+     "Assert": {
+      "Fn::Not": [
+       {
+        "Fn::Contains": [
+         [
+          "1",
+          "2",
+          "3",
+          "4",
+          "5"
+         ],
+         {
+          "Ref": "BootstrapVersion"
+         }
+        ]
+       }
+      ]
+     },
+     "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
+    }
+   ]
+  }
+ }
+}
diff --git a/...cdk/integ-runner/test/test-data/xxxxx.test-with-asset-in-nested-stack.js.snapshot/cdk.out b/...cdk/integ-runner/test/test-data/xxxxx.test-with-asset-in-nested-stack.js.snapshot/cdk.out
@@ -0,0 +1 @@
+{"version":"21.0.0"}
diff --git a/.../integ-runner/test/test-data/xxxxx.test-with-asset-in-nested-stack.js.snapshot/integ.json b/.../integ-runner/test/test-data/xxxxx.test-with-asset-in-nested-stack.js.snapshot/integ.json
@@ -0,0 +1,11 @@
+{
+  "version": "21.0.0",
+  "testCases": {
+    "TestWithNestedStack/DefaultTest": {
+      "stacks": ["test-stack"],
+      "diffAssets": true,
+      "assertionStack": "TestWithNestedStack/DefaultTest/DeployAssert",
+      "assertionStackName": "TestWithNestedStackDefaultTestDeployAssert9F80D3B6"
+    }
+  }
+}
diff --git a/...teg-runner/test/test-data/xxxxx.test-with-asset-in-nested-stack.js.snapshot/manifest.json b/...teg-runner/test/test-data/xxxxx.test-with-asset-in-nested-stack.js.snapshot/manifest.json
@@ -0,0 +1,141 @@
+{
+  "version": "21.0.0",
+  "artifacts": {
+    "test-stack.assets": {
+      "type": "cdk:asset-manifest",
+      "properties": {
+        "file": "test-stack.assets.json",
+        "requiresBootstrapStackVersion": 6,
+        "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version"
+      }
+    },
+    "test-stack": {
+      "type": "aws:cloudformation:stack",
+      "environment": "aws://unknown-account/unknown-region",
+      "properties": {
+        "templateFile": "test-stack.template.json",
+        "validateOnSynth": false,
+        "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
+        "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
+        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e329bc2347c70a89ff078c2c93444d2b59a21f04cbf91afff7e9e5cab307b381.json",
+        "requiresBootstrapStackVersion": 6,
+        "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
+        "additionalDependencies": [
+          "test-stack.assets"
+        ],
+        "lookupRole": {
+          "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}",
+          "requiresBootstrapStackVersion": 8,
+          "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version"
+        }
+      },
+      "dependencies": [
+        "test-stack.assets"
+      ],
+      "metadata": {
+        "/test-stack/MyFunction/ServiceRole/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "MyFunctionServiceRole3C357FF2"
+          }
+        ],
+        "/test-stack/MyFunction/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "MyFunction3BAA72D1"
+          }
+        ],
+        "/test-stack/nested/MyTopic/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "MyTopic86869434"
+          }
+        ],
+        "/test-stack/nested/MyNestedFunction/ServiceRole/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "MyNestedFunctionServiceRole56B518B8"
+          }
+        ],
+        "/test-stack/nested/MyNestedFunction/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "MyNestedFunctionA0269B76"
+          }
+        ],
+        "/test-stack/nested.NestedStack/nested.NestedStackResource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "nestedNestedStacknestedNestedStackResource3DD143BF"
+          }
+        ],
+        "/test-stack/BootstrapVersion": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "BootstrapVersion"
+          }
+        ],
+        "/test-stack/CheckBootstrapVersion": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "CheckBootstrapVersion"
+          }
+        ]
+      },
+      "displayName": "test-stack"
+    },
+    "TestWithNestedStackDefaultTestDeployAssert9F80D3B6.assets": {
+      "type": "cdk:asset-manifest",
+      "properties": {
+        "file": "TestWithNestedStackDefaultTestDeployAssert9F80D3B6.assets.json",
+        "requiresBootstrapStackVersion": 6,
+        "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version"
+      }
+    },
+    "TestWithNestedStackDefaultTestDeployAssert9F80D3B6": {
+      "type": "aws:cloudformation:stack",
+      "environment": "aws://unknown-account/unknown-region",
+      "properties": {
+        "templateFile": "TestWithNestedStackDefaultTestDeployAssert9F80D3B6.template.json",
+        "validateOnSynth": false,
+        "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
+        "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
+        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
+        "requiresBootstrapStackVersion": 6,
+        "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
+        "additionalDependencies": [
+          "TestWithNestedStackDefaultTestDeployAssert9F80D3B6.assets"
+        ],
+        "lookupRole": {
+          "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}",
+          "requiresBootstrapStackVersion": 8,
+          "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version"
+        }
+      },
+      "dependencies": [
+        "TestWithNestedStackDefaultTestDeployAssert9F80D3B6.assets"
+      ],
+      "metadata": {
+        "/TestWithNestedStack/DefaultTest/DeployAssert/BootstrapVersion": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "BootstrapVersion"
+          }
+        ],
+        "/TestWithNestedStack/DefaultTest/DeployAssert/CheckBootstrapVersion": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "CheckBootstrapVersion"
+          }
+        ]
+      },
+      "displayName": "TestWithNestedStack/DefaultTest/DeployAssert"
+    },
+    "Tree": {
+      "type": "cdk:tree",
+      "properties": {
+        "file": "tree.json"
+      }
+    }
+  }
+}