[8.10] [Fleet] Add secrets package API integration test (#164583) (#1…

…64666)

# Backport

This will backport the following commits from `main` to `8.10`:
- [[Fleet] Add secrets package API integration test
(#164583)](#164583)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Jill
Guyonnet","email":"jill.guyonnet@elastic.co"},"sourceCommit":{"committedDate":"2023-08-24T07:37:34Z","message":"[Fleet]
Add secrets package API integration test (#164583)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/162045\r\n\r\nThis PR adds an
API integration test for the following scenario:\r\n- Given an
integration with some non secret (plain text) vars that\r\nbecome secret
in a newer version;\r\n- When Fleet has an agent policy with this
integration and upgrades from\r\nthe old to the newer version;\r\n- Then
the vars that have become secrets should correctly be stored
as\r\nsecret values.\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: Julia Bardi
<90178898+juliaElastic@users.noreply.github.com>","sha":"766ff8fa614d6b62b750c0eef9c1d129b2187e4f","branchLabelMapping":{"^v8.11.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Fleet","v8.10.0","v8.11.0"],"number":164583,"url":"https://github.com/elastic/kibana/pull/164583","mergeCommit":{"message":"[Fleet]
Add secrets package API integration test (#164583)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/162045\r\n\r\nThis PR adds an
API integration test for the following scenario:\r\n- Given an
integration with some non secret (plain text) vars that\r\nbecome secret
in a newer version;\r\n- When Fleet has an agent policy with this
integration and upgrades from\r\nthe old to the newer version;\r\n- Then
the vars that have become secrets should correctly be stored
as\r\nsecret values.\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: Julia Bardi
<90178898+juliaElastic@users.noreply.github.com>","sha":"766ff8fa614d6b62b750c0eef9c1d129b2187e4f"}},"sourceBranch":"main","suggestedTargetBranches":["8.10"],"targetPullRequestStates":[{"branch":"8.10","label":"v8.10.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.11.0","labelRegex":"^v8.11.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/164583","number":164583,"mergeCommit":{"message":"[Fleet]
Add secrets package API integration test (#164583)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/162045\r\n\r\nThis PR adds an
API integration test for the following scenario:\r\n- Given an
integration with some non secret (plain text) vars that\r\nbecome secret
in a newer version;\r\n- When Fleet has an agent policy with this
integration and upgrades from\r\nthe old to the newer version;\r\n- Then
the vars that have become secrets should correctly be stored
as\r\nsecret values.\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: Julia Bardi
<90178898+juliaElastic@users.noreply.github.com>","sha":"766ff8fa614d6b62b750c0eef9c1d129b2187e4f"}}]}]
BACKPORT-->

Co-authored-by: Jill Guyonnet <jill.guyonnet@elastic.co>

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.0.0/data_stream/log/agent/input/input.yml.hbs

@@ -1,2 +1,4 @@
 package_var_secret: {{package_var_secret}}
+package_var_non_secret: {{package_var_non_secret}}
 input_var_secret: {{input_var_secret}}
+input_var_non_secret: {{input_var_non_secret}}

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.0.0/data_stream/log/agent/stream/stream.yml.hbs

@@ -1,4 +1,7 @@
 config.version: "2"
 package_var_secret: {{package_var_secret}}
+package_var_non_secret: {{package_var_non_secret}}
 input_var_secret: {{input_var_secret}}
+input_var_non_secret: {{input_var_non_secret}}
 stream_var_secret: {{stream_var_secret}}
+stream_var_non_secret: {{stream_var_non_secret}}

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.0.0/data_stream/log/manifest.yml

@@ -10,3 +10,8 @@ streams:
         multi: false
         show_user: true
         secret: true
+      - name: stream_var_non_secret
+        type: text
+        title: Stream Var Non Secret
+        multi: false
+        show_user: true

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.0.0/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: secrets
 title: Package with secrets
-description: This integration package has 3 secrets.
+description: This integration package has 3 secret and 3 non secret vars.
 version: 1.0.0
 categories: []
 # Options are experimental, beta, ga
@@ -32,6 +32,12 @@ vars:
     required: true
     show_user: true
     secret: true
+  - name: package_var_non_secret
+    type: text
+    title: Package Var Non Secret
+    multi: false
+    required: true
+    show_user: true
 policy_templates:
   - name: secrets
     title: This 
@@ -48,4 +54,9 @@ policy_templates:
             title: Input Var Secret
             multi: false
             show_user: true
-            secret: true
+            secret: true
+          - name: input_var_non_secret
+            type: text
+            title: Input Var Non Secret
+            multi: false
+            show_user: true

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.1.0/data_stream/log/agent/input/input.yml.hbs

@@ -0,0 +1,4 @@
+package_var_secret: {{package_var_secret}}
+package_var_non_secret: {{package_var_non_secret}}
+input_var_secret: {{input_var_secret}}
+input_var_non_secret: {{input_var_non_secret}}

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.1.0/data_stream/log/agent/stream/stream.yml.hbs

@@ -0,0 +1,7 @@
+config.version: "2"
+package_var_secret: {{package_var_secret}}
+package_var_non_secret: {{package_var_non_secret}}
+input_var_secret: {{input_var_secret}}
+input_var_non_secret: {{input_var_non_secret}}
+stream_var_secret: {{stream_var_secret}}
+stream_var_non_secret: {{stream_var_non_secret}}

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.1.0/data_stream/log/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.1.0/data_stream/log/manifest.yml

@@ -0,0 +1,18 @@
+title: Test stream
+type: logs
+streams:
+  - input: test_input
+    title: test input
+    vars:
+      - name: stream_var_secret
+        type: text
+        title: Stream Var Secret
+        multi: false
+        show_user: true
+        secret: true
+      - name: stream_var_non_secret
+        type: text
+        title: Stream Var Non Secret
+        multi: false
+        show_user: true
+        secret: true

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.1.0/docs/README.md

@@ -0,0 +1,3 @@
+# secrets
+
+This package has secrets

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.1.0/manifest.yml

@@ -0,0 +1,64 @@
+format_version: 1.0.0
+name: secrets
+title: Package with secrets
+description: This integration package has 3 secret and 3 non secret vars.
+version: 1.1.0
+categories: []
+# Options are experimental, beta, ga
+release: beta
+# The package type. The options for now are [integration, solution], more type might be added in the future.
+# The default type is integration and will be set if empty.
+type: integration
+license: basic
+owner:
+  github: elastic/fleet
+
+requirement:
+  elasticsearch:
+    versions: ">7.7.0"
+  kibana:
+    versions: ">7.7.0"
+
+icons:
+  - src: "/img/logo.svg"
+    size: "16x16"
+    type: "image/svg+xml"
+
+vars:
+  - name: package_var_secret
+    type: password
+    title: Package Var Secret
+    multi: false
+    required: true
+    show_user: true
+    secret: true
+  - name: package_var_non_secret
+    type: text
+    title: Package Var Non Secret
+    multi: false
+    required: true
+    show_user: true
+    secret: true
+policy_templates:
+  - name: secrets
+    title: This 
+    description: Test Package for Upgrading Package Policies
+    inputs:
+      - type: test_input
+        title: Test Input
+        description: Test Input
+        enabled: true
+        template_path: input.yml.hbs
+        vars:
+          - name: input_var_secret
+            type: text
+            title: Input Var Secret
+            multi: false
+            show_user: true
+            secret: true
+          - name: input_var_non_secret
+            type: text
+            title: Input Var Non Secret
+            multi: false
+            show_user: true
+            secret: true

x-pack/test/fleet_api_integration/apis/policy_secrets.ts

@@ -106,19 +106,22 @@ export default function (providerContext: FtrProviderContext) {
               enabled: true,
               vars: {
                 input_var_secret: 'input_secret_val',
+                input_var_non_secret: 'input_non_secret_val',
               },
               streams: {
                 'secrets.log': {
                   enabled: true,
                   vars: {
                     stream_var_secret: 'stream_secret_val',
+                    stream_var_non_secret: 'stream_non_secret_val',
                   },
                 },
               },
             },
           },
           vars: {
             package_var_secret: 'package_secret_val',
+            package_var_non_secret: 'package_non_secret_val',
           },
           package: {
             name: 'secrets',
@@ -128,6 +131,12 @@ export default function (providerContext: FtrProviderContext) {
         .expect(200);
     };
 
+    async function createPolicyWSecretVar() {
+      const { body: createResBody } = await createPolicyWithSecrets();
+      const createdPolicy = createResBody.item;
+      return createdPolicy;
+    }
+
     const createFleetServerAgent = async (
       agentPolicyId: string,
       hostname: string,
@@ -338,19 +347,22 @@ export default function (providerContext: FtrProviderContext) {
               enabled: true,
               vars: {
                 input_var_secret: 'input_secret_val',
+                input_var_non_secret: 'input_non_secret_val',
               },
               streams: {
                 'secrets.log': {
                   enabled: true,
                   vars: {
                     stream_var_secret: 'stream_secret_val',
+                    stream_var_non_secret: 'stream_non_secret_val',
                   },
                 },
               },
             },
           },
           vars: {
             package_var_secret: 'package_secret_val',
+            package_var_non_secret: 'package_non_secret_val',
           },
           package: {
             name: 'secrets',
@@ -376,18 +388,23 @@ export default function (providerContext: FtrProviderContext) {
         ])
       ).to.eql(true);
       expectedCompiledStream = {
-        'config.version': 2,
+        'config.version': '2',
         package_var_secret: secretVar(packageVarId),
+        package_var_non_secret: 'package_non_secret_val',
         input_var_secret: secretVar(inputVarId),
+        input_var_non_secret: 'input_non_secret_val',
         stream_var_secret: secretVar(streamVarId),
+        stream_var_non_secret: 'stream_non_secret_val',
       };
       expect(createdPackagePolicy.inputs[0].streams[0].compiled_stream).to.eql(
         expectedCompiledStream
       );
 
       expectedCompiledInput = {
         package_var_secret: secretVar(packageVarId),
+        package_var_non_secret: 'package_non_secret_val',
         input_var_secret: secretVar(inputVarId),
+        input_var_non_secret: 'input_non_secret_val',
       };
 
       expect(createdPackagePolicy.inputs[0].compiled_input).to.eql(expectedCompiledInput);
@@ -468,12 +485,17 @@ export default function (providerContext: FtrProviderContext) {
       expect(updatedPackagePolicy.inputs[0].streams[0].compiled_stream).to.eql({
         'config.version': 2,
         package_var_secret: secretVar(updatedPackageVarId),
+        package_var_non_secret: 'package_non_secret_val',
         input_var_secret: secretVar(inputVarId),
+        input_var_non_secret: 'input_non_secret_val',
         stream_var_secret: secretVar(streamVarId),
+        stream_var_non_secret: 'stream_non_secret_val',
       });
       expect(updatedPackagePolicy.inputs[0].compiled_input).to.eql({
         package_var_secret: secretVar(updatedPackageVarId),
+        package_var_non_secret: 'package_non_secret_val',
         input_var_secret: secretVar(inputVarId),
+        input_var_non_secret: 'input_non_secret_val',
       });
       expect(updatedPackagePolicy.vars.package_var_secret.value.isSecretRef).to.eql(true);
       expect(updatedPackagePolicy.vars.package_var_secret.value.id).eql(updatedPackageVarId);
@@ -594,18 +616,10 @@ export default function (providerContext: FtrProviderContext) {
       expect(createdPolicy.vars.package_var_secret.value).eql('package_secret_val');
     });
 
-    async function createPolicyWSecretVar() {
-      const { body: createResBody } = await createPolicyWithSecrets();
-      const createdPolicy = createResBody.item;
-      return createdPolicy;
-    }
-
     it('should not store secrets if there are no fleet servers', async () => {
       await clearAgents();
 
-      const { body: createResBody } = await createPolicyWithSecrets();
-
-      const createdPolicy = createResBody.item;
+      const createdPolicy = await createPolicyWSecretVar();
 
       // secret should be in plain text i.e not a secret refrerence
       expect(createdPolicy.vars.package_var_secret.value).eql('package_secret_val');
@@ -645,5 +659,76 @@ export default function (providerContext: FtrProviderContext) {
 
       expect(createdPolicy.vars.package_var_secret.value.isSecretRef).eql(true);
     });
+
+    it('should store new secrets after package upgrade', async () => {
+      const createdPolicy = await createPolicyWSecretVar();
+
+      // Install newer version of secrets package
+      await supertest
+        .post('/api/fleet/epm/packages/secrets/1.1.0')
+        .set('kbn-xsrf', 'xxxx')
+        .send({ force: true })
+        .expect(200);
+
+      // Upgrade package policy
+      await supertest
+        .post(`/api/fleet/package_policies/upgrade`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          packagePolicyIds: [createdPolicy.id],
+        })
+        .expect(200);
+
+      // Fetch policy again
+      const res = await supertest.get(`/api/fleet/package_policies/${createdPolicy.id}`);
+      const upgradedPolicy = res.body.item;
+
+      const packageSecretVarId = upgradedPolicy.vars.package_var_secret.value.id;
+      const packageNonSecretVarId = upgradedPolicy.vars.package_var_non_secret.value.id;
+      const inputSecretVarId = upgradedPolicy.inputs[0].vars.input_var_secret.value.id;
+      const inputNonSecretVarId = upgradedPolicy.inputs[0].vars.input_var_non_secret.value.id;
+      const streamSecretVarId = upgradedPolicy.inputs[0].streams[0].vars.stream_var_secret.value.id;
+      const streamNonSecretVarId =
+        upgradedPolicy.inputs[0].streams[0].vars.stream_var_non_secret.value.id;
+
+      expect(
+        arrayIdsEqual(upgradedPolicy.secret_references, [
+          { id: packageSecretVarId },
+          { id: packageNonSecretVarId },
+          { id: inputSecretVarId },
+          { id: inputNonSecretVarId },
+          { id: streamSecretVarId },
+          { id: streamNonSecretVarId },
+        ])
+      ).to.eql(true);
+
+      expect(upgradedPolicy.inputs[0].compiled_input).to.eql({
+        package_var_secret: secretVar(packageSecretVarId),
+        package_var_non_secret: secretVar(packageNonSecretVarId),
+        input_var_secret: secretVar(inputSecretVarId),
+        input_var_non_secret: secretVar(inputNonSecretVarId),
+      });
+
+      expect(upgradedPolicy.inputs[0].streams[0].compiled_stream).to.eql({
+        'config.version': '2',
+        package_var_secret: secretVar(packageSecretVarId),
+        package_var_non_secret: secretVar(packageNonSecretVarId),
+        input_var_secret: secretVar(inputSecretVarId),
+        input_var_non_secret: secretVar(inputNonSecretVarId),
+        stream_var_secret: secretVar(streamSecretVarId),
+        stream_var_non_secret: secretVar(streamNonSecretVarId),
+      });
+
+      expect(upgradedPolicy.vars.package_var_secret.value.isSecretRef).to.eql(true);
+      expect(upgradedPolicy.vars.package_var_non_secret.value.isSecretRef).to.eql(true);
+      expect(upgradedPolicy.inputs[0].vars.input_var_secret.value.isSecretRef).to.eql(true);
+      expect(upgradedPolicy.inputs[0].vars.input_var_non_secret.value.isSecretRef).to.eql(true);
+      expect(upgradedPolicy.inputs[0].streams[0].vars.stream_var_secret.value.isSecretRef).to.eql(
+        true
+      );
+      expect(
+        upgradedPolicy.inputs[0].streams[0].vars.stream_var_non_secret.value.isSecretRef
+      ).to.eql(true);
+    });
   });
 }