Updating package registry snapshot distribution version (#89776)

x-pack/plugins/fleet/server/errors/handlers.ts

@@ -44,7 +44,9 @@ interface LegacyESClientError {
   path?: string;
   query?: string | undefined;
   body?: {
-    error: object;
+    error: {
+      type: string;
+    };
     status: number;
   };
   statusCode?: number;

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts

@@ -19,6 +19,7 @@ import { getInstallation } from '../../packages';
 import { deleteTransforms, deleteTransformRefs } from './remove';
 import { getAsset } from './common';
 import { appContextService } from '../../../app_context';
+import { isLegacyESClientError } from '../../../../errors';
 
 interface TransformInstallation {
   installationName: string;
@@ -116,17 +117,27 @@ async function handleTransformInstall({
   callCluster: CallESAsCurrentUser;
   transform: TransformInstallation;
 }): Promise<EsAssetReference> {
-  // defer validation on put if the source index is not available
-  await callCluster('transport.request', {
-    method: 'PUT',
-    path: `/_transform/${transform.installationName}`,
-    query: 'defer_validation=true',
-    body: transform.content,
-  });
-
+  try {
+    // defer validation on put if the source index is not available
+    await callCluster('transport.request', {
+      method: 'PUT',
+      path: `/_transform/${transform.installationName}`,
+      query: 'defer_validation=true',
+      body: transform.content,
+    });
+  } catch (err) {
+    // swallow the error if the transform already exists.
+    const isAlreadyExistError =
+      isLegacyESClientError(err) && err?.body?.error?.type === 'resource_already_exists_exception';
+    if (!isAlreadyExistError) {
+      throw err;
+    }
+  }
   await callCluster('transport.request', {
     method: 'POST',
     path: `/_transform/${transform.installationName}/_start`,
+    // Ignore error if the transform is already started
+    ignore: [409],
   });
 
   return { id: transform.installationName, type: ElasticsearchAssetType.transform };

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts

@@ -17,6 +17,7 @@ jest.mock('./common', () => {
   };
 });
 
+import { errors as LegacyESErrors } from 'elasticsearch';
 import { installTransform } from './install';
 import { ILegacyScopedClusterClient, SavedObject, SavedObjectsClientContract } from 'kibana/server';
 import { ElasticsearchAssetType, Installation, RegistryPackage } from '../../../../types';
@@ -217,13 +218,15 @@ describe('test transform install', () => {
         {
           method: 'POST',
           path: '/_transform/endpoint.metadata-default-0.16.0-dev.0/_start',
+          ignore: [409],
         },
       ],
       [
         'transport.request',
         {
           method: 'POST',
           path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start',
+          ignore: [409],
         },
       ],
     ]);
@@ -345,6 +348,7 @@ describe('test transform install', () => {
         {
           method: 'POST',
           path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start',
+          ignore: [409],
         },
       ],
     ]);
@@ -492,4 +496,106 @@ describe('test transform install', () => {
       ],
     ]);
   });
+
+  test('ignore already exists error if saved object and ES transforms are out of sync', async () => {
+    const previousInstallation: Installation = ({
+      installed_es: [],
+    } as unknown) as Installation;
+
+    const currentInstallation: Installation = ({
+      installed_es: [
+        {
+          id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0',
+          type: ElasticsearchAssetType.transform,
+        },
+      ],
+    } as unknown) as Installation;
+    (getAsset as jest.MockedFunction<typeof getAsset>).mockReturnValueOnce(
+      Buffer.from('{"content": "data"}', 'utf8')
+    );
+    (getInstallation as jest.MockedFunction<typeof getInstallation>)
+      .mockReturnValueOnce(Promise.resolve(previousInstallation))
+      .mockReturnValueOnce(Promise.resolve(currentInstallation));
+
+    (getInstallationObject as jest.MockedFunction<
+      typeof getInstallationObject
+    >).mockReturnValueOnce(
+      Promise.resolve(({
+        attributes: { installed_es: [] },
+      } as unknown) as SavedObject<Installation>)
+    );
+    legacyScopedClusterClient.callAsCurrentUser = jest.fn();
+
+    legacyScopedClusterClient.callAsCurrentUser.mockImplementation(
+      async (endpoint, clientParams, options) => {
+        if (
+          endpoint === 'transport.request' &&
+          clientParams?.method === 'PUT' &&
+          clientParams?.path === '/_transform/endpoint.metadata_current-default-0.16.0-dev.0'
+        ) {
+          const err: LegacyESErrors._Abstract & { body?: any } = new LegacyESErrors.BadRequest();
+          err.body = {
+            error: { type: 'resource_already_exists_exception' },
+          };
+          throw err;
+        }
+      }
+    );
+    await installTransform(
+      ({
+        name: 'endpoint',
+        version: '0.16.0-dev.0',
+        data_streams: [
+          {
+            type: 'metrics',
+            dataset: 'endpoint.metadata_current',
+            title: 'Endpoint Metadata',
+            release: 'experimental',
+            package: 'endpoint',
+            ingest_pipeline: 'default',
+            elasticsearch: {
+              'index_template.mappings': {
+                dynamic: false,
+              },
+            },
+            path: 'metadata_current',
+          },
+        ],
+      } as unknown) as RegistryPackage,
+      ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
+      legacyScopedClusterClient.callAsCurrentUser,
+      savedObjectsClient
+    );
+
+    expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
+      [
+        'transport.request',
+        {
+          method: 'PUT',
+          path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0',
+          query: 'defer_validation=true',
+          body: '{"content": "data"}',
+        },
+      ],
+      [
+        'transport.request',
+        {
+          method: 'POST',
+          path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start',
+          ignore: [409],
+        },
+      ],
+    ]);
+    expect(savedObjectsClient.update.mock.calls).toEqual([
+      [
+        'epm-packages',
+        'endpoint',
+        {
+          installed_es: [
+            { id: 'endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' },
+          ],
+        },
+      ],
+    ]);
+  });
 });

x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_with_agents_setup.ts

@@ -104,7 +104,7 @@ export default function (providerContext: FtrProviderContext) {
         const agentPolicy = action.data.policy;
         expect(agentPolicy.id).to.be(policyId);
         // should have system inputs
-        expect(agentPolicy.inputs).length(2);
+        expect(agentPolicy.inputs).length(3);
         // should have default output
         expect(agentPolicy.outputs.default).not.empty();
       });

x-pack/test/fleet_api_integration/config.ts

@@ -11,9 +11,11 @@ import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
 import { defineDockerServersConfig } from '@kbn/test';
 
 // Docker image to use for Fleet API integration tests.
-// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit
+// This hash comes from the latest successful build of the Snapshot Distribution of the Package Registry, for
+// example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1.
+// It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry.
 export const dockerImage =
-  'docker.elastic.co/package-registry/distribution:fb58d670bafbac7e9e28baf6d6f99ba65cead548';
+  'docker.elastic.co/package-registry/distribution:5314869e2f6bc01d37b8652f7bda89248950b3a4';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));