From f3a9c763dfa08972b8106fac248456b4b3f20086 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 4 Feb 2021 05:40:18 -0800 Subject: [PATCH] Updating package registry snapshot distribution version (#89776) --- .../plugins/fleet/server/errors/handlers.ts | 4 +- .../epm/elasticsearch/transform/install.ts | 27 +++-- .../elasticsearch/transform/transform.test.ts | 106 ++++++++++++++++++ .../agent_policy_with_agents_setup.ts | 2 +- x-pack/test/fleet_api_integration/config.ts | 6 +- 5 files changed, 133 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/server/errors/handlers.ts b/x-pack/plugins/fleet/server/errors/handlers.ts index 45b79c4a6ebb9b..77db050309a60c 100644 --- a/x-pack/plugins/fleet/server/errors/handlers.ts +++ b/x-pack/plugins/fleet/server/errors/handlers.ts @@ -44,7 +44,9 @@ interface LegacyESClientError { path?: string; query?: string | undefined; body?: { - error: object; + error: { + type: string; + }; status: number; }; statusCode?: number; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index ea934eb563fabd..57e1090f8954b4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -19,6 +19,7 @@ import { getInstallation } from '../../packages'; import { deleteTransforms, deleteTransformRefs } from './remove'; import { getAsset } from './common'; import { appContextService } from '../../../app_context'; +import { isLegacyESClientError } from '../../../../errors'; interface TransformInstallation { installationName: string; @@ -116,17 +117,27 @@ async function handleTransformInstall({ callCluster: CallESAsCurrentUser; transform: TransformInstallation; }): Promise { - // defer validation on put if the source index is not available - await callCluster('transport.request', { - method: 'PUT', - path: `/_transform/${transform.installationName}`, - query: 'defer_validation=true', - body: transform.content, - }); - + try { + // defer validation on put if the source index is not available + await callCluster('transport.request', { + method: 'PUT', + path: `/_transform/${transform.installationName}`, + query: 'defer_validation=true', + body: transform.content, + }); + } catch (err) { + // swallow the error if the transform already exists. + const isAlreadyExistError = + isLegacyESClientError(err) && err?.body?.error?.type === 'resource_already_exists_exception'; + if (!isAlreadyExistError) { + throw err; + } + } await callCluster('transport.request', { method: 'POST', path: `/_transform/${transform.installationName}/_start`, + // Ignore error if the transform is already started + ignore: [409], }); return { id: transform.installationName, type: ElasticsearchAssetType.transform }; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 861692d23d9ac0..bd944391b5f23d 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -17,6 +17,7 @@ jest.mock('./common', () => { }; }); +import { errors as LegacyESErrors } from 'elasticsearch'; import { installTransform } from './install'; import { ILegacyScopedClusterClient, SavedObject, SavedObjectsClientContract } from 'kibana/server'; import { ElasticsearchAssetType, Installation, RegistryPackage } from '../../../../types'; @@ -217,6 +218,7 @@ describe('test transform install', () => { { method: 'POST', path: '/_transform/endpoint.metadata-default-0.16.0-dev.0/_start', + ignore: [409], }, ], [ @@ -224,6 +226,7 @@ describe('test transform install', () => { { method: 'POST', path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', + ignore: [409], }, ], ]); @@ -345,6 +348,7 @@ describe('test transform install', () => { { method: 'POST', path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', + ignore: [409], }, ], ]); @@ -492,4 +496,106 @@ describe('test transform install', () => { ], ]); }); + + test('ignore already exists error if saved object and ES transforms are out of sync', async () => { + const previousInstallation: Installation = ({ + installed_es: [], + } as unknown) as Installation; + + const currentInstallation: Installation = ({ + installed_es: [ + { + id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown) as Installation; + (getAsset as jest.MockedFunction).mockReturnValueOnce( + Buffer.from('{"content": "data"}', 'utf8') + ); + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + (getInstallationObject as jest.MockedFunction< + typeof getInstallationObject + >).mockReturnValueOnce( + Promise.resolve(({ + attributes: { installed_es: [] }, + } as unknown) as SavedObject) + ); + legacyScopedClusterClient.callAsCurrentUser = jest.fn(); + + legacyScopedClusterClient.callAsCurrentUser.mockImplementation( + async (endpoint, clientParams, options) => { + if ( + endpoint === 'transport.request' && + clientParams?.method === 'PUT' && + clientParams?.path === '/_transform/endpoint.metadata_current-default-0.16.0-dev.0' + ) { + const err: LegacyESErrors._Abstract & { body?: any } = new LegacyESErrors.BadRequest(); + err.body = { + error: { type: 'resource_already_exists_exception' }, + }; + throw err; + } + } + ); + await installTransform( + ({ + name: 'endpoint', + version: '0.16.0-dev.0', + data_streams: [ + { + type: 'metrics', + dataset: 'endpoint.metadata_current', + title: 'Endpoint Metadata', + release: 'experimental', + package: 'endpoint', + ingest_pipeline: 'default', + elasticsearch: { + 'index_template.mappings': { + dynamic: false, + }, + }, + path: 'metadata_current', + }, + ], + } as unknown) as RegistryPackage, + ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], + legacyScopedClusterClient.callAsCurrentUser, + savedObjectsClient + ); + + expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + [ + 'transport.request', + { + method: 'PUT', + path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', + query: 'defer_validation=true', + body: '{"content": "data"}', + }, + ], + [ + 'transport.request', + { + method: 'POST', + path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', + ignore: [409], + }, + ], + ]); + expect(savedObjectsClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'endpoint', + { + installed_es: [ + { id: 'endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' }, + ], + }, + ], + ]); + }); }); diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_with_agents_setup.ts b/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_with_agents_setup.ts index b8bd83739fea48..1cc96b59c460d5 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_with_agents_setup.ts +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_with_agents_setup.ts @@ -104,7 +104,7 @@ export default function (providerContext: FtrProviderContext) { const agentPolicy = action.data.policy; expect(agentPolicy.id).to.be(policyId); // should have system inputs - expect(agentPolicy.inputs).length(2); + expect(agentPolicy.inputs).length(3); // should have default output expect(agentPolicy.outputs.default).not.empty(); }); diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts index 596b9319064a5b..444b8c3a687765 100644 --- a/x-pack/test/fleet_api_integration/config.ts +++ b/x-pack/test/fleet_api_integration/config.ts @@ -11,9 +11,11 @@ import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; import { defineDockerServersConfig } from '@kbn/test'; // Docker image to use for Fleet API integration tests. -// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit +// This hash comes from the latest successful build of the Snapshot Distribution of the Package Registry, for +// example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. +// It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution:fb58d670bafbac7e9e28baf6d6f99ba65cead548'; + 'docker.elastic.co/package-registry/distribution:5314869e2f6bc01d37b8652f7bda89248950b3a4'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));