From 208e86e66a7001a1f93f0dd0d937af5cc4deb9db Mon Sep 17 00:00:00 2001 From: John Schulz Date: Thu, 12 Nov 2020 11:05:17 -0500 Subject: [PATCH] [Ingest Manager] Lift up registry/{stream,extract} functions (#83239) ## Summary * Move stream utility functions from `server/services/epm/registry/streams.ts` to `server/services/epm/streams.ts` * They're only used in registry at the moment but aren't specific to registry * Move archive extraction functions from `server/services/epm/registry/extract.ts` to `server/services/epm/archive.ts` * The Registry isn't the only service/code which needs to extract packages. Continue consolidating archive-related code under archive vs registry --- .../server/services/epm/{registry => archive}/extract.ts | 8 ++------ x-pack/plugins/fleet/server/services/epm/archive/index.ts | 8 +++++++- .../fleet/server/services/epm/archive/validation.ts | 3 ++- .../services/epm/elasticsearch/ingest_pipeline/install.ts | 3 +-- .../plugins/fleet/server/services/epm/packages/assets.ts | 6 +++--- .../fleet/server/services/epm/registry/index.test.ts | 5 ++--- .../plugins/fleet/server/services/epm/registry/index.ts | 4 +--- .../fleet/server/services/epm/registry/requests.ts | 2 +- .../fleet/server/services/epm/{registry => }/streams.ts | 0 9 files changed, 19 insertions(+), 20 deletions(-) rename x-pack/plugins/fleet/server/services/epm/{registry => archive}/extract.ts (95%) rename x-pack/plugins/fleet/server/services/epm/{registry => }/streams.ts (100%) diff --git a/x-pack/plugins/fleet/server/services/epm/registry/extract.ts b/x-pack/plugins/fleet/server/services/epm/archive/extract.ts similarity index 95% rename from x-pack/plugins/fleet/server/services/epm/registry/extract.ts rename to x-pack/plugins/fleet/server/services/epm/archive/extract.ts index b79218638ce247..6ac81a25dfc21e 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/extract.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/extract.ts @@ -6,12 +6,8 @@ import tar from 'tar'; import yauzl from 'yauzl'; -import { bufferToStream, streamToBuffer } from './streams'; - -export interface ArchiveEntry { - path: string; - buffer?: Buffer; -} +import { bufferToStream, streamToBuffer } from '../streams'; +import { ArchiveEntry } from './index'; export async function untarBuffer( buffer: Buffer, diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts index 810740d697fcbc..6d1150b3ac8bd5 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts @@ -14,10 +14,16 @@ import { setArchiveFilelist, deleteArchiveFilelist, } from './cache'; -import { ArchiveEntry, getBufferExtractor } from '../registry/extract'; +import { getBufferExtractor } from './extract'; import { parseAndVerifyArchiveEntries } from './validation'; export * from './cache'; +export { untarBuffer, unzipBuffer, getBufferExtractor } from './extract'; + +export interface ArchiveEntry { + path: string; + buffer?: Buffer; +} export async function getArchivePackage({ archiveBuffer, diff --git a/x-pack/plugins/fleet/server/services/epm/archive/validation.ts b/x-pack/plugins/fleet/server/services/epm/archive/validation.ts index d9d451544a9538..992020cb073add 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/validation.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/validation.ts @@ -15,7 +15,8 @@ import { RegistryVarsEntry, } from '../../../../common/types'; import { PackageInvalidArchiveError } from '../../../errors'; -import { ArchiveEntry, pkgToPkgKey } from '../registry'; +import { ArchiveEntry } from './index'; +import { pkgToPkgKey } from '../registry'; const MANIFESTS: Record = {}; const MANIFEST_NAME = 'manifest.yml'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index c5c9e8ac2c01ba..b6988f64843d09 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -11,8 +11,7 @@ import { ElasticsearchAssetType, InstallablePackage, } from '../../../../types'; -import { ArchiveEntry } from '../../registry'; -import { getAsset, getPathParts } from '../../archive'; +import { ArchiveEntry, getAsset, getPathParts } from '../../archive'; import { CallESAsCurrentUser } from '../../../../types'; import { saveInstalledEsRefs } from '../../packages/install'; import { getInstallationObject } from '../../packages'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts index 50d8f2f4d2fb22..80e1cbba6484b6 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts @@ -6,7 +6,7 @@ import { InstallablePackage } from '../../../types'; import * as Registry from '../registry'; -import { getArchiveFilelist, getAsset } from '../archive'; +import { ArchiveEntry, getArchiveFilelist, getAsset } from '../archive'; // paths from RegistryPackage are routes to the assets on EPR // e.g. `/package/nginx/1.2.0/data_stream/access/fields/fields.yml` @@ -51,14 +51,14 @@ export async function getAssetsData( packageInfo: InstallablePackage, filter = (path: string): boolean => true, datasetName?: string -): Promise { +): Promise { // TODO: Needs to be called to fill the cache but should not be required await Registry.ensureCachedArchiveInfo(packageInfo.name, packageInfo.version, 'registry'); // Gather all asset data const assets = getAssets(packageInfo, filter, datasetName); - const entries: Registry.ArchiveEntry[] = assets.map((path) => { + const entries: ArchiveEntry[] = assets.map((path) => { const buffer = getAsset(path); return { path, buffer }; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts index 1208ffdaefe4af..aea28b5d56ab9d 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts @@ -5,9 +5,8 @@ */ import { AssetParts } from '../../../types'; -import { getPathParts } from '../archive'; -import { getBufferExtractor, splitPkgKey } from './index'; -import { untarBuffer, unzipBuffer } from './extract'; +import { getBufferExtractor, getPathParts, untarBuffer, unzipBuffer } from '../archive'; +import { splitPkgKey } from './index'; const testPaths = [ { diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index c35e91bdf580bd..aef1bb75619cdf 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -24,13 +24,11 @@ import { unpackArchiveToCache, } from '../archive'; import { fetchUrl, getResponse, getResponseStream } from './requests'; -import { streamToBuffer } from './streams'; +import { streamToBuffer } from '../streams'; import { getRegistryUrl } from './registry_url'; import { appContextService } from '../..'; import { PackageNotFoundError, PackageCacheError } from '../../../errors'; -export { ArchiveEntry, getBufferExtractor } from './extract'; - export interface SearchParams { category?: CategoryId; experimental?: boolean; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/requests.ts b/x-pack/plugins/fleet/server/services/epm/registry/requests.ts index 2b9c3495657903..c8d158c8afaaaa 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/requests.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/requests.ts @@ -6,7 +6,7 @@ import fetch, { FetchError, Response, RequestInit } from 'node-fetch'; import pRetry from 'p-retry'; -import { streamToString } from './streams'; +import { streamToString } from '../streams'; import { appContextService } from '../../app_context'; import { RegistryError, RegistryConnectionError, RegistryResponseError } from '../../../errors'; import { getProxyAgent, getRegistryProxyUrl } from './proxy'; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/streams.ts b/x-pack/plugins/fleet/server/services/epm/streams.ts similarity index 100% rename from x-pack/plugins/fleet/server/services/epm/registry/streams.ts rename to x-pack/plugins/fleet/server/services/epm/streams.ts