[EPM] More realistic datasource SO. Error if package not installed.

x-pack/legacy/plugins/epm/server/datasources/create.ts

@@ -7,77 +7,95 @@
 import { SavedObjectsClientContract } from 'src/core/server/';
 import { Asset, Datasource, InputType } from '../../../ingest/server/libs/types';
 import { SAVED_OBJECT_TYPE_DATASOURCES } from '../../common/constants';
-import { AssetReference } from '../../common/types';
 import { CallESAsCurrentUser } from '../lib/cluster_access';
 import { installPipelines } from '../lib/elasticsearch/ingest_pipeline/ingest_pipelines';
 import { installTemplates } from '../lib/elasticsearch/template/install';
+import { AssetReference, InstallationStatus, RegistryPackage } from '../../common/types';
+import { getPackageInfo } from '../packages';
 import * as Registry from '../registry';
 
+const pkgToPkgKey = ({ name, version }: RegistryPackage) => `${name}-${version}`;
+export class PackageNotInstalledError extends Error {
+  constructor(pkgkey: string) {
+    super(`${pkgkey} is not installed`);
+  }
+}
+
 export async function createDatasource(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgkey: string;
   callCluster: CallESAsCurrentUser;
 }) {
   const { savedObjectsClient, pkgkey, callCluster } = options;
+  const info = await getPackageInfo({ savedObjectsClient, pkgkey });
+
+  if (info.status !== InstallationStatus.installed) {
+    throw new PackageNotInstalledError(pkgkey);
+  }
+
   const toSave = await installPipelines({ pkgkey, callCluster });
   // TODO: Clean up
-  const info = await Registry.fetchInfo(pkgkey);
-  await installTemplates(info, callCluster);
+  const pkg = await Registry.fetchInfo(pkgkey);
 
-  await saveDatasourceReferences({
-    savedObjectsClient,
-    pkgkey,
-    toSave,
-  });
+  await Promise.all([
+    installTemplates(pkg, callCluster),
+    saveDatasourceReferences({
+      savedObjectsClient,
+      pkg,
+      toSave,
+    }),
+  ]);
 
   return toSave;
 }
 
-export async function saveDatasourceReferences(options: {
+async function saveDatasourceReferences(options: {
   savedObjectsClient: SavedObjectsClientContract;
-  pkgkey: string;
+  pkg: RegistryPackage;
   toSave: AssetReference[];
 }) {
-  const { savedObjectsClient, pkgkey, toSave } = options;
-  const savedObject = await getDatasourceObject({ savedObjectsClient, pkgkey });
-  const savedRefs = savedObject?.attributes.package.assets;
-  const mergeRefsReducer = (current: Asset[] = [], pending: Asset) => {
-    const hasRef = current.find(c => c.id === pending.id && c.type === pending.type);
-    if (!hasRef) current.push(pending);
+  const { savedObjectsClient, pkg, toSave } = options;
+  const savedDatasource = await getDatasource({ savedObjectsClient, pkg });
+  const savedAssets = savedDatasource?.package.assets;
+  const assetsReducer = (current: Asset[] = [], pending: Asset) => {
+    const hasAsset = current.find(c => c.id === pending.id && c.type === pending.type);
+    if (!hasAsset) current.push(pending);
     return current;
   };
 
-  const toInstall = (toSave as Asset[]).reduce(mergeRefsReducer, savedRefs);
-  const datasource: Datasource = createFakeDatasource(pkgkey, toInstall);
+  const toInstall = (toSave as Asset[]).reduce(assetsReducer, savedAssets);
+  const datasource: Datasource = createFakeDatasource(pkg, toInstall);
   await savedObjectsClient.create<Datasource>(SAVED_OBJECT_TYPE_DATASOURCES, datasource, {
-    id: pkgkey,
+    id: pkgToPkgKey(pkg),
     overwrite: true,
   });
 
   return toInstall;
 }
 
-export async function getDatasourceObject(options: {
+async function getDatasource(options: {
   savedObjectsClient: SavedObjectsClientContract;
-  pkgkey: string;
+  pkg: RegistryPackage;
 }) {
-  const { savedObjectsClient, pkgkey } = options;
-  return savedObjectsClient
-    .get<Datasource>(SAVED_OBJECT_TYPE_DATASOURCES, pkgkey)
+  const { savedObjectsClient, pkg } = options;
+  const datasource = await savedObjectsClient
+    .get<Datasource>(SAVED_OBJECT_TYPE_DATASOURCES, pkgToPkgKey(pkg))
     .catch(e => undefined);
+
+  return datasource?.attributes;
 }
 
-function createFakeDatasource(pkgkey: string, assets: Asset[] = []): Datasource {
+function createFakeDatasource(pkg: RegistryPackage, assets: Asset[] = []): Datasource {
   return {
-    id: pkgkey,
+    id: pkgToPkgKey(pkg),
     name: 'name',
     read_alias: 'read_alias',
     package: {
-      name: 'name',
-      version: '1.0.1, 1.3.1',
-      description: 'description',
-      title: 'title',
-      assets: assets as Asset[],
+      name: pkg.name,
+      version: pkg.version,
+      description: pkg.description,
+      title: pkg.title,
+      assets,
     },
     streams: [
       {

x-pack/legacy/plugins/epm/server/datasources/handlers.ts

@@ -4,11 +4,12 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import Boom from 'boom';
+import { createDatasource, PackageNotInstalledError } from './index';
 import { getClusterAccessor } from '../lib/cluster_access';
 import { PluginContext } from '../plugin';
 import { getClient } from '../saved_objects';
 import { Request, ResponseToolkit } from '../types';
-import { createDatasource } from './index';
 
 // TODO: duplicated from packages/handlers.ts. unduplicate.
 interface Extra extends ResponseToolkit {
@@ -25,9 +26,20 @@ export async function handleRequestInstallDatasource(req: CreateDatasourceReques
   const { pkgkey } = req.params;
   const savedObjectsClient = getClient(req);
   const callCluster = getClusterAccessor(extra.context.esClient, req);
-  return createDatasource({
-    savedObjectsClient,
-    pkgkey,
-    callCluster,
-  });
+
+  try {
+    const result = await createDatasource({
+      savedObjectsClient,
+      pkgkey,
+      callCluster,
+    });
+
+    return result;
+  } catch (error) {
+    if (error instanceof PackageNotInstalledError) {
+      throw new Boom(error, { statusCode: 403 });
+    } else {
+      return error;
+    }
+  }
 }

x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/install.ts

@@ -8,7 +8,6 @@ import { safeLoad } from 'js-yaml';
 import { AssetReference, Dataset, RegistryPackage } from '../../../../common/types';
 import { CallESAsCurrentUser } from '../../../../server/lib/cluster_access';
 import { getAssetsData } from '../../../packages/assets';
-import * as Registry from '../../../registry';
 import { Field } from '../../fields/field';
 import { generateMappings, generateTemplateName, getTemplate } from './template';
 
@@ -22,19 +21,13 @@ const isFields = (path: string) => {
  * For each dataset, the fields.yml files are extracted. If there are multiple
  * in one datasets, they are merged together into 1 and then converted to a template
  * The template is currently loaded with the pkgey-package-dataset
- * @param callCluster
- * @param pkgkey
  */
-export async function installTemplates(p: RegistryPackage, callCluster: CallESAsCurrentUser) {
-  const pkgkey = p.name + '-' + p.version;
-  // TODO: Needs to be called to fill the cache but should not be required
-  await Registry.getArchiveInfo(pkgkey);
-
+export async function installTemplates(pkg: RegistryPackage, callCluster: CallESAsCurrentUser) {
   const promises: Array<Promise<AssetReference>> = [];
 
-  for (const dataset of p.datasets) {
+  for (const dataset of pkg.datasets) {
     // Fetch all assset entries for this dataset
-    const assetEntries = getAssetsData(p, isFields, dataset.name);
+    const assetEntries = await getAssetsData(pkg, isFields, dataset.name);
 
     // Merge all the fields of a dataset together and create an Elasticsearch index template
     let datasetFields: Field[] = [];
@@ -45,7 +38,7 @@ export async function installTemplates(p: RegistryPackage, callCluster: CallESAs
       }
     }
 
-    const promise = installTemplate({ callCluster, fields: datasetFields, p, dataset });
+    const promise = installTemplate({ callCluster, fields: datasetFields, pkg, dataset });
     promises.push(promise);
   }
 
@@ -55,16 +48,16 @@ export async function installTemplates(p: RegistryPackage, callCluster: CallESAs
 async function installTemplate({
   callCluster,
   fields,
-  p,
+  pkg,
   dataset,
 }: {
   callCluster: CallESAsCurrentUser;
   fields: Field[];
-  p: RegistryPackage;
+  pkg: RegistryPackage;
   dataset: Dataset;
 }): Promise<AssetReference> {
   const mappings = generateMappings(fields);
-  const templateName = generateTemplateName(p.name, dataset.name, dataset.type);
+  const templateName = generateTemplateName(pkg.name, dataset.name, dataset.type);
   const template = getTemplate(templateName + '-*', mappings);
   // TODO: Check return values for errors
   await callCluster('indices.putTemplate', {

x-pack/legacy/plugins/epm/server/packages/assets.ts

@@ -4,8 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { RegistryPackage } from '../../common/types';
 import * as Registry from '../registry';
+import { cacheHas } from '../registry/cache';
+import { RegistryPackage } from '../../common/types';
+
+// paths from RegistryPackage are routes to the assets on EPR
+// paths for ArchiveEntry are routes to the assets in the archive
+// RegistryPackage paths have a `/package/` prefix compared to ArchiveEntry paths
+const EPR_PATH_PREFIX = '/package/';
 
 export function getAssets(
   packageInfo: RegistryPackage,
@@ -24,7 +30,7 @@ export function getAssets(
     if (dataSet !== '') {
       // TODO: Filter for dataset path
       const comparePath =
-        '/package/' + packageInfo.name + '-' + packageInfo.version + '/dataset/' + dataSet;
+        EPR_PATH_PREFIX + packageInfo.name + '-' + packageInfo.version + '/dataset/' + dataSet;
       if (!path.includes(comparePath)) {
         continue;
       }
@@ -40,26 +46,23 @@ export function getAssets(
   return assets;
 }
 
-export function getAssetsData(
+export async function getAssetsData(
   packageInfo: RegistryPackage,
   filter = (path: string): boolean => true,
   dataSet: string = ''
-): Registry.ArchiveEntry[] {
+): Promise<Registry.ArchiveEntry[]> {
+  // TODO: Needs to be called to fill the cache but should not be required
+  const pkgkey = packageInfo.name + '-' + packageInfo.version;
+  if (!cacheHas(pkgkey)) await Registry.getArchiveInfo(pkgkey);
+
   // Gather all asset data
   const assets = getAssets(packageInfo, filter, dataSet);
+  const entries: Registry.ArchiveEntry[] = assets.map(path => {
+    const archivePath = path.replace(EPR_PATH_PREFIX, '');
+    const buffer = Registry.getAsset(archivePath);
 
-  const entries: Registry.ArchiveEntry[] = [];
-
-  for (const asset of assets) {
-    const subPath = asset.substring(9);
-    const buf = Registry.getAsset(subPath);
-
-    const entry: Registry.ArchiveEntry = {
-      path: asset,
-      buffer: buf,
-    };
-    entries.push(entry);
-  }
+    return { path, buffer };
+  });
 
   return entries;
 }