[Logs+] Add title / icons to installed packages API (#161777)

## Summary

Implements #160935.

This PR adds the `title` and `icons` (also `description` since it was
there) properties to the installed packages API, these are extracted
from the matching manifest assets (so that we don't have to call the
registry, and can query for multiple items at once). Discover has also
been updated to use these new properties.

List:

<img width="343" alt="Screenshot 2023-07-12 at 17 13 52"
src="https://github.com/elastic/kibana/assets/471693/6929b17d-5317-4332-96ea-0ee85c19aebe">

With selection:

<img width="319" alt="Screenshot 2023-07-12 at 17 14 10"
src="https://github.com/elastic/kibana/assets/471693/550d050f-5b33-42d9-9139-99164d64a770">

I've made this robust against a matching manifest asset not being found
- I don't think this is actually possible, but just incase.

## Testing

- When accessing the log explorer profile within Discover pretty titles
should be displayed in the dataset selector
- When accessing the log explorer profile within Discover icons should
be displayed in the dataset selector

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/discover_log_explorer/common/datasets/models/dataset.ts

@@ -10,7 +10,7 @@ import { DataViewSpec } from '@kbn/data-views-plugin/common';
 import { IndexPattern } from '@kbn/io-ts-utils';
 import { DatasetId, DatasetType, IntegrationType } from '../types';
 
-type IntegrationBase = Pick<IntegrationType, 'name' | 'version'>;
+type IntegrationBase = Pick<IntegrationType, 'name' | 'title' | 'icons' | 'version'>;
 interface DatasetDeps extends DatasetType {
   iconType?: IconType;
 }
@@ -29,13 +29,15 @@ export class Dataset {
     this.title = dataset.title ?? dataset.name;
     this.parentIntegration = parentIntegration && {
       name: parentIntegration.name,
+      title: parentIntegration.title,
+      icons: parentIntegration.icons,
       version: parentIntegration.version,
     };
   }
 
   getFullTitle(): string {
-    return this.parentIntegration?.name
-      ? `[${this.parentIntegration.name}] ${this.title}`
+    return this.parentIntegration?.title
+      ? `[${this.parentIntegration.title}] ${this.title}`
       : this.title;
   }
 

x-pack/plugins/discover_log_explorer/common/datasets/models/integration.ts

@@ -11,13 +11,19 @@ import { IntegrationId, IntegrationType } from '../types';
 export class Integration {
   id: IntegrationId;
   name: IntegrationType['name'];
+  title?: IntegrationType['title'];
+  description?: IntegrationType['description'];
+  icons?: IntegrationType['icons'];
   status: IntegrationType['status'];
   version: IntegrationType['version'];
   datasets: Dataset[];
 
   private constructor(integration: Integration) {
     this.id = integration.id;
     this.name = integration.name;
+    this.title = integration.title ?? integration.name;
+    this.description = integration.description;
+    this.icons = integration.icons;
     this.status = integration.status;
     this.version = integration.version;
     this.datasets = integration.datasets;

x-pack/plugins/discover_log_explorer/common/datasets/types.ts

@@ -25,12 +25,29 @@ const integrationStatusRT = rt.keyof({
   install_failed: null,
 });
 
-export const integrationRT = rt.type({
-  name: rt.string,
-  status: integrationStatusRT,
-  version: rt.string,
-  dataStreams: rt.array(datasetRT),
-});
+export const integrationRT = rt.intersection([
+  rt.type({
+    name: rt.string,
+    status: integrationStatusRT,
+    version: rt.string,
+    dataStreams: rt.array(datasetRT),
+  }),
+  rt.partial({
+    title: rt.union([rt.string, rt.undefined]),
+    icons: rt.union([
+      rt.array(
+        rt.type({
+          src: rt.string,
+          title: rt.string,
+          size: rt.string,
+          type: rt.string,
+        })
+      ),
+      rt.undefined,
+    ]),
+    description: rt.union([rt.string, rt.undefined]),
+  }),
+]);
 
 export type DatasetId = `dataset-${string}`;
 export type IntegrationId = `integration-${string}-${string}`;

x-pack/plugins/discover_log_explorer/public/components/dataset_selector/sub_components/datasets_popover.tsx

@@ -60,6 +60,7 @@ export const DatasetsPopover = ({
             <PackageIcon
               packageName={parentIntegration.name}
               version={parentIntegration.version}
+              icons={parentIntegration.icons}
               size="m"
               tryApi
             />

x-pack/plugins/discover_log_explorer/public/components/dataset_selector/utils.tsx

@@ -55,19 +55,19 @@ export const buildIntegrationsTree = ({
 }: IntegrationsTreeParams) => {
   return integrations.reduce(
     (integrationsTree: IntegrationsTree, integration, pos) => {
-      const { name, version, datasets } = integration;
+      const { name, title, version, datasets, icons } = integration;
       const isLastIntegration = pos === integrations.length - 1;
 
       integrationsTree.items.push({
-        name,
-        icon: <PackageIcon packageName={name} version={version} size="m" tryApi />,
+        name: title,
+        icon: <PackageIcon packageName={name} version={version} size="m" icons={icons} tryApi />,
         panel: integration.id,
         ...(isLastIntegration && { buttonRef: spyRef }),
       });
 
       integrationsTree.panels.push({
         id: integration.id,
-        title: name,
+        title,
         width: DATA_VIEW_POPOVER_CONTENT_WIDTH,
         items: datasets.map((dataset) => ({
           name: dataset.title,

x-pack/plugins/fleet/common/types/rest_spec/epm.ts

@@ -51,7 +51,7 @@ export interface GetPackagesResponse {
   response?: PackageList;
 }
 
-interface InstalledPackage {
+export interface InstalledPackage {
   name: string;
   version: string;
   status: EpmPackageInstallStatus;

x-pack/plugins/fleet/server/services/epm/packages/get.test.ts

@@ -10,7 +10,11 @@ import type { SavedObjectsClientContract, SavedObjectsFindResult } from '@kbn/co
 import { SavedObjectsErrorHelpers } from '@kbn/core/server';
 import { savedObjectsClientMock } from '@kbn/core/server/mocks';
 
-import { PACKAGES_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../../common';
+import {
+  ASSETS_SAVED_OBJECT_TYPE,
+  PACKAGES_SAVED_OBJECT_TYPE,
+  PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+} from '../../../../common';
 import type { RegistryPackage } from '../../../../common/types';
 import type { PackagePolicySOAttributes } from '../../../types';
 
@@ -435,24 +439,51 @@ owner: elastic`,
     it('Passes the correct parameters to the SavedObjects client', async () => {
       const soClient = savedObjectsClientMock.create();
 
-      soClient.find.mockResolvedValue({
-        saved_objects: [
-          {
-            type: 'epm-packages',
-            id: 'elastic_agent',
-            attributes: {
-              es_index_patterns: {
-                apm_server_logs: 'logs-elastic_agent.apm_server-*',
-                apm_server_metrics: 'metrics-elastic_agent.apm_server-*',
+      soClient.find.mockImplementation(async (options) => {
+        if (options.type === PACKAGES_SAVED_OBJECT_TYPE) {
+          return {
+            total: 5,
+            saved_objects: [
+              {
+                type: 'epm-packages',
+                id: 'elastic_agent',
+                attributes: {
+                  es_index_patterns: {
+                    apm_server_logs: 'logs-elastic_agent.apm_server-*',
+                    apm_server_metrics: 'metrics-elastic_agent.apm_server-*',
+                  },
+                  name: 'elastic_agent',
+                  version: '1.7.0',
+                  install_status: 'installed',
+                },
+                references: [],
+                sort: ['elastic_agent'],
               },
-              name: 'elastic_agent',
-              version: '1.7.0',
-              install_status: 'installed',
-            },
-            references: [],
-          },
-        ],
-      } as any);
+            ],
+          } as any;
+        } else if (options.type === ASSETS_SAVED_OBJECT_TYPE) {
+          return {
+            total: 5,
+            saved_objects: [
+              {
+                type: 'epm-packages-assets',
+                id: '338b6f9e-e126-5f1e-abb9-afe017d4788b',
+                attributes: {
+                  package_name: 'elastic_agent',
+                  package_version: '1.8.0',
+                  install_source: 'upload',
+                  asset_path: 'elastic_agent-1.8.0/manifest.yml',
+                  media_type: 'text/yaml; charset=utf-8',
+                  data_utf8:
+                    'name: elastic_agent\ntitle: Elastic Agent\nversion: 1.8.0\ndescription: Collect logs and metrics from Elastic Agents.\ntype: integration\nformat_version: 1.0.0\nlicense: basic\ncategories: ["elastic_stack"]\nconditions:\n  kibana.version: "^8.7.1"\nowner:\n  github: elastic/elastic-agent\nicons:\n  - src: /img/logo_elastic_agent.svg\n    title: logo Elastic Agent\n    size: 64x64\n    type: image/svg+xml\nscreenshots:\n  - src: /img/elastic_agent_overview.png\n    title: Elastic Agent Overview\n    size: 2560×1234\n    type: image/png\n  - src: /img/elastic_agent_metrics.png\n    title: Elastic Agent Metrics\n    size: 2560×1234\n    type: image/png\n  - src: /img/elastic_agent_info.png\n    title: Elastic Agent Information\n    size: 2560×1234\n    type: image/png\n  - src: /img/elastic_agent_integrations.png\n    title: Elastic Agent Integrations\n    size: 2560×1234\n    type: image/png\n',
+                  data_base64: '',
+                },
+                references: [],
+              },
+            ],
+          } as any;
+        }
+      });
 
       await getInstalledPackages({
         savedObjectsClient: soClient,
@@ -550,26 +581,51 @@ owner: elastic`,
     it('Formats items correctly', async () => {
       const soClient = savedObjectsClientMock.create();
 
-      soClient.find.mockResolvedValue({
-        total: 5,
-        saved_objects: [
-          {
-            type: 'epm-packages',
-            id: 'elastic_agent',
-            attributes: {
-              es_index_patterns: {
-                apm_server_logs: 'logs-elastic_agent.apm_server-*',
-                apm_server_metrics: 'metrics-elastic_agent.apm_server-*',
+      soClient.find.mockImplementation(async (options) => {
+        if (options.type === PACKAGES_SAVED_OBJECT_TYPE) {
+          return {
+            total: 5,
+            saved_objects: [
+              {
+                type: 'epm-packages',
+                id: 'elastic_agent',
+                attributes: {
+                  es_index_patterns: {
+                    apm_server_logs: 'logs-elastic_agent.apm_server-*',
+                    apm_server_metrics: 'metrics-elastic_agent.apm_server-*',
+                  },
+                  name: 'elastic_agent',
+                  version: '1.8.0',
+                  install_status: 'installed',
+                },
+                references: [],
+                sort: ['elastic_agent'],
               },
-              name: 'elastic_agent',
-              version: '1.7.0',
-              install_status: 'installed',
-            },
-            references: [],
-            sort: ['elastic_agent'],
-          },
-        ],
-      } as any);
+            ],
+          } as any;
+        } else if (options.type === ASSETS_SAVED_OBJECT_TYPE) {
+          return {
+            total: 5,
+            saved_objects: [
+              {
+                type: 'epm-packages-assets',
+                id: '338b6f9e-e126-5f1e-abb9-afe017d4788b',
+                attributes: {
+                  package_name: 'elastic_agent',
+                  package_version: '1.8.0',
+                  install_source: 'upload',
+                  asset_path: 'elastic_agent-1.8.0/manifest.yml',
+                  media_type: 'text/yaml; charset=utf-8',
+                  data_utf8:
+                    'name: elastic_agent\ntitle: Elastic Agent\nversion: 1.8.0\ndescription: Collect logs and metrics from Elastic Agents.\ntype: integration\nformat_version: 1.0.0\nlicense: basic\ncategories: ["elastic_stack"]\nconditions:\n  kibana.version: "^8.7.1"\nowner:\n  github: elastic/elastic-agent\nicons:\n  - src: /img/logo_elastic_agent.svg\n    title: logo Elastic Agent\n    size: 64x64\n    type: image/svg+xml\nscreenshots:\n  - src: /img/elastic_agent_overview.png\n    title: Elastic Agent Overview\n    size: 2560×1234\n    type: image/png\n  - src: /img/elastic_agent_metrics.png\n    title: Elastic Agent Metrics\n    size: 2560×1234\n    type: image/png\n  - src: /img/elastic_agent_info.png\n    title: Elastic Agent Information\n    size: 2560×1234\n    type: image/png\n  - src: /img/elastic_agent_integrations.png\n    title: Elastic Agent Integrations\n    size: 2560×1234\n    type: image/png\n',
+                  data_base64: '',
+                },
+                references: [],
+              },
+            ],
+          } as any;
+        }
+      });
 
       const results = await getInstalledPackages({
         savedObjectsClient: soClient,
@@ -586,7 +642,17 @@ owner: elastic`,
             dataStreams: [{ name: 'logs-elastic_agent.apm_server-*', title: 'apm_server_logs' }],
             name: 'elastic_agent',
             status: 'installed',
-            version: '1.7.0',
+            version: '1.8.0',
+            title: 'Elastic Agent',
+            description: 'Collect logs and metrics from Elastic Agents.',
+            icons: [
+              {
+                size: '64x64',
+                src: '/img/logo_elastic_agent.svg',
+                title: 'logo Elastic Agent',
+                type: 'image/svg+xml',
+              },
+            ],
           },
         ],
         searchAfter: ['elastic_agent'],

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -4,6 +4,8 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
+import * as yaml from 'js-yaml';
 import pMap from 'p-map';
 import type { SavedObjectsClientContract, SavedObjectsFindOptions } from '@kbn/core/server';
 import semverGte from 'semver/functions/gte';
@@ -18,6 +20,7 @@ import { buildNode as buildFunctionNode } from '@kbn/es-query/src/kuery/node_typ
 import { buildNode as buildWildcardNode } from '@kbn/es-query/src/kuery/node_types/wildcard';
 
 import {
+  ASSETS_SAVED_OBJECT_TYPE,
   installationStatuses,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   SO_SEARCH_LIMIT,
@@ -28,6 +31,8 @@ import type {
   Installable,
   PackageDataStreamTypes,
   PackageList,
+  InstalledPackage,
+  PackageSpecManifest,
 } from '../../../../common/types';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import type {
@@ -45,6 +50,7 @@ import {
 } from '../../../errors';
 import { appContextService } from '../..';
 import * as Registry from '../registry';
+import type { PackageAsset } from '../archive/storage';
 import { getEsPackage } from '../archive/storage';
 import { getArchivePackage } from '../archive';
 import { normalizeKuery } from '../../saved_object';
@@ -189,8 +195,25 @@ export async function getInstalledPackages(options: GetInstalledPackagesOptions)
     };
   });
 
+  const integrationManifests =
+    integrations.length > 0
+      ? await getInstalledPackageManifests(savedObjectsClient, integrations)
+      : new Map<string, PackageSpecManifest>();
+
+  const integrationsWithManifestContent = integrations.map((integration) => {
+    const { name, version } = integration;
+    const integrationAsset = integrationManifests.get(`${name}-${version}/manifest.yml`);
+
+    return {
+      ...integration,
+      title: integrationAsset?.title ?? undefined,
+      description: integrationAsset?.description ?? undefined,
+      icons: integrationAsset?.icons ?? undefined,
+    };
+  });
+
   return {
-    items: integrations,
+    items: integrationsWithManifestContent,
     total: packageSavedObjects.total,
     searchAfter: packageSavedObjects.saved_objects.at(-1)?.sort, // Enable ability to use searchAfter in subsequent queries
   };
@@ -306,6 +329,42 @@ export async function getInstalledPackageSavedObjects(
   return result;
 }
 
+export async function getInstalledPackageManifests(
+  savedObjectsClient: SavedObjectsClientContract,
+  installedPackages: InstalledPackage[]
+) {
+  const pathFilters = installedPackages.map((installedPackage) => {
+    const { name, version } = installedPackage;
+    return nodeBuilder.is(
+      `${ASSETS_SAVED_OBJECT_TYPE}.attributes.asset_path`,
+      `${name}-${version}/manifest.yml`
+    );
+  });
+
+  const result = await savedObjectsClient.find<PackageAsset>({
+    type: ASSETS_SAVED_OBJECT_TYPE,
+    filter: nodeBuilder.or(pathFilters),
+  });
+
+  const parsedManifests = result.saved_objects.reduce<Map<string, PackageSpecManifest>>(
+    (acc, asset) => {
+      acc.set(asset.attributes.asset_path, yaml.load(asset.attributes.data_utf8));
+      return acc;
+    },
+    new Map()
+  );
+
+  for (const savedObject of result.saved_objects) {
+    auditLoggingService.writeCustomSoAuditLog({
+      action: 'find',
+      id: savedObject.id,
+      savedObjectType: ASSETS_SAVED_OBJECT_TYPE,
+    });
+  }
+
+  return parsedManifests;
+}
+
 function getInstalledPackageSavedObjectDataStreams(
   indexPatterns: Record<string, string>,
   dataStreamType?: string