[Ingest] Full agent config schema & API

x-pack/plugins/ingest_manager/common/constants/routes.ts

@@ -34,6 +34,7 @@ export const AGENT_CONFIG_API_ROUTES = {
   CREATE_PATTERN: `${AGENT_CONFIG_API_ROOT}`,
   UPDATE_PATTERN: `${AGENT_CONFIG_API_ROOT}/{agentConfigId}`,
   DELETE_PATTERN: `${AGENT_CONFIG_API_ROOT}/delete`,
+  FULL_INFO_PATTERN: `${AGENT_CONFIG_API_ROOT}/{agentConfigId}/full`,
 };
 
 // Agent API routes

x-pack/plugins/ingest_manager/common/types/models/agent_config.ts

@@ -5,7 +5,13 @@
  */
 
 import { SavedObjectAttributes } from '../../../../../../src/core/public';
-import { Datasource } from './datasource';
+import {
+  Datasource,
+  DatasourcePackage,
+  DatasourceInput,
+  DatasourceInputStream,
+} from './datasource';
+import { Output } from './output';
 
 export enum AgentConfigStatus {
   Active = 'active',
@@ -26,3 +32,27 @@ export interface AgentConfig extends NewAgentConfig, SavedObjectAttributes {
   updated_on: string;
   updated_by: string;
 }
+
+export type FullAgentConfigDatasource = Pick<Datasource, 'name' | 'namespace' | 'enabled'> & {
+  package?: Pick<DatasourcePackage, 'name' | 'version'>;
+  use_output: string;
+  inputs: Array<
+    Omit<DatasourceInput, 'streams'> & {
+      streams: Array<
+        Omit<DatasourceInputStream, 'config'> & {
+          [key: string]: any;
+        }
+      >;
+    }
+  >;
+};
+
+export interface FullAgentConfig {
+  id: string;
+  outputs: {
+    [key: string]: Omit<Output, 'is_default' | 'config' | 'name' | 'id'> & {
+      [key: string]: any;
+    };
+  };
+  datasources: FullAgentConfigDatasource[];
+}

x-pack/plugins/ingest_manager/common/types/models/datasource.ts

@@ -4,34 +4,40 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+export interface DatasourcePackage {
+  assets: Array<{
+    id: string;
+    type: string;
+  }>;
+  description: string;
+  name: string;
+  title: string;
+  version: string;
+}
+
+export interface DatasourceInputStream {
+  id: string;
+  enabled: boolean;
+  dataset: string;
+  processors?: string[];
+  config?: Record<string, any>;
+}
+
+export interface DatasourceInput {
+  type: string;
+  enabled: boolean;
+  processors?: string[];
+  streams: DatasourceInputStream[];
+}
+
 export interface NewDatasource {
   name: string;
   namespace?: string;
   config_id: string;
   enabled: boolean;
-  package?: {
-    assets: Array<{
-      id: string;
-      type: string;
-    }>;
-    description: string;
-    name: string;
-    title: string;
-    version: string;
-  };
+  package?: DatasourcePackage;
   output_id: string;
-  inputs: Array<{
-    type: string;
-    enabled: boolean;
-    processors?: string[];
-    streams: Array<{
-      id: string;
-      enabled: boolean;
-      dataset: string;
-      processors?: string[];
-      config?: Record<string, any>;
-    }>;
-  }>;
+  inputs: DatasourceInput[];
 }
 
 export type Datasource = NewDatasource & { id: string };

x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { AgentConfig, NewAgentConfig } from '../models';
+import { AgentConfig, NewAgentConfig, FullAgentConfig } from '../models';
 import { ListWithKuery } from './common';
 
 export interface GetAgentConfigsRequest {
@@ -57,3 +57,14 @@ export type DeleteAgentConfigsResponse = Array<{
   id: string;
   success: boolean;
 }>;
+
+export interface GetFullAgentConfigRequest {
+  params: {
+    agentConfigId: string;
+  };
+}
+
+export interface GetFullAgentConfigResponse {
+  item: FullAgentConfig;
+  success: boolean;
+}

x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts

@@ -12,13 +12,15 @@ import {
   CreateAgentConfigRequestSchema,
   UpdateAgentConfigRequestSchema,
   DeleteAgentConfigsRequestSchema,
+  GetFullAgentConfigRequestSchema,
 } from '../../types';
 import {
   GetAgentConfigsResponse,
   GetOneAgentConfigResponse,
   CreateAgentConfigResponse,
   UpdateAgentConfigResponse,
   DeleteAgentConfigsResponse,
+  GetFullAgentConfigResponse,
 } from '../../../common';
 
 export const getAgentConfigsHandler: RequestHandler<
@@ -144,3 +146,35 @@ export const deleteAgentConfigsHandler: RequestHandler<
     });
   }
 };
+
+export const getFullAgentConfig: RequestHandler<TypeOf<
+  typeof GetFullAgentConfigRequestSchema.params
+>> = async (context, request, response) => {
+  const soClient = context.core.savedObjects.client;
+
+  try {
+    const fullAgentConfig = await agentConfigService.getFullConfig(
+      soClient,
+      request.params.agentConfigId
+    );
+    if (fullAgentConfig) {
+      const body: GetFullAgentConfigResponse = {
+        item: fullAgentConfig,
+        success: true,
+      };
+      return response.ok({
+        body,
+      });
+    } else {
+      return response.customError({
+        statusCode: 404,
+        body: { message: 'Agent config not found' },
+      });
+    }
+  } catch (e) {
+    return response.customError({
+      statusCode: 500,
+      body: { message: e.message },
+    });
+  }
+};

x-pack/plugins/ingest_manager/server/routes/agent_config/index.ts

@@ -11,13 +11,15 @@ import {
   CreateAgentConfigRequestSchema,
   UpdateAgentConfigRequestSchema,
   DeleteAgentConfigsRequestSchema,
+  GetFullAgentConfigRequestSchema,
 } from '../../types';
 import {
   getAgentConfigsHandler,
   getOneAgentConfigHandler,
   createAgentConfigHandler,
   updateAgentConfigHandler,
   deleteAgentConfigsHandler,
+  getFullAgentConfig,
 } from './handlers';
 
 export const registerRoutes = (router: IRouter) => {
@@ -70,4 +72,14 @@ export const registerRoutes = (router: IRouter) => {
     },
     deleteAgentConfigsHandler
   );
+
+  // Get one full agent config
+  router.get(
+    {
+      path: AGENT_CONFIG_API_ROUTES.FULL_INFO_PATTERN,
+      validate: GetFullAgentConfigRequestSchema,
+      options: { tags: [`access:${PLUGIN_ID}`] },
+    },
+    getFullAgentConfig
+  );
 };

x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts

@@ -5,7 +5,7 @@
  */
 import { TypeOf } from '@kbn/config-schema';
 import { RequestHandler } from 'kibana/server';
-import { datasourceService } from '../../services';
+import { appContextService, datasourceService, agentConfigService } from '../../services';
 import {
   GetDatasourcesRequestSchema,
   GetOneDatasourceRequestSchema,
@@ -70,8 +70,12 @@ export const createDatasourceHandler: RequestHandler<
   TypeOf<typeof CreateDatasourceRequestSchema.body>
 > = async (context, request, response) => {
   const soClient = context.core.savedObjects.client;
+  const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined;
   try {
     const datasource = await datasourceService.create(soClient, request.body);
+    await agentConfigService.assignDatasources(soClient, datasource.config_id, [datasource.id], {
+      user,
+    });
     return response.ok({
       body: { item: datasource, success: true },
     });

x-pack/plugins/ingest_manager/server/services/agent_config.ts

@@ -4,10 +4,17 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { SavedObjectsClientContract } from 'kibana/server';
-import { flatten } from 'lodash';
 import { AuthenticatedUser } from '../../../security/server';
 import { DEFAULT_AGENT_CONFIG, AGENT_CONFIG_SAVED_OBJECT_TYPE } from '../constants';
-import { NewAgentConfig, AgentConfig, AgentConfigStatus, ListWithKuery } from '../types';
+import {
+  Datasource,
+  NewAgentConfig,
+  AgentConfig,
+  FullAgentConfig,
+  FullAgentConfigDatasource,
+  AgentConfigStatus,
+  ListWithKuery,
+} from '../types';
 import { DeleteAgentConfigsResponse } from '../../common';
 import { datasourceService } from './datasource';
 import { outputService } from './output';
@@ -249,25 +256,38 @@ class AgentConfigService {
     return result;
   }
 
-  private storedDatasourceToAgentStreams(datasources: AgentConfig['datasources'] = []): any[] {
-    return flatten(
-      // @ts-ignore
-      datasources.map((ds: any) => {
-        return ds.streams.map((stream: any) => ({
-          ...stream.input,
-          id: stream.id,
-          type: stream.input.type as any,
-          output: { use_output: stream.output_id },
-          ...(stream.config || {}),
-        }));
-      })
-    );
-  }
+  private storedDatasourceToAgentDatasource = (
+    datasource: Datasource
+  ): FullAgentConfigDatasource => {
+    const { name, namespace, enabled, package: pkg, output_id, inputs } = datasource;
+    return {
+      name,
+      namespace,
+      enabled,
+      package: pkg
+        ? {
+            name: pkg.name,
+            version: pkg.version,
+          }
+        : undefined,
+      use_output: output_id,
+      inputs: inputs
+        .filter(input => input.enabled)
+        .map(input => ({
+          ...input,
+          streams: input.streams.map(stream => ({
+            ...stream,
+            config: undefined,
+            ...(stream.config || {}),
+          })),
+        })),
+    };
+  };
 
   public async getFullConfig(
     soClient: SavedObjectsClientContract,
     id: string
-  ): Promise<any | null> {
+  ): Promise<FullAgentConfig | null> {
     let config;
 
     try {
@@ -288,19 +308,18 @@ class AgentConfigService {
         // TEMPORARY as we only support a default output
         ...[
           await outputService.get(soClient, await outputService.getDefaultOutputId(soClient)),
-        ].reduce((outputs, { config: outputConfig, ...output }) => {
-          outputs[output.is_default ? 'default' : output.id] = {
+        ].reduce((outputs, { config: outputConfig, is_default, name, id: outputId, ...output }) => {
+          outputs[name] = {
             ...output,
             type: output.type as any,
             ...outputConfig,
           };
           return outputs;
         }, {} as any),
       },
-      streams:
-        config.datasources && config.datasources.length
-          ? this.storedDatasourceToAgentStreams(config.datasources)
-          : [],
+      datasources: (config.datasources as Datasource[]).map(ds =>
+        this.storedDatasourceToAgentDatasource(ds)
+      ),
     };
 
     return agentConfig;

x-pack/plugins/ingest_manager/server/types/index.tsx

@@ -16,6 +16,8 @@ export {
   AgentAction,
   Datasource,
   NewDatasource,
+  FullAgentConfigDatasource,
+  FullAgentConfig,
   AgentConfig,
   NewAgentConfig,
   AgentConfigStatus,

x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts

@@ -31,3 +31,9 @@ export const DeleteAgentConfigsRequestSchema = {
     agentConfigIds: schema.arrayOf(schema.string()),
   }),
 };
+
+export const GetFullAgentConfigRequestSchema = {
+  params: schema.object({
+    agentConfigId: schema.string(),
+  }),
+};