From 5c212bd19ab02123d0b0a5134c634fd160a4abeb Mon Sep 17 00:00:00 2001 From: Su Date: Thu, 25 Aug 2022 11:37:54 -0700 Subject: [PATCH] support legacy client for data source Signed-off-by: Su --- .../opensearch/legacy/cluster_client.ts | 2 +- .../fetcher/index_patterns_fetcher.ts | 6 +- .../field_capabilities/field_capabilities.ts | 4 +- .../fetcher/lib/opensearch_api.ts | 13 +- .../data/server/index_patterns/routes.ts | 15 ++- .../data_source/server/client/client_pool.ts | 15 +-- .../server/client/configure_client.ts | 54 ++++++-- .../data_source/server/data_source_service.ts | 82 ++++++++---- .../server/legacy/client_config.ts | 28 +++++ .../server/legacy/configure_legacy_client.ts | 118 ++++++++++++++++++ .../data_source/server/legacy/index.ts | 6 + src/plugins/data_source/server/plugin.ts | 25 +++- src/plugins/data_source/server/types.ts | 29 ++++- .../server/routes/resolve_index.ts | 30 ++--- 14 files changed, 341 insertions(+), 86 deletions(-) create mode 100644 src/plugins/data_source/server/legacy/client_config.ts create mode 100644 src/plugins/data_source/server/legacy/configure_legacy_client.ts create mode 100644 src/plugins/data_source/server/legacy/index.ts diff --git a/src/core/server/opensearch/legacy/cluster_client.ts b/src/core/server/opensearch/legacy/cluster_client.ts index bb8884fd3952..ab8bd4f8ceb0 100644 --- a/src/core/server/opensearch/legacy/cluster_client.ts +++ b/src/core/server/opensearch/legacy/cluster_client.ts @@ -65,7 +65,7 @@ const noop = () => undefined; * OpenSearch JS client. * @param options Options that affect the way we call the API and process the result. */ -const callAPI = async ( +export const callAPI = async ( client: Client, endpoint: string, clientParams: Record = {}, diff --git a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts index a53e06cdea0a..9144e505133c 100644 --- a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts +++ b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts @@ -28,7 +28,7 @@ * under the License. */ -import { LegacyAPICaller, OpenSearchClient } from 'opensearch-dashboards/server'; +import { LegacyAPICaller } from 'opensearch-dashboards/server'; import { getFieldCapabilities, resolveTimePattern, createNoMatchingIndicesError } from './lib'; @@ -48,9 +48,9 @@ interface FieldSubType { } export class IndexPatternsFetcher { - private _callDataCluster: LegacyAPICaller | OpenSearchClient; + private _callDataCluster: LegacyAPICaller; - constructor(callDataCluster: LegacyAPICaller | OpenSearchClient) { + constructor(callDataCluster: LegacyAPICaller) { this._callDataCluster = callDataCluster; } diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts index 57d59ce5a486..0aee1222a361 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts @@ -30,7 +30,7 @@ import { defaults, keyBy, sortBy } from 'lodash'; -import { LegacyAPICaller, OpenSearchClient } from 'opensearch-dashboards/server'; +import { LegacyAPICaller } from 'opensearch-dashboards/server'; import { callFieldCapsApi } from '../opensearch_api'; import { FieldCapsResponse, readFieldCapsResponse } from './field_caps_response'; import { mergeOverrides } from './overrides'; @@ -47,7 +47,7 @@ import { FieldDescriptor } from '../../index_patterns_fetcher'; * @return {Promise>} */ export async function getFieldCapabilities( - callCluster: LegacyAPICaller | OpenSearchClient, + callCluster: LegacyAPICaller, indices: string | string[] = [], metaFields: string[] = [], fieldCapsOptions?: { allowNoIndices: boolean } diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/opensearch_api.ts b/src/plugins/data/server/index_patterns/fetcher/lib/opensearch_api.ts index 4400925e4dd4..cd746b43eb39 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/opensearch_api.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/opensearch_api.ts @@ -94,22 +94,11 @@ export async function callIndexAliasApi( * @return {Promise} */ export async function callFieldCapsApi( - callCluster: LegacyAPICaller | OpenSearchClient, + callCluster: LegacyAPICaller, indices: string[] | string, fieldCapsOptions: { allowNoIndices: boolean } = { allowNoIndices: false } ) { try { - if ('transport' in callCluster) { - return ( - await callCluster.fieldCaps({ - index: indices, - fields: '*', - ignore_unavailable: true, - allow_no_indices: fieldCapsOptions.allowNoIndices, - }) - ).body as FieldCapsResponse; - } - return (await callCluster('fieldCaps', { index: indices, fields: '*', diff --git a/src/plugins/data/server/index_patterns/routes.ts b/src/plugins/data/server/index_patterns/routes.ts index f41c15ccb381..36715b5e681a 100644 --- a/src/plugins/data/server/index_patterns/routes.ts +++ b/src/plugins/data/server/index_patterns/routes.ts @@ -29,7 +29,11 @@ */ import { schema } from '@osd/config-schema'; -import { HttpServiceSetup, RequestHandlerContext } from 'opensearch-dashboards/server'; +import { + HttpServiceSetup, + LegacyAPICaller, + RequestHandlerContext, +} from 'opensearch-dashboards/server'; import { IndexPatternsFetcher } from './fetcher'; export function registerRoutes(http: HttpServiceSetup) { @@ -153,9 +157,8 @@ export function registerRoutes(http: HttpServiceSetup) { const decideClient = async (context: RequestHandlerContext, request: any) => { const dataSourceId = request.query.data_source; - if (dataSourceId) { - return await context.dataSource.opensearch.getClient(dataSourceId); - } - - return context.core.opensearch.legacy.client.callAsCurrentUser; + return dataSourceId + ? ((await context.dataSource.opensearch.legacy.getClient(dataSourceId) + .callAPI) as LegacyAPICaller) + : context.core.opensearch.legacy.client.callAsCurrentUser; }; diff --git a/src/plugins/data_source/server/client/client_pool.ts b/src/plugins/data_source/server/client/client_pool.ts index fe6f20e51ae4..612e9ff93708 100644 --- a/src/plugins/data_source/server/client/client_pool.ts +++ b/src/plugins/data_source/server/client/client_pool.ts @@ -4,13 +4,14 @@ */ import { Client } from '@opensearch-project/opensearch'; +import { Client as LegacyClient } from 'elasticsearch'; import LRUCache from 'lru-cache'; import { Logger } from 'src/core/server'; import { DataSourcePluginConfigType } from '../../config'; export interface OpenSearchClientPoolSetup { - getClientFromPool: (id: string) => Client | undefined; - addClientToPool: (endpoint: string, client: Client) => void; + getClientFromPool: (id: string) => Client | LegacyClient | undefined; + addClientToPool: (endpoint: string, client: Client | LegacyClient) => void; } /** @@ -21,14 +22,14 @@ export interface OpenSearchClientPoolSetup { */ export class OpenSearchClientPool { // LRU cache - // key: data source endpoint url - // value: OpenSearch client object - private cache?: LRUCache; + // key: data source endpoint, prefixed with identifier for legacy + // value: OpenSearch client object | Legacy client object + private cache?: LRUCache; private isClosed = false; constructor(private logger: Logger) {} - public async setup(config: DataSourcePluginConfigType) { + public async setup(config: DataSourcePluginConfigType): Promise { const logger = this.logger; const { size } = config.clientPool; @@ -53,7 +54,7 @@ export class OpenSearchClientPool { return this.cache!.get(endpoint); }; - const addClientToPool = (endpoint: string, client: Client) => { + const addClientToPool = (endpoint: string, client: Client | LegacyClient) => { this.cache!.set(endpoint, client); }; diff --git a/src/plugins/data_source/server/client/configure_client.ts b/src/plugins/data_source/server/client/configure_client.ts index e8af67809981..ea861deb2fed 100644 --- a/src/plugins/data_source/server/client/configure_client.ts +++ b/src/plugins/data_source/server/client/configure_client.ts @@ -5,7 +5,7 @@ import { Client } from '@opensearch-project/opensearch'; import { - Logger, + LegacyAPICaller, SavedObject, SavedObjectsClientContract, SavedObjectsErrorHelpers, @@ -18,17 +18,18 @@ import { import { DataSourceAttributes } from '../../common/data_sources'; import { DataSourcePluginConfigType } from '../../config'; import { CryptographyClient } from '../cryptography'; +import { DataSourceClientParams } from '../types'; import { parseClientOptions } from './client_config'; import { OpenSearchClientPoolSetup } from './client_pool'; -export const configureClient = async ( - dataSourceId: string, - savedObjects: SavedObjectsClientContract, - cryptographyClient: CryptographyClient, - openSearchClientPoolSetup: OpenSearchClientPoolSetup, - config: DataSourcePluginConfigType, - logger: Logger -): Promise => { +export const configureClient = async ({ + dataSourceId, + savedObjects, + config, + logger, + cryptographyClient, + openSearchClientPoolSetup, +}: DataSourceClientParams): Promise => { const dataSource = await getDataSource(dataSourceId, savedObjects); const rootClient = getRootClient(dataSource.attributes, config, openSearchClientPoolSetup); @@ -120,7 +121,7 @@ const getRootClient = ( const endpoint = dataSourceAttr.endpoint; const cachedClient = getClientFromPool(endpoint); if (cachedClient) { - return cachedClient; + return cachedClient as Client; } else { const clientOptions = parseClientOptions(config, endpoint); @@ -147,3 +148,36 @@ const getBasicAuthClient = ( headers: { authorization: null }, }); }; + +// const getLegacyClient = (client: Client, isLegacy: boolean, logger: Logger): Client => { +// if (isLegacy) { +// logger.warn( +// 'Legacy OpenSearch client is used. Please migrate your code to use the new OpenSearch client.' +// ); +// return new LegacyScopedClusterClient(callAsCurrentUser, callAsCurrentUser); +// } else { +// return client; +// } +// }; + +/** + * Calls specified endpoint with provided clientParams on behalf of the + * user initiated request to the OpenSearch Dashboards server (via HTTP request headers). + * See {@link LegacyAPICaller}. + * + * @param endpoint - String descriptor of the endpoint e.g. `cluster.getSettings` or `ping`. + * @param clientParams - A dictionary of parameters that will be passed directly to the OpenSearch JS client. + * @param options - Options that affect the way we call the API and process the result. + */ +// const callAsCurrentUser: LegacyAPICaller = async ( +// client: Client, +// endpoint: string, +// clientParams: Record = {}, +// options?: LegacyCallAPIOptions +// ) => { +// return await (callAPI.bind(null, client as EsClient) as LegacyAPICaller)( +// endpoint, +// clientParams, +// options +// ); +// }; diff --git a/src/plugins/data_source/server/data_source_service.ts b/src/plugins/data_source/server/data_source_service.ts index b7071962f5ab..01ba4558018e 100644 --- a/src/plugins/data_source/server/data_source_service.ts +++ b/src/plugins/data_source/server/data_source_service.ts @@ -3,17 +3,42 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Logger, OpenSearchClient, SavedObjectsClientContract } from '../../../../src/core/server'; +import { + LegacyCallAPIOptions, + Logger, + OpenSearchClient, + SavedObjectsClientContract, +} from '../../../../src/core/server'; import { DataSourcePluginConfigType } from '../config'; import { OpenSearchClientPool, configureClient } from './client'; import { CryptographyClient } from './cryptography'; +import { configureLegacyClient } from './legacy'; + +export interface LegacyClientCallAPIParams { + endpoint: string; + clientParams?: Record; + options?: LegacyCallAPIOptions; +} + +export interface DataSourceClientParams { + dataSourceId: string; + // this saved objects client is used to fetch data source on behalf of users, caller should pass scoped saved objects client + savedObjects: SavedObjectsClientContract; + cryptographyClient: CryptographyClient; +} + export interface DataSourceServiceSetup { - getDataSourceClient: ( - dataSourceId: string, - // this saved objects client is used to fetch data source on behalf of users, caller should pass scoped saved objects client - savedObjects: SavedObjectsClientContract, - cryptographyClient: CryptographyClient - ) => Promise; + getDataSourceClient: (params: DataSourceClientParams) => Promise; + + getDataSourceLegacyClient: ( + params: DataSourceClientParams + ) => { + callAPI: ( + endpoint: string, + clientParams?: Record, + options?: LegacyCallAPIOptions + ) => Promise; + }; } export class DataSourceService { private readonly openSearchClientPool: OpenSearchClientPool; @@ -22,25 +47,40 @@ export class DataSourceService { this.openSearchClientPool = new OpenSearchClientPool(logger); } - async setup(config: DataSourcePluginConfigType) { - const openSearchClientPoolSetup = await this.openSearchClientPool.setup(config); + async setup(config: DataSourcePluginConfigType): Promise { + const basicParams = { + openSearchClientPoolSetup: await this.openSearchClientPool.setup(config), + config, + logger: this.logger, + }; const getDataSourceClient = async ( - dataSourceId: string, - savedObjects: SavedObjectsClientContract, - cryptographyClient: CryptographyClient + dataSourceClientParams: DataSourceClientParams ): Promise => { - return configureClient( - dataSourceId, - savedObjects, - cryptographyClient, - openSearchClientPoolSetup, - config, - this.logger - ); + return configureClient({ + ...basicParams, + ...dataSourceClientParams, + }); + }; + + const getDataSourceLegacyClient = (dataSourceClientParams: DataSourceClientParams) => { + return { + callAPI: ( + endpoint: string, + clientParams?: Record, + options?: LegacyCallAPIOptions + ) => + configureLegacyClient( + { + ...basicParams, + ...dataSourceClientParams, + }, + { endpoint, clientParams, options } + ), + }; }; - return { getDataSourceClient }; + return { getDataSourceClient, getDataSourceLegacyClient }; } start() {} diff --git a/src/plugins/data_source/server/legacy/client_config.ts b/src/plugins/data_source/server/legacy/client_config.ts new file mode 100644 index 000000000000..d9b1cc704e3a --- /dev/null +++ b/src/plugins/data_source/server/legacy/client_config.ts @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ConfigOptions } from 'elasticsearch'; +import { DataSourcePluginConfigType } from '../../config'; + +/** + * Parse the client options from given data source config and endpoint + * + * @param config The config to generate the client options from. + * @param endpoint endpoint url of data source + */ +export function parseClientOptions( + // TODO: will use client configs, that comes from a merge result of user config and default legacy client config, + config: DataSourcePluginConfigType, + endpoint: string +): ConfigOptions { + const configOptions: ConfigOptions = { + host: endpoint, + ssl: { + rejectUnauthorized: true, + }, + }; + + return configOptions; +} diff --git a/src/plugins/data_source/server/legacy/configure_legacy_client.ts b/src/plugins/data_source/server/legacy/configure_legacy_client.ts new file mode 100644 index 000000000000..79ce3baf8bfd --- /dev/null +++ b/src/plugins/data_source/server/legacy/configure_legacy_client.ts @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Client } from 'elasticsearch'; +// eslint-disable-next-line @osd/eslint/no-restricted-paths +import { callAPI } from '../../../../../src/core/server/opensearch/legacy/cluster_client'; +import { + Headers, + LegacyAPICaller, + SavedObject, + SavedObjectsClientContract, +} from '../../../../../src/core/server'; +import { UsernamePasswordTypedContent } from '../../common/credentials/types'; +import { DataSourceAttributes } from '../../common/data_sources'; +import { DataSourcePluginConfigType } from '../../config'; +import { CryptographyClient } from '../cryptography'; +import { DataSourceClientParams } from '../types'; +import { OpenSearchClientPoolSetup } from '../client/client_pool'; +import { getCredential, getDataSource } from '../client/configure_client'; +import { LegacyClientCallAPIParams } from '../data_source_service'; +import { parseClientOptions } from './client_config'; + +const LEGACY_CLIENT_PREFIX = 'legacy'; + +export const configureLegacyClient = async ( + { + dataSourceId, + savedObjects, + config, + logger, + cryptographyClient, + openSearchClientPoolSetup, + }: DataSourceClientParams, + callApiParams: LegacyClientCallAPIParams +) => { + const dataSource = await getDataSource(dataSourceId, savedObjects); + const rootClient = getRootClient(dataSource.attributes, config, openSearchClientPoolSetup); + const queryClient = await getQueryClient( + rootClient, + dataSource, + savedObjects, + cryptographyClient, + callApiParams + ); + return queryClient; +}; + +/** + * Create a child client object with given auth info. + * + * @param rootClient root client for the connection with given data source endpoint. + * @param dataSource data source saved object + * @param savedObjects scoped saved object client + * @returns child client. + */ +const getQueryClient = async ( + rootClient: Client, + dataSource: SavedObject, + savedObjects: SavedObjectsClientContract, + cryptographyClient: CryptographyClient, + callApiParams: LegacyClientCallAPIParams +) => { + if (dataSource.attributes.noAuth) { + return legacyClientWrapper(rootClient, callApiParams); + } else { + const credential = await getCredential( + dataSource.references[0].id, + savedObjects, + cryptographyClient + ); + return legacyClientWrapper(rootClient, callApiParams, credential); + } +}; + +/** + * Gets a root client object of the OpenSearch endpoint. + * Will attempt to get from cache, if cache miss, create a new one and load into cache. + * + * @param dataSourceAttr data source saved objects attributes. + * @param config data source config + * @returns OpenSearch client for the given data source endpoint. + */ +const getRootClient = ( + dataSourceAttr: DataSourceAttributes, + config: DataSourcePluginConfigType, + { getClientFromPool, addClientToPool }: OpenSearchClientPoolSetup +): Client => { + const endpoint = dataSourceAttr.endpoint; + const identifier = LEGACY_CLIENT_PREFIX + endpoint; + const cachedClient = getClientFromPool(identifier); + if (cachedClient) { + return cachedClient as Client; + } else { + const configOptions = parseClientOptions(config, endpoint); + const client = new Client(configOptions); + addClientToPool(identifier, client); + + return client; + } +}; + +const legacyClientWrapper = async ( + rootClient: Client, + { endpoint, clientParams = {}, options }: LegacyClientCallAPIParams, + credential?: UsernamePasswordTypedContent +) => { + if (credential) { + const headers: Headers = { + authorization: + 'Basic ' + Buffer.from(`${credential.username}:${credential.password}`).toString('base64'), + }; + clientParams.headers = Object.assign({}, clientParams.headers, headers); + } + + return await (callAPI.bind(null, rootClient) as LegacyAPICaller)(endpoint, clientParams, options); +}; diff --git a/src/plugins/data_source/server/legacy/index.ts b/src/plugins/data_source/server/legacy/index.ts new file mode 100644 index 000000000000..b7c8d7a26ad7 --- /dev/null +++ b/src/plugins/data_source/server/legacy/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { configureLegacyClient } from './configure_legacy_client'; diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index 49d5418c86bc..674af932c5b7 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -15,6 +15,7 @@ import { Logger, IContextProvider, RequestHandler, + LegacyAPICaller, } from '../../../../src/core/server'; import { DataSourceService, DataSourceServiceSetup } from './data_source_service'; import { DataSourcePluginSetup, DataSourcePluginStart } from './types'; @@ -91,11 +92,11 @@ export class DataSourcePlugin implements Plugin { try { - return dataSourceService.getDataSourceClient( + return dataSourceService.getDataSourceClient({ dataSourceId, - context.core.savedObjects.client, - cryptographyClient - ); + savedObjects: context.core.savedObjects.client, + cryptographyClient, + }); } catch (error: any) { logger.error( `Fail to get data source client for dataSourceId: [${dataSourceId}]. Detail: ${error.messages}` @@ -103,6 +104,22 @@ export class DataSourcePlugin implements Plugin { + try { + return dataSourceService.getDataSourceLegacyClient({ + dataSourceId, + savedObjects: context.core.savedObjects.client, + cryptographyClient, + }); + } catch (error: any) { + logger.error( + `Fail to get legacy data source client for dataSourceId: [${dataSourceId}]. Detail: ${error.messages}` + ); + throw new OpenSearchClientError(error.message); + } + }, + }, }, }; }; diff --git a/src/plugins/data_source/server/types.ts b/src/plugins/data_source/server/types.ts index bad309b4b871..330d93e0009f 100644 --- a/src/plugins/data_source/server/types.ts +++ b/src/plugins/data_source/server/types.ts @@ -3,11 +3,38 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { OpenSearchClient } from 'src/core/server'; +import { + LegacyCallAPIOptions, + Logger, + OpenSearchClient, + SavedObjectsClientContract, +} from 'src/core/server'; +import { DataSourcePluginConfigType } from '../config'; +import { OpenSearchClientPoolSetup } from './client/client_pool'; +import { CryptographyClient } from './cryptography'; +export interface DataSourceClientParams { + dataSourceId: string; + savedObjects: SavedObjectsClientContract; + cryptographyClient: CryptographyClient; + openSearchClientPoolSetup: OpenSearchClientPoolSetup; + config: DataSourcePluginConfigType; + logger: Logger; +} export interface DataSourcePluginRequestContext { opensearch: { getClient: (dataSourceId: string) => Promise; + legacy: { + getClient: ( + dataSourceId: string + ) => { + callAPI: ( + endpoint: string, + clientParams: Record, + options?: LegacyCallAPIOptions + ) => Promise; + }; + }; }; } declare module 'src/core/server' { diff --git a/src/plugins/index_pattern_management/server/routes/resolve_index.ts b/src/plugins/index_pattern_management/server/routes/resolve_index.ts index 510eeb367da8..6b5285f70692 100644 --- a/src/plugins/index_pattern_management/server/routes/resolve_index.ts +++ b/src/plugins/index_pattern_management/server/routes/resolve_index.ts @@ -29,7 +29,7 @@ */ import { schema } from '@osd/config-schema'; -import { IRouter } from 'src/core/server'; +import { IRouter, LegacyAPICaller } from 'src/core/server'; export function registerResolveIndexRoute(router: IRouter): void { router.get( @@ -59,25 +59,17 @@ export function registerResolveIndexRoute(router: IRouter): void { : null; const dataSourceId = req.query.data_source; - if (dataSourceId) { - const result = await ( - await context.dataSource.opensearch.getClient(dataSourceId) - ).indices.resolveIndex({ - name: encodeURIComponent(req.params.query), - expand_wildcards: req.query.expand_wildcards, - }); - return res.ok({ body: result.body }); - } + const caller = dataSourceId + ? ((await context.dataSource.opensearch.legacy.getClient(dataSourceId) + .callAPI) as LegacyAPICaller) + : context.core.opensearch.legacy.client.callAsCurrentUser; - const result = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'transport.request', - { - method: 'GET', - path: `/_resolve/index/${encodeURIComponent(req.params.query)}${ - queryString ? '?' + new URLSearchParams(queryString).toString() : '' - }`, - } - ); + const result = await caller('transport.request', { + method: 'GET', + path: `/_resolve/index/${encodeURIComponent(req.params.query)}${ + queryString ? '?' + new URLSearchParams(queryString).toString() : '' + }`, + }); return res.ok({ body: result }); } );