diff --git a/server/adaptors/integrations/__test__/builder.test.ts b/server/adaptors/integrations/__test__/builder.test.ts index 81af1d6f6..33aff3497 100644 --- a/server/adaptors/integrations/__test__/builder.test.ts +++ b/server/adaptors/integrations/__test__/builder.test.ts @@ -5,7 +5,7 @@ import { SavedObjectsClientContract } from '../../../../../../src/core/server'; import { IntegrationInstanceBuilder } from '../integrations_builder'; -import { Integration } from '../repository/integration'; +import { IntegrationReader } from '../repository/integration'; const mockSavedObjectsClient: SavedObjectsClientContract = ({ bulkCreate: jest.fn(), @@ -16,7 +16,7 @@ const mockSavedObjectsClient: SavedObjectsClientContract = ({ update: jest.fn(), } as unknown) as SavedObjectsClientContract; -const sampleIntegration: Integration = ({ +const sampleIntegration: IntegrationReader = ({ deepCheck: jest.fn().mockResolvedValue(true), getAssets: jest.fn().mockResolvedValue({ savedObjects: [ @@ -34,7 +34,7 @@ const sampleIntegration: Integration = ({ name: 'integration-template', type: 'integration-type', }), -} as unknown) as Integration; +} as unknown) as IntegrationReader; describe('IntegrationInstanceBuilder', () => { let builder: IntegrationInstanceBuilder; @@ -93,13 +93,23 @@ describe('IntegrationInstanceBuilder', () => { ], }; - // Mock the implementation of the methods in the Integration class - sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); - sampleIntegration.getAssets = jest.fn().mockResolvedValue({ savedObjects: remappedAssets }); - sampleIntegration.getConfig = jest.fn().mockResolvedValue({ + const mockTemplate: Partial = { name: 'integration-template', type: 'integration-type', - }); + assets: { + savedObjects: { + name: 'assets', + version: '1.0.0', + }, + }, + }; + + // Mock the implementation of the methods in the Integration class + sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: mockTemplate }); + sampleIntegration.getAssets = jest + .fn() + .mockResolvedValue({ ok: true, value: { savedObjects: remappedAssets } }); + sampleIntegration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: mockTemplate }); // Mock builder sub-methods const remapIDsSpy = jest.spyOn(builder, 'remapIDs'); @@ -121,22 +131,24 @@ describe('IntegrationInstanceBuilder', () => { dataSource: 'instance-datasource', name: 'instance-name', }; - sampleIntegration.deepCheck = jest.fn().mockResolvedValue(false); + sampleIntegration.deepCheck = jest + .fn() + .mockResolvedValue({ ok: false, error: new Error('Mock error') }); - await expect(builder.build(sampleIntegration, options)).rejects.toThrowError( - 'Integration is not valid' - ); + await expect(builder.build(sampleIntegration, options)).rejects.toThrowError('Mock error'); }); - it('should reject with an error if getAssets throws an error', async () => { + it('should reject with an error if getAssets rejects', async () => { const options = { dataSource: 'instance-datasource', name: 'instance-name', }; const errorMessage = 'Failed to get assets'; - sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); - sampleIntegration.getAssets = jest.fn().mockRejectedValue(new Error(errorMessage)); + sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: {} }); + sampleIntegration.getAssets = jest + .fn() + .mockResolvedValue({ ok: false, error: new Error(errorMessage) }); await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(errorMessage); }); @@ -153,22 +165,14 @@ describe('IntegrationInstanceBuilder', () => { }, ]; const errorMessage = 'Failed to post assets'; - sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); - sampleIntegration.getAssets = jest.fn().mockResolvedValue({ savedObjects: remappedAssets }); + sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: {} }); + sampleIntegration.getAssets = jest + .fn() + .mockResolvedValue({ ok: true, value: { savedObjects: remappedAssets } }); builder.postAssets = jest.fn().mockRejectedValue(new Error(errorMessage)); await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(errorMessage); }); - - it('should reject with an error if getConfig returns null', async () => { - const options = { - dataSource: 'instance-datasource', - name: 'instance-name', - }; - sampleIntegration.getConfig = jest.fn().mockResolvedValue(null); - - await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(); - }); }); describe('remapIDs', () => { @@ -264,8 +268,11 @@ describe('IntegrationInstanceBuilder', () => { it('should build an integration instance', async () => { const integration = { getConfig: jest.fn().mockResolvedValue({ - name: 'integration-template', - type: 'integration-type', + ok: true, + value: { + name: 'integration-template', + type: 'integration-type', + }, }), }; const refs = [ @@ -291,7 +298,7 @@ describe('IntegrationInstanceBuilder', () => { }; const instance = await builder.buildInstance( - (integration as unknown) as Integration, + (integration as unknown) as IntegrationReader, refs, options ); @@ -319,7 +326,7 @@ describe('IntegrationInstanceBuilder', () => { }; await expect( - builder.buildInstance((integration as unknown) as Integration, refs, options) + builder.buildInstance((integration as unknown) as IntegrationReader, refs, options) ).rejects.toThrowError(); }); }); diff --git a/server/adaptors/integrations/__test__/local_repository.test.ts b/server/adaptors/integrations/__test__/local_repository.test.ts index 6b0ddc72f..f1bfeb9b2 100644 --- a/server/adaptors/integrations/__test__/local_repository.test.ts +++ b/server/adaptors/integrations/__test__/local_repository.test.ts @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Repository } from '../repository/repository'; -import { Integration } from '../repository/integration'; +import { RepositoryReader } from '../repository/repository'; +import { IntegrationReader } from '../repository/integration'; import path from 'path'; import * as fs from 'fs/promises'; @@ -20,15 +20,25 @@ describe('The local repository', () => { return Promise.resolve(null); } // Otherwise, all directories must be integrations - const integ = new Integration(integPath); - await expect(integ.check()).resolves.toBe(true); + const integ = new IntegrationReader(integPath); + expect(integ.getConfig()).resolves.toHaveProperty('ok', true); }) ); }); it('Should pass deep validation for all local integrations.', async () => { - const repository: Repository = new Repository(path.join(__dirname, '../__data__/repository')); - const integrations: Integration[] = await repository.getIntegrationList(); - await Promise.all(integrations.map((i) => expect(i.deepCheck()).resolves.toBeTruthy())); + const repository: RepositoryReader = new RepositoryReader( + path.join(__dirname, '../__data__/repository') + ); + const integrations: IntegrationReader[] = await repository.getIntegrationList(); + await Promise.all( + integrations.map(async (i) => { + const result = await i.deepCheck(); + if (!result.ok) { + console.error(result.error); + } + expect(result.ok).toBe(true); + }) + ); }); }); diff --git a/server/adaptors/integrations/__test__/kibana_backend.test.ts b/server/adaptors/integrations/__test__/manager.test.ts similarity index 70% rename from server/adaptors/integrations/__test__/kibana_backend.test.ts rename to server/adaptors/integrations/__test__/manager.test.ts index 63d62764c..75b89e520 100644 --- a/server/adaptors/integrations/__test__/kibana_backend.test.ts +++ b/server/adaptors/integrations/__test__/manager.test.ts @@ -3,17 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IntegrationsKibanaBackend } from '../integrations_kibana_backend'; +import { IntegrationsManager } from '../integrations_manager'; import { SavedObject, SavedObjectsClientContract } from '../../../../../../src/core/server/types'; -import { Repository } from '../repository/repository'; +import { RepositoryReader } from '../repository/repository'; import { IntegrationInstanceBuilder } from '../integrations_builder'; -import { Integration } from '../repository/integration'; +import { IntegrationReader } from '../repository/integration'; import { SavedObjectsFindResponse } from '../../../../../../src/core/server'; describe('IntegrationsKibanaBackend', () => { let mockSavedObjectsClient: jest.Mocked; - let mockRepository: jest.Mocked; - let backend: IntegrationsKibanaBackend; + let mockRepository: jest.Mocked; + let backend: IntegrationsManager; beforeEach(() => { mockSavedObjectsClient = { @@ -26,7 +26,7 @@ describe('IntegrationsKibanaBackend', () => { getIntegration: jest.fn(), getIntegrationList: jest.fn(), } as any; - backend = new IntegrationsKibanaBackend(mockSavedObjectsClient, mockRepository); + backend = new IntegrationsManager(mockSavedObjectsClient, mockRepository); }); describe('deleteIntegrationInstance', () => { @@ -147,24 +147,28 @@ describe('IntegrationsKibanaBackend', () => { describe('getIntegrationTemplates', () => { it('should get integration templates by name', async () => { const query = { name: 'template1' }; - const integration = { getConfig: jest.fn().mockResolvedValue({ name: 'template1' }) }; - mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + const integration = { + getConfig: jest.fn().mockResolvedValue({ ok: true, value: { name: 'template1' } }), + }; + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); const result = await backend.getIntegrationTemplates(query); expect(mockRepository.getIntegration).toHaveBeenCalledWith(query.name); expect(integration.getConfig).toHaveBeenCalled(); - expect(result).toEqual({ hits: [await integration.getConfig()] }); + expect(result).toEqual({ hits: [{ name: 'template1' }] }); }); it('should get all integration templates', async () => { const integrationList = [ - { getConfig: jest.fn().mockResolvedValue({ name: 'template1' }) }, - { getConfig: jest.fn().mockResolvedValue(null) }, - { getConfig: jest.fn().mockResolvedValue({ name: 'template2' }) }, + { getConfig: jest.fn().mockResolvedValue({ ok: true, value: { name: 'template1' } }) }, + { getConfig: jest.fn().mockResolvedValue({ ok: false, error: new Error() }) }, + { getConfig: jest.fn().mockResolvedValue({ ok: true, value: { name: 'template2' } }) }, ]; mockRepository.getIntegrationList.mockResolvedValue( - (integrationList as unknown) as Integration[] + (integrationList as unknown) as IntegrationReader[] ); const result = await backend.getIntegrationTemplates(); @@ -174,7 +178,7 @@ describe('IntegrationsKibanaBackend', () => { expect(integrationList[1].getConfig).toHaveBeenCalled(); expect(integrationList[2].getConfig).toHaveBeenCalled(); expect(result).toEqual({ - hits: [await integrationList[0].getConfig(), await integrationList[2].getConfig()], + hits: [{ name: 'template1' }, { name: 'template2' }], }); }); }); @@ -224,7 +228,7 @@ describe('IntegrationsKibanaBackend', () => { build: jest.fn().mockResolvedValue({ name, dataset: 'nginx', namespace: 'prod' }), }; const createdInstance = { name, dataset: 'nginx', namespace: 'prod' }; - mockRepository.getIntegration.mockResolvedValue((template as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue((template as unknown) as IntegrationReader); mockSavedObjectsClient.create.mockResolvedValue(({ result: 'created', } as unknown) as SavedObject); @@ -263,7 +267,7 @@ describe('IntegrationsKibanaBackend', () => { build: jest.fn().mockRejectedValue(new Error('Failed to build instance')), }; backend.instanceBuilder = (instanceBuilder as unknown) as IntegrationInstanceBuilder; - mockRepository.getIntegration.mockResolvedValue((template as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue((template as unknown) as IntegrationReader); await expect( backend.loadIntegrationInstance(templateName, name, 'datasource') @@ -277,9 +281,11 @@ describe('IntegrationsKibanaBackend', () => { const staticPath = 'path/to/static'; const assetData = Buffer.from('asset data'); const integration = { - getStatic: jest.fn().mockResolvedValue(assetData), + getStatic: jest.fn().mockResolvedValue({ ok: true, value: assetData }), }; - mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); const result = await backend.getStatic(templateName, staticPath); @@ -288,7 +294,7 @@ describe('IntegrationsKibanaBackend', () => { expect(result).toEqual(assetData); }); - it('should reject with a 404 if asset is not found', async () => { + it('should reject with a 404 if integration is not found', async () => { const templateName = 'template1'; const staticPath = 'path/to/static'; mockRepository.getIntegration.mockResolvedValue(null); @@ -298,6 +304,136 @@ describe('IntegrationsKibanaBackend', () => { 404 ); }); + + it('should reject with a 404 if static data is not found', async () => { + const templateName = 'template1'; + const staticPath = 'path/to/static'; + mockRepository.getIntegration.mockResolvedValue({ + getStatic: jest.fn().mockResolvedValue({ + ok: false, + error: { message: 'Not found', code: 'ENOENT' }, + }), + } as any); + + await expect(backend.getStatic(templateName, staticPath)).rejects.toHaveProperty( + 'statusCode', + 404 + ); + }); + }); + + describe('getSchemas', () => { + it('should get schema data', async () => { + const templateName = 'template1'; + const schemaData = { mappings: { test: {} } }; + const integration = { + getSchemas: jest.fn().mockResolvedValue({ ok: true, value: schemaData }), + }; + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); + + const result = await backend.getSchemas(templateName); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(templateName); + expect(integration.getSchemas).toHaveBeenCalled(); + expect(result).toEqual(schemaData); + }); + + it('should reject with a 404 if integration is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue(null); + + await expect(backend.getSchemas(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + + it('should reject with a 404 if schema data is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue({ + getSchemas: jest.fn().mockResolvedValue({ + ok: false, + error: { message: 'Not found', code: 'ENOENT' }, + }), + } as any); + + await expect(backend.getSchemas(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + }); + + describe('getAssets', () => { + it('should get asset data', async () => { + const templateName = 'template1'; + const assetData = { savedObjects: [{ test: true }] }; + const integration = { + getAssets: jest.fn().mockResolvedValue({ ok: true, value: assetData }), + }; + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); + + const result = await backend.getAssets(templateName); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(templateName); + expect(integration.getAssets).toHaveBeenCalled(); + expect(result).toEqual(assetData); + }); + + it('should reject with a 404 if integration is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue(null); + + await expect(backend.getAssets(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + + it('should reject with a 404 if asset data is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue({ + getAssets: jest.fn().mockResolvedValue({ + ok: false, + error: { message: 'Not found', code: 'ENOENT' }, + }), + } as any); + + await expect(backend.getAssets(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + }); + + describe('getSampleData', () => { + it('should get sample data', async () => { + const templateName = 'template1'; + const sampleData = { sampleData: [{ test: true }] }; + const integration = { + getSampleData: jest.fn().mockResolvedValue({ ok: true, value: sampleData }), + }; + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); + + const result = await backend.getSampleData(templateName); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(templateName); + expect(integration.getSampleData).toHaveBeenCalled(); + expect(result).toEqual(sampleData); + }); + + it('should reject with a 404 if integration is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue(null); + + await expect(backend.getSampleData(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + + it('should reject with a 404 if sample data is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue({ + getSampleData: jest.fn().mockResolvedValue({ + ok: false, + error: { message: 'Not found', code: 'ENOENT' }, + }), + } as any); + + await expect(backend.getSampleData(templateName)).rejects.toHaveProperty('statusCode', 404); + }); }); describe('getAssetStatus', () => { diff --git a/server/adaptors/integrations/__test__/validators.test.ts b/server/adaptors/integrations/__test__/validators.test.ts index ba573c4c4..6c09b595b 100644 --- a/server/adaptors/integrations/__test__/validators.test.ts +++ b/server/adaptors/integrations/__test__/validators.test.ts @@ -3,9 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { validateTemplate, validateInstance, ValidationResult } from '../validators'; +import { validateTemplate, validateInstance } from '../validators'; -const validTemplate: IntegrationTemplate = { +const validTemplate: IntegrationConfig = { name: 'test', version: '1.0.0', license: 'Apache-2.0', @@ -29,7 +29,7 @@ const validInstance: IntegrationInstance = { describe('validateTemplate', () => { it('Returns a success value for a valid Integration Template', () => { - const result: ValidationResult = validateTemplate(validTemplate); + const result: Result = validateTemplate(validTemplate); expect(result.ok).toBe(true); expect((result as any).value).toBe(validTemplate); }); @@ -38,7 +38,7 @@ describe('validateTemplate', () => { const sample: any = structuredClone(validTemplate); sample.license = undefined; - const result: ValidationResult = validateTemplate(sample); + const result: Result = validateTemplate(sample); expect(result.ok).toBe(false); expect((result as any).error).toBeInstanceOf(Error); @@ -48,7 +48,7 @@ describe('validateTemplate', () => { const sample: any = structuredClone(validTemplate); sample.components[0].name = 'not-logs'; - const result: ValidationResult = validateTemplate(sample); + const result: Result = validateTemplate(sample); expect(result.ok).toBe(false); expect((result as any).error).toBeInstanceOf(Error); @@ -62,7 +62,7 @@ describe('validateTemplate', () => { describe('validateInstance', () => { it('Returns true for a valid Integration Instance', () => { - const result: ValidationResult = validateInstance(validInstance); + const result: Result = validateInstance(validInstance); expect(result.ok).toBe(true); expect((result as any).value).toBe(validInstance); }); @@ -71,7 +71,7 @@ describe('validateInstance', () => { const sample: any = structuredClone(validInstance); sample.templateName = undefined; - const result: ValidationResult = validateInstance(sample); + const result: Result = validateInstance(sample); expect(result.ok).toBe(false); expect((result as any).error).toBeInstanceOf(Error); diff --git a/server/adaptors/integrations/integrations_adaptor.ts b/server/adaptors/integrations/integrations_adaptor.ts index cf7f4853e..574a4d25d 100644 --- a/server/adaptors/integrations/integrations_adaptor.ts +++ b/server/adaptors/integrations/integrations_adaptor.ts @@ -24,9 +24,7 @@ export interface IntegrationsAdaptor { getStatic: (templateName: string, path: string) => Promise; - getSchemas: ( - templateName: string - ) => Promise<{ mappings: { [key: string]: unknown }; schemas: { [key: string]: unknown } }>; + getSchemas: (templateName: string) => Promise<{ mappings: { [key: string]: unknown } }>; getAssets: (templateName: string) => Promise<{ savedObjects?: unknown }>; diff --git a/server/adaptors/integrations/integrations_builder.ts b/server/adaptors/integrations/integrations_builder.ts index b12e1a132..7a8026cea 100644 --- a/server/adaptors/integrations/integrations_builder.ts +++ b/server/adaptors/integrations/integrations_builder.ts @@ -6,7 +6,7 @@ import { v4 as uuidv4 } from 'uuid'; import { uuidRx } from 'public/components/custom_panels/redux/panel_slice'; import { SavedObjectsClientContract } from '../../../../../src/core/server'; -import { Integration } from './repository/integration'; +import { IntegrationReader } from './repository/integration'; import { SavedObjectsBulkCreateObject } from '../../../../../src/core/public'; interface BuilderOptions { @@ -21,15 +21,21 @@ export class IntegrationInstanceBuilder { this.client = client; } - async build(integration: Integration, options: BuilderOptions): Promise { + build(integration: IntegrationReader, options: BuilderOptions): Promise { const instance = integration .deepCheck() .then((result) => { - if (!result) { - return Promise.reject(new Error('Integration is not valid')); + if (!result.ok) { + return Promise.reject(result.error); } + return integration.getAssets(); + }) + .then((assets) => { + if (!assets.ok) { + return Promise.reject(assets.error); + } + return assets.value; }) - .then(() => integration.getAssets()) .then((assets) => this.remapIDs(assets.savedObjects!)) .then((assets) => this.remapDataSource(assets, options.dataSource)) .then((assets) => this.postAssets(assets)) @@ -86,14 +92,19 @@ export class IntegrationInstanceBuilder { } async buildInstance( - integration: Integration, + integration: IntegrationReader, refs: AssetReference[], options: BuilderOptions ): Promise { - const config: IntegrationTemplate = (await integration.getConfig())!; + const config: Result = await integration.getConfig(); + if (!config.ok) { + return Promise.reject( + new Error('Attempted to create instance with invalid template', config.error) + ); + } return Promise.resolve({ name: options.name, - templateName: config.name, + templateName: config.value.name, dataSource: options.dataSource, creationDate: new Date().toISOString(), assets: refs, diff --git a/server/adaptors/integrations/integrations_kibana_backend.ts b/server/adaptors/integrations/integrations_manager.ts similarity index 69% rename from server/adaptors/integrations/integrations_kibana_backend.ts rename to server/adaptors/integrations/integrations_manager.ts index f28c883ec..d365e48ee 100644 --- a/server/adaptors/integrations/integrations_kibana_backend.ts +++ b/server/adaptors/integrations/integrations_manager.ts @@ -4,20 +4,21 @@ */ import path from 'path'; -import { addRequestToMetric } from '../../../server/common/metrics/metrics_helper'; +import { addRequestToMetric } from '../../common/metrics/metrics_helper'; import { IntegrationsAdaptor } from './integrations_adaptor'; import { SavedObject, SavedObjectsClientContract } from '../../../../../src/core/server/types'; import { IntegrationInstanceBuilder } from './integrations_builder'; -import { Repository } from './repository/repository'; +import { RepositoryReader } from './repository/repository'; -export class IntegrationsKibanaBackend implements IntegrationsAdaptor { +export class IntegrationsManager implements IntegrationsAdaptor { client: SavedObjectsClientContract; instanceBuilder: IntegrationInstanceBuilder; - repository: Repository; + repository: RepositoryReader; - constructor(client: SavedObjectsClientContract, repository?: Repository) { + constructor(client: SavedObjectsClientContract, repository?: RepositoryReader) { this.client = client; - this.repository = repository ?? new Repository(path.join(__dirname, '__data__/repository')); + this.repository = + repository ?? new RepositoryReader(path.join(__dirname, '__data__/repository')); this.instanceBuilder = new IntegrationInstanceBuilder(this.client); } @@ -53,17 +54,33 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { return result; }; + // Internal; use getIntegrationTemplates. + _getAllIntegrationTemplates = async (): Promise => { + const integrationList = await this.repository.getIntegrationList(); + const configResults = await Promise.all(integrationList.map((x) => x.getConfig())); + const configs = configResults.filter((cfg) => cfg.ok) as Array<{ value: IntegrationConfig }>; + return Promise.resolve({ hits: configs.map((cfg) => cfg.value) }); + }; + + // Internal; use getIntegrationTemplates. + _getIntegrationTemplatesByName = async ( + name: string + ): Promise => { + const integration = await this.repository.getIntegration(name); + const config = await integration?.getConfig(); + if (!config || !config.ok) { + return Promise.resolve({ hits: [] }); + } + return Promise.resolve({ hits: [config.value] }); + }; + getIntegrationTemplates = async ( query?: IntegrationTemplateQuery ): Promise => { if (query?.name) { - const integration = await this.repository.getIntegration(query.name); - const config = await integration?.getConfig(); - return Promise.resolve({ hits: config ? [config] : [] }); + return this._getIntegrationTemplatesByName(query.name); } - const integrationList = await this.repository.getIntegrationList(); - const configList = await Promise.all(integrationList.map((x) => x.getConfig())); - return Promise.resolve({ hits: configList.filter((x) => x !== null) as IntegrationTemplate[] }); + return this._getAllIntegrationTemplates(); }; getIntegrationInstances = async ( @@ -159,17 +176,25 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { }; getStatic = async (templateName: string, staticPath: string): Promise => { - const data = await (await this.repository.getIntegration(templateName))?.getStatic(staticPath); - if (!data) { + const integration = await this.repository.getIntegration(templateName); + if (integration === null) { return Promise.reject({ - message: `Asset ${staticPath} not found`, + message: `Template ${templateName} not found`, statusCode: 404, }); } - return Promise.resolve(data); + const data = await integration.getStatic(staticPath); + if (data.ok) { + return data.value; + } + const is404 = (data.error as { code?: string }).code === 'ENOENT'; + return Promise.reject({ + message: data.error.message, + statusCode: is404 ? 404 : 500, + }); }; - getSchemas = async (templateName: string): Promise => { + getSchemas = async (templateName: string): Promise<{ mappings: { [key: string]: unknown } }> => { const integration = await this.repository.getIntegration(templateName); if (integration === null) { return Promise.reject({ @@ -177,7 +202,15 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { statusCode: 404, }); } - return Promise.resolve(integration.getSchemas()); + const result = await integration.getSchemas(); + if (result.ok) { + return result.value; + } + const is404 = (result.error as { code?: string }).code === 'ENOENT'; + return Promise.reject({ + message: result.error.message, + statusCode: is404 ? 404 : 500, + }); }; getAssets = async (templateName: string): Promise<{ savedObjects?: any }> => { @@ -188,7 +221,15 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { statusCode: 404, }); } - return Promise.resolve(integration.getAssets()); + const assets = await integration.getAssets(); + if (assets.ok) { + return assets.value; + } + const is404 = (assets.error as { code?: string }).code === 'ENOENT'; + return Promise.reject({ + message: assets.error.message, + statusCode: is404 ? 404 : 500, + }); }; getSampleData = async (templateName: string): Promise<{ sampleData: object[] | null }> => { @@ -199,6 +240,14 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { statusCode: 404, }); } - return Promise.resolve(integration.getSampleData()); + const sampleData = await integration.getSampleData(); + if (sampleData.ok) { + return sampleData.value; + } + const is404 = (sampleData.error as { code?: string }).code === 'ENOENT'; + return Promise.reject({ + message: sampleData.error.message, + statusCode: is404 ? 404 : 500, + }); }; } diff --git a/server/adaptors/integrations/repository/__test__/integration.test.ts b/server/adaptors/integrations/repository/__test__/integration.test.ts index 2002ad04a..7ffbb176b 100644 --- a/server/adaptors/integrations/repository/__test__/integration.test.ts +++ b/server/adaptors/integrations/repository/__test__/integration.test.ts @@ -4,15 +4,16 @@ */ import * as fs from 'fs/promises'; -import { Integration } from '../integration'; +import { IntegrationReader } from '../integration'; import { Dirent, Stats } from 'fs'; import * as path from 'path'; +import { FileSystemCatalogDataAdaptor } from '../fs_data_adaptor'; jest.mock('fs/promises'); describe('Integration', () => { - let integration: Integration; - const sampleIntegration: IntegrationTemplate = { + let integration: IntegrationReader; + const sampleIntegration: IntegrationConfig = { name: 'sample', version: '2.0.0', license: 'Apache-2.0', @@ -32,36 +33,8 @@ describe('Integration', () => { }; beforeEach(() => { - integration = new Integration('./sample'); - }); - - describe('check', () => { - it('should return false if the directory does not exist', async () => { - const spy = jest.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => false } as Stats); - - const result = await integration.check(); - - expect(spy).toHaveBeenCalled(); - expect(result).toBe(false); - }); - - it('should return true if the directory exists and getConfig returns a valid template', async () => { - jest.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => true } as Stats); - integration.getConfig = jest.fn().mockResolvedValue(sampleIntegration); - - const result = await integration.check(); - - expect(result).toBe(true); - }); - - it('should return false if the directory exists but getConfig returns null', async () => { - jest.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => true } as Stats); - integration.getConfig = jest.fn().mockResolvedValue(null); - - const result = await integration.check(); - - expect(result).toBe(false); - }); + integration = new IntegrationReader('./sample'); + jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); }); describe('getLatestVersion', () => { @@ -94,39 +67,46 @@ describe('Integration', () => { }); describe('getConfig', () => { + it('should return an error if the directory does not exist', async () => { + const spy = jest + .spyOn(fs, 'lstat') + .mockResolvedValueOnce({ isDirectory: () => false } as Stats); + + const result = await integration.getConfig(); + + expect(spy).toHaveBeenCalled(); + expect(result.ok).toBe(false); + }); + it('should return the parsed config template if it is valid', async () => { jest.spyOn(fs, 'readFile').mockResolvedValue(JSON.stringify(sampleIntegration)); + jest.spyOn(fs, 'lstat').mockResolvedValueOnce({ isDirectory: () => true } as Stats); const result = await integration.getConfig(sampleIntegration.version); - expect(result).toEqual(sampleIntegration); + expect(result).toEqual({ ok: true, value: sampleIntegration }); }); - it('should return null and log validation errors if the config template is invalid', async () => { + it('should return an error if the config template is invalid', async () => { const invalidTemplate = { ...sampleIntegration, version: 2 }; jest.spyOn(fs, 'readFile').mockResolvedValue(JSON.stringify(invalidTemplate)); - const logValidationErrorsMock = jest.spyOn(console, 'error'); const result = await integration.getConfig(sampleIntegration.version); - expect(result).toBeNull(); - expect(logValidationErrorsMock).toHaveBeenCalled(); + expect(result.ok).toBe(false); }); - it('should return null and log syntax errors if the config file has syntax errors', async () => { + it('should return an error if the config file has syntax errors', async () => { jest.spyOn(fs, 'readFile').mockResolvedValue('Invalid JSON'); - const logSyntaxErrorsMock = jest.spyOn(console, 'error'); const result = await integration.getConfig(sampleIntegration.version); - expect(result).toBeNull(); - expect(logSyntaxErrorsMock).toHaveBeenCalledWith(expect.any(String), expect.any(SyntaxError)); + expect(result.ok).toBe(false); }); - it('should return null and log errors if the integration config does not exist', async () => { - integration.directory = './non-existing-directory'; - const logErrorsMock = jest.spyOn(console, 'error'); - jest.spyOn(fs, 'readFile').mockImplementation((..._args) => { + it('should return an error if the integration config does not exist', async () => { + integration.directory = './empty-directory'; + const readFileMock = jest.spyOn(fs, 'readFile').mockImplementation((..._args) => { // Can't find any information on how to mock an actual file not found error, // But at least according to the current implementation this should be equivalent. const error: any = new Error('ENOENT: File not found'); @@ -136,37 +116,38 @@ describe('Integration', () => { const result = await integration.getConfig(sampleIntegration.version); - expect(jest.spyOn(fs, 'readFile')).toHaveBeenCalled(); - expect(logErrorsMock).toHaveBeenCalledWith(expect.any(String)); - expect(result).toBeNull(); + expect(readFileMock).toHaveBeenCalled(); + expect(result.ok).toBe(false); }); }); describe('getAssets', () => { it('should return linked saved object assets when available', async () => { - integration.getConfig = jest.fn().mockResolvedValue(sampleIntegration); + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleIntegration }); jest.spyOn(fs, 'readFile').mockResolvedValue('{"name":"asset1"}\n{"name":"asset2"}'); const result = await integration.getAssets(sampleIntegration.version); - expect(result.savedObjects).toEqual([{ name: 'asset1' }, { name: 'asset2' }]); + expect(result.ok).toBe(true); + expect((result as any).value.savedObjects).toStrictEqual([ + { name: 'asset1' }, + { name: 'asset2' }, + ]); }); - it('should reject a return if the provided version has no config', async () => { - integration.getConfig = jest.fn().mockResolvedValue(null); + it('should return an error if the provided version has no config', async () => { + integration.getConfig = jest.fn().mockResolvedValue({ ok: false, error: new Error() }); - expect(integration.getAssets()).rejects.toThrowError(); + expect(integration.getAssets()).resolves.toHaveProperty('ok', false); }); - it('should log an error if the saved object assets are invalid', async () => { - const logErrorsMock = jest.spyOn(console, 'error'); - integration.getConfig = jest.fn().mockResolvedValue(sampleIntegration); + it('should return an error if the saved object assets are invalid', async () => { + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleIntegration }); jest.spyOn(fs, 'readFile').mockResolvedValue('{"unclosed":'); const result = await integration.getAssets(sampleIntegration.version); - expect(logErrorsMock).toHaveBeenCalledWith(expect.any(String), expect.any(Error)); - expect(result.savedObjects).toBeUndefined(); + expect(result.ok).toBe(false); }); }); @@ -178,7 +159,7 @@ describe('Integration', () => { { name: 'component2', version: '2.0.0' }, ], }; - integration.getConfig = jest.fn().mockResolvedValue(sampleConfig); + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleConfig }); const mappingFile1 = 'component1-1.0.0.mapping.json'; const mappingFile2 = 'component2-2.0.0.mapping.json'; @@ -190,7 +171,8 @@ describe('Integration', () => { const result = await integration.getSchemas(); - expect(result).toEqual({ + expect(result.ok).toBe(true); + expect((result as any).value).toStrictEqual({ mappings: { component1: { mapping: 'mapping1' }, component2: { mapping: 'mapping2' }, @@ -207,22 +189,20 @@ describe('Integration', () => { ); }); - it('should reject with an error if the config is null', async () => { - integration.getConfig = jest.fn().mockResolvedValue(null); + it('should reject with an error if the config is invalid', async () => { + integration.getConfig = jest.fn().mockResolvedValue({ ok: false, error: new Error() }); - await expect(integration.getSchemas()).rejects.toThrowError( - 'Attempted to get assets of invalid config' - ); + await expect(integration.getSchemas()).resolves.toHaveProperty('ok', false); }); it('should reject with an error if a mapping file is invalid', async () => { const sampleConfig = { components: [{ name: 'component1', version: '1.0.0' }], }; - integration.getConfig = jest.fn().mockResolvedValue(sampleConfig); + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleConfig }); jest.spyOn(fs, 'readFile').mockRejectedValueOnce(new Error('Could not load schema')); - await expect(integration.getSchemas()).rejects.toThrowError('Could not load schema'); + await expect(integration.getSchemas()).resolves.toHaveProperty('ok', false); }); }); @@ -231,21 +211,56 @@ describe('Integration', () => { const readFileMock = jest .spyOn(fs, 'readFile') .mockResolvedValue(Buffer.from('logo data', 'ascii')); - expect(await integration.getStatic('/logo.png')).toStrictEqual( - Buffer.from('logo data', 'ascii') - ); + + const result = await integration.getStatic('logo.png'); + + expect(result.ok).toBe(true); + expect((result as any).value).toStrictEqual(Buffer.from('logo data', 'ascii')); expect(readFileMock).toBeCalledWith(path.join('sample', 'static', 'logo.png')); }); - it('should return null and log an error if the static file is not found', async () => { - const logErrorsMock = jest.spyOn(console, 'error'); + it('should return an error if the static file is not found', async () => { jest.spyOn(fs, 'readFile').mockImplementation((..._args) => { const error: any = new Error('ENOENT: File not found'); error.code = 'ENOENT'; return Promise.reject(error); }); - expect(await integration.getStatic('/logo.png')).toBeNull(); - expect(logErrorsMock).toBeCalledWith(expect.any(String)); + expect(integration.getStatic('/logo.png')).resolves.toHaveProperty('ok', false); + }); + }); + + describe('getSampleData', () => { + it('should return sample data', async () => { + const sampleConfig = { sampleData: { path: 'sample.json' } }; + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleConfig }); + const readFileMock = jest.spyOn(fs, 'readFile').mockResolvedValue('[{"sample": true}]'); + + const result = await integration.getSampleData(); + + expect(result.ok).toBe(true); + expect((result as any).value.sampleData).toStrictEqual([{ sample: true }]); + expect(readFileMock).toBeCalledWith(path.join('sample', 'data', 'sample.json'), { + encoding: 'utf-8', + }); + }); + + it("should return null if there's no sample data", async () => { + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: {} }); + + const result = await integration.getSampleData(); + + expect(result.ok).toBe(true); + expect((result as any).value.sampleData).toBeNull(); + }); + + it('should catch and fail gracefully on invalid sample data', async () => { + const sampleConfig = { sampleData: { path: 'sample.json' } }; + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleConfig }); + jest.spyOn(fs, 'readFile').mockResolvedValue('[{"closingBracket": false]'); + + const result = await integration.getSampleData(); + + expect(result.ok).toBe(false); }); }); }); diff --git a/server/adaptors/integrations/repository/__test__/repository.test.ts b/server/adaptors/integrations/repository/__test__/repository.test.ts index 913968f49..d66fc5e86 100644 --- a/server/adaptors/integrations/repository/__test__/repository.test.ts +++ b/server/adaptors/integrations/repository/__test__/repository.test.ts @@ -4,36 +4,33 @@ */ import * as fs from 'fs/promises'; -import { Repository } from '../repository'; -import { Integration } from '../integration'; +import { RepositoryReader } from '../repository'; +import { IntegrationReader } from '../integration'; import { Dirent, Stats } from 'fs'; import path from 'path'; jest.mock('fs/promises'); describe('Repository', () => { - let repository: Repository; + let repository: RepositoryReader; beforeEach(() => { - repository = new Repository('path/to/directory'); + repository = new RepositoryReader('path/to/directory'); }); describe('getIntegrationList', () => { it('should return an array of Integration instances', async () => { - // Mock fs.readdir to return a list of folders jest.spyOn(fs, 'readdir').mockResolvedValue((['folder1', 'folder2'] as unknown) as Dirent[]); - - // Mock fs.lstat to return a directory status jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); - - // Mock Integration check method to always return true - jest.spyOn(Integration.prototype, 'check').mockResolvedValue(true); + jest + .spyOn(IntegrationReader.prototype, 'getConfig') + .mockResolvedValue({ ok: true, value: {} as any }); const integrations = await repository.getIntegrationList(); expect(integrations).toHaveLength(2); - expect(integrations[0]).toBeInstanceOf(Integration); - expect(integrations[1]).toBeInstanceOf(Integration); + expect(integrations[0]).toBeInstanceOf(IntegrationReader); + expect(integrations[1]).toBeInstanceOf(IntegrationReader); }); it('should filter out null values from the integration list', async () => { @@ -41,19 +38,21 @@ describe('Repository', () => { // Mock fs.lstat to return a mix of directories and files jest.spyOn(fs, 'lstat').mockImplementation(async (toLstat) => { - if (toLstat === path.join('path', 'to', 'directory', 'folder1')) { + if (toLstat.toString().startsWith(path.join('path', 'to', 'directory', 'folder1'))) { return { isDirectory: () => true } as Stats; } else { return { isDirectory: () => false } as Stats; } }); - jest.spyOn(Integration.prototype, 'check').mockResolvedValue(true); + jest + .spyOn(IntegrationReader.prototype, 'getConfig') + .mockResolvedValue({ ok: true, value: {} as any }); const integrations = await repository.getIntegrationList(); expect(integrations).toHaveLength(1); - expect(integrations[0]).toBeInstanceOf(Integration); + expect(integrations[0]).toBeInstanceOf(IntegrationReader); }); it('should handle errors and return an empty array', async () => { @@ -67,15 +66,20 @@ describe('Repository', () => { describe('getIntegration', () => { it('should return an Integration instance if it exists and passes the check', async () => { - jest.spyOn(Integration.prototype, 'check').mockResolvedValue(true); + jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); + jest + .spyOn(IntegrationReader.prototype, 'getConfig') + .mockResolvedValue({ ok: true, value: {} as any }); const integration = await repository.getIntegration('integrationName'); - expect(integration).toBeInstanceOf(Integration); + expect(integration).toBeInstanceOf(IntegrationReader); }); - it('should return null if the integration does not exist or fails the check', async () => { - jest.spyOn(Integration.prototype, 'check').mockResolvedValue(false); + it('should return null if the integration does not exist or fails checks', async () => { + jest + .spyOn(IntegrationReader.prototype, 'getConfig') + .mockResolvedValue({ ok: false, error: new Error() }); const integration = await repository.getIntegration('invalidIntegration'); diff --git a/server/adaptors/integrations/repository/catalog_data_adaptor.ts b/server/adaptors/integrations/repository/catalog_data_adaptor.ts new file mode 100644 index 000000000..6373fee4d --- /dev/null +++ b/server/adaptors/integrations/repository/catalog_data_adaptor.ts @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +type IntegrationPart = 'assets' | 'data' | 'schemas' | 'static'; + +interface CatalogDataAdaptor { + /** + * Reads a Json or NDJson file from the data source. + * + * @param filename The name of the file to read. + * @param type Optional. The type of integration part to read ('assets', 'data', 'schemas', or 'static'). + * @returns A Promise that resolves with the content of the file as a string. + */ + readFile: (filename: string, type?: IntegrationPart) => Promise>; + + /** + * Reads a file from the data source as raw binary data. + * + * @param filename The name of the file to read. + * @param type Optional. The type of integration part to read ('assets', 'data', 'schemas', or 'static'). + * @returns A Promise that resolves with the content of the file as a Buffer. + */ + readFileRaw: (filename: string, type?: IntegrationPart) => Promise>; + + /** + * Reads the contents of a repository directory from the data source to find integrations. + * + * @param dirname The name of the directory to read. + * @returns A Promise that resolves with an array of filenames within the directory. + */ + findIntegrations: (dirname?: string) => Promise>; + + /** + * Reads the contents of an integration version to find available versions. + * + * @param dirname The name of the directory to read. + * @returns A Promise that resolves with an array of filenames within the directory. + */ + findIntegrationVersions: (dirname?: string) => Promise>; + + /** + * Determine whether a directory is an integration, repository, or otherwise. + * + * @param dirname The path to check. + * @returns A Promise that resolves with a boolean indicating whether the path is a directory or not. + */ + getDirectoryType: (dirname?: string) => Promise<'integration' | 'repository' | 'unknown'>; + + /** + * Creates a new CatalogDataAdaptor instance with the specified subdirectory appended to the current directory. + * Useful for exploring nested data without needing to know the instance type. + * + * @param subdirectory The path to append to the current directory. + * @returns A new CatalogDataAdaptor instance. + */ + join: (subdirectory: string) => CatalogDataAdaptor; +} diff --git a/server/adaptors/integrations/repository/fs_data_adaptor.ts b/server/adaptors/integrations/repository/fs_data_adaptor.ts new file mode 100644 index 000000000..df1c6781a --- /dev/null +++ b/server/adaptors/integrations/repository/fs_data_adaptor.ts @@ -0,0 +1,151 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as fs from 'fs/promises'; +import path from 'path'; + +/** + * Helper function to compare version numbers. + * Assumes that the version numbers are valid, produces undefined behavior otherwise. + * + * @param a Left-hand number + * @param b Right-hand number + * @returns -1 if a > b, 1 if a < b, 0 otherwise. + */ +function compareVersions(a: string, b: string): number { + const aParts = a.split('.').map(Number.parseInt); + const bParts = b.split('.').map(Number.parseInt); + + for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) { + const aValue = i < aParts.length ? aParts[i] : 0; + const bValue = i < bParts.length ? bParts[i] : 0; + + if (aValue > bValue) { + return -1; // a > b + } else if (aValue < bValue) { + return 1; // a < b + } + } + + return 0; // a == b +} + +function tryParseNDJson(content: string): object[] | null { + try { + const objects = []; + for (const line of content.split('\n')) { + if (line.trim() === '') { + // Other OSD ndjson parsers skip whitespace lines + continue; + } + objects.push(JSON.parse(line)); + } + return objects; + } catch (err: any) { + return null; + } +} + +/** + * A CatalogDataAdaptor that reads from the local filesystem. + * Used to read Integration information when the user uploads their own catalog. + */ +export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor { + directory: string; + + /** + * Creates a new FileSystemCatalogDataAdaptor instance. + * + * @param directory The base directory from which to read files. This is not sanitized. + */ + constructor(directory: string) { + this.directory = directory; + } + + async readFile(filename: string, type?: IntegrationPart): Promise> { + let content; + try { + content = await fs.readFile(path.join(this.directory, type ?? '.', filename), { + encoding: 'utf-8', + }); + } catch (err: any) { + return { ok: false, error: err }; + } + // First try to parse as JSON, then NDJSON, then fail. + try { + const parsed = JSON.parse(content); + return { ok: true, value: parsed }; + } catch (err: any) { + const parsed = tryParseNDJson(content); + if (parsed) { + return { ok: true, value: parsed }; + } + return { + ok: false, + error: new Error('Unable to parse file as JSON or NDJson', { cause: err }), + }; + } + } + + async readFileRaw(filename: string, type?: IntegrationPart): Promise> { + try { + const buffer = await fs.readFile(path.join(this.directory, type ?? '.', filename)); + return { ok: true, value: buffer }; + } catch (err: any) { + return { ok: false, error: err }; + } + } + + async findIntegrations(dirname: string = '.'): Promise> { + try { + const files = await fs.readdir(path.join(this.directory, dirname)); + return { ok: true, value: files }; + } catch (err: any) { + return { ok: false, error: err }; + } + } + + async findIntegrationVersions(dirname: string = '.'): Promise> { + let files; + const integPath = path.join(this.directory, dirname); + try { + files = await fs.readdir(integPath); + } catch (err: any) { + return { ok: false, error: err }; + } + const versions: string[] = []; + + for (const file of files) { + // TODO handle nested repositories -- assumes integrations are 1 level deep + if (path.extname(file) === '.json' && file.startsWith(`${path.basename(integPath)}-`)) { + const version = file.substring(path.basename(integPath).length + 1, file.length - 5); + if (!version.match(/^\d+(\.\d+)*$/)) { + continue; + } + versions.push(version); + } + } + + versions.sort((a, b) => compareVersions(a, b)); + return { ok: true, value: versions }; + } + + async getDirectoryType(dirname?: string): Promise<'integration' | 'repository' | 'unknown'> { + const isDir = (await fs.lstat(path.join(this.directory, dirname ?? '.'))).isDirectory(); + if (!isDir) { + return 'unknown'; + } + // Sloppily just check for one mandatory integration directory to distinguish. + // Improve if we need to distinguish a repository with an integration named "schemas". + const hasSchemas = ( + await fs.lstat(path.join(this.directory, dirname ?? '.', 'schemas')) + ).isDirectory(); + return hasSchemas ? 'integration' : 'repository'; + } + + join(filename: string): FileSystemCatalogDataAdaptor { + return new FileSystemCatalogDataAdaptor(path.join(this.directory, filename)); + } +} diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index 21f187bde..fca1aef5c 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -3,107 +3,51 @@ * SPDX-License-Identifier: Apache-2.0 */ -import * as fs from 'fs/promises'; import path from 'path'; import { validateTemplate } from '../validators'; - -/** - * Helper function to compare version numbers. - * Assumes that the version numbers are valid, produces undefined behavior otherwise. - * - * @param a Left-hand number - * @param b Right-hand number - * @returns -1 if a > b, 1 if a < b, 0 otherwise. - */ -function compareVersions(a: string, b: string): number { - const aParts = a.split('.').map(Number.parseInt); - const bParts = b.split('.').map(Number.parseInt); - - for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) { - const aValue = i < aParts.length ? aParts[i] : 0; - const bValue = i < bParts.length ? bParts[i] : 0; - - if (aValue > bValue) { - return -1; // a > b - } else if (aValue < bValue) { - return 1; // a < b - } - } - - return 0; // a == b -} - -/** - * Helper function to check if the given path is a directory - * - * @param dirPath The directory to check. - * @returns True if the path is a directory. - */ -async function isDirectory(dirPath: string): Promise { - try { - const stats = await fs.stat(dirPath); - return stats.isDirectory(); - } catch { - return false; - } -} +import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; /** * The Integration class represents the data for Integration Templates. * It is backed by the repository file system. * It includes accessor methods for integration configs, as well as helpers for nested components. */ -export class Integration { +export class IntegrationReader { + reader: CatalogDataAdaptor; directory: string; name: string; - constructor(directory: string) { + constructor(directory: string, reader?: CatalogDataAdaptor) { this.directory = directory; this.name = path.basename(directory); + this.reader = reader ?? new FileSystemCatalogDataAdaptor(directory); } /** - * Check the integration for validity. - * This is not a deep check, but a quick check to verify that the integration is a valid directory and has a config file. - * - * @returns true if the integration is valid. - */ - async check(): Promise { - if (!(await isDirectory(this.directory))) { - return false; - } - return (await this.getConfig()) !== null; - } - - /** - * Like check(), but thoroughly checks all nested integration dependencies. + * Like getConfig(), but thoroughly checks all nested integration dependencies for validity. * - * @returns true if the integration is valid. + * @returns a Result indicating whether the integration is valid. */ - async deepCheck(): Promise { - if (!(await this.check())) { - console.error('check failed'); - return false; + async deepCheck(): Promise> { + const configResult = await this.getConfig(); + if (!configResult.ok) { + return configResult; } try { - // An integration must have at least one mapping const schemas = await this.getSchemas(); - if (Object.keys(schemas.mappings).length === 0) { - return false; + if (!schemas.ok || Object.keys(schemas.value.mappings).length === 0) { + return { ok: false, error: new Error('The integration has no schemas available') }; } - // An integration must have at least one asset const assets = await this.getAssets(); - if (Object.keys(assets).length === 0) { - return false; + if (!assets.ok || Object.keys(assets).length === 0) { + return { ok: false, error: new Error('An integration must have at least one asset') }; } } catch (err: any) { - // Any loading errors are considered invalid - console.error('Deep check failed for exception', err); - return false; + return { ok: false, error: err }; } - return true; + return configResult; } /** @@ -114,22 +58,12 @@ export class Integration { * @returns A string with the latest version, or null if no versions are available. */ async getLatestVersion(): Promise { - const files = await fs.readdir(this.directory); - const versions: string[] = []; - - for (const file of files) { - if (path.extname(file) === '.json' && file.startsWith(`${this.name}-`)) { - const version = file.substring(this.name.length + 1, file.length - 5); - if (!version.match(/^\d+(\.\d+)*$/)) { - continue; - } - versions.push(version); - } + const versions = await this.reader.findIntegrationVersions(); + if (!versions.ok) { + console.error(versions.error); + return null; } - - versions.sort((a, b) => compareVersions(a, b)); - - return versions.length > 0 ? versions[0] : null; + return versions.value.length > 0 ? versions.value[0] : null; } /** @@ -138,36 +72,27 @@ export class Integration { * @param version The version of the config to retrieve. * @returns The config if a valid config matching the version is present, otherwise null. */ - async getConfig(version?: string): Promise { + async getConfig(version?: string): Promise> { + if ((await this.reader.getDirectoryType()) !== 'integration') { + return { ok: false, error: new Error(`${this.directory} is not a valid integration`) }; + } + const maybeVersion: string | null = version ? version : await this.getLatestVersion(); if (maybeVersion === null) { - return null; + return { + ok: false, + error: new Error(`No valid config matching version ${version} is available`), + }; } const configFile = `${this.name}-${maybeVersion}.json`; - const configPath = path.join(this.directory, configFile); - try { - const config = await fs.readFile(configPath, { encoding: 'utf-8' }); - const possibleTemplate = JSON.parse(config); - const template = validateTemplate(possibleTemplate); - if (template.ok) { - return template.value; - } - console.error(template.error); - return null; - } catch (err: any) { - if (err instanceof SyntaxError) { - console.error(`Syntax errors in ${configFile}`, err); - return null; - } - if (err instanceof Error && (err as { code?: string }).code === 'ENOENT') { - console.error(`Attempted to retrieve non-existent config ${configFile}`); - return null; - } - throw new Error('Could not load integration', { cause: err }); + const config = await this.reader.readFile(configFile); + if (!config.ok) { + return config; } + return validateTemplate(config.value); } /** @@ -181,30 +106,27 @@ export class Integration { */ async getAssets( version?: string - ): Promise<{ - savedObjects?: object[]; - }> { - const config = await this.getConfig(version); - if (config === null) { - return Promise.reject(new Error('Attempted to get assets of invalid config')); + ): Promise< + Result<{ + savedObjects?: object[]; + }> + > { + const configResult = await this.getConfig(version); + if (!configResult.ok) { + return configResult; } - const result: { savedObjects?: object[] } = {}; + const config = configResult.value; + + const resultValue: { savedObjects?: object[] } = {}; if (config.assets.savedObjects) { - const sobjPath = path.join( - this.directory, - 'assets', - `${config.assets.savedObjects.name}-${config.assets.savedObjects.version}.ndjson` - ); - try { - const ndjson = await fs.readFile(sobjPath, { encoding: 'utf-8' }); - const asJson = '[' + ndjson.trim().replace(/\n/g, ',') + ']'; - const parsed = JSON.parse(asJson); - result.savedObjects = parsed; - } catch (err: any) { - console.error("Failed to load saved object assets, proceeding as if it's absent", err); + const sobjPath = `${config.assets.savedObjects.name}-${config.assets.savedObjects.version}.ndjson`; + const assets = await this.reader.readFile(sobjPath, 'assets'); + if (!assets.ok) { + return assets; } + resultValue.savedObjects = assets.value as object[]; } - return result; + return { ok: true, value: resultValue }; } /** @@ -217,38 +139,41 @@ export class Integration { */ async getSampleData( version?: string - ): Promise<{ - sampleData: object[] | null; - }> { - const config = await this.getConfig(version); - if (config === null) { - return Promise.reject(new Error('Attempted to get assets of invalid config')); + ): Promise< + Result<{ + sampleData: object[] | null; + }> + > { + const configResult = await this.getConfig(version); + if (!configResult.ok) { + return configResult; } - const result: { sampleData: object[] | null } = { sampleData: null }; + const config = configResult.value; + + const resultValue: { sampleData: object[] | null } = { sampleData: null }; if (config.sampleData) { - const sobjPath = path.join(this.directory, 'data', config.sampleData?.path); - try { - const jsonContent = await fs.readFile(sobjPath, { encoding: 'utf-8' }); - const parsed = JSON.parse(jsonContent) as object[]; - for (const value of parsed) { - if (!('@timestamp' in value)) { - continue; - } - // Randomly scatter timestamps across last 10 minutes - // Assume for now that the ordering of events isn't important, can change to a sequence if needed - // Also doesn't handle fields like `observedTimestamp` if present - Object.assign(value, { - '@timestamp': new Date( - Date.now() - Math.floor(Math.random() * 1000 * 60 * 10) - ).toISOString(), - }); + const jsonContent = await this.reader.readFile(config.sampleData.path, 'data'); + if (!jsonContent.ok) { + return jsonContent; + } + for (const value of jsonContent.value as object[]) { + if (!('@timestamp' in value)) { + continue; + } + // Randomly scatter timestamps across last 10 minutes + // Assume for now that the ordering of events isn't important, can change to a sequence if needed + // Also doesn't handle fields like `observedTimestamp` if present + const newTime = new Date( + Date.now() - Math.floor(Math.random() * 1000 * 60 * 10) + ).toISOString(); + Object.assign(value, { '@timestamp': newTime }); + if ('observedTimestamp' in value) { + Object.assign(value, { observedTimestamp: newTime }); } - result.sampleData = parsed; - } catch (err: any) { - console.error("Failed to load saved object assets, proceeding as if it's absent", err); } + resultValue.sampleData = jsonContent.value as object[]; } - return result; + return { ok: true, value: resultValue }; } /** @@ -263,32 +188,29 @@ export class Integration { */ async getSchemas( version?: string - ): Promise<{ - mappings: { [key: string]: any }; - }> { - const config = await this.getConfig(version); - if (config === null) { - return Promise.reject(new Error('Attempted to get assets of invalid config')); + ): Promise< + Result<{ + mappings: { [key: string]: any }; + }> + > { + const configResult = await this.getConfig(version); + if (!configResult.ok) { + return configResult; } - const result: { mappings: { [key: string]: any } } = { + const config = configResult.value; + + const resultValue: { mappings: { [key: string]: object } } = { mappings: {}, }; - try { - for (const component of config.components) { - const schemaFile = `${component.name}-${component.version}.mapping.json`; - const rawSchema = await fs.readFile(path.join(this.directory, 'schemas', schemaFile), { - encoding: 'utf-8', - }); - const parsedSchema = JSON.parse(rawSchema); - result.mappings[component.name] = parsedSchema; + for (const component of config.components) { + const schemaFile = `${component.name}-${component.version}.mapping.json`; + const schema = await this.reader.readFile(schemaFile, 'schemas'); + if (!schema.ok) { + return schema; } - } catch (err: any) { - // It's not clear that an invalid schema can be recovered from. - // For integrations to function, we need schemas to be valid. - console.error('Error loading schema', err); - return Promise.reject(new Error('Could not load schema', { cause: err })); + resultValue.mappings[component.name] = schema.value; } - return result; + return { ok: true, value: resultValue }; } /** @@ -297,16 +219,7 @@ export class Integration { * @param staticPath The path of the static to retrieve. * @returns A buffer with the static's data if present, otherwise null. */ - async getStatic(staticPath: string): Promise { - const fullStaticPath = path.join(this.directory, 'static', staticPath); - try { - return await fs.readFile(fullStaticPath); - } catch (err: any) { - if (err instanceof Error && (err as { code?: string }).code === 'ENOENT') { - console.error(`Static not found: ${staticPath}`); - return null; - } - throw err; - } + async getStatic(staticPath: string): Promise> { + return await this.reader.readFileRaw(staticPath, 'static'); } } diff --git a/server/adaptors/integrations/repository/repository.ts b/server/adaptors/integrations/repository/repository.ts index 00d241327..08200d474 100644 --- a/server/adaptors/integrations/repository/repository.ts +++ b/server/adaptors/integrations/repository/repository.ts @@ -3,39 +3,43 @@ * SPDX-License-Identifier: Apache-2.0 */ -import * as fs from 'fs/promises'; import * as path from 'path'; -import { Integration } from './integration'; +import { IntegrationReader } from './integration'; +import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; -export class Repository { +export class RepositoryReader { + reader: CatalogDataAdaptor; directory: string; - constructor(directory: string) { + constructor(directory: string, reader?: CatalogDataAdaptor) { this.directory = directory; + this.reader = reader ?? new FileSystemCatalogDataAdaptor(directory); } - async getIntegrationList(): Promise { - try { - const folders = await fs.readdir(this.directory); - const integrations = Promise.all( - folders.map(async (folder) => { - const integPath = path.join(this.directory, folder); - if (!(await fs.lstat(integPath)).isDirectory()) { - return null; - } - const integ = new Integration(integPath); - return (await integ.check()) ? integ : null; - }) - ); - return (await integrations).filter((x) => x !== null) as Integration[]; - } catch (error) { - console.error(`Error reading integration directories in: ${this.directory}`, error); + async getIntegrationList(): Promise { + // TODO in the future, we want to support traversing nested directory structures. + const folders = await this.reader.findIntegrations(); + if (!folders.ok) { + console.error(`Error reading integration directories in: ${this.directory}`, folders.error); return []; } + const integrations = await Promise.all( + folders.value.map((i) => this.getIntegration(path.basename(i))) + ); + return integrations.filter((x) => x !== null) as IntegrationReader[]; } - async getIntegration(name: string): Promise { - const integ = new Integration(path.join(this.directory, name)); - return (await integ.check()) ? integ : null; + async getIntegration(name: string): Promise { + if ((await this.reader.getDirectoryType(name)) !== 'integration') { + console.error(`Requested integration '${name}' does not exist`); + return null; + } + const integ = new IntegrationReader(name, this.reader.join(name)); + const checkResult = await integ.getConfig(); + if (!checkResult.ok) { + console.error(`Integration '${name}' is invalid:`, checkResult.error); + return null; + } + return integ; } } diff --git a/server/adaptors/integrations/types.ts b/server/adaptors/integrations/types.ts index c12476909..c74829d30 100644 --- a/server/adaptors/integrations/types.ts +++ b/server/adaptors/integrations/types.ts @@ -3,7 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -interface IntegrationTemplate { +type Result = { ok: true; value: T } | { ok: false; error: E }; + +interface IntegrationConfig { name: string; version: string; displayName?: string; @@ -46,7 +48,7 @@ interface DisplayAsset { } interface IntegrationTemplateSearchResult { - hits: IntegrationTemplate[]; + hits: IntegrationConfig[]; } interface IntegrationTemplateQuery { diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index b40871eef..7486a38ed 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -5,8 +5,6 @@ import Ajv, { JSONSchemaType } from 'ajv'; -export type ValidationResult = { ok: true; value: T } | { ok: false; error: E }; - const ajv = new Ajv(); const staticAsset: JSONSchemaType = { @@ -19,7 +17,7 @@ const staticAsset: JSONSchemaType = { additionalProperties: false, }; -const templateSchema: JSONSchemaType = { +const templateSchema: JSONSchemaType = { type: 'object', properties: { name: { type: 'string' }, @@ -118,11 +116,11 @@ const instanceValidator = ajv.compile(instanceSchema); * this is a more conventional wrapper that simplifies calling. * * @param data The data to be validated as an IntegrationTemplate. - * @return A ValidationResult indicating whether the validation was successful or not. + * @return A Result indicating whether the validation was successful or not. * If validation succeeds, returns an object with 'ok' set to true and the validated data. * If validation fails, returns an object with 'ok' set to false and an Error object describing the validation error. */ -export const validateTemplate = (data: unknown): ValidationResult => { +export const validateTemplate = (data: unknown): Result => { if (!templateValidator(data)) { return { ok: false, error: new Error(ajv.errorsText(templateValidator.errors)) }; } @@ -143,11 +141,11 @@ export const validateTemplate = (data: unknown): ValidationResult => { +export const validateInstance = (data: unknown): Result => { if (!instanceValidator(data)) { return { ok: false, diff --git a/server/routes/integrations/integrations_router.ts b/server/routes/integrations/integrations_router.ts index 5a4813127..46fe47768 100644 --- a/server/routes/integrations/integrations_router.ts +++ b/server/routes/integrations/integrations_router.ts @@ -13,7 +13,7 @@ import { OpenSearchDashboardsRequest, OpenSearchDashboardsResponseFactory, } from '../../../../../src/core/server/http/router'; -import { IntegrationsKibanaBackend } from '../../adaptors/integrations/integrations_kibana_backend'; +import { IntegrationsManager } from '../../adaptors/integrations/integrations_manager'; /** * Handle an `OpenSearchDashboardsRequest` using the provided `callback` function. @@ -54,7 +54,7 @@ const getAdaptor = ( context: RequestHandlerContext, _request: OpenSearchDashboardsRequest ): IntegrationsAdaptor => { - return new IntegrationsKibanaBackend(context.core.savedObjects.client); + return new IntegrationsManager(context.core.savedObjects.client); }; export function registerIntegrationsRoute(router: IRouter) {