diff --git a/common/constants/integrations.ts b/common/constants/integrations.ts index 9c8ca02995..9df90f0047 100644 --- a/common/constants/integrations.ts +++ b/common/constants/integrations.ts @@ -5,7 +5,3 @@ export const OPENSEARCH_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/integrations/index'; export const ASSET_FILTER_OPTIONS = ['index-pattern', 'search', 'visualization', 'dashboard']; -// TODO get this list dynamically from the API -export const INTEGRATION_TEMPLATE_OPTIONS = ['nginx']; -// TODO get this list dynamically from the API -export const INTEGRATION_CATEOGRY_OPTIONS = ['communication', 'http', 'cloud', 'container', 'logs']; diff --git a/public/components/integrations/components/add_integration_flyout.tsx b/public/components/integrations/components/add_integration_flyout.tsx index bfdfaa23e5..29cc8921e6 100644 --- a/public/components/integrations/components/add_integration_flyout.tsx +++ b/public/components/integrations/components/add_integration_flyout.tsx @@ -91,7 +91,7 @@ export const doPropertyValidation = ( }; export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { - const { onClose, onCreate, integrationName, integrationType } = props; + const { onClose, onCreate, integrationName, integrationType, http } = props; const [isDataSourceValid, setDataSourceValid] = useState(null); @@ -131,11 +131,13 @@ export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { const fetchDataSourceMappings = async ( targetDataSource: string ): Promise<{ [key: string]: { properties: any } } | null> => { - return fetch(`/api/console/proxy?path=${targetDataSource}/_mapping&method=GET`, { - method: 'POST', - headers: [['osd-xsrf', 'true']], - }) - .then((response) => response.json()) + return http + .post('/api/console/proxy', { + query: { + path: `${targetDataSource}/_mapping`, + method: 'GET', + }, + }) .then((response) => { // Un-nest properties by a level for caller convenience Object.keys(response).forEach((key) => { @@ -152,8 +154,8 @@ export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { const fetchIntegrationMappings = async ( targetName: string ): Promise<{ [key: string]: { template: { mappings: { properties?: any } } } } | null> => { - return fetch(`/api/integrations/repository/${targetName}/schema`) - .then((response) => response.json()) + return http + .get(`/api/integrations/repository/${targetName}/schema`) .then((response) => { if (response.statusCode && response.statusCode !== 200) { throw new Error('Failed to retrieve Integration schema', { cause: response }); diff --git a/public/components/integrations/components/added_integration_overview_page.tsx b/public/components/integrations/components/added_integration_overview_page.tsx index 89026239c4..ef9623b64c 100644 --- a/public/components/integrations/components/added_integration_overview_page.tsx +++ b/public/components/integrations/components/added_integration_overview_page.tsx @@ -16,6 +16,7 @@ import { HttpStart } from '../../../../../../src/core/public'; export interface AddedIntegrationsTableProps { loading: boolean; data: AddedIntegrationsList; + setData: React.Dispatch>; http: HttpStart; } @@ -57,7 +58,7 @@ export function AddedIntegrationOverviewPage(props: AddedIntegrationOverviewPage {IntegrationHeader()} - {AddedIntegrationsTable({ data, loading: false, http })} + {AddedIntegrationsTable({ data, setData, loading: false, http })} ); diff --git a/public/components/integrations/components/added_integration_table.tsx b/public/components/integrations/components/added_integration_table.tsx index 3072bbf39c..c75e4557b1 100644 --- a/public/components/integrations/components/added_integration_table.tsx +++ b/public/components/integrations/components/added_integration_table.tsx @@ -18,17 +18,11 @@ import { import _ from 'lodash'; import React, { useState } from 'react'; import { AddedIntegrationsTableProps } from './added_integration_overview_page'; -import { - ASSET_FILTER_OPTIONS, - INTEGRATION_TEMPLATE_OPTIONS, -} from '../../../../common/constants/integrations'; import { DeleteModal } from '../../../../public/components/common/helpers/delete_modal'; import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { useToast } from '../../../../public/components/common/toast'; export function AddedIntegrationsTable(props: AddedIntegrationsTableProps) { - const integrations = props.data.hits; - const { http } = props; const { setToast } = useToast(); @@ -94,6 +88,9 @@ export function AddedIntegrationsTable(props: AddedIntegrationsTableProps) { .delete(`${INTEGRATIONS_BASE}/store/${integrationInstance}`) .then(() => { setToast(`${name} integration successfully deleted!`, 'success'); + props.setData({ + hits: props.data.hits.filter((i) => i.id !== integrationInstance), + }); }) .catch((err) => { setToast(`Error deleting ${name} or its assets`, 'danger'); @@ -103,7 +100,7 @@ export function AddedIntegrationsTable(props: AddedIntegrationsTableProps) { }); } - const getModal = (integrationInstanceId, name) => { + const getModal = (integrationInstanceId: string, name: string) => { setModalLayout( { @@ -120,26 +117,28 @@ export function AddedIntegrationsTable(props: AddedIntegrationsTableProps) { setIsModalVisible(true); }; + const integTemplateNames = [...new Set(integrations.map((i) => i.templateName))].sort(); + const search = { box: { incremental: true, }, filters: [ { - type: 'field_value_selection', + type: 'field_value_selection' as const, field: 'templateName', name: 'Type', multiSelect: false, - options: INTEGRATION_TEMPLATE_OPTIONS.map((i) => ({ - value: i, - name: i, - view: i, + options: integTemplateNames.map((name) => ({ + name, + value: name, + view: name, })), }, ], }; - const entries = integrations.map((integration) => { + const entries = props.data.hits.map((integration) => { const id = integration.id; const templateName = integration.templateName; const creationDate = integration.creationDate; diff --git a/public/components/integrations/components/available_integration_card_view.tsx b/public/components/integrations/components/available_integration_card_view.tsx index ebcf6f1aa0..b3e3880d6a 100644 --- a/public/components/integrations/components/available_integration_card_view.tsx +++ b/public/components/integrations/components/available_integration_card_view.tsx @@ -28,6 +28,7 @@ import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { badges } from './integration_category_badge_group'; export function AvailableIntegrationsCardView(props: AvailableIntegrationsCardViewProps) { + const http = props.http; const [toggleIconIdSelected, setToggleIconIdSelected] = useState('1'); const getImage = (url?: string) => { @@ -72,7 +73,9 @@ export function AvailableIntegrationsCardView(props: AvailableIntegrationsCardVi void; renderCateogryFilters: () => React.JSX.Element; + http: HttpStart; } export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOverviewPageProps) { @@ -75,32 +76,18 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver setIsPopoverOpen(false); }; - const [items, setItems] = useState( - INTEGRATION_CATEOGRY_OPTIONS.map((x) => { - return { name: x }; - }) - ); + const [items, setItems] = useState([] as Array<{ name: string; checked: boolean }>); - function updateItem(index) { + function updateItem(index: number) { if (!items[index]) { return; } - const newItems = [...items]; - - switch (newItems[index].checked) { - case 'on': - newItems[index].checked = undefined; - break; - - default: - newItems[index].checked = 'on'; - } - + newItems[index].checked = !items[index].checked; setItems(newItems); } - const helper = items.filter((item) => item.checked === 'on').map((x) => x.name); + const helper = items.filter((item) => item.checked).map((x) => x.name); const button = ( item.checked === 'on')} - numActiveFilters={items.filter((item) => item.checked === 'on').length} + hasActiveFilters={!!items.find((item) => item.checked)} + numActiveFilters={items.filter((item) => item.checked).length} > Categories @@ -126,7 +113,20 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver }, []); async function handleDataRequest() { - http.get(`${INTEGRATIONS_BASE}/repository`).then((exists) => setData(exists.data)); + http.get(`${INTEGRATIONS_BASE}/repository`).then((exists) => { + setData(exists.data); + + let newItems = exists.data.hits + .flatMap((hit: { components: Array<{ name: string }> }) => hit.components) + .map((component: { name: string }) => component.name); + newItems = [...new Set(newItems)].sort().map((newItem) => { + return { + name: newItem, + checked: false, + }; + }); + setItems(newItems); + }); } async function addIntegrationRequest(name: string) { @@ -163,7 +163,7 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver
{items.map((item, index) => ( updateItem(index)} > @@ -192,6 +192,7 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver query, setQuery, renderCateogryFilters, + http, }) : AvailableIntegrationsTable({ loading: false, diff --git a/public/components/integrations/components/integration.tsx b/public/components/integrations/components/integration.tsx index ad1a1dfe65..01d288e485 100644 --- a/public/components/integrations/components/integration.tsx +++ b/public/components/integrations/components/integration.tsx @@ -48,36 +48,28 @@ export function Integration(props: AvailableIntegrationProps) { ): Promise<{ [key: string]: { properties: any } } | null> => { const version = payload.template.mappings._meta.version; if (componentName !== integration.type) { - return fetch( - `/api/console/proxy?path=_component_template/ss4o_${componentName}_${version}_template&method=POST`, - { - method: 'POST', - headers: [ - ['osd-xsrf', 'true'], - ['Content-Type', 'application/json'], - ], + return http + .post('/api/console/proxy', { body: JSON.stringify(payload), - } - ) - .then((response) => response.json()) + query: { + path: `_component_template/ss4o_${componentName}_${version}_template`, + method: 'POST', + }, + }) .catch((err: any) => { console.error(err); return err; }); } else { payload.index_patterns = [dataSourceName]; - return fetch( - `/api/console/proxy?path=_index_template/${componentName}_${version}&method=POST`, - { - method: 'POST', - headers: [ - ['osd-xsrf', 'true'], - ['Content-Type', 'application/json'], - ], + return http + .post('/api/console/proxy', { body: JSON.stringify(payload), - } - ) - .then((response) => response.json()) + query: { + path: `_index_template/${componentName}_${version}`, + method: 'POST', + }, + }) .catch((err: any) => { console.error(err); return err; @@ -86,11 +78,7 @@ export function Integration(props: AvailableIntegrationProps) { }; const createDataSourceMappings = async (targetDataSource: string): Promise => { - const data = await fetch( - `${INTEGRATIONS_BASE}/repository/${integrationTemplateId}/schema` - ).then((response) => { - return response.json(); - }); + const data = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}/schema`); let error = null; const mappings = data.data.mappings; mappings[integration.type].composed_of = mappings[integration.type].composed_of.map( @@ -147,8 +135,8 @@ export function Integration(props: AvailableIntegrationProps) { if (Object.keys(integration).length === 0) { return; } - fetch(`${INTEGRATIONS_BASE}/repository/${integration.name}/schema`) - .then((response) => response.json()) + http + .get(`${INTEGRATIONS_BASE}/repository/${integration.name}/schema`) .then((parsedResponse) => { if (parsedResponse.statusCode && parsedResponse.statusCode !== 200) { throw new Error('Request for schema failed: ' + parsedResponse.message); @@ -165,8 +153,8 @@ export function Integration(props: AvailableIntegrationProps) { if (Object.keys(integration).length === 0) { return; } - fetch(`${INTEGRATIONS_BASE}/repository/${integration.name}/assets`) - .then((response) => response.json()) + http + .get(`${INTEGRATIONS_BASE}/repository/${integration.name}/assets`) .then((parsedResponse) => { if (parsedResponse.statusCode && parsedResponse.statusCode !== 200) { throw new Error('Request for assets failed: ' + parsedResponse.message); @@ -224,14 +212,14 @@ export function Integration(props: AvailableIntegrationProps) { data.sampleData .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) .join('\n') + '\n'; - fetch(`/api/console/proxy?path=${dataSource}/_bulk&method=POST`, { - method: 'POST', - body: requestBody, - headers: [ - ['osd-xsrf', 'true'], - ['Content-Type', 'application/json; charset=utf-8'], - ], - }) + http + .post('/api/console/proxy', { + body: requestBody, + query: { + path: `${dataSource}/_bulk`, + method: 'POST', + }, + }) .catch((err) => { console.error(err); setToast('Failed to load sample data', 'danger'); @@ -298,7 +286,7 @@ export function Integration(props: AvailableIntegrationProps) { {IntegrationDetails({ integration })} - {IntegrationScreenshots({ integration })} + {IntegrationScreenshots({ integration, http })} {renderTabs()} diff --git a/public/components/integrations/components/integration_screenshots_panel.tsx b/public/components/integrations/components/integration_screenshots_panel.tsx index 4c993a895c..d05f38ca5e 100644 --- a/public/components/integrations/components/integration_screenshots_panel.tsx +++ b/public/components/integrations/components/integration_screenshots_panel.tsx @@ -10,6 +10,7 @@ import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; export function IntegrationScreenshots(props: any) { const config = props.integration; + const http = props.http; let screenshots; if (config.statics.gallery) { screenshots = config.statics.gallery; @@ -23,7 +24,9 @@ export function IntegrationScreenshots(props: any) { return ( Made with NaturalEarth\", \"created_at\":\"2017-04-26T17:12:15.978370\", \"fields\":[ { \"description\":\"ISO 3166-1 alpha-2 Code\", \"name\":\"iso2\", \"type\":\"id\" }, { \"description\":\"ISO 3166-1 alpha-3 Code\", \"name\":\"iso3\", \"type\":\"id\" }, { \"description\":\"Name\", \"name\":\"name\", \"type\":\"name\" } ], \"format\":{ \"type\":\"geojson\" }, \"id\":\"world_countries\", \"isEMS\":true, \"layerId\":\"elastic_maps_service.World Countries\", \"name\":\"World Countries\", \"origin\":\"elastic_maps_service\" }, \"showAllShapes\":true, \"wms\":{ \"enabled\":false, \"options\":{ \"attribution\":\"\", \"format\":\"image/png\", \"layers\":\"\", \"styles\":\"\", \"transparent\":true, \"version\":\"\" }, \"selectedTmsLayer\":{ \"attribution\":\"Map data © OpenStreetMap contributors\", \"id\":\"road_map\", \"maxZoom\":14, \"minZoom\":0, \"origin\":\"elastic_maps_service\" }, \"url\":\"\" } } }" }, "coreMigrationVersion": "7.0.0", "id": "Apache-access-unique-IPs-map-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzMzcsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Operating systems breakdown ", "uiStateJSON": "{}", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": { \"field\": \"communication.source.address.keyword\" }, \"schema\": \"metric\", \"type\": \"cardinality\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"field\": \"http.user_agent.os.name\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"segment\", \"type\": \"terms\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"field\": \"http.user_agent.os.version\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"segment\", \"type\": \"terms\" } ], \"listeners\": {}, \"params\": { \"addLegend\": true, \"addTooltip\": true, \"distinctColors\": true, \"isDonut\": true, \"legendPosition\": \"bottom\", \"palette\": { \"name\": \"kibana_palette\", \"type\": \"palette\" }, \"shareYAxis\": true }, \"title\": \"Apache operating systems ECS\", \"type\": \"pie\" }" }, "coreMigrationVersion": "7.0.0", "id": "Apache-operating-systems-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDAsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Error logs over time ", "uiStateJSON": "{}", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": {}, \"schema\": \"metric\", \"type\": \"count\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"extended_bounds\": {}, \"field\": \"@timestamp\", \"interval\": \"auto\", \"min_doc_count\": 1 }, \"schema\": \"segment\", \"type\": \"date_histogram\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"field\": \"severity.text.keyword\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"group\", \"type\": \"terms\" } ], \"listeners\": {}, \"params\": { \"addLegend\": true, \"addTimeMarker\": false, \"addTooltip\": true, \"defaultYExtents\": false, \"legendPosition\": \"right\", \"mode\": \"stacked\", \"scale\": \"linear\", \"setYExtents\": false, \"shareYAxis\": true, \"times\": [], \"yAxis\": {} }, \"title\": \"Apache error logs over time ECS\", \"type\": \"histogram\" }" }, "coreMigrationVersion": "7.10.0", "id": "Apache-error-logs-over-time-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-errors-log-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDEsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Top URLs by response code ", "uiStateJSON": "{ \"vis\": { \"colors\": { \"200\": \"#7EB26D\", \"404\": \"#EF843C\" } } }", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": { }, \"schema\": \"metric\", \"type\": \"count\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"customLabel\": \"URL\", \"field\": \"http.url\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"split\", \"type\": \"terms\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"field\": \"http.response.status_code\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"segment\", \"type\": \"terms\" } ], \"listeners\": { }, \"params\": { \"addLegend\": true, \"addTooltip\": true, \"distinctColors\": true, \"isDonut\": false, \"legendPosition\": \"right\", \"palette\": { \"name\": \"kibana_palette\", \"type\": \"palette\" }, \"row\": false, \"shareYAxis\": true }, \"title\": \"Apache response codes of top URLs ECS\", \"type\": \"pie\" }" }, "coreMigrationVersion": "8.0.0", "id": "Apache-response-codes-of-top-URLs-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzMzgsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Response codes over time ", "uiStateJSON": "{ \"vis\": { \"colors\": { \"200\": \"#629E51\", \"404\": \"#EF843C\" } } }", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": {}, \"schema\": \"metric\", \"type\": \"count\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"extended_bounds\": {}, \"field\": \"@timestamp\", \"interval\": \"auto\", \"min_doc_count\": 1 }, \"schema\": \"segment\", \"type\": \"date_histogram\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"field\": \"http.response.status_code\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"group\", \"type\": \"terms\" } ], \"listeners\": { }, \"params\": { \"addLegend\": true, \"addTimeMarker\": false, \"addTooltip\": true, \"defaultYExtents\": false, \"legendPosition\": \"right\", \"mode\": \"stacked\", \"scale\": \"linear\", \"setYExtents\": false, \"shareYAxis\": true, \"times\": [ ], \"yAxis\": { } }, \"title\": \"Apache response codes over time ECS\", \"type\": \"histogram\" }" }, "coreMigrationVersion": "8.0.0", "id": "Apache-response-codes-over-time-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDIsMV0=" } +{ "attributes": { "description": "Filebeat Apache module dashboard", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" }, "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "refreshInterval": { "pause": true, "value": 0 }, "timeFrom": "now-15m", "timeRestore": true, "panelsJSON": "[ { \"embeddableConfig\": { \"enhancements\": {}, \"mapBounds\": { \"bottom_right\": { \"lat\": -3.864254615721396, \"lon\": 205.3125 }, \"top_left\": { \"lat\": 67.7427590666639, \"lon\": -205.6640625 } }, \"mapCenter\": [ 40.713955826286046, -0.17578125 ], \"mapCollar\": { \"bottom_right\": { \"lat\": -39.667755, \"lon\": 180 }, \"top_left\": { \"lat\": 90, \"lon\": -180 }, \"zoom\": 2 }, \"mapZoom\": 2 }, \"gridData\": { \"h\": 12, \"i\": \"1\", \"w\": 48, \"x\": 0, \"y\": 0 }, \"panelIndex\": \"1\", \"panelRefName\": \"panel_1\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": {} }, \"gridData\": { \"h\": 12, \"i\": \"2\", \"w\": 32, \"x\": 0, \"y\": 20 }, \"panelIndex\": \"2\", \"panelRefName\": \"panel_2\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": {} }, \"gridData\": { \"h\": 12, \"i\": \"3\", \"w\": 16, \"x\": 32, \"y\": 20 }, \"panelIndex\": \"3\", \"panelRefName\": \"panel_3\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": { } }, \"gridData\": { \"h\": 8, \"i\": \"4\", \"w\": 8, \"x\": 40, \"y\": 12 }, \"panelIndex\": \"4\", \"panelRefName\": \"panel_4\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": { } }, \"gridData\": { \"h\": 8, \"i\": \"5\", \"w\": 48, \"x\": 0, \"y\": 32 }, \"panelIndex\": \"5\", \"panelRefName\": \"panel_5\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": { } }, \"gridData\": { \"h\": 8, \"i\": \"6\", \"w\": 40, \"x\": 0, \"y\": 12 }, \"panelIndex\": \"6\", \"panelRefName\": \"panel_6\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"columns\": [ \"communication.source.address\", \"severity.number\", \"severity.text\", \"body\" ], \"enhancements\": { }, \"sort\": [ \"@timestamp\", \"desc\" ] }, \"gridData\": { \"h\": 12, \"i\": \"7\", \"w\": 48, \"x\": 0, \"y\": 50 }, \"panelIndex\": \"7\", \"panelRefName\": \"panel_7\", \"type\": \"search\", \"version\": \"7.3.0\" } ]", "title": "Access and error logs ECS", "version": 1 }, "coreMigrationVersion": "7.9.3", "id": "Filebeat-Apache-Dashboard-ecs", "migrationVersion": { "dashboard": "7.9.3" }, "references": [ { "id": "Apache-access-unique-IPs-map-ecs", "name": "panel_1", "type": "visualization" }, { "id": "Apache-response-codes-of-top-URLs-ecs", "name": "panel_2", "type": "visualization" }, { "id": "Apache-browsers-ecs", "name": "panel_3", "type": "visualization" }, { "id": "Apache-operating-systems-ecs", "name": "panel_4", "type": "visualization" }, { "id": "Apache-error-logs-over-time-ecs", "name": "panel_5", "type": "visualization" }, { "id": "Apache-response-codes-over-time-ecs", "name": "panel_6", "type": "visualization" }, { "id": "Apache-errors-log-ecs", "name": "panel_7", "type": "search" } ], "type": "dashboard", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDUsMV0=" } +{"exportedCount":10,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/apache/data/sample.json b/server/adaptors/integrations/__data__/repository/apache/data/sample.json new file mode 100644 index 0000000000..5d41e04c90 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/apache/data/sample.json @@ -0,0 +1,206 @@ +[ + { + "observedTimestamp": "2023-07-21T16:52:08.000Z", + "http": { + "response": { + "status_code": 406, + "bytes": 6141 + }, + "url": "/strategize", + "flavor": "1.1", + "request": { + "method": "GET" + }, + "user_agent": { + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", + "name": "Chrome", + "version": "114.0.0", + "os": { + "name": "Mac OS X", + "full": "Mac OS X 10.15.7", + "version": "10.15.7", + "device": { + "name": "Mac" + } + } + } + }, + "attributes": { + "data_stream": { + "dataset": "apache.access", + "namespace": "production", + "type": "logs" + } + }, + "event": { + "result": "success", + "category": "web", + "name": "access", + "type": "access", + "domain": "apache.access", + "kind": "event" + }, + "communication": { + "source": { + "address": "127.0.0.1", + "ip": "42.204.151.42", + "geo": { + "country": "China", + "country_iso_code": "CN" + } + } + }, + "body": "15.248.1.132 - - [21/Jun/2023:21:35:24 +0000] \"GET / HTTP/1.1\" 403 45 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36\"", + "traceId": "d09d293a27c9a754f2bf0196b5a1c9bc", + "spanId": "18ba0e515e42dad0", + "@timestamp": "2023-07-21T16:52:08.000Z" + }, + { + "observedTimestamp": "2023-07-21T16:52:08.000Z", + "http": { + "response": { + "status_code": 406, + "bytes": 6141 + }, + "url": "/strategize", + "flavor": "1.1", + "request": { + "method": "GET" + }, + "user_agent": { + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", + "name": "Chrome", + "version": "114.0.0", + "os": { + "name": "Mac OS X", + "full": "Mac OS X 10.15.7", + "version": "10.15.7", + "device": { + "name": "Mac" + } + } + } + }, + "attributes": { + "data_stream": { + "dataset": "apache.access", + "namespace": "production", + "type": "logs" + } + }, + "event": { + "result": "success", + "category": "web", + "name": "access", + "type": "access", + "domain": "apache.access", + "kind": "event" + }, + "communication": { + "source": { + "address": "127.0.0.1", + "ip": "42.204.151.42", + "geo": { + "country": "China", + "country_iso_code": "CN" + } + } + }, + + "body": "15.248.1.132 - - [21/Jun/2023:21:35:24 +0000] \"GET / HTTP/1.1\" 403 45 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36\"", + "traceId": "d09d293a27c9a754f2bf0196b5a1c9bc", + "spanId": "18ba0e515e42dad0", + "@timestamp": "2023-07-21T16:52:08.000Z" + }, + { + "observedTimestamp": "2023-07-25:52:08.000Z", + "http": { + "response": { + "status_code": 400, + "bytes": 6141 + }, + "url": "/strategize", + "flavor": "1.1", + "request": { + "method": "GET" + }, + "user_agent": { + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", + "name": "Chrome", + "version": "114.0.0", + "os": { + "name": "Mac OS X", + "full": "Mac OS X 10.15.7", + "version": "10.15.7", + "device": { + "name": "Mac" + } + } + } + }, + "attributes": { + "data_stream": { + "dataset": "apache.access", + "namespace": "production", + "type": "logs" + } + }, + "event": { + "result": "success", + "category": "web", + "name": "access", + "type": "access", + "domain": "apache.access", + "kind": "event" + }, + "communication": { + "source": { + "address": "127.0.0.1", + "ip": "42.204.151.42", + "geo": { + "country": "United States", + "country_iso_code": "US" + } + } + }, + "body": "15.248.1.132 - - [21/Jun/2023:21:35:24 +0000] \"GET / HTTP/1.1\" 403 45 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36\"", + "traceId": "d09d293a27c9a754f2bf0196b5a1c9bc", + "spanId": "18ba0e515e42dad0", + "@timestamp": "2023-07-21T16:52:08.000Z" + }, + { + "attributes": { + "data_stream": { + "dataset": "apache.error", + "namespace": "production", + "type": "logs" + } + }, + "observedTimestamp": "2023-07-21T16:52:08.000Z", + "@timestamp": "2023-07-21T16:52:08.000Z", + "severity": { + "text": "cgid:error" + }, + "communication": { + "source": { + "address": "127.0.0.1", + "ip": "42.204.151.42", + "geo": { + "country": "France", + "country_iso_code": "FR" + } + } + }, + "event": { + "result": "error", + "category": "web", + "name": "error", + "type": "error", + "domain": "apache.error", + "kind": "error" + }, + "traceId": "d09d293a27c9a754f2bf0196b5a1c9bc", + "spanId": "18ba0e515e42dad0", + "body": "[Sat Aug 12 04:05:51 2006] [notice] Apache/1.3.11 (Unix) mod_perl/1.21 configured -- resuming normal operations" + } +] \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md b/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md new file mode 100644 index 0000000000..f9f64bcdd6 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md @@ -0,0 +1,149 @@ +# Ingestion Pipeline +To set up an ingestion pipeline, I used Docker to run Fluent-bit and an Apache fake log generator, along with an instance of OpenSearch. + +## FluentBit and OpenSearch Setup +First, create a `docker-compose.yml` instance like [this](https://github.com/opensearch-project/data-prepper/blob/93d06db5cad280e2e4c53e12dfb47c7cbaa7b364/examples/log-ingestion/docker-compose.yaml). This will pull FluentBit and OpenSearch Docker images and run them in `opensearch-net` Docker network. + +Next, use an Apache log generator to generate sample logs. I used `mingrammer/flog` in my `docker-compose.yml` file: +``` + apache: + image: mingrammer/flog + command: "--loop -d 5s -f apache_combined" + container_name: apache + networks: + - opensearch-net # All of the containers will join the same Docker bridge network + links: + - fluentbit + logging: + driver: "fluentd" + options: + fluentd-address: 127.0.0.1:24224 + tag: apache.access + fluentd-async: "true" +``` + +Then, create a `fluent-bit.conf` as follows: +``` +[SERVICE] + Parsers_File parsers.conf + +[INPUT] + Name forward + Port 24224 + +[FILTER] + Name parser + Match apache.access + Key_Name log + Parser apache + +[FILTER] + Name lua + Match apache.access + Script otel-converter.lua + call convert_to_otel + +[OUTPUT] + Name opensearch + Match apache.access + Host opensearch + Port 9200 + Index ss4o_logs-apache-prod-sample + Suppress_Type_Name On +``` +This creates a Fluent-Bit pipeline that forwards your generated apache logs through a parser to OpenSearch. + +Create a `parsers.conf` file as follows: +``` +[PARSER] + Name apache + Format regex + Regex ^(?[^ ]*) [^ ]* (?[^ ]*) \[(?