Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/metrics multiline display #899

Merged
merged 9 commits into from
Aug 15, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 147 additions & 37 deletions public/components/custom_panels/helpers/utils.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import dateMath from '@elastic/datemath';
import { ShortDate } from '@elastic/eui';
import { DurationRange } from '@elastic/eui/src/components/date_picker/types';
import _, { isEmpty } from 'lodash';
import _, { castArray, forEach, isEmpty } from 'lodash';
import { Moment } from 'moment-timezone';
import React from 'react';
import { Layout } from 'react-grid-layout';
Expand All @@ -30,6 +30,7 @@ import { SavedObjectsActions } from '../../../services/saved_objects/saved_objec
import { ObservabilitySavedVisualization } from '../../../services/saved_objects/saved_object_client/types';
import { getDefaultVisConfig } from '../../event_analytics/utils';
import { Visualization } from '../../visualizations/visualization';
import { MetricType } from '../../../../common/types/metrics';

/*
* "Utils" This file contains different reused functions in operational panels
Expand Down Expand Up @@ -58,11 +59,23 @@ export const convertDateTime = (datetime: string, isStart = true, formatted = tr
} else {
returnTime = dateMath.parse(datetime, { roundUp: true });
}

if (formatted) return returnTime!.utc().format(PPL_DATE_FORMAT);
return returnTime;
};

export const convertDateTimeToEpoch = (datetime: string, isStart = true, formatted = true) => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we re-use or merge convertDateTime function here?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can take a look into this

let returnTime: undefined | Moment;
if (isStart) {
returnTime = dateMath.parse(datetime);
} else {
returnTime = dateMath.parse(datetime, { roundUp: true });
}

const myDate = new Date(returnTime._d); // Your timezone!
const epochTime = myDate.getTime() / 1000.0;
return Math.round(epochTime);
};

// Merges new layout into visualizations
export const mergeLayoutAndVisualizations = (
layout: Layout[],
Expand Down Expand Up @@ -202,14 +215,19 @@ export const getQueryResponse = (
setIsLoading: React.Dispatch<React.SetStateAction<boolean>>,
setIsError: React.Dispatch<React.SetStateAction<VizContainerError>>,
filterQuery = '',
timestampField = 'timestamp'
timestampField = 'timestamp',
metricVisualization = false
) => {
setIsLoading(true);
setIsError({} as VizContainerError);

let finalQuery = '';
try {
finalQuery = queryAccumulator(query, timestampField, startTime, endTime, filterQuery);
if (!metricVisualization) {
finalQuery = queryAccumulator(query, timestampField, startTime, endTime, filterQuery);
} else {
finalQuery = query;
}
} catch (error) {
const errorMessage = 'Issue in building final query';
setIsError({ errorMessage });
Expand Down Expand Up @@ -311,44 +329,107 @@ const createCatalogVisualizationMetaData = (
};
};

const updateCatalogVisualizationQuery = ({
catalogSourceName,
catalogTableName,
aggregation,
attributesGroupBy,
startTime,
endTime,
spanParam,
}: {
catalogSourceName: string;
catalogTableName: string;
aggregation: string;
attributesGroupBy: string[];
startTime: string;
endTime: string;
spanParam: string | undefined;
}) => {
// source=my_prometheus.query_range('avg by(attribue1, attribuyte2) (prometheus_requests_total)', 1686694425, 1686700130, 14)
kavithacm marked this conversation as resolved.
Show resolved Hide resolved
const attributesGroupString = attributesGroupBy.toString();
const startEpochTime = convertDateTimeToEpoch(startTime);
const endEpochTime = convertDateTimeToEpoch(endTime, false);
const promQuery =
attributesGroupBy.length === 0
? catalogTableName
: `${aggregation} by(${attributesGroupString}) (${catalogTableName})`;

return `source = ${catalogSourceName}.query_range('${promQuery}', ${startEpochTime}, ${endEpochTime}, '${spanParam}')`;
};

// Creates a catalogVisualization for a runtime catalog based PPL query and runs getQueryResponse
export const renderCatalogVisualization = async (
http: CoreStart['http'],
pplService: PPLService,
catalogSource: string,
startTime: string,
endTime: string,
filterQuery: string,
spanParam: string | undefined,
setVisualizationTitle: React.Dispatch<React.SetStateAction<string>>,
setVisualizationType: React.Dispatch<React.SetStateAction<string>>,
setVisualizationData: React.Dispatch<React.SetStateAction<Plotly.Data[]>>,
setVisualizationMetaData: React.Dispatch<React.SetStateAction<undefined>>,
setIsLoading: React.Dispatch<React.SetStateAction<boolean>>,
setIsError: React.Dispatch<React.SetStateAction<VizContainerError>>,
spanResolution?: string
) => {
export const renderCatalogVisualization = async ({
http,
pplService,
catalogSource,
startTime,
endTime,
filterQuery,
spanParam,
setVisualizationTitle,
setVisualizationType,
setVisualizationData,
setVisualizationMetaData,
setIsLoading,
setIsError,
spanResolution,
queryMetaData,
}: {
http: CoreStart['http'];
pplService: PPLService;
catalogSource: string;
startTime: string;
endTime: string;
filterQuery: string;
spanParam: string | undefined;
setVisualizationTitle: React.Dispatch<React.SetStateAction<string>>;
setVisualizationType: React.Dispatch<React.SetStateAction<string>>;
setVisualizationData: React.Dispatch<React.SetStateAction<Plotly.Data[]>>;
setVisualizationMetaData: React.Dispatch<React.SetStateAction<undefined>>;
setIsLoading: React.Dispatch<React.SetStateAction<boolean>>;
setIsError: React.Dispatch<React.SetStateAction<VizContainerError>>;
spanResolution?: string;
queryMetaData?: MetricType;
pjfitzgibbons marked this conversation as resolved.
Show resolved Hide resolved
}) => {
setIsLoading(true);
setIsError({} as VizContainerError);

const visualizationType = 'line';
const visualizationTimeField = '@timestamp';
let visualizationQuery = `source = ${catalogSource} | stats avg(@value) by span(${visualizationTimeField},1h)`;

if (spanParam !== undefined) {
visualizationQuery = updateQuerySpanInterval(
visualizationQuery,
visualizationTimeField,
spanParam
);
}
const catalogSourceName = catalogSource.split('.')[0];
const catalogTableName = catalogSource.split('.')[1];

const defaultAggregation = 'avg'; // pass in attributes to this function
pjfitzgibbons marked this conversation as resolved.
Show resolved Hide resolved
// const attributes: string[] = ['instance', 'job']; // pass in attributes to this function
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should remove commented code here too

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this was for reference, ideally it should be deleted

const attributes: string[] = [];

const visualizationQuery = updateCatalogVisualizationQuery({
catalogSourceName,
catalogTableName,
aggregation: defaultAggregation,
attributesGroupBy: attributes,
startTime,
endTime,
spanParam,
});

const visualizationMetaData = createCatalogVisualizationMetaData(
catalogSource,
visualizationQuery,
visualizationType,
visualizationTimeField
);

visualizationMetaData.user_configs = {
layoutConfig: {
height: 390,
margin: { t: 5 },
legend: { orientation: 'h', yanchor: 'top', x: 0.0, y: -0.4 },
},
};

setVisualizationTitle(catalogSource);
setVisualizationType(visualizationType);

Expand All @@ -364,7 +445,8 @@ export const renderCatalogVisualization = async (
setIsLoading,
setIsError,
filterQuery,
visualizationTimeField
visualizationTimeField,
true
);
};

Expand Down Expand Up @@ -397,9 +479,7 @@ export const parseSavedVisualizations = (
timeField: visualization.savedVisualization.selected_timestamp.name,
selected_date_range: visualization.savedVisualization.selected_date_range,
selected_fields: visualization.savedVisualization.selected_fields,
user_configs: visualization.savedVisualization.user_configs
? JSON.parse(visualization.savedVisualization.user_configs)
: {},
pjfitzgibbons marked this conversation as resolved.
Show resolved Hide resolved
user_configs: visualization.savedVisualization.user_configs || {},
sub_type: visualization.savedVisualization.hasOwnProperty('sub_type')
? visualization.savedVisualization.sub_type
: '',
Expand Down Expand Up @@ -462,16 +542,43 @@ export const isPPLFilterValid = (
return true;
};

export const processMetricsData = (schema: any, dataConfig: any) => {
if (isEmpty(schema)) return {};
if (
schema.length === 3 &&
schema.every((schemaField) => ['@labels', '@value', '@timestamp'].includes(schemaField.name))
pjfitzgibbons marked this conversation as resolved.
Show resolved Hide resolved
) {
return prepareMetricsData(schema, dataConfig);
}
return {};
};

export const prepareMetricsData = (schema: any, dataConfig: any) => {
const metricBreakdown: any[] = [];
pjfitzgibbons marked this conversation as resolved.
Show resolved Hide resolved
const metricSeries: any[] = [];
const metricDimension: any[] = [];

forEach(schema, (field) => {
if (field.name === '@timestamp')
metricDimension.push({ name: '@timestamp', label: '@timestamp' });
if (field.name === '@labels') metricBreakdown.push({ name: '@labels', customLabel: '@labels' });
if (field.name === '@value') metricSeries.push({ name: '@value', label: '@value' });
});

return {
breakdowns: metricBreakdown,
series: metricSeries,
dimensions: metricDimension,
span: {},
};
};

// Renders visualization in the vizualization container component
export const displayVisualization = (metaData: any, data: any, type: string) => {
if (metaData === undefined || isEmpty(metaData)) {
return <></>;
}

if (metaData.user_configs !== undefined && metaData.user_configs !== '') {
metaData.user_configs = JSON.parse(metaData.user_configs);
}

const dataConfig = { ...(metaData.user_configs?.dataConfig || {}) };
const hasBreakdowns = !_.isEmpty(dataConfig.breakdowns);
const realTimeParsedStats = {
Expand All @@ -487,13 +594,16 @@ export const displayVisualization = (metaData: any, data: any, type: string) =>
);
}

const finalDataConfig = {
let finalDataConfig = {
...dataConfig,
...realTimeParsedStats,
dimensions: finalDimensions,
breakdowns,
};

// add metric specific overriding
finalDataConfig = { ...finalDataConfig, ...processMetricsData(data.schema, finalDataConfig) };

const mixedUserConfigs = {
availabilityConfig: {
...(metaData.user_configs?.availabilityConfig || {}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import {
} from '../../helpers/utils';
import './visualization_container.scss';
import { VizContainerError } from '../../../../../common/types/custom_panels';
import { coreRefs } from '../../../../framework/core_refs';

/*
* Visualization container - This module is a placeholder to add visualizations in react-grid-layout
Expand Down Expand Up @@ -217,21 +218,21 @@ export const VisualizationContainer = ({

const loadVisaulization = async () => {
if (catalogVisualization)
await renderCatalogVisualization(
await renderCatalogVisualization({
http,
pplService,
savedVisualizationId,
fromTime,
toTime,
pplFilterValue,
catalogSource: savedVisualizationId,
startTime: fromTime,
endTime: toTime,
filterQuery: pplFilterValue,
spanParam,
setVisualizationTitle,
setVisualizationType,
setVisualizationData,
setVisualizationMetaData,
setIsLoading,
setIsError
);
setIsError,
});
else
await renderSavedVisualization(
http,
Expand Down
Loading
Loading