From 0ef1c3d735a83f1716d1f03793c00579886779f9 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 18 Jun 2021 09:31:29 +0100 Subject: [PATCH] [ML] Anomaly detection job custom_settings improvements (#102099) * [ML] Anomaly detection job custom_settings improvements * filter improvements * translations * fixing types * fixing tests * one more test fix * fixing bug with expanded row Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../anomaly_detection_jobs/summary_job.ts | 1 + .../job_details/extract_job_details.js | 28 ++++++++--- .../components/job_details/job_details.js | 6 ++- .../jobs_list_view/jobs_list_view.js | 50 +++++++++++++++---- .../jobs/jobs_list/components/utils.js | 39 ++++++++++----- .../ml/server/models/job_service/jobs.ts | 1 + .../hooks/use_installed_security_jobs.test.ts | 1 + .../common/components/ml_popover/api.mock.ts | 8 +++ .../hooks/use_security_jobs.test.ts | 1 + .../hooks/use_security_jobs_helpers.test.tsx | 1 + .../hooks/use_security_jobs_helpers.tsx | 1 + .../__snapshots__/jobs_table.test.tsx.snap | 3 ++ .../jobs_table_filters.test.tsx.snap | 3 ++ 13 files changed, 114 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts index 624056fdf3b825..e9e89a3c99771c 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts @@ -36,6 +36,7 @@ export interface MlSummaryJob { earliestStartTimestampMs?: number; awaitingNodeAssignment: boolean; alertingRules?: MlAnomalyDetectionAlertRule[]; + jobTags: Record; } export interface AuditMessage { diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/extract_job_details.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/extract_job_details.js index 673484f08e1964..dea8fdd30e3727 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/extract_job_details.js +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/extract_job_details.js @@ -26,19 +26,33 @@ export function extractJobDetails(job, basePath, refreshJobList) { items: filterObjects(job, true).map(formatValues), }; + const { job_tags: tags, custom_urls: urls, ...settings } = job.custom_settings ?? {}; const customUrl = { id: 'customUrl', title: i18n.translate('xpack.ml.jobsList.jobDetails.customUrlsTitle', { defaultMessage: 'Custom URLs', }), position: 'right', - items: [], + items: urls ? urls.map((cu) => [cu.url_name, cu.url_value, cu.time_range]) : [], + }; + + const customSettings = { + id: 'analysisConfig', + title: i18n.translate('xpack.ml.jobsList.jobDetails.customSettingsTitle', { + defaultMessage: 'Custom settings', + }), + position: 'right', + items: settings ? filterObjects(settings, true, true) : [], + }; + + const jobTags = { + id: 'analysisConfig', + title: i18n.translate('xpack.ml.jobsList.jobDetails.jobTagsTitle', { + defaultMessage: 'Job tags', + }), + position: 'right', + items: tags ? filterObjects(tags) : [], }; - if (job.custom_settings && job.custom_settings.custom_urls) { - customUrl.items.push( - ...job.custom_settings.custom_urls.map((cu) => [cu.url_name, cu.url_value, cu.time_range]) - ); - } const node = { id: 'node', @@ -213,6 +227,8 @@ export function extractJobDetails(job, basePath, refreshJobList) { analysisConfig, analysisLimits, dataDescription, + customSettings, + jobTags, datafeed, counts, modelSizeStats, diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_details.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_details.js index 812d156421c162..b514c8433daf48 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_details.js +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_details.js @@ -41,7 +41,7 @@ export class JobDetailsUI extends Component { } render() { - const { job } = this.state; + const job = this.state.job ?? this.props.job; const { services: { http: { basePath }, @@ -67,6 +67,8 @@ export class JobDetailsUI extends Component { analysisConfig, analysisLimits, dataDescription, + customSettings, + jobTags, datafeed, counts, modelSizeStats, @@ -85,7 +87,7 @@ export class JobDetailsUI extends Component { content: ( ), time: job.open_time, diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list_view/jobs_list_view.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list_view/jobs_list_view.js index 214b7616cf9272..bf8db538bc8ae6 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list_view/jobs_list_view.js +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list_view/jobs_list_view.js @@ -18,7 +18,7 @@ import { EuiSpacer, EuiTitle, } from '@elastic/eui'; -import { isEqual } from 'lodash'; +import { isEqual, debounce } from 'lodash'; import { ml } from '../../../../services/ml_api_service'; import { checkForAutoStartDatafeed, filterJobs, loadFullJob } from '../utils'; @@ -43,6 +43,11 @@ import { JobListMlAnomalyAlertFlyout } from '../../../../../alerting/ml_alerting let deletingJobsRefreshTimeout = null; +const filterJobsDebounce = debounce((jobsSummaryList, filterClauses, callback) => { + const ss = filterJobs(jobsSummaryList, filterClauses); + callback(ss); +}, 500); + // 'isManagementTable' bool prop to determine when to configure table for use in Kibana management page export class JobsListView extends Component { constructor(props) { @@ -221,7 +226,7 @@ export class JobsListView extends Component { refreshSelectedJobs() { const selectedJobsIds = this.state.selectedJobs.map((j) => j.id); - const filteredJobIds = this.state.filteredJobsSummaryList.map((j) => j.id); + const filteredJobIds = (this.state.filteredJobsSummaryList ?? []).map((j) => j.id); // refresh the jobs stored as selected // only select those which are also in the filtered list @@ -232,9 +237,17 @@ export class JobsListView extends Component { this.setState({ selectedJobs }); } - setFilters = (query) => { - const filterClauses = (query && query.ast && query.ast.clauses) || []; - const filteredJobsSummaryList = filterJobs(this.state.jobsSummaryList, filterClauses); + setFilters = async (query) => { + if (query === null) { + this.setState( + { filteredJobsSummaryList: this.state.jobsSummaryList, filterClauses: [] }, + () => { + this.refreshSelectedJobs(); + } + ); + + return; + } this.props.onJobsViewStateUpdate( { @@ -244,11 +257,30 @@ export class JobsListView extends Component { this._isFiltersSet === false ); - this._isFiltersSet = true; + const filterClauses = (query && query.ast && query.ast.clauses) || []; - this.setState({ filteredJobsSummaryList, filterClauses }, () => { - this.refreshSelectedJobs(); - }); + if (filterClauses.length === 0) { + this.setState({ filteredJobsSummaryList: this.state.jobsSummaryList, filterClauses }, () => { + this.refreshSelectedJobs(); + }); + return; + } + + if (this._isFiltersSet === true) { + filterJobsDebounce(this.state.jobsSummaryList, filterClauses, (jobsSummaryList) => { + this.setState({ filteredJobsSummaryList: jobsSummaryList, filterClauses }, () => { + this.refreshSelectedJobs(); + }); + }); + } else { + // first use after page load, do not debounce. + const filteredJobsSummaryList = filterJobs(this.state.jobsSummaryList, filterClauses); + this.setState({ filteredJobsSummaryList, filterClauses }, () => { + this.refreshSelectedJobs(); + }); + } + + this._isFiltersSet = true; }; onRefreshClick = () => { diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js index 5b8fa5c672c6e0..f004fb6bad49dc 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js @@ -347,12 +347,18 @@ export function filterJobs(jobs, clauses) { // if it's an array of job ids if (c.field === 'id') { js = jobs.filter((job) => c.value.indexOf(jobProperty(job, c.field)) >= 0); - } else { + } else if (c.field === 'groups') { // the groups value is an array of group ids js = jobs.filter((job) => jobProperty(job, c.field).some((g) => c.value.indexOf(g) >= 0)); + } else if (c.field === 'job_tags') { + js = jobTagFilter(jobs, c.value); } } else { - js = jobs.filter((job) => jobProperty(job, c.field) === c.value); + if (c.field === 'job_tags') { + js = js = jobTagFilter(jobs, [c.value]); + } else { + js = jobs.filter((job) => jobProperty(job, c.field) === c.value); + } } } @@ -369,6 +375,25 @@ export function filterJobs(jobs, clauses) { return filteredJobs; } +function jobProperty(job, prop) { + const propMap = { + job_state: 'jobState', + datafeed_state: 'datafeedState', + groups: 'groups', + id: 'id', + job_tags: 'jobTags', + }; + return job[propMap[prop]]; +} + +function jobTagFilter(jobs, value) { + return jobs.filter((job) => { + const tags = jobProperty(job, 'job_tags'); + return Object.entries(tags) + .map((t) => t.join(':')) + .find((t) => value.some((t1) => t1 === t)); + }); +} // check to see if a job has been stored in mlJobService.tempJobCloningObjects // if it has, return an object with the minimum properties needed for the // start datafeed modal. @@ -390,13 +415,3 @@ export function checkForAutoStartDatafeed() { }; } } - -function jobProperty(job, prop) { - const propMap = { - job_state: 'jobState', - datafeed_state: 'datafeedState', - groups: 'groups', - id: 'id', - }; - return job[propMap[prop]]; -} diff --git a/x-pack/plugins/ml/server/models/job_service/jobs.ts b/x-pack/plugins/ml/server/models/job_service/jobs.ts index b7f8ce569641ed..22bac1cb08e190 100644 --- a/x-pack/plugins/ml/server/models/job_service/jobs.ts +++ b/x-pack/plugins/ml/server/models/job_service/jobs.ts @@ -218,6 +218,7 @@ export function jobsProvider( deleting: job.deleting || undefined, awaitingNodeAssignment: isJobAwaitingNodeAssignment(job), alertingRules: job.alerting_rules, + jobTags: job.custom_settings?.job_tags ?? {}, }; if (jobIds.find((j) => j === tempJob.id)) { tempJob.fullJob = job; diff --git a/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.test.ts b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.test.ts index a8f4d09cd78732..403b33d9c08f78 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.test.ts +++ b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.test.ts @@ -55,6 +55,7 @@ describe('useInstalledSecurityJobs', () => { id: 'siem-api-rare_process_linux_ecs', isSingleMetricViewerJob: true, jobState: 'closed', + jobTags: {}, latestTimestampMs: 1557434782207, memory_status: 'hard_limit', processed_record_count: 582251, diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts b/x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts index ac057dff156210..28d0ae179508d2 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts @@ -48,6 +48,7 @@ export const mockOpenedJob: MlSummaryJob = { nodeName: 'siem-es', processed_record_count: 3425264, awaitingNodeAssignment: false, + jobTags: {}, }; export const mockJobsSummaryResponse: MlSummaryJob[] = [ @@ -67,6 +68,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [ earliestTimestampMs: 1554327458406, isSingleMetricViewerJob: true, awaitingNodeAssignment: false, + jobTags: {}, }, { id: 'siem-api-rare_process_linux_ecs', @@ -83,6 +85,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [ earliestTimestampMs: 1557353420495, isSingleMetricViewerJob: true, awaitingNodeAssignment: false, + jobTags: {}, }, { id: 'siem-api-rare_process_windows_ecs', @@ -97,6 +100,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [ datafeedState: 'stopped', isSingleMetricViewerJob: true, awaitingNodeAssignment: false, + jobTags: {}, }, { id: 'siem-api-suspicious_login_activity_ecs', @@ -111,6 +115,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [ datafeedState: 'stopped', isSingleMetricViewerJob: true, awaitingNodeAssignment: false, + jobTags: {}, }, ]; @@ -520,6 +525,7 @@ export const mockSecurityJobs: SecurityJob[] = [ isInstalled: true, isElasticJob: true, awaitingNodeAssignment: false, + jobTags: {}, }, { id: 'rare_process_by_host_linux_ecs', @@ -539,6 +545,7 @@ export const mockSecurityJobs: SecurityJob[] = [ isInstalled: true, isElasticJob: true, awaitingNodeAssignment: false, + jobTags: {}, }, { datafeedId: '', @@ -558,5 +565,6 @@ export const mockSecurityJobs: SecurityJob[] = [ isInstalled: false, isElasticJob: true, awaitingNodeAssignment: false, + jobTags: {}, }, ]; diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs.test.ts b/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs.test.ts index 3731bebd92624e..3c91baa920da7d 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs.test.ts +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs.test.ts @@ -62,6 +62,7 @@ describe('useSecurityJobs', () => { isInstalled: true, isSingleMetricViewerJob: true, jobState: 'closed', + jobTags: {}, latestTimestampMs: 1557434782207, memory_status: 'hard_limit', moduleId: '', diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.test.tsx b/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.test.tsx index 8250807355648a..7a488847cd583a 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.test.tsx @@ -44,6 +44,7 @@ describe('useSecurityJobsHelpers', () => { isInstalled: false, isSingleMetricViewerJob: false, jobState: '', + jobTags: {}, memory_status: '', moduleId: 'siem_auditbeat', processed_record_count: 0, diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx b/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx index 70a2a7c87225f7..fe3803c88e4f77 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx @@ -44,6 +44,7 @@ export const moduleToSecurityJob = ( isInstalled: false, isElasticJob: true, awaitingNodeAssignment: false, + jobTags: {}, }; }; diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/__snapshots__/jobs_table.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/__snapshots__/jobs_table.test.tsx.snap index d64fb474c1fb30..2d2525b92deb1a 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/__snapshots__/jobs_table.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/__snapshots__/jobs_table.test.tsx.snap @@ -46,6 +46,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = ` "isInstalled": true, "isSingleMetricViewerJob": true, "jobState": "closed", + "jobTags": Object {}, "latestResultsTimestampMs": 1571022900000, "latestTimestampMs": 1571022859393, "memory_status": "ok", @@ -73,6 +74,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = ` "isInstalled": true, "isSingleMetricViewerJob": true, "jobState": "closed", + "jobTags": Object {}, "memory_status": "ok", "moduleId": "siem_auditbeat", "processed_record_count": 0, @@ -96,6 +98,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = ` "isInstalled": false, "isSingleMetricViewerJob": false, "jobState": "", + "jobTags": Object {}, "memory_status": "", "moduleId": "siem_winlogbeat", "processed_record_count": 0, diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/filters/__snapshots__/jobs_table_filters.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/filters/__snapshots__/jobs_table_filters.test.tsx.snap index 7367dbf7bdc0a3..d66740ee5bb0eb 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/filters/__snapshots__/jobs_table_filters.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/filters/__snapshots__/jobs_table_filters.test.tsx.snap @@ -49,6 +49,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = ` "isInstalled": true, "isSingleMetricViewerJob": true, "jobState": "closed", + "jobTags": Object {}, "latestResultsTimestampMs": 1571022900000, "latestTimestampMs": 1571022859393, "memory_status": "ok", @@ -76,6 +77,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = ` "isInstalled": true, "isSingleMetricViewerJob": true, "jobState": "closed", + "jobTags": Object {}, "memory_status": "ok", "moduleId": "siem_auditbeat", "processed_record_count": 0, @@ -99,6 +101,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = ` "isInstalled": false, "isSingleMetricViewerJob": false, "jobState": "", + "jobTags": Object {}, "memory_status": "", "moduleId": "siem_winlogbeat", "processed_record_count": 0,