From 6aacfbd25dc38ef4717745203b9048168ca68ea3 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Thu, 2 Jul 2020 12:04:28 -0700 Subject: [PATCH] [Data Plugin] Allow server-side date formatters to accept custom timezone When Advanced Settings shows the date format timezone to be "Browser," this means nothing to field formatters in the server-side context. The field formatters need a way to accept custom format parameters. This allows a server-side module that creates a FieldFormatMap to set a timezone as a custom parameter. When custom formatting parameters exist, they get combined with the defaults. --- .../constants/base_formatters.ts | 2 - .../common/field_formats/converters/index.ts | 1 - .../field_formats/field_formats_registry.ts | 20 ++- .../data/common/field_formats/index.ts | 1 - .../converters/date_nanos.test.ts | 0 .../field_formats/converters/date_nanos.ts | 11 +- .../public/field_formats/converters/index.ts | 1 + .../data/public/field_formats/index.ts | 2 +- src/plugins/data/public/index.ts | 3 +- .../converters/date_nanos_server.test.ts | 129 ++++++++++++++++ .../converters/date_nanos_server.ts | 141 ++++++++++++++++++ .../server/field_formats/converters/index.ts | 1 + .../field_formats/field_formats_service.ts | 8 +- src/plugins/data/server/index.ts | 2 - 14 files changed, 305 insertions(+), 17 deletions(-) rename src/plugins/data/{common => public}/field_formats/converters/date_nanos.test.ts (100%) rename src/plugins/data/{common => public}/field_formats/converters/date_nanos.ts (96%) create mode 100644 src/plugins/data/server/field_formats/converters/date_nanos_server.test.ts create mode 100644 src/plugins/data/server/field_formats/converters/date_nanos_server.ts diff --git a/src/plugins/data/common/field_formats/constants/base_formatters.ts b/src/plugins/data/common/field_formats/constants/base_formatters.ts index 921c50571f727..99c24496cf220 100644 --- a/src/plugins/data/common/field_formats/constants/base_formatters.ts +++ b/src/plugins/data/common/field_formats/constants/base_formatters.ts @@ -23,7 +23,6 @@ import { BoolFormat, BytesFormat, ColorFormat, - DateNanosFormat, DurationFormat, IpFormat, NumberFormat, @@ -40,7 +39,6 @@ export const baseFormatters: FieldFormatInstanceType[] = [ BoolFormat, BytesFormat, ColorFormat, - DateNanosFormat, DurationFormat, IpFormat, NumberFormat, diff --git a/src/plugins/data/common/field_formats/converters/index.ts b/src/plugins/data/common/field_formats/converters/index.ts index cc9fae7fc9965..f71ddf5f781f7 100644 --- a/src/plugins/data/common/field_formats/converters/index.ts +++ b/src/plugins/data/common/field_formats/converters/index.ts @@ -19,7 +19,6 @@ export { UrlFormat } from './url'; export { BytesFormat } from './bytes'; -export { DateNanosFormat } from './date_nanos'; export { RelativeDateFormat } from './relative_date'; export { DurationFormat } from './duration'; export { IpFormat } from './ip'; diff --git a/src/plugins/data/common/field_formats/field_formats_registry.ts b/src/plugins/data/common/field_formats/field_formats_registry.ts index 74a942b51583d..f76bdc530bc82 100644 --- a/src/plugins/data/common/field_formats/field_formats_registry.ts +++ b/src/plugins/data/common/field_formats/field_formats_registry.ts @@ -40,6 +40,7 @@ export class FieldFormatsRegistry { protected defaultMap: Record = {}; protected metaParamsOptions: Record = {}; protected getConfig?: FieldFormatsGetConfigFn; + protected customParams: Record = {}; // overriden on the public contract public deserialize: (mapping: SerializedFieldFormat) => IFieldFormat = () => { return new (FieldFormat.from(identity))(); @@ -57,6 +58,13 @@ export class FieldFormatsRegistry { this.metaParamsOptions = metaParamsOptions; } + /* + * Allow use-case specific params that are reflected in getInstance / getDefaultInstancePlain + */ + setCustomParams(params: Record) { + this.customParams = params; + } + /** * Get the id of the default type for this field type * using the format:defaultTypeMap config map @@ -157,7 +165,11 @@ export class FieldFormatsRegistry { * @return {FieldFormat} */ getInstance = memoize( - (formatId: FieldFormatId, params: Record = {}): FieldFormat => { + (formatId: FieldFormatId, instanceParams: Record = {}): FieldFormat => { + const params = { + ...instanceParams, + ...this.customParams, + }; const ConcreteFieldFormat = this.getType(formatId); if (!ConcreteFieldFormat) { @@ -182,8 +194,12 @@ export class FieldFormatsRegistry { */ getDefaultInstancePlain(fieldType: KBN_FIELD_TYPES, esTypes?: ES_FIELD_TYPES[]): FieldFormat { const conf = this.getDefaultConfig(fieldType, esTypes); + const defaultParams = { + ...conf.params, + ...this.customParams, + }; - return this.getInstance(conf.id, conf.params); + return this.getInstance(conf.id, defaultParams); } /** * Returns a cache key built by the given variables for caching in memoized diff --git a/src/plugins/data/common/field_formats/index.ts b/src/plugins/data/common/field_formats/index.ts index 104ff030873aa..d622af2f663a1 100644 --- a/src/plugins/data/common/field_formats/index.ts +++ b/src/plugins/data/common/field_formats/index.ts @@ -27,7 +27,6 @@ export { BoolFormat, BytesFormat, ColorFormat, - DateNanosFormat, DurationFormat, IpFormat, NumberFormat, diff --git a/src/plugins/data/common/field_formats/converters/date_nanos.test.ts b/src/plugins/data/public/field_formats/converters/date_nanos.test.ts similarity index 100% rename from src/plugins/data/common/field_formats/converters/date_nanos.test.ts rename to src/plugins/data/public/field_formats/converters/date_nanos.test.ts diff --git a/src/plugins/data/common/field_formats/converters/date_nanos.ts b/src/plugins/data/public/field_formats/converters/date_nanos.ts similarity index 96% rename from src/plugins/data/common/field_formats/converters/date_nanos.ts rename to src/plugins/data/public/field_formats/converters/date_nanos.ts index 3fa2b1c276cd7..3345d49cac30e 100644 --- a/src/plugins/data/common/field_formats/converters/date_nanos.ts +++ b/src/plugins/data/public/field_formats/converters/date_nanos.ts @@ -18,11 +18,14 @@ */ import { i18n } from '@kbn/i18n'; -import moment, { Moment } from 'moment'; import { memoize, noop } from 'lodash'; -import { KBN_FIELD_TYPES } from '../../kbn_field_types/types'; -import { FieldFormat } from '../field_format'; -import { TextContextTypeConvert, FIELD_FORMAT_IDS } from '../types'; +import moment, { Moment } from 'moment'; +import { + FieldFormat, + FIELD_FORMAT_IDS, + KBN_FIELD_TYPES, + TextContextTypeConvert, +} from '../../../common'; /** * Analyse the given moment.js format pattern for the fractional sec part (S,SS,SSS...) diff --git a/src/plugins/data/public/field_formats/converters/index.ts b/src/plugins/data/public/field_formats/converters/index.ts index c51111092beca..f5f154084242f 100644 --- a/src/plugins/data/public/field_formats/converters/index.ts +++ b/src/plugins/data/public/field_formats/converters/index.ts @@ -18,3 +18,4 @@ */ export { DateFormat } from './date'; +export { DateNanosFormat } from './date_nanos'; diff --git a/src/plugins/data/public/field_formats/index.ts b/src/plugins/data/public/field_formats/index.ts index 015d5b39561bb..4525959fb864d 100644 --- a/src/plugins/data/public/field_formats/index.ts +++ b/src/plugins/data/public/field_formats/index.ts @@ -18,5 +18,5 @@ */ export { FieldFormatsService, FieldFormatsSetup, FieldFormatsStart } from './field_formats_service'; -export { DateFormat } from './converters'; +export { DateFormat, DateNanosFormat } from './converters'; export { baseFormattersPublic } from './constants'; diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 89b0d7e0303b9..fb685821058f0 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -157,7 +157,6 @@ import { BoolFormat, BytesFormat, ColorFormat, - DateNanosFormat, DurationFormat, IpFormat, NumberFormat, @@ -170,7 +169,7 @@ import { TruncateFormat, } from '../common/field_formats'; -import { DateFormat } from './field_formats'; +import { DateNanosFormat, DateFormat } from './field_formats'; export { baseFormattersPublic } from './field_formats'; // Field formats helpers namespace: diff --git a/src/plugins/data/server/field_formats/converters/date_nanos_server.test.ts b/src/plugins/data/server/field_formats/converters/date_nanos_server.test.ts new file mode 100644 index 0000000000000..83e24cb76c864 --- /dev/null +++ b/src/plugins/data/server/field_formats/converters/date_nanos_server.test.ts @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import moment from 'moment-timezone'; +import { DateNanosFormat, analysePatternForFract, formatWithNanos } from './date_nanos_server'; + +describe('Date Nanos Format', () => { + let convert: Function; + let mockConfig: Record; + + beforeEach(() => { + mockConfig = {}; + mockConfig.dateNanosFormat = 'MMMM Do YYYY, HH:mm:ss.SSSSSSSSS'; + mockConfig['dateFormat:tz'] = 'Browser'; + + const getConfig = (key: string) => mockConfig[key]; + const date = new DateNanosFormat({}, getConfig); + + convert = date.convert.bind(date); + }); + + test('should inject fractional seconds into formatted timestamp', () => { + [ + { + input: '2019-05-20T14:04:56.357001234Z', + pattern: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS', + expected: 'May 20, 2019 @ 14:04:56.357001234', + }, + { + input: '2019-05-05T14:04:56.357111234Z', + pattern: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS', + expected: 'May 5, 2019 @ 14:04:56.357111234', + }, + { + input: '2019-05-05T14:04:56.357Z', + pattern: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS', + expected: 'May 5, 2019 @ 14:04:56.357000000', + }, + { + input: '2019-05-05T14:04:56Z', + pattern: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS', + expected: 'May 5, 2019 @ 14:04:56.000000000', + }, + { + input: '2019-05-05T14:04:56.201900001Z', + pattern: 'MMM D, YYYY @ HH:mm:ss SSSS', + expected: 'May 5, 2019 @ 14:04:56 2019', + }, + { + input: '2019-05-05T14:04:56.201900001Z', + pattern: 'SSSSSSSSS', + expected: '201900001', + }, + ].forEach((fixture) => { + const fracPattern = analysePatternForFract(fixture.pattern); + const momentDate = moment(fixture.input).utc(); + const value = formatWithNanos(momentDate, fixture.input, fracPattern); + expect(value).toBe(fixture.expected); + }); + }); + + test('decoding an undefined or null date should return an empty string', () => { + expect(convert(null)).toBe('-'); + expect(convert(undefined)).toBe('-'); + }); + + test('should clear the memoization cache after changing the date', () => { + function setDefaultTimezone() { + moment.tz.setDefault(mockConfig['dateFormat:tz']); + } + + const dateTime = '2019-05-05T14:04:56.201900001Z'; + + mockConfig['dateFormat:tz'] = 'America/Chicago'; + setDefaultTimezone(); + const chicagoTime = convert(dateTime); + + mockConfig['dateFormat:tz'] = 'America/Phoenix'; + setDefaultTimezone(); + const phoenixTime = convert(dateTime); + + expect(chicagoTime).not.toBe(phoenixTime); + }); + + test('should return the value itself when it cannot successfully be formatted', () => { + const dateMath = 'now+1M/d'; + expect(convert(dateMath)).toBe(dateMath); + }); +}); + +describe('analysePatternForFract', () => { + test('analysePatternForFract using timestamp format containing fractional seconds', () => { + expect(analysePatternForFract('MMM, YYYY @ HH:mm:ss.SSS')).toMatchInlineSnapshot(` + Object { + "length": 3, + "pattern": "MMM, YYYY @ HH:mm:ss.SSS", + "patternEscaped": "MMM, YYYY @ HH:mm:ss.[SSS]", + "patternNanos": "SSS", + } + `); + }); + + test('analysePatternForFract using timestamp format without fractional seconds', () => { + expect(analysePatternForFract('MMM, YYYY @ HH:mm:ss')).toMatchInlineSnapshot(` + Object { + "length": 0, + "pattern": "MMM, YYYY @ HH:mm:ss", + "patternEscaped": "", + "patternNanos": "", + } + `); + }); +}); diff --git a/src/plugins/data/server/field_formats/converters/date_nanos_server.ts b/src/plugins/data/server/field_formats/converters/date_nanos_server.ts new file mode 100644 index 0000000000000..c5828fd9c51b9 --- /dev/null +++ b/src/plugins/data/server/field_formats/converters/date_nanos_server.ts @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { i18n } from '@kbn/i18n'; +import { memoize, noop } from 'lodash'; +import moment, { Moment } from 'moment-timezone'; +import { + FieldFormat, + FIELD_FORMAT_IDS, + KBN_FIELD_TYPES, + TextContextTypeConvert, +} from '../../../common'; + +/** + * Analyse the given moment.js format pattern for the fractional sec part (S,SS,SSS...) + * returning length, match, pattern and an escaped pattern, that excludes the fractional + * part when formatting with moment.js -> e.g. [SSS] + */ +export function analysePatternForFract(pattern: string) { + const fracSecMatch = pattern.match('S+') as any; // extract fractional seconds sub-pattern + const fracSecMatchStr = fracSecMatch ? fracSecMatch[0] : ''; + + return { + length: fracSecMatchStr.length, + patternNanos: fracSecMatchStr, + pattern, + patternEscaped: fracSecMatchStr ? pattern.replace(fracSecMatch, `[${fracSecMatch}]`) : '', + }; +} + +/** + * Format a given moment.js date object + * Since momentjs would loose the exact value for fractional seconds with a higher resolution than + * milliseconds, the fractional pattern is replaced by the fractional value of the raw timestamp + */ +export function formatWithNanos( + dateMomentObj: Moment, + valRaw: string, + fracPatternObj: Record +) { + if (fracPatternObj.length <= 3) { + // S,SS,SSS is formatted correctly by moment.js + return dateMomentObj.format(fracPatternObj.pattern); + } else { + // Beyond SSS the precise value of the raw datetime string is used + const valFormatted = dateMomentObj.format(fracPatternObj.patternEscaped); + // Extract fractional value of ES formatted timestamp, zero pad if necessary: + // 2020-05-18T20:45:05.957Z -> 957000000 + // 2020-05-18T20:45:05.957000123Z -> 957000123 + // we do not need to take care of the year 10000 bug since max year of date_nanos is 2262 + const valNanos = valRaw + .substr(20, valRaw.length - 21) // remove timezone(Z) + .padEnd(9, '0') // pad shorter fractionals + .substr(0, fracPatternObj.patternNanos.length); + return valFormatted.replace(fracPatternObj.patternNanos, valNanos); + } +} + +export class DateNanosFormat extends FieldFormat { + static id = FIELD_FORMAT_IDS.DATE_NANOS; + static title = i18n.translate('data.fieldFormats.date_nanos.title', { + defaultMessage: 'Date nanos', + }); + static fieldType = KBN_FIELD_TYPES.DATE; + + private memoizedConverter: Function = noop; + private memoizedPattern: string = ''; + private timeZone: string = ''; + + getParamDefaults() { + return { + pattern: this.getConfig!('dateNanosFormat'), + fallbackPattern: this.getConfig!('dateFormat'), + timezone: this.getConfig!('dateFormat:tz'), + }; + } + + textConvert: TextContextTypeConvert = (val) => { + // don't give away our ref to converter so + // we can hot-swap when config changes + const pattern = this.param('pattern'); + const timezone = this.param('timezone'); + const fractPattern = analysePatternForFract(pattern); + const fallbackPattern = this.param('patternFallback'); + + const timezoneChanged = this.timeZone !== timezone; + const datePatternChanged = this.memoizedPattern !== pattern; + if (timezoneChanged || datePatternChanged) { + this.timeZone = timezone; + this.memoizedPattern = pattern; + + this.memoizedConverter = memoize((value: any) => { + if (value === null || value === undefined) { + return '-'; + } + + /* On the server, importing moment returns a new instance. Unlike on + * the client side, it doesn't have the dateFormat:tz configuration + * baked in. + * We need to set the timezone manually here. The date is taken in as + * UTC and converted into the desired timezone. */ + let date; + if (this.timeZone === 'Browser') { + // Assume a warning has been logged that this can be unpredictable. It + // would be too verbose to log anything here. + date = moment.utc(val); + } else { + date = moment.utc(val).tz(this.timeZone); + } + + if (typeof value !== 'string' && date.isValid()) { + // fallback for max/min aggregation, where unixtime in ms is returned as a number + // aggregations in Elasticsearch generally just return ms + return date.format(fallbackPattern); + } else if (date.isValid()) { + return formatWithNanos(date, value, fractPattern); + } else { + return value; + } + }); + } + + return this.memoizedConverter(val); + }; +} diff --git a/src/plugins/data/server/field_formats/converters/index.ts b/src/plugins/data/server/field_formats/converters/index.ts index f5c69df972869..1c6b827e2fbb5 100644 --- a/src/plugins/data/server/field_formats/converters/index.ts +++ b/src/plugins/data/server/field_formats/converters/index.ts @@ -18,3 +18,4 @@ */ export { DateFormat } from './date_server'; +export { DateNanosFormat } from './date_nanos_server'; diff --git a/src/plugins/data/server/field_formats/field_formats_service.ts b/src/plugins/data/server/field_formats/field_formats_service.ts index 70584efbee0a0..cafb88de4b893 100644 --- a/src/plugins/data/server/field_formats/field_formats_service.ts +++ b/src/plugins/data/server/field_formats/field_formats_service.ts @@ -23,10 +23,14 @@ import { baseFormatters, } from '../../common/field_formats'; import { IUiSettingsClient } from '../../../../core/server'; -import { DateFormat } from './converters'; +import { DateFormat, DateNanosFormat } from './converters'; export class FieldFormatsService { - private readonly fieldFormatClasses: FieldFormatInstanceType[] = [DateFormat, ...baseFormatters]; + private readonly fieldFormatClasses: FieldFormatInstanceType[] = [ + DateFormat, + DateNanosFormat, + ...baseFormatters, + ]; public setup() { return { diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index 6a4eb38b552ff..c9f94ee25bd35 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -86,7 +86,6 @@ import { BoolFormat, BytesFormat, ColorFormat, - DateNanosFormat, DurationFormat, IpFormat, NumberFormat, @@ -105,7 +104,6 @@ export const fieldFormats = { BoolFormat, BytesFormat, ColorFormat, - DateNanosFormat, DurationFormat, IpFormat, NumberFormat,