Add timestamp processor

CHANGELOG.next.asciidoc

@@ -178,6 +178,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `decode_base64_field` processor for decoding base64 field. {pull}11914[11914]
 - Add aws overview dashboard. {issue}11007[11007] {pull}12175[12175]
 - Add `decompress_gzip_field` processor. {pull}12733[12733]
+- Add `timestamp` processor for parsing time fields. {pull}12699[12699]
 
 *Auditbeat*
 

Makefile

@@ -7,7 +7,7 @@ PYTHON_ENV?=$(BUILD_DIR)/python-env
 VIRTUALENV_PARAMS?=
 FIND=find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*"
 GOLINT=golint
-GOLINT_REPO=github.com/golang/lint/golint
+GOLINT_REPO=golang.org/x/lint/golint
 REVIEWDOG=reviewdog
 REVIEWDOG_OPTIONS?=-diff "git diff master"
 REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog

NOTICE.txt

@@ -9,6 +9,66 @@ Third party libraries used by the Elastic Beats project:
 ==========================================================================
 
 
+--------------------------------------------------------------------
+Dependency: 4d63.com/tz
+Revision: bd6cee76f7319745d44ff235e42124b3891e293d
+License type (autodetected): MIT
+./vendor/4d63.com/tz/LICENSE:
+--------------------------------------------------------------------
+MIT License
+
+Copyright (c) 2018 Leigh McCulloch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+zoneinfo.go generated from /lib/time/zoneinfo.zip from Go.
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 --------------------------------------------------------------------
 Dependency: cloud.google.com/go
 Revision: 632e11ec7c242a85e1931e4a8fdda2002316569f

filebeat/cmd/root.go

@@ -27,8 +27,9 @@ import (
 	cmd "github.com/elastic/beats/libbeat/cmd"
 	"github.com/elastic/beats/libbeat/cmd/instance"
 
-	// Import the script processor.
+	// Import processors.
 	_ "github.com/elastic/beats/libbeat/processors/script"
+	_ "github.com/elastic/beats/libbeat/processors/timestamp"
 )
 
 // Name of this beat

filebeat/docs/index.asciidoc

@@ -20,6 +20,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
 :has_docker_label_ex:
 :has_decode_csv_fields_processor:
 :has_script_processor:
+:has_timestamp_processor:
 :has_modules_command:
 :has_registry:
 :deb_os:

journalbeat/cmd/root.go

@@ -23,8 +23,9 @@ import (
 	cmd "github.com/elastic/beats/libbeat/cmd"
 	"github.com/elastic/beats/libbeat/cmd/instance"
 
-	// Import the script processor.
+	// Import processors.
 	_ "github.com/elastic/beats/libbeat/processors/script"
+	_ "github.com/elastic/beats/libbeat/processors/timestamp"
 )
 
 // Name of this beat

journalbeat/docs/index.asciidoc

@@ -15,6 +15,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
 :beat_default_index_prefix: {beatname_lc}
 :has_decode_csv_fields_processor:
 :has_script_processor:
+:has_timestamp_processor:
 :deb_os:
 :rpm_os:
 :linux_os:

libbeat/docs/processors-using.asciidoc

@@ -202,30 +202,34 @@ The supported processors are:
  * <<add-docker-metadata,`add_docker_metadata`>>
  * <<add-fields, `add_fields`>>
  * <<add-host-metadata,`add_host_metadata`>>
- * <<add-observer-metadata,`add_observer_metadata`>>
  * <<add-kubernetes-metadata,`add_kubernetes_metadata`>>
  * <<add-labels, `add_labels`>>
  * <<add-locale,`add_locale`>>
+ * <<add-observer-metadata,`add_observer_metadata`>>
  * <<add-process-metadata,`add_process_metadata`>>
  * <<add-tags, `add_tags`>>
  * <<community-id,`community_id`>>
  * <<convert,`convert`>>
+ * <<decode-base64-field,`decode_base64_field`>>
 ifdef::has_decode_csv_fields_processor[]
  * <<decode-csv-fields,`decode_csv_fields`>>
 endif::[]
  * <<decode-json-fields,`decode_json_fields`>>
  * <<decode-base64-field,`decode_base64_field`>>
  * <<decompress-gzip-field,`decompress_gzip_field`>>
  * <<dissect, `dissect`>>
- * <<extract-array,`extract_array`>>
  * <<processor-dns, `dns`>>
  * <<drop-event,`drop_event`>>
  * <<drop-fields,`drop_fields`>>
+ * <<extract-array,`extract_array`>>
  * <<include-fields,`include_fields`>>
  * <<rename-fields,`rename`>>
 ifdef::has_script_processor[]
  * <<processor-script,`script`>>
 endif::[]
+ifdef::has_timestamp_processor[]
+ * <<processor-timestamp,`timestamp`>>
+endif::[]
 
 [[conditions]]
 ==== Conditions
@@ -1066,6 +1070,49 @@ See <<conditions>> for a list of supported conditions.
 NOTE: If you define an empty list of fields under `drop_fields`, then no fields
 are dropped.
 
+[[extract-array]]
+=== Extract array
+
+experimental[]
+
+The `extract_array` processor populates fields with values read from an array
+field. The following example will populate `source.ip` with the first element of
+the `my_array` field, `destination.ip` with the second element, and
+`network.transport` with the third.
+
+[source,yaml]
+-----------------------------------------------------
+processors:
+ - extract_array:
+     field: my_array
+     mappings:
+        source.ip: 0
+        destination.ip: 1
+        network.transport: 2
+-----------------------------------------------------
+
+The following settings are supported:
+
+`field`:: The array field whose elements are to be extracted.
+`mappings`:: Maps each field name to an array index. Use 0 for the first element in
+             the array. Multiple fields can be mapped to the same array element.
+`ignore_missing`:: (Optional) Whether to ignore events where the array field is
+                   missing. The default is `false`, which will fail processing
+                   of an event if the specified field does not exist. Set it to
+                   `true` to ignore this condition.
+`overwrite_keys`:: Whether the target fields specified in the mapping are
+                   overwritten if they already exist. The default is `false`,
+                   which will fail processing if a target field already exists.
+`fail_on_error`:: (Optional) If set to `true` and an error happens, changes to
+                  the event are reverted, and the original event is returned. If
+                  set to `false`, processing continues despite errors.
+                  Default is `true`.
+`omit_empty`:: (Optional) Whether empty values are extracted from the array. If
+                  set to `true`, instead of the target field being set to an
+                  empty value, it is left unset. The empty string (`""`), an
+                  empty array (`[]`) or an empty object (`{}`) are considered
+                  empty values. Default is `false`.
+
 [[include-fields]]
 === Keep fields from events
 
@@ -1820,45 +1867,70 @@ thrown.
 |===
 endif::[]
 
-[[extract-array]]
-=== Extract array
+ifdef::has_timestamp_processor[]
+[[processor-timestamp]]
+=== Timestamp
 
-experimental[]
+beta[]
 
-The `extract_array` processor populates fields with values read from an array
-field. The following example will populate `source.ip` with the first element of
-the `my_array` field, `destination.ip` with the second element, and
-`network.transport` with the third.
+The `timestamp` processor parses a timestamp from a field. By default the
+timestamp processor writes the parsed result to the `@timestamp` field. You can
+specify a different field by setting the `target_field` parameter. The timestamp
+value is parsed according to the `layouts` parameter. Multiple layouts can be
+specified and they will be used sequentially to attempt parsing the timestamp
+field.
 
-[source,yaml]
------------------------------------------------------
-processors:
- - extract_array:
-     field: my_array
-     mappings:
-        source.ip: 0
-        destination.ip: 1
-        network.transport: 2
------------------------------------------------------
+NOTE: The timestamp layouts used by this processor are different than the
+      formats supported by date processors in Logstash and Elasticsearch Ingest
+      Node.
 
-The following settings are supported:
+The `layouts` are described using a reference time that is based on this
+specific time:
 
-`field`:: The array field whose elements are to be extracted.
-`mappings`:: Maps each field name to an array index. Use 0 for the first element in
-             the array. Multiple fields can be mapped to the same array element.
-`ignore_missing`:: (Optional) Whether to ignore events where the array field is
-                   missing. The default is `false`, which will fail processing
-                   of an event if the specified field does not exist. Set it to
-                   `true` to ignore this condition.
-`overwrite_keys`:: Whether the target fields specified in the mapping are
-                   overwritten if they already exist. The default is `false`,
-                   which will fail processing if a target field already exists.
-`fail_on_error`:: (Optional) If set to `true` and an error happens, changes to
-                  the event are reverted, and the original event is returned. If
-                  set to `false`, processing continues despite errors.
-                  Default is `true`.
-`omit_empty`:: (Optional) Whether empty values are extracted from the array. If
-                  set to `true`, instead of the target field being set to an
-                  empty value, it is left unset. The empty string (`""`), an
-                  empty array (`[]`) or an empty object (`{}`) are considered
-                  empty values. Default is `false`.
+    Mon Jan 2 15:04:05 MST 2006
+
+Since MST is GMT-0700, the reference time is:
+
+    01/02 03:04:05PM '06 -0700
+
+To define your own layout, rewrite the reference time in a format that matches
+the timestamps you expect to parse. For more layout examples and details see the
+https://godoc.org/time#pkg-constants[Go time package documentation].
+
+If a layout does not contain a year then the current year in the specified
+`timezone` is added to the time value.
+
+.Timestamp options
+[options="header"]
+|======
+| Name             | Required | Default    | Description                                                                                 |
+| `field`          | yes      |            | Source field containing the time to be parsed.                                              |
+| `target_field`   | no       | @timestamp | Target field for the parsed time value. The target value is always written as UTC.          |
+| `layouts`        | yes      |            | Timestamp layouts that define the expected time value format.                               |
+| `timezone`       | no       | UTC        | Timezone (e.g. America/New_York) to use when parsing a timestamp not containing a timezone. |
+| `ignore_missing` | no       | false      | Ignore errors when the source field is missing.                                             |
+| `ignore_failure` | no       | false      | Ignore all errors produced by the processor.                                                |
+| `test`           | no       |            | A list of timestamps that must parse successfully when loading the processor.               |
+| `id`             | no       |            | An identifier for this processor instance. Useful for debugging.                            |
+|======
+
+Here is an example that parses the `start_time` field and writes the result
+to the `@timestamp` field then deletes the `start_time` field. When the
+processor is loaded it will immediately validate that the two `test` timestamps
+parse with this configuration.
+
+[source,yaml]
+----
+processors:
+- timestamp:
+    field: start_time
+    layouts:
+      - '2006-01-02T15:04:05Z'
+      - '2006-01-02T15:04:05.999Z'
+    test:
+      - '2019-06-22T16:33:51Z'
+      - '2019-11-18T04:59:51.123Z'
+- drop_fields:
+    fields: [start_time]
+----
+endif::[]

libbeat/processors/script/javascript/module/processor/chain.go

@@ -126,6 +126,7 @@ func newJSProcessor(fn jsFunction) *jsProcessor {
 func (p *jsProcessor) run(event javascript.Event) error {
 	p.call.Arguments[0] = event.JSObject()
 	p.fn(p.call)
+	p.call.Arguments[0] = nil
 	return nil
 }
 

libbeat/processors/script/javascript/module/processor/processor.go

@@ -38,6 +38,7 @@ import (
 	"github.com/elastic/beats/libbeat/processors/dns"
 	"github.com/elastic/beats/libbeat/processors/extract_array"
 	"github.com/elastic/beats/libbeat/processors/script/javascript"
+	"github.com/elastic/beats/libbeat/processors/timestamp"
 )
 
 // Create constructors for most of the Beat processors.
@@ -61,6 +62,7 @@ var constructors = map[string]processors.Constructor{
 	"DNS":                   dns.New,
 	"ExtractArray":          extract_array.New,
 	"Rename":                actions.NewRenameFields,
+	"Timestamp":             timestamp.New,
 	"TruncateFields":        actions.NewTruncateFields,
 }
 

libbeat/processors/timestamp/config.go

@@ -0,0 +1,35 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package timestamp
+
+type config struct {
+	Field          string   `config:"field" validate:"required"`   // Source field containing time time to be parsed.
+	TargetField    string   `config:"target_field"`                // Target field for the parsed time value. The target value is always written as UTC. Defaults to @timestamp.
+	Layouts        []string `config:"layouts" validate:"required"` // Timestamp layouts that define the expected time value format.
+	Timezone       string   `config:"timezone"`                    // Timezone (e.g. America/New_York) to use when parsing a timestamp not containing a timezone.
+	IgnoreMissing  bool     `config:"ignore_missing"`              // Ignore errors when the source field is missing.
+	IgnoreFailure  bool     `config:"ignore_failure"`              // Ignore errors when parsing the timestamp.
+	TestTimestamps []string `config:"test"`                        // A list of timestamps that must parse successfully when loading the processor.
+	ID             string   `config:"id"`                          // An identifier for this processor. Useful for debugging.
+}
+
+func defaultConfig() config {
+	return config{
+		TargetField: "@timestamp",
+	}
+}