Add per-field metadata.

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -1229,11 +1229,11 @@ public void testFieldCaps() throws IOException {
         assertEquals(2, ratingResponse.size());
 
         FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities(
-            "rating", "keyword", true, true, new String[]{"index2"}, null, null);
+            "rating", "keyword", true, true, new String[]{"index2"}, null, null, Collections.emptyMap());
         assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword"));
 
         FieldCapabilities expectedLongCapabilities = new FieldCapabilities(
-            "rating", "long", true, true, new String[]{"index1"}, null, null);
+            "rating", "long", true, true, new String[]{"index1"}, null, null, Collections.emptyMap());
         assertEquals(expectedLongCapabilities, ratingResponse.get("long"));
 
         // Check the capabilities for the 'field' field.
@@ -1242,7 +1242,7 @@ public void testFieldCaps() throws IOException {
         assertEquals(1, fieldResponse.size());
 
         FieldCapabilities expectedTextCapabilities = new FieldCapabilities(
-            "field", "text", true, false);
+            "field", "text", true, false, Collections.emptyMap());
         assertEquals(expectedTextCapabilities, fieldResponse.get("text"));
     }
 

docs/reference/mapping/params.asciidoc

@@ -8,23 +8,24 @@ parameters that are used by <<mapping-types,field mappings>>:
 The following mapping parameters are common to some or all field datatypes:
 
 * <<analyzer,`analyzer`>>
-* <<normalizer, `normalizer`>>
 * <<mapping-boost,`boost`>>
 * <<coerce,`coerce`>>
 * <<copy-to,`copy_to`>>
 * <<doc-values,`doc_values`>>
 * <<dynamic,`dynamic`>>
+* <<eager-global-ordinals,`eager_global_ordinals`>>
 * <<enabled,`enabled`>>
 * <<fielddata,`fielddata`>>
-* <<eager-global-ordinals,`eager_global_ordinals`>>
+* <<multi-fields,`fields`>>
 * <<mapping-date-format,`format`>>
 * <<ignore-above,`ignore_above`>>
 * <<ignore-malformed,`ignore_malformed`>>
 * <<index-options,`index_options`>>
 * <<index-phrases,`index_phrases`>>
 * <<index-prefixes,`index_prefixes`>>
 * <<mapping-index,`index`>>
-* <<multi-fields,`fields`>>
+* <<mapping-field-meta,`meta`>>
+* <<normalizer, `normalizer`>>
 * <<norms,`norms`>>
 * <<null-value,`null_value`>>
 * <<position-increment-gap,`position_increment_gap`>>
@@ -37,8 +38,6 @@ The following mapping parameters are common to some or all field datatypes:
 
 include::params/analyzer.asciidoc[]
 
-include::params/normalizer.asciidoc[]
-
 include::params/boost.asciidoc[]
 
 include::params/coerce.asciidoc[]
@@ -49,10 +48,10 @@ include::params/doc-values.asciidoc[]
 
 include::params/dynamic.asciidoc[]
 
-include::params/enabled.asciidoc[]
-
 include::params/eager-global-ordinals.asciidoc[]
 
+include::params/enabled.asciidoc[]
+
 include::params/fielddata.asciidoc[]
 
 include::params/format.asciidoc[]
@@ -69,8 +68,12 @@ include::params/index-phrases.asciidoc[]
 
 include::params/index-prefixes.asciidoc[]
 
+include::params/meta.asciidoc[]
+
 include::params/multi-fields.asciidoc[]
 
+include::params/normalizer.asciidoc[]
+
 include::params/norms.asciidoc[]
 
 include::params/null-value.asciidoc[]

docs/reference/mapping/params/meta.asciidoc

@@ -0,0 +1,31 @@
+[[mapping-field-meta]]
+=== `meta`
+
+Metadata attached to the field. This metadata is opaque to Elasticsearch, it is
+only useful for multiple applications that work on the same indices to share
+meta information about fields such as units
+
+[source,console]
+------------
+PUT my_index
+{
+  "mappings": {
+    "properties": {
+      "latency": {
+        "type": "long",
+        "meta": {
+          "unit": "ms"
+        }
+      }
+    }
+  }
+}
+------------
+// TEST
+
+NOTE: Field metadata enforces at most 5 entries, that keys have a length that
+is less than or equal to 20, and that values are strings whose length is less
+than or equal to 50.
+
+NOTE: Field metadata is updatable by submitting a mapping update. The metadata
+of the update will override the metadata of the existing field.

docs/reference/mapping/types/boolean.asciidoc

@@ -120,3 +120,6 @@ The following parameters are accepted by `boolean` fields:
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
     (default).
 
+<<mapping-field-meta,`meta`>>::
+
+    Metadata about the field.

docs/reference/mapping/types/date.asciidoc

@@ -137,3 +137,7 @@ The following parameters are accepted by `date` fields:
     Whether the field value should be stored and retrievable separately from
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
     (default).
+
+<<mapping-field-meta,`meta`>>::
+
+    Metadata about the field.

docs/reference/mapping/types/keyword.asciidoc

@@ -109,6 +109,10 @@ The following parameters are accepted by `keyword` fields:
     when building a query for this field.
     Accepts `true` or `false` (default).
 
+<<mapping-field-meta,`meta`>>::
+
+    Metadata about the field.
+
 NOTE: Indexes imported from 2.x do not support `keyword`. Instead they will
 attempt to downgrade `keyword` into `string`. This allows you to merge modern
 mappings with legacy mappings. Long lived indexes will have to be recreated

docs/reference/mapping/types/numeric.asciidoc

@@ -124,6 +124,10 @@ The following parameters are accepted by numeric types:
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
     (default).
 
+<<mapping-field-meta,`meta`>>::
+
+    Metadata about the field.
+
 [[scaled-float-params]]
 ==== Parameters for `scaled_float`
 

docs/reference/mapping/types/text.asciidoc

@@ -143,3 +143,7 @@ The following parameters are accepted by `text` fields:
 
     Whether term vectors should be stored for an <<mapping-index,`analyzed`>>
     field. Defaults to `no`.
+
+<<mapping-field-meta,`meta`>>::
+
+    Metadata about the field.

docs/reference/search/field-caps.asciidoc

@@ -78,6 +78,12 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
   The list of indices where this field is not aggregatable, or null if all 
   indices have the same definition for the field.
 
+`meta`::
+  Merged metadata across all indices as a map of string keys to arrays of values.
+  A value length of 1 indicates that all indices had the same value for this key,
+  while a length of 2 or more indicates that not all indices had the same value
+  for this key.
+
 
 [[search-field-caps-api-example]]
 ==== {api-examples-title}

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -35,6 +36,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
 import static org.hamcrest.Matchers.containsString;
@@ -353,4 +355,33 @@ public void testRejectIndexOptions() throws IOException {
         MapperParsingException e = expectThrows(MapperParsingException.class, () -> parser.parse("type", new CompressedXContent(mapping)));
         assertThat(e.getMessage(), containsString("index_options not allowed in field [foo] of type [scaled_float]"));
     }
+
+    public void testMeta() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc")
+                .startObject("properties").startObject("field").field("type", "scaled_float")
+                .field("meta", Collections.singletonMap("foo", "bar"))
+                .field("scaling_factor", 10.0)
+                .endObject().endObject().endObject().endObject());
+
+        DocumentMapper mapper = indexService.mapperService().merge("_doc",
+                new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc")
+                .startObject("properties").startObject("field").field("type", "scaled_float")
+                .field("scaling_factor", 10.0)
+                .endObject().endObject().endObject().endObject());
+        mapper = indexService.mapperService().merge("_doc",
+                new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE);
+        assertEquals(mapping2, mapper.mappingSource().toString());
+
+        String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc")
+                .startObject("properties").startObject("field").field("type", "scaled_float")
+                .field("meta", Collections.singletonMap("baz", "quux"))
+                .field("scaling_factor", 10.0)
+                .endObject().endObject().endObject().endObject());
+        mapper = indexService.mapperService().merge("_doc",
+                new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE);
+        assertEquals(mapping3, mapper.mappingSource().toString());
+    }
 }

rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml

@@ -317,4 +317,3 @@ setup:
   - match: {fields.misc.unmapped.searchable:            false}
   - match: {fields.misc.unmapped.aggregatable:          false}
   - match: {fields.misc.unmapped.indices:               ["test2", "test3"]}
-

rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/20_meta.yml

@@ -0,0 +1,65 @@
+---
+"Merge metadata across multiple indices":
+
+  - skip:
+      version: " - 7.99.99"
+      reason: Metadata support was added in 7.6
+
+  - do:
+        indices.create:
+          index: test1
+          body:
+              mappings:
+                properties:
+                  latency:
+                    type: long
+                    meta:
+                      unit: ms
+                      metric_type: gauge
+
+  - do:
+        indices.create:
+          index: test2
+          body:
+              mappings:
+                properties:
+                  latency:
+                    type: long
+                    meta:
+                      unit: ns
+                      metric_type: gauge
+
+  - do:
+        indices.create:
+          index: test3
+
+  - do:
+      field_caps:
+        index: test3
+        fields: [latency]
+
+  - is_false: fields.latency.long.meta.unit
+
+  - do:
+      field_caps:
+        index: test1
+        fields: [latency]
+
+  - match: {fields.latency.long.meta.unit:              ["ms"]}
+  - match: {fields.latency.long.meta.metric_type:       ["gauge"]}
+
+  - do:
+      field_caps:
+        index: test1,test3
+        fields: [latency]
+
+  - match: {fields.latency.long.meta.unit:              ["ms"]}
+  - match: {fields.latency.long.meta.metric_type:       ["gauge"]}
+
+  - do:
+      field_caps:
+        index: test1,test2,test3
+        fields: [latency]
+
+  - match: {fields.latency.long.meta.unit:              ["ms", "ns"]}
+  - match: {fields.latency.long.meta.metric_type:       ["gauge"]}

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml

@@ -108,3 +108,39 @@
 
   - match: { error.type: "illegal_argument_exception" }
   - match: { error.reason: "Types cannot be provided in put mapping requests, unless the include_type_name parameter is set to true." }
+
+---
+"Update per-field metadata":
+
+  - skip:
+      version:     " - 7.99.99"
+      reason:      "Per-field meta was introduced in 7.6"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          mappings:
+            properties:
+              foo:
+                type: keyword
+                meta:
+                  bar: baz
+
+  - do:
+      indices.put_mapping:
+        index: test_index
+        body:
+          properties:
+            foo:
+              type: keyword
+              meta:
+                baz: quux
+
+  - do:
+      indices.get_mapping:
+        index: test_index
+
+  - is_false: test_index.mappings.properties.foo.meta.bar
+  - match: { test_index.mappings.properties.foo.meta.baz: "quux" }
+