Wildcard field optimised for wildcard queries

docs/reference/mapping/types.asciidoc

@@ -7,7 +7,7 @@ document:
 [float]
 === Core datatypes
 
-string::         <<text,`text`>> and <<keyword,`keyword`>>
+string::         <<text,`text`>>, <<keyword,`keyword`>> and <<wildcard,`wildcard`>>
 <<number>>::     `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`
 <<date>>::       `date`
 <<date_nanos>>:: `date_nanos`
@@ -131,3 +131,5 @@ include::types/token-count.asciidoc[]
 include::types/shape.asciidoc[]
 
 include::types/constant-keyword.asciidoc[]
+
+include::types/wildcard.asciidoc[]

docs/reference/mapping/types/wildcard.asciidoc

@@ -0,0 +1,53 @@
+[role="xpack"]
+[testenv="basic"]
+[[wildcard]]
+=== Wildcard datatype
+++++
+<titleabbrev>Wildcard</titleabbrev>
+++++
+
+A `wildcard` field stores values optimised for wildcard grep-like queries.
+Wildcard queries are possible on other field types but suffer from constraints:
+* `text` fields limit matching of any wildcard expressions to individual tokens rather than the original whole value held in a field
+* `keyword` fields are untokenized but slow at performing wildcard queries (especially patterns with leading wildcards).
+
+Internally the `wildcard` field indexes the whole field value using ngrams and stores the full string.
+The index is used as a rough filter to cut down the number of values that are then checked by retrieving and checking the full values.
+This field is especially well suited to run grep-like queries on log lines. Storage costs are typically lower than those of `keyword`
+fields but search speeds for exact matches on full terms are slower.
+
+You index and search a wildcard field as follows
+
+[source,console]
+--------------------------------------------------
+PUT my_index
+{
+  "mappings": {
+    "properties": {
+      "my_wildcard": {
+        "type": "wildcard"
+      }
+    }
+  }
+}
+
+PUT my_index/_doc/1
+{
+  "my_wildcard" : "This string can be quite lengthy"
+}
+
+POST my_index/_doc/_search
+{
+  "query": {
+      "wildcard" : "*quite*lengthy"
+  }
+}
+
+
+--------------------------------------------------
+
+
+==== Limitations
+
+* `wildcard` fields are untokenized like keyword fields, so do not support queries that rely on word positions such as phrase queries.
+

server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java

@@ -21,8 +21,6 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortedSetSortField;
-import org.apache.lucene.search.SortedSetSelector;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.Index;
@@ -54,20 +52,7 @@ public BinaryDVAtomicFieldData loadDirect(LeafReaderContext context) throws Exce
     public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested,
             boolean reverse) {
         XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
-        /**
-         * Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
-         * returns a custom sort field otherwise.
-         */
-        if (nested != null ||
-                (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN) ||
-                (source.sortMissingFirst(missingValue) == false && source.sortMissingLast(missingValue) == false)) {
-            return new SortField(getFieldName(), source, reverse);
-        }
-        SortField sortField = new SortedSetSortField(fieldName, reverse,
-            sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN);
-        sortField.setMissingValue(source.sortMissingLast(missingValue) ^ reverse ?
-            SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST);
-        return sortField;
+        return new SortField(getFieldName(), source, reverse);
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java

@@ -613,6 +613,16 @@ public boolean isFlattenedAllowed() {
     public boolean isVectorsAllowed() {
         return allowForAllLicenses();
     }
+
+
+    /**
+     * Determine if Wildcard support should be enabled.
+     * <p>
+     *  Wildcard is available for all license types except {@link OperationMode#MISSING}
+     */
+    public synchronized boolean isWildcardAllowed() {
+        return status.active;
+    }    
 
     public boolean isOdbcAllowed() {
         return isAllowedByLicense(OperationMode.PLATINUM);

x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml

@@ -0,0 +1,218 @@
+setup:
+  - skip:
+      features: headers
+      version: " - 7.9.99"
+      reason: "wildcard fields were added from 8.0"
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          settings:
+            number_of_replicas: 0
+          mappings:
+            properties:
+              my_wildcard:
+                 type: wildcard
+  - do:
+      index:
+        index: test-index
+        id: 1
+        body:
+          my_wildcard: hello world
+  - do:
+      index:
+        index: test-index
+        id: 2
+        body:
+          my_wildcard: goodbye world
+
+  - do:
+      indices.refresh: {}
+
+---
+"Short prefix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hel*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Long prefix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hello wor*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Short unrooted query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Long unrooted query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello worl*" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Short suffix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ld" }
+
+
+  - match: {hits.total.value: 2}
+
+---
+"Long suffix query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*ello world" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"No wildcard wildcard query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "hello world" }
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Term query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            term:
+              my_wildcard: "hello world"
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Terms query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            terms:
+              my_wildcard: ["hello world", "does not exist"]
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Prefix query on wildcard field":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            prefix:
+              my_wildcard:
+                value: "hell*"
+
+
+  - match: {hits.total.value: 1}
+
+---
+"Sequence fail":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*world*hello*" }
+
+
+  - match: {hits.total.value: 0}
+
+---
+"Aggs work":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*world*" }
+          aggs:
+            top_vals:
+              terms: {field: "my_wildcard" }
+
+
+  - match: {hits.total.value: 2}
+  - length: { aggregations.top_vals.buckets: 2 }
+
+---
+"Sort works":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          sort: [ { "my_wildcard": "desc" } ]
+
+  - match: { hits.total.value: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.1._id: "2" }
+
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          sort: [ { "my_wildcard": "asc" } ]
+
+  - match: { hits.total.value: 2 }
+  - length: { hits.hits: 2 }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.1._id: "1" }
+
+

x-pack/plugin/wildcard/build.gradle

@@ -0,0 +1,18 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'wildcard'
+  description 'A plugin for a keyword field type with efficient wildcard search'
+  classname 'org.elasticsearch.xpack.wildcard.Wildcard'
+  extendedPlugins = ['x-pack-core']
+}
+archivesBaseName = 'x-pack-wildcard'
+
+dependencies {
+  compileOnly project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+integTest.enabled = false

x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/Wildcard.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.wildcard;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.wildcard.mapper.WildcardFieldMapper;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class Wildcard extends Plugin implements MapperPlugin {
+
+
+    public Wildcard(Settings settings) {
+    }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
+        mappers.put(WildcardFieldMapper.CONTENT_TYPE, new WildcardFieldMapper.TypeParser());
+        return Collections.unmodifiableMap(mappers);
+    }
+}