Merge branch 'master' into wait-for-pending-tasks

* master:
  Use trial license in docs tests (elastic#34673)
  Scripting: Convert script fields to use script context (elastic#34164)
  TEST: Mute testDedupByPrimaryTerm
  ingest: processor stats (elastic#34202)

client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java

@@ -47,7 +47,7 @@ public void testGetStoredScript() throws Exception {
             Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()));
 
         PutStoredScriptRequest request =
-            new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource);
+            new PutStoredScriptRequest(id, "score", new BytesArray("{}"), XContentType.JSON, scriptSource);
         assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync));
 
         GetStoredScriptRequest getRequest = new GetStoredScriptRequest("calculate-score");
@@ -66,7 +66,7 @@ public void testDeleteStoredScript() throws Exception {
                 Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()));
 
         PutStoredScriptRequest request =
-            new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource);
+            new PutStoredScriptRequest(id, "score", new BytesArray("{}"), XContentType.JSON, scriptSource);
         assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync));
 
         DeleteStoredScriptRequest deleteRequest = new DeleteStoredScriptRequest(id);
@@ -89,7 +89,7 @@ public void testPutScript() throws Exception {
                 Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()));
 
         PutStoredScriptRequest request =
-            new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource);
+            new PutStoredScriptRequest(id, "score", new BytesArray("{}"), XContentType.JSON, scriptSource);
         assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync));
 
         Map<String, Object> script = getAsMap("/_scripts/" + id);

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java

@@ -307,7 +307,7 @@ public void onFailure(Exception e) {
 
     private void putStoredScript(String id, StoredScriptSource scriptSource) throws IOException {
         PutStoredScriptRequest request =
-            new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource);
+            new PutStoredScriptRequest(id, "score", new BytesArray("{}"), XContentType.JSON, scriptSource);
         assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync));
     }
 }

docs/build.gradle

@@ -36,8 +36,11 @@ buildRestTests.expectedUnconvertedCandidates = [
 ]
 
 integTestCluster {
-  /* Enable regexes in painless so our tests don't complain about example
-   * snippets that use them. */
+  if ("zip".equals(integTestCluster.distribution)) {
+    setting 'xpack.license.self_generated.type', 'trial'
+  }
+
+  // enable regexes in painless so our tests don't complain about example snippets that use them
   setting 'script.painless.regex.enabled', 'true'
   Closure configFile = {
     extraConfigFile it, "src/test/cluster/config/$it"

docs/painless/painless-contexts/painless-field-context.asciidoc

@@ -14,18 +14,15 @@ a customized value for each document in the results of a query.
         Contains the fields of the specified document where each field is a
         `List` of values.
 
-{ref}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
+{ref}/mapping-source-field.html[`params['_source']`] (`Map`, read-only)::
         Contains extracted JSON in a `Map` and `List` structure for the fields
         existing in a stored document.
 
-`_score` (`double` read-only)::
-        The original score of the specified document.
-
 *Return*
 
 `Object`::
         The customized value for each document.
 
 *API*
 
-The standard <<painless-api-reference, Painless API>> is available.
+The standard <<painless-api-reference, Painless API>> is available.

docs/reference/licensing/get-license.asciidoc

@@ -39,7 +39,7 @@ For more information, see
 [float]
 ==== Examples
 
-The following example provides information about a basic license:
+The following example provides information about a trial license:
 
 [source,js]
 --------------------------------------------------
@@ -53,9 +53,11 @@ GET _xpack/license
   "license" : {
     "status" : "active",
     "uid" : "cbff45e7-c553-41f7-ae4f-9205eabd80xx",
-    "type" : "basic",
-    "issue_date" : "2018-02-22T23:12:05.550Z",
-    "issue_date_in_millis" : 1519341125550,
+    "type" : "trial",
+    "issue_date" : "2018-10-20T22:05:12.332Z",
+    "issue_date_in_millis" : 1540073112332,
+    "expiry_date" : "2018-11-19T22:05:12.332Z",
+    "expiry_date_in_millis" : 1542665112332,
     "max_nodes" : 1000,
     "issued_to" : "test",
     "issuer" : "elasticsearch",
@@ -65,8 +67,10 @@ GET _xpack/license
 --------------------------------------------------
 // TESTRESPONSE[s/"cbff45e7-c553-41f7-ae4f-9205eabd80xx"/$body.license.uid/]
 // TESTRESPONSE[s/"basic"/$body.license.type/]
-// TESTRESPONSE[s/"2018-02-22T23:12:05.550Z"/$body.license.issue_date/]
-// TESTRESPONSE[s/1519341125550/$body.license.issue_date_in_millis/]
+// TESTRESPONSE[s/"2018-10-20T22:05:12.332Z"/$body.license.issue_date/]
+// TESTRESPONSE[s/1540073112332/$body.license.issue_date_in_millis/]
+// TESTRESPONSE[s/"2018-11-19T22:05:12.332Z"/$body.license.expiry_date/]
+// TESTRESPONSE[s/1542665112332/$body.license.expiry_date_in_millis/]
 // TESTRESPONSE[s/1000/$body.license.max_nodes/]
 // TESTRESPONSE[s/"test"/$body.license.issued_to/]
 // TESTRESPONSE[s/"elasticsearch"/$body.license.issuer/]

docs/reference/rest-api/info.asciidoc

@@ -57,24 +57,25 @@ Example response:
    },
    "license" : {
       "uid" : "893361dc-9749-4997-93cb-xxx",
-      "type" : "basic",
-      "mode" : "basic",
-      "status" : "active"
+      "type" : "trial",
+      "mode" : "trial",
+      "status" : "active",
+      "expiry_date_in_millis" : 1542665112332
    },
    "features" : {
       "graph" : {
          "description" : "Graph Data Exploration for the Elastic Stack",
-         "available" : false,
+         "available" : true,
          "enabled" : true
       },
       "logstash" : {
          "description" : "Logstash management component for X-Pack",
-         "available" : false,
+         "available" : true,
          "enabled" : true
       },
       "ml" : {
          "description" : "Machine Learning for the Elastic Stack",
-         "available" : false,
+         "available" : true,
          "enabled" : true,
          "native_code_info" : {
            "version" : "7.0.0-alpha1-SNAPSHOT",
@@ -93,12 +94,12 @@ Example response:
       },
       "security" : {
          "description" : "Security for the Elastic Stack",
-         "available" : false,
-         "enabled" : true
+         "available" : true,
+         "enabled" : false
       },
       "watcher" : {
          "description" : "Alerting, Notification and Automation for the Elastic Stack",
-         "available" : false,
+         "available" : true,
          "enabled" : true
       }
    },
@@ -108,7 +109,7 @@ Example response:
 // TESTRESPONSE[s/"hash" : "2798b1a3ce779b3611bb53a0082d4d741e4d3168",/"hash" : "$body.build.hash",/]
 // TESTRESPONSE[s/"date" : "2015-04-07T13:34:42Z"/"date" : "$body.build.date"/]
 // TESTRESPONSE[s/"uid" : "893361dc-9749-4997-93cb-xxx",/"uid": "$body.license.uid",/]
-// TESTRESPONSE[s/"expiry_date_in_millis" : 1914278399999/"expiry_date_in_millis" : "$body.license.expiry_date_in_millis"/]
+// TESTRESPONSE[s/"expiry_date_in_millis" : 1542665112332/"expiry_date_in_millis" : "$body.license.expiry_date_in_millis"/]
 // TESTRESPONSE[s/"version" : "7.0.0-alpha1-SNAPSHOT",/"version": "$body.features.ml.native_code_info.version",/]
 // TESTRESPONSE[s/"build_hash" : "99a07c016d5a73"/"build_hash": "$body.features.ml.native_code_info.build_hash"/]
 // So much s/// but at least we test that the layout is close to matching....

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionFieldScript.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import org.apache.lucene.expressions.Expression;
+import org.apache.lucene.expressions.SimpleBindings;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.elasticsearch.script.FieldScript;
+import org.elasticsearch.script.GeneralScriptException;
+
+import java.io.IOException;
+
+public class ExpressionFieldScript implements FieldScript.LeafFactory {
+    private final Expression exprScript;
+    private final DoubleValuesSource source;
+
+    ExpressionFieldScript(Expression e, SimpleBindings b) {
+        this.exprScript = e;
+        this.source = exprScript.getDoubleValuesSource(b);
+    }
+
+    @Override
+    public FieldScript newInstance(final LeafReaderContext leaf) throws IOException {
+        return new FieldScript() {
+
+            // Fake the scorer until setScorer is called.
+            DoubleValues values = source.getValues(leaf, null);
+
+            @Override
+            public Object execute() {
+                try {
+                    return values.doubleValue();
+                } catch (Exception exception) {
+                    throw new GeneralScriptException("Error evaluating " + exprScript, exception);
+                }
+            }
+
+            @Override
+            public void setDocument(int d) {
+                try {
+                    values.advanceExact(d);
+                } catch (IOException e) {
+                    throw new IllegalStateException("Can't advance to doc using " + exprScript, e);
+                }
+            }
+        };
+    }
+}

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.script.BucketAggregationScript;
 import org.elasticsearch.script.BucketAggregationSelectorScript;
 import org.elasticsearch.script.ClassPermission;
+import org.elasticsearch.script.FieldScript;
 import org.elasticsearch.script.FilterScript;
 import org.elasticsearch.script.NumberSortScript;
 import org.elasticsearch.script.ScoreScript;
@@ -139,6 +140,9 @@ public boolean execute() {
         } else if (context.instanceClazz.equals(NumberSortScript.class)) {
             NumberSortScript.Factory factory = (p, lookup) -> newSortScript(expr, lookup, p);
             return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(FieldScript.class)) {
+            FieldScript.Factory factory = (p, lookup) -> newFieldScript(expr, lookup, p);
+            return context.factoryClazz.cast(factory);
         }
         throw new IllegalArgumentException("expression engine does not know how to handle script context [" + context.name + "]");
     }
@@ -289,6 +293,23 @@ private AggregationScript.LeafFactory newAggregationScript(Expression expr, Sear
         return new ExpressionAggregationScript(expr, bindings, specialValue);
     }
 
+    private FieldScript.LeafFactory newFieldScript(Expression expr, SearchLookup lookup, @Nullable Map<String, Object> vars) {
+        SimpleBindings bindings = new SimpleBindings();
+        for (String variable : expr.variables) {
+            try {
+                if (vars != null && vars.containsKey(variable)) {
+                    bindFromParams(vars, bindings, variable);
+                } else {
+                    final ValueSource valueSource = getDocValueSource(variable, lookup);
+                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                }
+            } catch (Exception e) {
+                throw convertToScriptException("link error", expr.sourceText, variable, e);
+            }
+        }
+        return new ExpressionFieldScript(expr, bindings);
+    }
+
     /**
      * This is a hack for filter scripts, which must return booleans instead of doubles as expression do.
      * See https://github.com/elastic/elasticsearch/issues/26429.

modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java → modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java

@@ -24,31 +24,31 @@
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
-import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.script.FieldScript;
 import org.elasticsearch.script.ScriptException;
-import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.text.ParseException;
 import java.util.Collections;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.anyInt;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class ExpressionTests extends ESTestCase {
+public class ExpressionFieldScriptTests extends ESTestCase {
     private ExpressionScriptEngine service;
     private SearchLookup lookup;
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
 
-        NumberFieldType fieldType = new NumberFieldType(NumberType.DOUBLE);
+        NumberFieldMapper.NumberFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
         MapperService mapperService = mock(MapperService.class);
         when(mapperService.fullName("field")).thenReturn(fieldType);
         when(mapperService.fullName("alias")).thenReturn(fieldType);
@@ -68,18 +68,11 @@ public void setUp() throws Exception {
         lookup = new SearchLookup(mapperService, ignored -> fieldData, null);
     }
 
-    private SearchScript.LeafFactory compile(String expression) {
-        SearchScript.Factory factory = service.compile(null, expression, SearchScript.CONTEXT, Collections.emptyMap());
+    private FieldScript.LeafFactory compile(String expression) {
+        FieldScript.Factory factory = service.compile(null, expression, FieldScript.CONTEXT, Collections.emptyMap());
         return factory.newFactory(Collections.emptyMap(), lookup);
     }
 
-    public void testNeedsScores() {
-        assertFalse(compile("1.2").needs_score());
-        assertFalse(compile("doc['field'].value").needs_score());
-        assertTrue(compile("1/_score").needs_score());
-        assertTrue(compile("doc['field'].value * _score").needs_score());
-    }
-
     public void testCompileError() {
         ScriptException e = expectThrows(ScriptException.class, () -> {
             compile("doc['field'].value * *@#)(@$*@#$ + 4");
@@ -95,18 +88,18 @@ public void testLinkError() {
     }
 
     public void testFieldAccess() throws IOException {
-        SearchScript script = compile("doc['field'].value").newInstance(null);
+        FieldScript script = compile("doc['field'].value").newInstance(null);
         script.setDocument(1);
 
-        double result = script.runAsDouble();
-        assertEquals(2.718, result, 0.0);
+        Object result = script.execute();
+        assertThat(result, equalTo(2.718));
     }
 
     public void testFieldAccessWithFieldAlias() throws IOException {
-        SearchScript script = compile("doc['alias'].value").newInstance(null);
+        FieldScript script = compile("doc['alias'].value").newInstance(null);
         script.setDocument(1);
 
-        double result = script.runAsDouble();
-        assertEquals(2.718, result, 0.0);
+        Object result = script.execute();
+        assertThat(result, equalTo(2.718));
     }
 }

modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java

@@ -70,7 +70,7 @@ public void testAllOpsDisabledIndexedScripts() throws IOException {
                     .setIndices("test").setTypes("scriptTest").get();
             fail("search script should have been rejected");
         } catch(Exception e) {
-            assertThat(e.toString(), containsString("cannot execute scripts using [search] context"));
+            assertThat(e.toString(), containsString("cannot execute scripts using [field] context"));
         }
         try {
             client().prepareSearch("test")

modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java

@@ -27,7 +27,6 @@
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptException;
-import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -67,7 +66,6 @@ protected Settings scriptEngineSettings() {
      */
     protected Map<ScriptContext<?>, List<Whitelist>> scriptContexts() {
         Map<ScriptContext<?>, List<Whitelist>> contexts = new HashMap<>();
-        contexts.put(SearchScript.CONTEXT, Whitelist.BASE_WHITELISTS);
         contexts.put(PainlessTestScript.CONTEXT, Whitelist.BASE_WHITELISTS);
         return contexts;
     }

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yml

@@ -38,12 +38,12 @@
       catch: bad_request
       put_script:
         id: "1"
-        context: "search"
+        context: "score"
         body: { "script": {"lang": "painless", "source": "_score * foo bar + doc['myParent.weight'].value"} }
 
   - do:
       catch: /compile error/
       put_script:
         id: "1"
-        context: "search"
+        context: "score"
         body: { "script": {"lang": "painless", "source": "_score * foo bar + doc['myParent.weight'].value"} }