diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle index d357fcf9a6e10..f49609c4e801f 100644 --- a/x-pack/plugin/async-search/build.gradle +++ b/x-pack/plugin/async-search/build.gradle @@ -13,6 +13,25 @@ archivesBaseName = 'x-pack-async-search' compileJava.options.compilerArgs << "-Xlint:-rawtypes" compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + // add all sub-projects of the qa sub-project gradle.projectsEvaluated { project.subprojects @@ -28,6 +47,7 @@ dependencies { compileOnly project(path: xpackModule('core'), configuration: 'default') testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackModule('ilm')) + testCompile project(path: xpackModule('async')) } dependencyLicenses { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java index 1c7dc44eb68bc..f21e3628c6878 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -7,49 +7,28 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; -import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.function.Supplier; -import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; -import static org.elasticsearch.xpack.search.AsyncSearchMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; +import static org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; public final class AsyncSearch extends Plugin implements ActionPlugin { - private final Settings settings; - - public AsyncSearch(Settings settings) { - this.settings = settings; - } @Override public List> getActions() { @@ -71,31 +50,6 @@ public List getRestHandlers(Settings settings, RestController restC ); } - @Override - public Collection createComponents(Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier) { - if (DiscoveryNode.isDataNode(environment.settings())) { - // only data nodes should be eligible to run the maintenance service. - AsyncTaskIndexService indexService = - new AsyncTaskIndexService<>(XPackPlugin.ASYNC_RESULTS_INDEX, clusterService, threadPool.getThreadContext(), client, - ASYNC_SEARCH_ORIGIN, AsyncSearchResponse::new, namedWriteableRegistry); - AsyncSearchMaintenanceService maintenanceService = - new AsyncSearchMaintenanceService(clusterService, nodeEnvironment.nodeId(), settings, threadPool, indexService); - return Collections.singletonList(maintenanceService); - } else { - return Collections.emptyList(); - } - } - @Override public List> getSettings() { return Collections.singletonList(ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING); diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java deleted file mode 100644 index e23cacd9bbddd..0000000000000 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.search; - -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; -import org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService; - -public class AsyncSearchMaintenanceService extends AsyncTaskMaintenanceService { - - /** - * Controls the interval at which the cleanup is scheduled. - * Defaults to 1h. It is an undocumented/expert setting that - * is mainly used by integration tests to make the garbage - * collection of search responses more reactive. - */ - public static final Setting ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING = - Setting.timeSetting("async_search.index_cleanup_interval", TimeValue.timeValueHours(1), Setting.Property.NodeScope); - - AsyncSearchMaintenanceService(ClusterService clusterService, - String localNodeId, - Settings nodeSettings, - ThreadPool threadPool, - AsyncTaskIndexService indexService) { - super(clusterService, XPackPlugin.ASYNC_RESULTS_INDEX, localNodeId, threadPool, indexService, - ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING.get(nodeSettings)); - } -} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index c74e3a2b6a323..651bde001ce9d 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.xpack.async.AsyncResultsIndexPlugin; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -30,10 +31,11 @@ import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest; import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; @@ -50,7 +52,7 @@ import java.util.concurrent.ExecutionException; import static org.elasticsearch.xpack.core.XPackPlugin.ASYNC_RESULTS_INDEX; -import static org.elasticsearch.xpack.search.AsyncSearchMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; +import static org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -79,7 +81,7 @@ public List getAggregations() { @Before public void startMaintenanceService() { - for (AsyncSearchMaintenanceService service : internalCluster().getDataNodeInstances(AsyncSearchMaintenanceService.class)) { + for (AsyncTaskMaintenanceService service : internalCluster().getDataNodeInstances(AsyncTaskMaintenanceService.class)) { if (service.lifecycleState() == Lifecycle.State.STOPPED) { // force the service to start again service.start(); @@ -91,7 +93,7 @@ public void startMaintenanceService() { @After public void stopMaintenanceService() { - for (AsyncSearchMaintenanceService service : internalCluster().getDataNodeInstances(AsyncSearchMaintenanceService.class)) { + for (AsyncTaskMaintenanceService service : internalCluster().getDataNodeInstances(AsyncTaskMaintenanceService.class)) { service.stop(); } } @@ -103,7 +105,7 @@ public void releaseQueryLatch() { @Override protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class, + return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, AsyncResultsIndexPlugin.class, IndexLifecycle.class, SearchTestPlugin.class, ReindexPlugin.class); } diff --git a/x-pack/plugin/async/build.gradle b/x-pack/plugin/async/build.gradle new file mode 100644 index 0000000000000..c8042ca59f1d9 --- /dev/null +++ b/x-pack/plugin/async/build.gradle @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-async' + description 'A module which handles common async operations' + classname 'org.elasticsearch.xpack.async.AsyncResultsIndexPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-async' + +dependencies { + compileOnly project(":server") + compileOnly project(path: xpackModule('core'), configuration: 'default') +} + +dependencyLicenses { + ignoreSha 'x-pack-core' +} + +integTest.enabled = false + diff --git a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java new file mode 100644 index 0000000000000..faa1628cfbd68 --- /dev/null +++ b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.async; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; +import org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; + +public class AsyncResultsIndexPlugin extends Plugin implements SystemIndexPlugin { + + protected final Settings settings; + + public AsyncResultsIndexPlugin(Settings settings) { + this.settings = settings; + } + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(XPackPlugin.ASYNC_RESULTS_INDEX, this.getClass().getSimpleName())); + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + List components = new ArrayList<>(); + if (DiscoveryNode.isDataNode(environment.settings())) { + // only data nodes should be eligible to run the maintenance service. + AsyncTaskIndexService indexService = new AsyncTaskIndexService<>( + XPackPlugin.ASYNC_RESULTS_INDEX, + clusterService, + threadPool.getThreadContext(), + client, + ASYNC_SEARCH_ORIGIN, + AsyncSearchResponse::new, + namedWriteableRegistry + ); + AsyncTaskMaintenanceService maintenanceService = new AsyncTaskMaintenanceService( + clusterService, + nodeEnvironment.nodeId(), + settings, + threadPool, + indexService + ); + components.add(maintenanceService); + } + return components; + } +} diff --git a/x-pack/plugin/async/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/async/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java b/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java new file mode 100644 index 0000000000000..c2f48fdc24851 --- /dev/null +++ b/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.async; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +public class AsyncResultsIndexPluginTests extends ESTestCase { + + public void testDummy() { + // This is a dummy test case to satisfy the conventions + AsyncResultsIndexPlugin plugin = new AsyncResultsIndexPlugin(Settings.EMPTY); + assertThat(plugin.getSystemIndexDescriptors(Settings.EMPTY), Matchers.hasSize(1)); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskMaintenanceService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskMaintenanceService.java index 5dfd77c0f46f7..7838ba5adfb33 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskMaintenanceService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskMaintenanceService.java @@ -15,6 +15,8 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.gateway.GatewayService; @@ -23,6 +25,7 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; @@ -34,7 +37,17 @@ * Since we will have several injected implementation of this class injected into different transports, and we bind components created * by {@linkplain org.elasticsearch.plugins.Plugin#createComponents} to their classes, we need to implement one class per binding. */ -public abstract class AsyncTaskMaintenanceService extends AbstractLifecycleComponent implements ClusterStateListener { +public class AsyncTaskMaintenanceService extends AbstractLifecycleComponent implements ClusterStateListener { + + /** + * Controls the interval at which the cleanup is scheduled. + * Defaults to 1h. It is an undocumented/expert setting that + * is mainly used by integration tests to make the garbage + * collection of search responses more reactive. + */ + public static final Setting ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING = + Setting.timeSetting("async_search.index_cleanup_interval", TimeValue.timeValueHours(1), Setting.Property.NodeScope); + private static final Logger logger = LogManager.getLogger(AsyncTaskMaintenanceService.class); private final ClusterService clusterService; @@ -48,17 +61,16 @@ public abstract class AsyncTaskMaintenanceService extends AbstractLifecycleCompo private volatile Scheduler.Cancellable cancellable; public AsyncTaskMaintenanceService(ClusterService clusterService, - String index, String localNodeId, + Settings nodeSettings, ThreadPool threadPool, - AsyncTaskIndexService indexService, - TimeValue delay) { + AsyncTaskIndexService indexService) { this.clusterService = clusterService; - this.index = index; + this.index = XPackPlugin.ASYNC_RESULTS_INDEX; this.localNodeId = localNodeId; this.threadPool = threadPool; this.indexService = indexService; - this.delay = delay; + this.delay = ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING.get(nodeSettings); } diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle index 53582002b653f..2e97788ee2ec8 100644 --- a/x-pack/plugin/eql/build.gradle +++ b/x-pack/plugin/eql/build.gradle @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import org.elasticsearch.gradle.info.BuildParams evaluationDependsOn(xpackModule('core')) @@ -34,6 +53,7 @@ dependencies { } compile "org.antlr:antlr4-runtime:${antlrVersion}" compileOnly project(path: xpackModule('ql'), configuration: 'default') + testCompile project(path: xpackModule('async')) testCompile project(':test:framework') testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')