From 719e75bb3fc8bdb3a8aaa7d2959b03244d6cbb46 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 25 Jan 2017 17:09:25 -0600 Subject: [PATCH] Add repository-url module and move URLRepository (#22752) This is related to #22116. URLRepository requires SocketPermission connect. This commit introduces a new module called "repository-url" where URLRepository will reside. With the new module, permissions can be removed from core. --- .../gradle/test/ClusterFormationTasks.groovy | 3 +- .../resources/checkstyle_suppressions.xml | 2 - .../common/settings/ClusterSettings.java | 4 - .../repositories/RepositoriesModule.java | 13 +- .../bwcompat/RestoreBackwardsCompatIT.java | 48 +++---- .../snapshots/RepositoriesIT.java | 24 ---- .../SharedClusterSnapshotRestoreIT.java | 57 -------- modules/repository-url/build.gradle | 29 ++++ .../blobstore/url/URLBlobContainer.java | 0 .../common/blobstore/url/URLBlobStore.java | 3 +- .../repository/url/URLRepositoryPlugin.java | 50 +++++++ .../repositories/url}/URLRepository.java | 10 +- .../blobstore/url/URLBlobStoreTests.java | 102 ++++++++++++++ .../RepositoryURLClientYamlTestSuiteIT.java | 40 ++++++ .../repositories/url/URLRepositoryTests.java | 81 +++++++++++ .../url/URLSnapshotRestoreTests.java | 131 ++++++++++++++++++ .../test/repository_url/10_basic.yaml | 32 +++++ .../test/repository_url/20_repository.yaml | 16 +++ qa/rolling-upgrade/build.gradle | 3 + .../snapshot.get_repository/10_basic.yaml | 8 +- settings.gradle | 1 + 21 files changed, 525 insertions(+), 132 deletions(-) create mode 100644 modules/repository-url/build.gradle rename {core => modules/repository-url}/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java (100%) rename {core => modules/repository-url}/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java (97%) create mode 100644 modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java rename {core/src/main/java/org/elasticsearch/repositories/uri => modules/repository-url/src/main/java/org/elasticsearch/repositories/url}/URLRepository.java (91%) create mode 100644 modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java create mode 100644 modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java create mode 100644 modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java create mode 100644 modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java create mode 100644 modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml create mode 100644 modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 756c05b07d523..3fc622ef5aa4f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -276,8 +276,7 @@ class ClusterFormationTasks { 'path.repo' : "${node.sharedDir}/repo", 'path.shared_data' : "${node.sharedDir}/", // Define a node attribute so we can test that it exists - 'node.attr.testattr' : 'test', - 'repositories.url.allowed_urls': 'http://snapshot.test*' + 'node.attr.testattr' : 'test' ] // we set min master nodes to the total number of nodes in the cluster and // basically skip initial state recovery to allow the cluster to form using a realistic master election diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 3e863b87bd901..be1ec1dc71f05 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -256,7 +256,6 @@ - @@ -437,7 +436,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index ff010f9f9494d..279be58e01d4d 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -81,7 +81,6 @@ import org.elasticsearch.node.Node; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.fs.FsRepository; -import org.elasticsearch.repositories.uri.URLRepository; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; @@ -348,9 +347,6 @@ public void apply(Settings value, Settings current, Settings previous) { Node.NODE_INGEST_SETTING, Node.NODE_ATTRIBUTES, Node.NODE_LOCAL_STORAGE_SETTING, - URLRepository.ALLOWED_URLS_SETTING, - URLRepository.REPOSITORIES_URL_SETTING, - URLRepository.SUPPORTED_PROTOCOLS_SETTING, TransportMasterNodeReadAction.FORCE_LOCAL_SETTING, AutoCreateIndex.AUTO_CREATE_INDEX_SETTING, BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX, diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java index d03e2c1ac349f..3073e1f0b0ef6 100644 --- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java +++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java @@ -19,24 +19,22 @@ package org.elasticsearch.repositories; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.binder.LinkedBindingBuilder; import org.elasticsearch.common.inject.multibindings.MapBinder; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.fs.FsRepository; -import org.elasticsearch.repositories.uri.URLRepository; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.SnapshotShardsService; import org.elasticsearch.snapshots.SnapshotsService; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * Sets up classes for Snapshot/Restore. */ @@ -47,7 +45,6 @@ public class RepositoriesModule extends AbstractModule { public RepositoriesModule(Environment env, List repoPlugins, NamedXContentRegistry namedXContentRegistry) { Map factories = new HashMap<>(); factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env, namedXContentRegistry)); - factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env, namedXContentRegistry)); for (RepositoryPlugin repoPlugin : repoPlugins) { Map newRepoTypes = repoPlugin.getRepositories(env, namedXContentRegistry); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index 6f8019bcdddc6..9dd9e9dd4a62d 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -26,21 +26,22 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; +import org.elasticsearch.common.io.FileTestUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.repositories.uri.URLRepository; +import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotRestoreException; +import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.VersionUtils; +import org.junit.BeforeClass; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -61,27 +62,19 @@ @ClusterScope(scope = Scope.TEST) public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { + private static Path repoPath; + @Override protected Settings nodeSettings(int nodeOrdinal) { - if (randomBoolean()) { - // Configure using path.repo - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(Environment.PATH_REPO_SETTING.getKey(), getBwcIndicesPath()) - .build(); - } else { - // Configure using url white list - try { - URI repoJarPatternUri = new URI("jar:" + getBwcIndicesPath().toUri().toString() + "*.zip!/repo/"); - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .putArray(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoJarPatternUri.toString()) - .build(); - } catch (URISyntaxException ex) { - throw new IllegalArgumentException(ex); - } + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(Environment.PATH_REPO_SETTING.getKey(), repoPath) + .build(); + } - } + @BeforeClass + public static void repoSetup() throws IOException { + repoPath = createTempDir("repositories"); } public void testRestoreOldSnapshots() throws Exception { @@ -151,13 +144,14 @@ private List listRepoVersions(String prefix) throws Exception { } private void createRepo(String prefix, String version, String repo) throws Exception { - Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip"); - URI repoFileUri = repoFile.toUri(); - URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/"); + Path repoFileFromBuild = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip"); + String repoFileName = repoFileFromBuild.getFileName().toString().split(".zip")[0]; + Path fsRepoPath = repoPath.resolve(repoFileName); + FileTestUtils.unzip(repoFileFromBuild, fsRepoPath, null); logger.info("--> creating repository [{}] for version [{}]", repo, version); assertAcked(client().admin().cluster().preparePutRepository(repo) - .setType("url").setSettings(Settings.builder() - .put("url", repoJarUri.toString()))); + .setType(MockRepository.TYPE).setSettings(Settings.builder() + .put(FsRepository.REPOSITORIES_LOCATION_SETTING.getKey(), fsRepoPath.getParent().relativize(fsRepoPath).resolve("repo").toString()))); } private void testOldSnapshot(String version, String repo, String snapshot) throws IOException { @@ -198,7 +192,7 @@ private void testOldSnapshot(String version, String repo, String snapshot) throw equalTo("{\"type1\":{\"_source\":{\"enabled\":0}}}"), equalTo("{\"type1\":{\"_source\":{\"enabled\":\"off\"}}}"), equalTo("{\"type1\":{\"_source\":{\"enabled\":\"no\"}}}") - )); + )); assertThat(template.aliases().size(), equalTo(3)); assertThat(template.aliases().get("alias1"), notNullValue()); assertThat(template.aliases().get("alias2").filter().string(), containsString(version)); diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java index 639b60d6d09ff..0f6dcec818fc9 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -142,30 +142,6 @@ public void testMisconfiguredRepository() throws Exception { } catch (RepositoryException ex) { assertThat(ex.toString(), containsString("location [" + location + "] doesn't match any of the locations specified by path.repo")); } - - String repoUrl = invalidRepoPath.toAbsolutePath().toUri().toURL().toString(); - String unsupportedUrl = repoUrl.replace("file:/", "netdoc:/"); - logger.info("--> trying creating url repository with unsupported url protocol"); - try { - client().admin().cluster().preparePutRepository("test-repo") - .setType("url").setSettings(Settings.builder().put("url", unsupportedUrl)) - .get(); - fail("Shouldn't be here"); - } catch (RepositoryException ex) { - assertThat(ex.toString(), - either(containsString("unsupported url protocol [netdoc]")) - .or(containsString("unknown protocol: netdoc"))); // newer versions of JDK 9 - } - - logger.info("--> trying creating url repository with location that is not registered in path.repo setting"); - try { - client().admin().cluster().preparePutRepository("test-repo") - .setType("url").setSettings(Settings.builder().put("url", invalidRepoPath.toUri().toURL())) - .get(); - fail("Shouldn't be here"); - } catch (RepositoryException ex) { - assertThat(ex.toString(), containsString("doesn't match any of the locations specified by path.repo")); - } } public void testRepositoryAckTimeout() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 9a4a302ef975a..308f79393a36e 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1442,63 +1442,6 @@ public void testDeleteRepositoryWhileSnapshotting() throws Exception { assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); } - public void testUrlRepository() throws Exception { - Client client = client(); - - logger.info("--> creating repository"); - Path repositoryLocation = randomRepoPath(); - assertAcked(client.admin().cluster().preparePutRepository("test-repo") - .setType("fs").setSettings(Settings.builder() - .put("location", repositoryLocation) - .put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); - - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); - - logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); - - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); - - logger.info("--> delete index"); - cluster().wipeIndices("test-idx"); - - logger.info("--> create read-only URL repository"); - assertAcked(client.admin().cluster().preparePutRepository("url-repo") - .setType("url").setSettings(Settings.builder() - .put("url", repositoryLocation.toUri().toURL()) - .put("list_directories", randomBoolean()))); - logger.info("--> restore index after deletion"); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet(); - assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); - - logger.info("--> list available shapshots"); - GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); - assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); - - logger.info("--> delete snapshot"); - DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get(); - assertAcked(deleteSnapshotResponse); - - logger.info("--> list available shapshot again, no snapshots should be returned"); - getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); - assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0)); - } - @TestLogging("_root:DEBUG") // this fails every now and then: https://github.com/elastic/elasticsearch/issues/18121 but without // more logs we cannot find out why public void testReadonlyRepository() throws Exception { diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle new file mode 100644 index 0000000000000..dc769e3c1f300 --- /dev/null +++ b/modules/repository-url/build.gradle @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Module for URL repository' + classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin' +} + +integTest { + cluster { + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' + } +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java similarity index 100% rename from core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java rename to modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java similarity index 97% rename from core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java rename to modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java index 2386f79c076f1..158ecff9b2b4e 100644 --- a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java @@ -55,7 +55,8 @@ public class URLBlobStore extends AbstractComponent implements BlobStore { public URLBlobStore(Settings settings, URL path) { super(settings); this.path = path; - this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); + this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", + new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); } /** diff --git a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java new file mode 100644 index 0000000000000..a28413d213a97 --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.repository.url; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.RepositoryPlugin; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.url.URLRepository; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin { + + @Override + public List> getSettings() { + return Arrays.asList( + URLRepository.ALLOWED_URLS_SETTING, + URLRepository.REPOSITORIES_URL_SETTING, + URLRepository.SUPPORTED_PROTOCOLS_SETTING + ); + } + + @Override + public Map getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) { + return Collections.singletonMap(URLRepository.TYPE, metadata -> new URLRepository(metadata, env, namedXContentRegistry)); + } +} diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java similarity index 91% rename from core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java rename to modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index fdeb27819bf08..c1128fd683a70 100644 --- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.repositories.uri; +package org.elasticsearch.repositories.url; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobPath; @@ -127,8 +127,12 @@ private URL checkURL(URL url) { // We didn't match white list - try to resolve against path.repo URL normalizedUrl = environment.resolveRepoURL(url); if (normalizedUrl == null) { - logger.warn("The specified url [{}] doesn't start with any repository paths specified by the path.repo setting or by {} setting: [{}] ", url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles()); - throw new RepositoryException(getMetadata().name(), "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + ALLOWED_URLS_SETTING.getKey()); + String logMessage = "The specified url [{}] doesn't start with any repository paths specified by the " + + "path.repo setting or by {} setting: [{}] "; + logger.warn(logMessage, url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles()); + String exceptionMessage = "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + + ALLOWED_URLS_SETTING.getKey(); + throw new RepositoryException(getMetadata().name(), exceptionMessage); } return normalizedUrl; } diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java new file mode 100644 index 0000000000000..4df83c58bd38b --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.blobstore.url; + +import com.sun.net.httpserver.HttpServer; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.test.ESTestCase; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.NoSuchFileException; + +@SuppressForbidden(reason = "use http server") +public class URLBlobStoreTests extends ESTestCase { + + private static HttpServer httpServer; + private static String blobName; + private static byte[] message = new byte[512]; + private URLBlobStore urlBlobStore; + + @BeforeClass + public static void startHttp() throws Exception { + for (int i = 0; i < message.length; ++i) { + message[i] = randomByte(); + } + blobName = randomAsciiOfLength(8); + + httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 6001), 0); + + httpServer.createContext("/indices/" + blobName, (s) -> { + s.sendResponseHeaders(200, message.length); + OutputStream responseBody = s.getResponseBody(); + responseBody.write(message); + responseBody.close(); + }); + + httpServer.start(); + } + + @AfterClass + public static void stopHttp() throws IOException { + httpServer.stop(0); + httpServer = null; + } + + @Before + public void storeSetup() throws MalformedURLException { + Settings settings = Settings.EMPTY; + String spec = "http://localhost:6001/"; + urlBlobStore = new URLBlobStore(settings, new URL(spec)); + } + + public void testURLBlobStoreCanReadBlob() throws IOException { + BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices")); + try (InputStream stream = container.readBlob(blobName)) { + byte[] bytes = new byte[message.length]; + int read = stream.read(bytes); + assertEquals(message.length, read); + assertArrayEquals(message, bytes); + } + } + + public void testNoBlobFound() throws IOException { + BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices")); + String incorrectBlobName = "incorrect_" + blobName; + try (InputStream ignored = container.readBlob(incorrectBlobName)) { + fail("Should have thrown NoSuchFileException exception"); + ignored.read(); + } catch (NoSuchFileException e) { + assertEquals(String.format("[%s] blob not found", incorrectBlobName), e.getMessage()); + } + } +} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..66fba622e602a --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.url; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +import java.io.IOException; + +public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException { + return ESClientYamlSuiteTestCase.createParameters(); + } +} + diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java new file mode 100644 index 0000000000000..ea274eeae602a --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.url; + +import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; + +public class URLRepositoryTests extends ESTestCase { + + public void testWhiteListingRepoURL() throws IOException { + String repoPath = createTempDir().resolve("repository").toUri().toURL().toString(); + Settings baseSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoPath) + .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath) + .build(); + RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings); + new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); + } + + public void testIfNotWhiteListedMustSetRepoURL() throws IOException { + String repoPath = createTempDir().resolve("repository").toUri().toURL().toString(); + Settings baseSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath) + .build(); + RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings); + try { + new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); + fail("RepositoryException should have been thrown."); + } catch (RepositoryException e) { + String msg = "[url] file url [" + repoPath + + "] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls"; + assertEquals(msg, e.getMessage()); + } + } + + public void testMustBeSupportedProtocol() throws IOException { + Path directory = createTempDir(); + String repoPath = directory.resolve("repository").toUri().toURL().toString(); + Settings baseSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(Environment.PATH_REPO_SETTING.getKey(), directory.toString()) + .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath) + .put(URLRepository.SUPPORTED_PROTOCOLS_SETTING.getKey(), "http,https") + .build(); + RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings); + try { + new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); + fail("RepositoryException should have been thrown."); + } catch (RepositoryException e) { + assertEquals("[url] unsupported url protocol [file] from URL [" + repoPath +"]", e.getMessage()); + } + } + +} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java new file mode 100644 index 0000000000000..f7fb7af055c62 --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java @@ -0,0 +1,131 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.url; + +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.plugin.repository.url.URLRepositoryPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.test.ESIntegTestCase; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) +public class URLSnapshotRestoreTests extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(URLRepositoryPlugin.class); + } + + public void testUrlRepository() throws Exception { + Client client = client(); + + logger.info("--> creating repository"); + Path repositoryLocation = randomRepoPath(); + assertAcked(client.admin().cluster().preparePutRepository("test-repo") + .setType(FsRepository.TYPE).setSettings(Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + + createIndex("test-idx"); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client + .admin() + .cluster() + .prepareCreateSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setIndices("test-idx") + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + int actualTotalShards = createSnapshotResponse.getSnapshotInfo().totalShards(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(actualTotalShards)); + + SnapshotState state = client + .admin() + .cluster() + .prepareGetSnapshots("test-repo") + .setSnapshots("test-snap") + .get() + .getSnapshots() + .get(0) + .state(); + assertThat(state, equalTo(SnapshotState.SUCCESS)); + + logger.info("--> delete index"); + cluster().wipeIndices("test-idx"); + + logger.info("--> create read-only URL repository"); + assertAcked(client.admin().cluster().preparePutRepository("url-repo") + .setType(URLRepository.TYPE).setSettings(Settings.builder() + .put(URLRepository.URL_SETTING.getKey(), repositoryLocation.toUri().toURL()) + .put("list_directories", randomBoolean()))); + logger.info("--> restore index after deletion"); + RestoreSnapshotResponse restoreSnapshotResponse = client + .admin() + .cluster() + .prepareRestoreSnapshot("url-repo", "test-snap") + .setWaitForCompletion(true) + .setIndices("test-idx") + .execute() + .actionGet(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); + + logger.info("--> list available shapshots"); + GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); + assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); + + logger.info("--> delete snapshot"); + DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get(); + assertAcked(deleteSnapshotResponse); + + logger.info("--> list available shapshot again, no snapshots should be returned"); + getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); + assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0)); + } +} diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml new file mode 100644 index 0000000000000..75e7873299869 --- /dev/null +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml @@ -0,0 +1,32 @@ +# Integration tests for URL Repository component +# +"URL Repository plugin loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.modules.0.name: repository-url } + +--- +setup: + + - do: + snapshot.create_repository: + repository: test_repo1 + body: + type: url + settings: + url: "http://snapshot.test1" + + - do: + snapshot.create_repository: + repository: test_repo2 + body: + type: url + settings: + url: "http://snapshot.test2" diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml new file mode 100644 index 0000000000000..39cfeee192c9b --- /dev/null +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml @@ -0,0 +1,16 @@ +"Repository can be registered": + + - do: + snapshot.create_repository: + repository: test_repo1 + body: + type: url + settings: + url: "http://snapshot.test1" + - is_true: acknowledged + + - do: + snapshot.get_repository: + repository: test_repo1 + + - is_true : test_repo1 diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 182e6a9f7d947..b03d12fbdf787 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -29,6 +29,7 @@ task oldClusterTest(type: RestIntegTestTask) { numBwcNodes = 2 numNodes = 2 clusterName = 'rolling-upgrade' + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' } systemProperty 'tests.rest.suite', 'old_cluster' } @@ -40,6 +41,7 @@ task mixedClusterTest(type: RestIntegTestTask) { clusterName = 'rolling-upgrade' unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() } dataDir = "${-> oldClusterTest.nodes[1].dataDir}" + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' } systemProperty 'tests.rest.suite', 'mixed_cluster' finalizedBy 'oldClusterTest#node0.stop' @@ -52,6 +54,7 @@ task upgradedClusterTest(type: RestIntegTestTask) { clusterName = 'rolling-upgrade' unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() } dataDir = "${-> oldClusterTest.nodes[0].dataDir}" + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' } systemProperty 'tests.rest.suite', 'upgraded_cluster' // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml index 5788453a73950..b944fe43791e4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml @@ -5,17 +5,17 @@ setup: snapshot.create_repository: repository: test_repo_get_1 body: - type: url + type: fs settings: - url: "http://snapshot.test1" + location: "test_repo_get_1_loc" - do: snapshot.create_repository: repository: test_repo_get_2 body: - type: url + type: fs settings: - url: "http://snapshot.test2" + location: "test_repo_get_1_loc" --- "Get all repositories": diff --git a/settings.gradle b/settings.gradle index 61443fb987bf3..5b6ff5d67b8c8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -31,6 +31,7 @@ List projects = [ 'modules:transport-netty4', 'modules:reindex', 'modules:percolator', + 'modules:repository-url', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', 'plugins:analysis-phonetic',