Skip to content

Commit

Permalink
Merge branch 'opensearch-project:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
dlin2028 authored May 8, 2024
2 parents d706f15 + bac93dc commit 60f27f3
Show file tree
Hide file tree
Showing 9 changed files with 200 additions and 263 deletions.
15 changes: 8 additions & 7 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ buildscript {
opensearch_build = version_tokens[0] + '.0'

common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT')

kafka_version = '3.7.0'
apache_cxf_version = '4.0.4'
open_saml_version = '4.3.2'
Expand Down Expand Up @@ -471,7 +472,7 @@ configurations {
resolutionStrategy {
force 'commons-codec:commons-codec:1.17.0'
force 'org.slf4j:slf4j-api:1.7.36'
force 'org.scala-lang:scala-library:2.13.13'
force 'org.scala-lang:scala-library:2.13.14'
force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
Expand All @@ -494,8 +495,8 @@ configurations {
// For integrationTest
force "org.apache.httpcomponents:httpclient:4.5.14"
force "org.apache.httpcomponents:httpcore:4.4.16"
force "com.google.errorprone:error_prone_annotations:2.27.0"
force "org.checkerframework:checker-qual:3.42.0"
force "com.google.errorprone:error_prone_annotations:2.27.1"
force "org.checkerframework:checker-qual:3.43.0"
force "ch.qos.logback:logback-classic:1.5.6"
}
}
Expand Down Expand Up @@ -605,12 +606,12 @@ dependencies {
runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
runtimeOnly 'commons-codec:commons-codec:1.17.0'
runtimeOnly 'org.cryptacular:cryptacular:1.2.6'
compileOnly 'com.google.errorprone:error_prone_annotations:2.27.0'
compileOnly 'com.google.errorprone:error_prone_annotations:2.27.1'
runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0'
runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
runtimeOnly 'org.ow2.asm:asm:9.7'

testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1'
testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.2'

//OpenSAML
implementation 'net.shibboleth.utilities:java-support:8.4.2'
Expand Down Expand Up @@ -649,7 +650,7 @@ dependencies {
runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1'
runtimeOnly 'org.apache.santuario:xmlsec:2.3.4'
runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}"
runtimeOnly 'org.checkerframework:checker-qual:3.42.0'
runtimeOnly 'org.checkerframework:checker-qual:3.43.0'
runtimeOnly "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2'

Expand Down Expand Up @@ -699,7 +700,7 @@ dependencies {
testRuntimeOnly ("org.springframework:spring-core:${spring_version}") {
exclude(group:'org.springframework', module: 'spring-jcl' )
}
testRuntimeOnly 'org.scala-lang:scala-library:2.13.13'
testRuntimeOnly 'org.scala-lang:scala-library:2.13.14'
testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5'
testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.2') {
exclude(group:'ch.qos.logback', module: 'logback-classic' )
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/
package org.opensearch.security.api;

import java.util.StringJoiner;

import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.Test;

import org.opensearch.security.DefaultObjectMapper;
import org.opensearch.security.dlic.rest.api.Endpoint;
import org.opensearch.test.framework.cluster.TestRestClient;

import static org.opensearch.security.api.PatchPayloadHelper.patch;
import static org.opensearch.security.api.PatchPayloadHelper.replaceOp;
import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.SECURITY_CONFIG_UPDATE;
import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ADMIN_ENABLED;
import static org.opensearch.security.support.ConfigConstants.SECURITY_UNSUPPORTED_RESTAPI_ALLOW_SECURITYCONFIG_MODIFICATION;

public class ConfigRestApiIntegrationTest extends AbstractApiIntegrationTest {

final static String REST_API_ADMIN_CONFIG_UPDATE = "rest-api-admin-config-update";

static {
clusterSettings.put(SECURITY_UNSUPPORTED_RESTAPI_ALLOW_SECURITYCONFIG_MODIFICATION, true).put(SECURITY_RESTAPI_ADMIN_ENABLED, true);
testSecurityConfig.withRestAdminUser(REST_ADMIN_USER, allRestAdminPermissions())
.withRestAdminUser(REST_API_ADMIN_CONFIG_UPDATE, restAdminPermission(Endpoint.CONFIG, SECURITY_CONFIG_UPDATE));
}

private String securityConfigPath(final String... path) {
final var fullPath = new StringJoiner("/").add(super.apiPath("securityconfig"));
if (path != null) for (final var p : path)
fullPath.add(p);
return fullPath.toString();
}

@Test
public void forbiddenForRegularUsers() throws Exception {
withUser(NEW_USER, client -> {
forbidden(() -> client.get(securityConfigPath()));
forbidden(() -> client.putJson(securityConfigPath("config"), EMPTY_BODY));
forbidden(() -> client.patch(securityConfigPath(), EMPTY_BODY));
verifyNotAllowedMethods(client);
});
}

@Test
public void partiallyAvailableForAdminUser() throws Exception {
withUser(ADMIN_USER_NAME, client -> ok(() -> client.get(securityConfigPath())));
withUser(ADMIN_USER_NAME, client -> {
badRequest(() -> client.putJson(securityConfigPath("xxx"), EMPTY_BODY));
forbidden(() -> client.putJson(securityConfigPath("config"), EMPTY_BODY));
forbidden(() -> client.patch(securityConfigPath(), EMPTY_BODY));
});
withUser(ADMIN_USER_NAME, this::verifyNotAllowedMethods);
}

@Test
public void availableForTlsAdminUser() throws Exception {
withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), client -> ok(() -> client.get(securityConfigPath())));
withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), this::verifyUpdate);
}

@Test
public void availableForRestAdminUser() throws Exception {
withUser(REST_ADMIN_USER, client -> ok(() -> client.get(securityConfigPath())));
withUser(REST_ADMIN_USER, this::verifyUpdate);
withUser(REST_API_ADMIN_CONFIG_UPDATE, this::verifyUpdate);
}

void verifyUpdate(final TestRestClient client) throws Exception {
badRequest(() -> client.putJson(securityConfigPath("xxx"), EMPTY_BODY));
verifyNotAllowedMethods(client);

final var configJson = ok(() -> client.get(securityConfigPath())).bodyAsJsonNode();
final var authFailureListeners = DefaultObjectMapper.objectMapper.createObjectNode();
authFailureListeners.set(
"ip_rate_limiting",
DefaultObjectMapper.objectMapper.createObjectNode()
.put("type", "ip")
.put("allowed_tries", 10)
.put("time_window_seconds", 3_600)
.put("block_expiry_seconds", 600)
.put("max_blocked_clients", 100_000)
.put("max_tracked_clients", 100_000)
);
authFailureListeners.set(
"internal_authentication_backend_limiting",
DefaultObjectMapper.objectMapper.createObjectNode()
.put("type", "username")
.put("authentication_backend", "intern")
.put("allowed_tries", 10)
.put("time_window_seconds", 3_600)
.put("block_expiry_seconds", 600)
.put("max_blocked_clients", 100_000)
.put("max_tracked_clients", 100_000)
);
final var dynamicConfigJson = (ObjectNode) configJson.get("config").get("dynamic");
dynamicConfigJson.set("auth_failure_listeners", authFailureListeners);
ok(() -> client.putJson(securityConfigPath("config"), DefaultObjectMapper.writeValueAsString(configJson.get("config"), false)));
ok(() -> client.patch(securityConfigPath(), patch(replaceOp("/config/dynamic/hosts_resolver_mode", "other"))));
}

void verifyNotAllowedMethods(final TestRestClient client) throws Exception {
methodNotAllowed(() -> client.postJson(securityConfigPath(), EMPTY_BODY));
methodNotAllowed(() -> client.delete(securityConfigPath()));
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
import static org.opensearch.security.api.PatchPayloadHelper.patch;
import static org.opensearch.security.api.PatchPayloadHelper.replaceOp;

public class DefaultApiAvailabilityIntegrationTest extends AbstractApiIntegrationTest {

Expand Down Expand Up @@ -49,18 +51,7 @@ private void verifySecurityConfigApi(final TestRestClient client) throws Excepti
methodNotAllowed(() -> client.putJson(apiPath("securityconfig"), EMPTY_BODY));
methodNotAllowed(() -> client.postJson(apiPath("securityconfig"), EMPTY_BODY));
methodNotAllowed(() -> client.delete(apiPath("securityconfig")));
forbidden(
() -> client.patch(
apiPath("securityconfig"),
(builder, params) -> builder.startArray()
.startObject()
.field("op", "replace")
.field("path", "/a/b/c")
.field("value", "other")
.endObject()
.endArray()
)
);
forbidden(() -> client.patch(apiPath("securityconfig"), patch(replaceOp("/a/b/c", "other"))));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

package org.opensearch.security.api;

import java.util.Locale;

import org.opensearch.core.xcontent.ToXContentObject;

interface PatchPayloadHelper extends ToXContentObject {

enum Op {
ADD,
REPLACE,
REMOVE;
}

static <T> ToXContentObject addOp(final String path, final T value) {
return operation(Op.ADD, path, value);
}

static <T> ToXContentObject replaceOp(final String path, final T value) {
return operation(Op.REPLACE, path, value);
}

static ToXContentObject removeOp(final String path) {
return operation(Op.REMOVE, path, null);
}

private static <T> ToXContentObject operation(final Op op, final String path, final T value) {
return (builder, params) -> {
final var opPath = path.startsWith("/") ? path : "/" + path;
builder.startObject().field("op", op.name().toLowerCase(Locale.ROOT)).field("path", opPath);
if (value != null) {
if (value instanceof ToXContentObject) {
builder.field("value", (ToXContentObject) value);
} else if (value instanceof String) {
builder.field("value", (String) value);
} else if (value instanceof Boolean) {
builder.field("value", (Boolean) value);
} else {
throw new IllegalArgumentException("Unsupported java type " + value.getClass());
}
}
return builder.endObject();
};
}

static ToXContentObject patch(final ToXContentObject... operations) {
return (builder, params) -> {
builder.startArray();
for (final var o : operations)
o.toXContent(builder, EMPTY_PARAMS);
return builder.endArray();
};
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,6 @@ public class LocalOpenSearchCluster {

private File snapshotDir;

private int nodeCounter = 0;

public LocalOpenSearchCluster(
String clusterName,
ClusterManager clusterManager,
Expand Down Expand Up @@ -302,10 +300,9 @@ private CompletableFuture<Void> startNodes(
Iterator<Integer> httpPortIterator = httpPorts.iterator();
List<CompletableFuture<StartStage>> futures = new ArrayList<>();

for (NodeSettings nodeSettings : nodeSettingList) {
Node node = new Node(nodeCounter, nodeSettings, transportPortIterator.next(), httpPortIterator.next());
for (var i = 0; i < nodeSettingList.size(); i++) {
Node node = new Node(i, nodeSettingList.get(i), transportPortIterator.next(), httpPortIterator.next());
futures.add(node.start());
nodeCounter += 1;
}
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
}
Expand Down Expand Up @@ -518,7 +515,6 @@ private Settings getOpenSearchSettings() {
.build();

if (nodeSettingsSupplier != null) {
// TODO node number
return Settings.builder().put(settings).put(nodeSettingsSupplier.get(nodeNumber)).build();
}
return settings;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
import org.opensearch.cluster.metadata.IndexAbstraction;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.util.IndexUtils;
import org.opensearch.core.index.Index;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.reindex.ReindexRequest;
Expand All @@ -91,7 +92,6 @@
import org.opensearch.security.support.SnapshotRestoreHelper;
import org.opensearch.security.support.WildcardMatcher;
import org.opensearch.snapshots.SnapshotInfo;
import org.opensearch.snapshots.SnapshotUtils;
import org.opensearch.transport.RemoteClusterService;
import org.opensearch.transport.TransportRequest;

Expand Down Expand Up @@ -694,7 +694,7 @@ private boolean getOrReplaceAllIndices(final Object request, final IndicesProvid
);
provider.provide(new String[] { "*" }, request, false);
} else {
final List<String> requestedResolvedIndices = SnapshotUtils.filterIndices(
final List<String> requestedResolvedIndices = IndexUtils.filterIndices(
snapshotInfo.indices(),
restoreRequest.indices(),
restoreRequest.indicesOptions()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,12 @@
import org.opensearch.SpecialPermission;
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
import org.opensearch.action.support.PlainActionFuture;
import org.opensearch.common.util.IndexUtils;
import org.opensearch.repositories.RepositoriesService;
import org.opensearch.repositories.Repository;
import org.opensearch.security.OpenSearchSecurityPlugin;
import org.opensearch.snapshots.SnapshotId;
import org.opensearch.snapshots.SnapshotInfo;
import org.opensearch.snapshots.SnapshotUtils;
import org.opensearch.threadpool.ThreadPool;

public class SnapshotRestoreHelper {
Expand All @@ -56,7 +56,7 @@ public static List<String> resolveOriginalIndices(RestoreSnapshotRequest restore
log.warn("snapshot repository '{}', snapshot '{}' not found", restoreRequest.repository(), restoreRequest.snapshot());
return null;
} else {
return SnapshotUtils.filterIndices(snapshotInfo.indices(), restoreRequest.indices(), restoreRequest.indicesOptions());
return IndexUtils.filterIndices(snapshotInfo.indices(), restoreRequest.indices(), restoreRequest.indicesOptions());
}

}
Expand Down
Loading

0 comments on commit 60f27f3

Please sign in to comment.