From ad374bde252bf27393c4f7bc5f053342dd08959a Mon Sep 17 00:00:00 2001 From: dblock Date: Fri, 24 Jun 2022 22:55:02 +0000 Subject: [PATCH 1/3] Calculate uncompressed data size for x-amz-decoded-content-length. Signed-off-by: dblock --- README.md | 10 ++++++ pom.xml | 2 +- src/main/java/RESTClientTest.java | 7 ++--- .../AWSRequestSigningApacheInterceptor.java | 31 +++++++++++++------ 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 2d2af65..a909e73 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ # Java-Client +``` +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +export AWS_SESSION_TOKEN= + +mvn install +mvn compile exec:java -Dexec.mainClass="RESTClientTest" + +``` + 1. step to repro issue: https://github.com/opensearch-project/OpenSearch/issues/3640 2. signing sigv4 for streams https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html diff --git a/pom.xml b/pom.xml index 896593e..4da99ea 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.amazonaws aws-java-sdk-core - 1.11.106 + 1.12.247 org.opensearch.client diff --git a/src/main/java/RESTClientTest.java b/src/main/java/RESTClientTest.java index 4f6e49b..6492a10 100644 --- a/src/main/java/RESTClientTest.java +++ b/src/main/java/RESTClientTest.java @@ -24,12 +24,9 @@ public class RESTClientTest { - private static String host = "https://xxxxxx"; // put your own end-point + private static String host = "https://search-dblock-test-opensearch-21-tu5gqrjd4vg4qazjsu6bps5zsy.us-west-2.es.amazonaws.com"; // put your own end-point private static String serviceName = "es"; - private static String region = "eu-west-1"; - - - + private static String region = "us-west-2"; public static void main(String[] args) throws IOException { RestHighLevelClient searchClient = searchClient(serviceName, region); diff --git a/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java b/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java index 38ab71b..6b563bc 100644 --- a/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java +++ b/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java @@ -22,16 +22,19 @@ import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; import org.apache.http.NameValuePair; +import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.BasicHttpEntity; import org.apache.http.message.BasicHeader; import org.apache.http.protocol.HttpContext; +import org.apache.http.util.EntityUtils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -92,26 +95,33 @@ public void process(final HttpRequest request, final HttpContext context) if (host != null) { signableRequest.setEndpoint(URI.create(host.toURI())); } - final HttpMethodName httpMethod = - HttpMethodName.fromValue(request.getRequestLine().getMethod()); + final HttpMethodName httpMethod = HttpMethodName.fromValue(request.getRequestLine().getMethod()); signableRequest.setHttpMethod(httpMethod); try { signableRequest.setResourcePath(uriBuilder.build().getRawPath()); } catch (URISyntaxException e) { throw new IOException("Invalid URI", e); } -// long decodedContentLength = request.get + + long contentLength = 0; if (request instanceof HttpEntityEnclosingRequest) { HttpEntityEnclosingRequest httpEntityEnclosingRequest = (HttpEntityEnclosingRequest) request; - if (httpEntityEnclosingRequest.getEntity() == null) { - signableRequest.setContent(new ByteArrayInputStream(new byte[0])); - } else { + if (httpEntityEnclosingRequest.getEntity() != null) { + + GzipDecompressingEntity decompressedEntity = new GzipDecompressingEntity(httpEntityEnclosingRequest.getEntity()); + contentLength = EntityUtils.toString(decompressedEntity).length(); + System.out.println(EntityUtils.toString(decompressedEntity)); signableRequest.setContent(httpEntityEnclosingRequest.getEntity().getContent()); + // signableRequest.setContent(new ByteArrayInputStream(EntityUtils.toString(decompressedEntity).getBytes())); } } signableRequest.setParameters(nvpToMapParams(uriBuilder.getQueryParams())); - signableRequest.setHeaders(headerArrayToMap(request.getAllHeaders())); + List
headers = new ArrayList<>(); + headers.addAll(Arrays.asList(request.getAllHeaders())); + headers.add(new BasicHeader("x-Amz-Decoded-Content-Length", String.valueOf(contentLength))); + // headers.add(new BasicHeader("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")); + signableRequest.setHeaders(headerArrayToMap(headers)); // Sign it signer.sign(signableRequest, awsCredentialsProvider.getCredentials()); @@ -127,6 +137,10 @@ public void process(final HttpRequest request, final HttpContext context) httpEntityEnclosingRequest.setEntity(basicHttpEntity); } } + + for(Header header : request.getAllHeaders()) { + System.out.println(header.getName() + ": " + header.getValue()); + } } /** @@ -147,7 +161,7 @@ private static Map> nvpToMapParams(final List headerArrayToMap(final Header[] headers) { + private static Map headerArrayToMap(List
headers) { Map headersMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (Header header : headers) { if (!skipHeader(header)) { @@ -178,7 +192,6 @@ private static Header[] mapToHeaderArray(final Map mapHeaders) { for (Map.Entry headerEntry : mapHeaders.entrySet()) { headers[i++] = new BasicHeader(headerEntry.getKey(), headerEntry.getValue()); } -// headers[i++] = new BasicHeader("x-amz-decoded-content-length", "28"); return headers; } } From c3bb46e17f85251b7384066dca8f4775fa30bf1f Mon Sep 17 00:00:00 2001 From: dblock Date: Fri, 24 Jun 2022 23:04:34 +0000 Subject: [PATCH 2/3] Do not skip header. Signed-off-by: dblock --- README.md | 16 +++------ .../AWSRequestSigningApacheInterceptor.java | 34 ++++++++----------- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index a909e73..1dd1e1b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Java-Client +To reproduce https://github.com/opensearch-project/OpenSearch/issues/3640: + +Create openSearch domain in (AWS) which support IAM based AuthN/AuthZ. + ``` export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= @@ -7,16 +11,6 @@ export AWS_SESSION_TOKEN= mvn install mvn compile exec:java -Dexec.mainClass="RESTClientTest" - ``` -1. step to repro issue: https://github.com/opensearch-project/OpenSearch/issues/3640 - -2. signing sigv4 for streams https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html - -3. for (a) with content-length replace rest-client ( patched jar https://github.com/jiten1551/Java-Client/tree/master/src/main/resources) jar in ~/.m2 repo of local workspace (jar contains fix that uses content length instead of transfer-encoding) and re-run the RestClientTest.java file -4. for (b) with transfer-encoding use original rest-client jar that use transfer encoding when compression enabled. - -### NOTE: for both to have ```x-amz-decoded-content-length``` add this header edit https://github.com/jiten1551/Java-Client/blob/master/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java#L181 - - +Toggle [`.setCompressionEnabled(true/false)`](https://github.com/dblock/Java-Client/blob/master/src/main/java/RESTClientTest.java#L103). \ No newline at end of file diff --git a/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java b/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java index 6b563bc..02e02e5 100644 --- a/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java +++ b/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java @@ -103,23 +103,31 @@ public void process(final HttpRequest request, final HttpContext context) throw new IOException("Invalid URI", e); } - long contentLength = 0; + long contentLength = -1L; if (request instanceof HttpEntityEnclosingRequest) { HttpEntityEnclosingRequest httpEntityEnclosingRequest = (HttpEntityEnclosingRequest) request; if (httpEntityEnclosingRequest.getEntity() != null) { - GzipDecompressingEntity decompressedEntity = new GzipDecompressingEntity(httpEntityEnclosingRequest.getEntity()); - contentLength = EntityUtils.toString(decompressedEntity).length(); - System.out.println(EntityUtils.toString(decompressedEntity)); + Header contentEncodingHeader = httpEntityEnclosingRequest.getFirstHeader("Content-Encoding"); + if (contentEncodingHeader != null && contentEncodingHeader.getValue() == "gzip") { + GzipDecompressingEntity decompressedEntity = new GzipDecompressingEntity(httpEntityEnclosingRequest.getEntity()); + contentLength = EntityUtils.toString(decompressedEntity).length(); + System.out.println(EntityUtils.toString(decompressedEntity)); + // signableRequest.setContent(new ByteArrayInputStream(EntityUtils.toString(decompressedEntity).getBytes())); + } + signableRequest.setContent(httpEntityEnclosingRequest.getEntity().getContent()); - // signableRequest.setContent(new ByteArrayInputStream(EntityUtils.toString(decompressedEntity).getBytes())); } } signableRequest.setParameters(nvpToMapParams(uriBuilder.getQueryParams())); List
headers = new ArrayList<>(); headers.addAll(Arrays.asList(request.getAllHeaders())); - headers.add(new BasicHeader("x-Amz-Decoded-Content-Length", String.valueOf(contentLength))); + + if (contentLength > 0) { + headers.add(new BasicHeader("x-Amz-Decoded-Content-Length", String.valueOf(contentLength))); + } + // headers.add(new BasicHeader("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")); signableRequest.setHeaders(headerArrayToMap(headers)); @@ -164,24 +172,12 @@ private static Map> nvpToMapParams(final List headerArrayToMap(List
headers) { Map headersMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (Header header : headers) { - if (!skipHeader(header)) { - headersMap.put(header.getName(), header.getValue()); - } + headersMap.put(header.getName(), header.getValue()); } // headersMap.put("Content-Length", "44"); return headersMap; } - /** - * @param header header line to check - * @return true if the given header should be excluded when signing - */ - private static boolean skipHeader(final Header header) { - return ("content-length".equalsIgnoreCase(header.getName()) - && "0".equals(header.getValue())) // Strip Content-Length: 0 - || "host".equalsIgnoreCase(header.getName());// Host comes from endpoint - } - /** * @param mapHeaders Map of header entries * @return modeled Header objects From 0fcae856724e350718c49c848e0b4cb0f819648b Mon Sep 17 00:00:00 2001 From: dblock Date: Mon, 27 Jun 2022 13:54:15 +0000 Subject: [PATCH 3/3] Added repro details. Signed-off-by: dblock --- README.md | 12 +++++++++++- .../http/AWSRequestSigningApacheInterceptor.java | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1dd1e1b..7204994 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ To reproduce https://github.com/opensearch-project/OpenSearch/issues/3640: Create openSearch domain in (AWS) which support IAM based AuthN/AuthZ. +Update the value of `host` and `region` in [RESTClientTest.java](/src/main/java/RESTClientTest.java#L27) to your endpoint. + ``` export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= @@ -13,4 +15,12 @@ mvn install mvn compile exec:java -Dexec.mainClass="RESTClientTest" ``` -Toggle [`.setCompressionEnabled(true/false)`](https://github.com/dblock/Java-Client/blob/master/src/main/java/RESTClientTest.java#L103). \ No newline at end of file +Toggle [`.setCompressionEnabled(true/false)`](src/main/java/RESTClientTest.java#L103). + +With compression disabled the code will create an index, add a document, then cleanup. + +With compression enabled the request will fail with a 403 forbidden and an invalid signature error. + +``` +{"message":"The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details."} +``` diff --git a/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java b/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java index 02e02e5..a83e7cc 100644 --- a/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java +++ b/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java @@ -125,7 +125,7 @@ public void process(final HttpRequest request, final HttpContext context) headers.addAll(Arrays.asList(request.getAllHeaders())); if (contentLength > 0) { - headers.add(new BasicHeader("x-Amz-Decoded-Content-Length", String.valueOf(contentLength))); + headers.add(new BasicHeader("x-amz-decoded-content-length", Long.toString(contentLength))); } // headers.add(new BasicHeader("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"));