Fix: generate correct AWS SIGv4 signature for compressed requests

README.md

@@ -1,12 +1,26 @@
 # Java-Client
 
-1. step to repro issue: https://github.com/opensearch-project/OpenSearch/issues/3640
+To reproduce https://github.com/opensearch-project/OpenSearch/issues/3640:
 
-2. signing sigv4 for streams https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+Create openSearch domain in (AWS) which support IAM based AuthN/AuthZ.
 
-3. for (a) with content-length replace rest-client ( patched jar https://github.com/jiten1551/Java-Client/tree/master/src/main/resources) jar in ~/.m2 repo of local workspace (jar contains fix that uses content length instead of transfer-encoding) and re-run the RestClientTest.java file
-4. for (b) with transfer-encoding use original rest-client jar that use transfer encoding when compression enabled.
+Update the value of `host` and `region` in [RESTClientTest.java](/src/main/java/RESTClientTest.java#L27) to your endpoint.
 
-### NOTE: for both to have ```x-amz-decoded-content-length``` add this header edit https://github.com/jiten1551/Java-Client/blob/master/src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java#L181
+```
+export AWS_ACCESS_KEY_ID=
+export AWS_SECRET_ACCESS_KEY=
+export AWS_SESSION_TOKEN=
 
+mvn install
+mvn compile exec:java -Dexec.mainClass="RESTClientTest"
+```
 
+Toggle [`.setCompressionEnabled(true/false)`](src/main/java/RESTClientTest.java#L103).
+
+With compression disabled the code will create an index, add a document, then cleanup.
+
+With compression enabled the request will fail with a 403 forbidden and an invalid signature error.
+
+```
+{"message":"The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details."}
+```

pom.xml

@@ -24,7 +24,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-core</artifactId>
-            <version>1.11.106</version>
+            <version>1.12.247</version>
         </dependency>
         <dependency>
             <groupId>org.opensearch.client</groupId>

src/main/java/RESTClientTest.java

@@ -24,12 +24,9 @@
 
 public class RESTClientTest {
 
-    private static String host = "https://xxxxxx"; // put your own end-point
+    private static String host = "https://search-dblock-test-opensearch-21-tu5gqrjd4vg4qazjsu6bps5zsy.us-west-2.es.amazonaws.com"; // put your own end-point
     private static String serviceName = "es";
-    private static String region = "eu-west-1";
-
-
-
+    private static String region = "us-west-2";
 
     public static void main(String[] args) throws IOException {
         RestHighLevelClient searchClient = searchClient(serviceName, region);

src/main/java/com/amazonaws/http/AWSRequestSigningApacheInterceptor.java

@@ -22,16 +22,19 @@
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpRequestInterceptor;
 import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.GzipDecompressingEntity;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.entity.BasicHttpEntity;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.EntityUtils;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -92,26 +95,41 @@ public void process(final HttpRequest request, final HttpContext context)
         if (host != null) {
             signableRequest.setEndpoint(URI.create(host.toURI()));
         }
-        final HttpMethodName httpMethod =
-                HttpMethodName.fromValue(request.getRequestLine().getMethod());
+        final HttpMethodName httpMethod = HttpMethodName.fromValue(request.getRequestLine().getMethod());
         signableRequest.setHttpMethod(httpMethod);
         try {
             signableRequest.setResourcePath(uriBuilder.build().getRawPath());
         } catch (URISyntaxException e) {
             throw new IOException("Invalid URI", e);
         }
-//        long decodedContentLength = request.get
+
+        long contentLength = -1L;
         if (request instanceof HttpEntityEnclosingRequest) {
             HttpEntityEnclosingRequest httpEntityEnclosingRequest =
                     (HttpEntityEnclosingRequest) request;
-            if (httpEntityEnclosingRequest.getEntity() == null) {
-                signableRequest.setContent(new ByteArrayInputStream(new byte[0]));
-            } else {
+            if (httpEntityEnclosingRequest.getEntity() != null) {
+
+                Header contentEncodingHeader = httpEntityEnclosingRequest.getFirstHeader("Content-Encoding");
+                if (contentEncodingHeader != null && contentEncodingHeader.getValue() == "gzip") {
+                    GzipDecompressingEntity decompressedEntity = new GzipDecompressingEntity(httpEntityEnclosingRequest.getEntity());
+                    contentLength = EntityUtils.toString(decompressedEntity).length();
+                    System.out.println(EntityUtils.toString(decompressedEntity));
+                    // signableRequest.setContent(new ByteArrayInputStream(EntityUtils.toString(decompressedEntity).getBytes()));
+                }
+
                 signableRequest.setContent(httpEntityEnclosingRequest.getEntity().getContent());
             }
         }
         signableRequest.setParameters(nvpToMapParams(uriBuilder.getQueryParams()));
-        signableRequest.setHeaders(headerArrayToMap(request.getAllHeaders()));
+        List<Header> headers = new ArrayList<>();
+        headers.addAll(Arrays.asList(request.getAllHeaders()));
+
+        if (contentLength > 0) {
+            headers.add(new BasicHeader("x-amz-decoded-content-length", Long.toString(contentLength)));
+        }
+
+        // headers.add(new BasicHeader("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"));
+        signableRequest.setHeaders(headerArrayToMap(headers));
 
         // Sign it
         signer.sign(signableRequest, awsCredentialsProvider.getCredentials());
@@ -127,6 +145,10 @@ public void process(final HttpRequest request, final HttpContext context)
                 httpEntityEnclosingRequest.setEntity(basicHttpEntity);
             }
         }
+
+        for(Header header : request.getAllHeaders()) {
+            System.out.println(header.getName() + ": " + header.getValue());
+        }
     }
 
     /**
@@ -147,27 +169,15 @@ private static Map<String, List<String>> nvpToMapParams(final List<NameValuePair
      * @param headers modeled Header objects
      * @return a Map of header entries
      */
-    private static Map<String, String> headerArrayToMap(final Header[] headers) {
+    private static Map<String, String> headerArrayToMap(List<Header> headers) {
         Map<String, String> headersMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
         for (Header header : headers) {
-            if (!skipHeader(header)) {
-                headersMap.put(header.getName(), header.getValue());
-            }
+            headersMap.put(header.getName(), header.getValue());
         }
 //        headersMap.put("Content-Length", "44");
         return headersMap;
     }
 
-    /**
-     * @param header header line to check
-     * @return true if the given header should be excluded when signing
-     */
-    private static boolean skipHeader(final Header header) {
-        return ("content-length".equalsIgnoreCase(header.getName())
-                && "0".equals(header.getValue())) // Strip Content-Length: 0
-                || "host".equalsIgnoreCase(header.getName());// Host comes from endpoint
-    }
-
     /**
      * @param mapHeaders Map of header entries
      * @return modeled Header objects
@@ -178,7 +188,6 @@ private static Header[] mapToHeaderArray(final Map<String, String> mapHeaders) {
         for (Map.Entry<String, String> headerEntry : mapHeaders.entrySet()) {
             headers[i++] = new BasicHeader(headerEntry.getKey(), headerEntry.getValue());
         }
-//        headers[i++] = new BasicHeader("x-amz-decoded-content-length", "28");
         return headers;
     }
 }