Merge pull request #655 from hmaheswa/hotfix_cors_presigned_put_uri

Automation of Inditex hotfix bz testing: CORS presigned put url

rgw/v2/lib/curl/resource_op.py

@@ -25,6 +25,7 @@ def __init__(self, user_info, ssh_con, curl_silent=True, ssl=None):
             self.prefix = f"curl --show-error --fail-with-body -v --aws-sigv4 aws:amz:us-east-1:s3 -u '{self.username}:{self.password}'"
         if ssl:
             self.prefix = self.prefix + " --insecure"
+        self.prefix_for_presigned_url = f"curl --show-error --fail-with-body -v -s"
 
     def command(
         self,
@@ -35,6 +36,7 @@ def command(
         url_suffix=None,
         raw_data_list=None,
         head_request=None,
+        presigned_url=None,
     ):
         """
         Args:
@@ -47,7 +49,10 @@ def command(
             head_request(bool): whether it is a head request or not specified with -I or --head
         Returns: command to be executed
         """
-        cmd = self.prefix
+        if presigned_url:
+            cmd = self.prefix_for_presigned_url
+        else:
+            cmd = self.prefix
 
         if http_method:
             cmd = f"{cmd} -X {http_method}"
@@ -57,7 +62,7 @@ def command(
         if headers:
             for key, val in headers.items():
                 # any underscores in the header will automatically be converted to hyphen by curl
-                header_string = f"{header_string} -H '{key}:{val}'"
+                header_string = f"{header_string} -H '{key}: {val}'"
         cmd = f"{cmd} {header_string}"
         if input_file:
             cmd = f"{cmd} -T {input_file}"
@@ -66,10 +71,12 @@ def command(
         if raw_data_list:
             for each_str in raw_data_list:
                 cmd = f"{cmd} -d '{each_str}'"
-
-        url = self.endpoint_url
-        if url_suffix:
-            url = f"'{url}/{url_suffix}'"
-        cmd = f"{cmd} {url}"
+        if presigned_url:
+            url = presigned_url
+        else:
+            url = self.endpoint_url
+            if url_suffix:
+                url = f"{url}/{url_suffix}"
+        cmd = f"{cmd} '{url}'"
         log.info(f"CURL command created: {cmd}")
         return cmd

rgw/v2/tests/curl/configs/test_cors_presigned_put_url_using_curl.yaml

@@ -0,0 +1,28 @@
+# script: test_cors_using_curl.py
+# polarion: CEPH-83604475
+# hotfix bz: https://bugzilla.redhat.com/show_bug.cgi?id=2299642
+config:
+  user_count: 1
+  bucket_count: 2
+  objects_count: 25
+  objects_size_range:
+    min: 5
+    max: 15
+  local_file_delete: true
+  test_ops:
+    create_bucket: true
+    create_object: true
+    user_remove: true
+    cors_origin: "http://www.cors-example.com"
+    cors_presigned_put_url: true
+    object_acl: "private"
+    policy_document:
+      "CORSRules":
+        [
+          {
+            "AllowedOrigins": ["http://www.cors-example.com"],
+            "AllowedHeaders": ["*"],
+            "AllowedMethods": ["PUT", "GET", "DELETE"],
+            "MaxAgeSeconds": 3000
+          },
+        ]

rgw/v2/tests/curl/reusable.py

@@ -185,6 +185,9 @@ def upload_object(
     append_data=False,
     append_msg=None,
     Transfer_Encoding=None,
+    extra_headers={},
+    request_options=False,
+    presigned_url=None,
 ):
     """
     upload object using curl
@@ -222,11 +225,28 @@ def upload_object(
         headers["Transfer-Encoding"] = Transfer_Encoding
     else:
         headers["Content-Length"] = config.obj_size
+    headers.update(extra_headers)
+    if request_options:
+        log.info("testing requests.options before the actual put request")
+        headers_options = headers.copy()
+        headers_options["Access-Control-Request-Method"] = "PUT"
+        command = curl_auth.command(
+            http_method="OPTIONS",
+            headers=headers_options,
+            input_file=s3_object_path,
+            # url_suffix=f"{bucket_name}/{s3_object_name}",
+            presigned_url=presigned_url,
+        )
+        upload_object_status = utils.exec_shell_cmd(command)
+        if upload_object_status is False:
+            raise TestExecError("requests.options failed for PUT")
+        log.info(f"object {s3_object_name} uploaded")
     command = curl_auth.command(
         http_method="PUT",
         headers=headers,
         input_file=s3_object_path,
         url_suffix=f"{bucket_name}/{s3_object_name}",
+        presigned_url=presigned_url,
     )
     upload_object_status = utils.exec_shell_cmd(command)
     if upload_object_status is False:

rgw/v2/tests/curl/test_cors_using_curl.py

@@ -5,6 +5,7 @@
     Note: Following yaml can be used
     test_cors_using_curl.yaml
     test_crlf_injection_curl.yaml
+    test_cors_presigned_put_url_using_curl.yaml
 Operation:
 1. Create bucket and put CORS policy on the bucket.
 2. for the object count mentioned , upload objects using CURL calls with specified ORIGIN
@@ -27,6 +28,7 @@
 from v2.lib.aws.resource_op import AWS
 from v2.lib.curl.resource_op import CURL
 from v2.lib.exceptions import RGWBaseException, TestExecError
+from v2.lib.s3.auth import Auth
 from v2.lib.s3.write_io_info import BasicIOInfoStructure, IOInfoInitialize
 from v2.tests.aws import reusable as aws_reusable
 from v2.tests.curl import reusable as curl_reusable
@@ -53,6 +55,8 @@ def test_exec(config, ssh_con):
     all_users_info = resource_op.create_users(no_of_users_to_create=config.user_count)
 
     for each_user in all_users_info:
+        auth = Auth(each_user, ssh_con, ssl=config.ssl, haproxy=config.haproxy)
+        rgw_s3_client = auth.do_auth_using_client()
         user_name = each_user["user_id"]
         log.info(user_name)
         cli_aws = AWS(ssl=config.ssl)
@@ -71,7 +75,7 @@ def test_exec(config, ssh_con):
             file_name = "cors.json"
             cors_policy = json.dumps(config.test_ops["policy_document"])
             log.info(cors_policy)
-            with open(file_name, "a") as f:
+            with open(file_name, "w") as f:
                 f.write(cors_policy)
             aws_reusable.put_bucket_cors(cli_aws, bucket_name, file_name, endpoint)
             cors_origin = config.test_ops["cors_origin"]
@@ -88,35 +92,58 @@ def test_exec(config, ssh_con):
                 log.info(f"s3 object name: {s3_object_name}")
                 s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
                 log.info(f"s3 object path: {s3_object_path}")
-                out = curl_reusable.put_cors_object(
-                    curl_auth,
-                    bucket_name,
-                    s3_object_name,
-                    TEST_DATA_PATH,
-                    config,
-                    cors_origin,
-                )
-                if config.test_ops.get("CRLF_injection", False):
-                    if "\r" not in out:
-                        log.info("\r is sanitized as expected")
-                    else:
-                        raise TestExecError("\r is still present in curl output")
-                else:
-                    objects_created_list.append(s3_object_name)
-                    curl_reusable.download_object(
+                if config.test_ops.get("cors_presigned_put_url", False):
+                    object_acl = config.test_ops["object_acl"]
+                    presigned_url_for_put = s3_reusable.generate_presigned_url(
+                        rgw_s3_client=rgw_s3_client,
+                        client_method="put_object",
+                        http_method="PUT",
+                        params={
+                            "Bucket": bucket_name,
+                            "Key": "test",
+                        },
+                    )
+                    curl_reusable.upload_object(
                         curl_auth,
                         bucket_name,
                         s3_object_name,
                         TEST_DATA_PATH,
-                        s3_object_path,
-                        cors_origin,
+                        config,
+                        Transfer_Encoding=config.test_ops.get("Transfer_Encoding"),
+                        extra_headers={"Origin": cors_origin, "x-amz-acl": object_acl},
+                        request_options=True,
+                        presigned_url=presigned_url_for_put,
                     )
-                    curl_reusable.delete_object(
+                else:
+                    out = curl_reusable.put_cors_object(
                         curl_auth,
                         bucket_name,
                         s3_object_name,
+                        TEST_DATA_PATH,
+                        config,
                         cors_origin,
                     )
+                    if config.test_ops.get("CRLF_injection", False):
+                        if "\r" not in out:
+                            log.info("\r is sanitized as expected")
+                        else:
+                            raise TestExecError("\r is still present in curl output")
+                    else:
+                        objects_created_list.append(s3_object_name)
+                        curl_reusable.download_object(
+                            curl_auth,
+                            bucket_name,
+                            s3_object_name,
+                            TEST_DATA_PATH,
+                            s3_object_path,
+                            cors_origin,
+                        )
+                        curl_reusable.delete_object(
+                            curl_auth,
+                            bucket_name,
+                            s3_object_name,
+                            cors_origin,
+                        )
             if config.local_file_delete is True:
                 log.info("deleting local file created after the upload")
                 utils.exec_shell_cmd(f"rm -rf  {s3_object_path}")

rgw/v2/tests/s3_swift/reusable.py

@@ -2758,3 +2758,14 @@ def get_object_attributes(
         else:
             log.info(f"{attr} verified successfully")
     log.info("GetObjectAttributes verified successfully")
+
+
+def generate_presigned_url(rgw_s3_client, client_method, http_method, params):
+    log.info(
+        f"generating presigned url for client_method {client_method}, http_method {http_method} with params {params}"
+    )
+    presigned_url = rgw_s3_client.generate_presigned_url(
+        ClientMethod=client_method, HttpMethod=http_method, Params=params
+    )
+    log.info(f"presigned_url: {presigned_url}")
+    return presigned_url