Automating MOD hotfix 404 again - testing complete and abort multipar…

…t upload race

Signed-off-by: Hemanth Sai Maheswarla <hemanthsaimaheswarla@Hemanths-MacBook-Pro.local>

rgw/v2/tests/s3_swift/configs/test_Mbuckets_with_Nobjects_multipart.yaml

@@ -1,4 +1,4 @@
-# upload type: non multipart
+# upload type: multipart
 # script: test_Mbuckets_with_Nobjects.py
 config:
   user_count: 1

rgw/v2/tests/s3_swift/configs/test_Mbuckets_with_Nobjects_multipart_upload_complete_abort_race.yaml

@@ -0,0 +1,25 @@
+# upload type: multipart
+# script: test_Mbuckets_with_Nobjects.py
+# bz: https://bugzilla.redhat.com/show_bug.cgi?id=2331908
+# polarion id: CEPH-83604471
+config:
+  user_count: 1
+  bucket_count: 2
+  objects_count: 50
+  split_size: 5
+  objects_size_range:
+    min: 10M
+    max: 15M
+  test_ops:
+    test_multipart_race_complete_abort: true
+    create_bucket: true
+    create_object: true
+    upload_type: multipart
+    download_object: true
+    delete_bucket_object: false
+    sharding:
+      enable: false
+      max_shards: 0
+    compression:
+      enable: false
+      type: zlib

rgw/v2/tests/s3_swift/reusable.py

@@ -510,6 +510,7 @@ def upload_mutipart_object(
     append_data=False,
     append_msg=None,
     abort_multipart=False,
+    complete_abort_race=False,
 ):
     log.info("s3 object name: %s" % s3_object_name)
     s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
@@ -606,7 +607,22 @@ def upload_mutipart_object(
     # log.info('parts_info so far: %s'% parts_info)
     if len(parts_list) == part_number:
         log.info("all parts upload completed")
-        mpu.complete(MultipartUpload=parts_info)
+        if complete_abort_race:
+            log.info("triggering complete and abort multipart upload at the same time")
+            t1 = Thread(
+                target=mpu.complete,
+                kwargs={"MultipartUpload": parts_info},
+            )
+            t2 = Thread(target=mpu.abort, kwargs={})
+
+            t1.start()
+            time.sleep(0.01)
+            t2.start()
+
+            t1.join()
+            t2.join()
+        else:
+            mpu.complete(MultipartUpload=parts_info)
         log.info("multipart upload complete for key: %s" % s3_object_name)
     if config.test_ops.get("test_get_object_attributes"):
         return object_parts_info

rgw/v2/tests/s3_swift/test_Mbuckets_with_Nobjects.py

@@ -22,6 +22,7 @@
     test_Mbuckets_with_Nobjects_get_object_attributes.yaml
     test_Mbuckets_with_Nobjects_get_object_attributes_checksum_sha256.yaml
     test_Mbuckets_with_Nobjects_get_object_attributes_multipart.yaml
+    test_Mbuckets_with_Nobjects_multipart_upload_complete_abort_race.yaml
 
 Operation:
 	Creates M bucket and N objects
@@ -346,6 +347,18 @@ def test_exec(config, ssh_con):
                                     s3_object_name=s3_object_name,
                                     object_parts_info=object_parts_info,
                                 )
+                            elif config.test_ops.get(
+                                "test_multipart_race_complete_abort"
+                            ):
+                                reusable.upload_mutipart_object(
+                                    s3_object_name,
+                                    bucket,
+                                    TEST_DATA_PATH,
+                                    config,
+                                    each_user,
+                                    abort_multipart=abort_multipart,
+                                    complete_abort_race=True,
+                                )
                             else:
                                 reusable.upload_mutipart_object(
                                     s3_object_name,
@@ -841,6 +854,31 @@ def test_exec(config, ssh_con):
                                     f"bucket check fix did not removed orphan objects on a bucket {bucket.name}"
                                 )
 
+                if config.test_ops.get("test_multipart_race_complete_abort"):
+                    utils.exec_shell_cmd(f"radosgw-admin gc process --include-all")
+                    log.info(
+                        "sleeping for 10 seconds before verifying objects download"
+                    )
+                    time.sleep(10)
+                    log.info(f"downloading all objects in bucket: {bucket.name}")
+                    objects = s3lib.resource_op(
+                        {"obj": bucket, "resource": "objects", "args": None}
+                    )
+                    log.info(f"objects :{objects}")
+                    all_objects = s3lib.resource_op(
+                        {"obj": objects, "resource": "all", "args": None}
+                    )
+                    log.info(f"all objects: {all_objects}")
+                    for obj in all_objects:
+                        log.info(f"object_name: {obj.key}")
+                        reusable.download_object(
+                            obj.key,
+                            bucket,
+                            TEST_DATA_PATH,
+                            f"{TEST_DATA_PATH}/{obj.key}",
+                            config,
+                        )
+
                 if config.test_ops.get("delete_bucket") is True:
                     reusable.delete_bucket(bucket)