Automating MOD hotfix 404 again - testing complete and abort multipar…

…t upload race

Signed-off-by: Hemanth Sai Maheswarla <hemanthsaimaheswarla@Hemanths-MacBook-Pro.local>

rgw/v2/tests/s3_swift/configs/test_Mbuckets_with_Nobjects_multipart.yaml

@@ -1,4 +1,4 @@
-# upload type: non multipart
+# upload type: multipart
 # script: test_Mbuckets_with_Nobjects.py
 config:
   user_count: 1

rgw/v2/tests/s3_swift/configs/test_Mbuckets_with_Nobjects_multipart_upload_complete_abort_race.yaml

@@ -0,0 +1,23 @@
+# upload type: multipart
+# script: test_Mbuckets_with_Nobjects.py
+config:
+  user_count: 1
+  bucket_count: 2
+  objects_count: 50
+  split_size: 5
+  objects_size_range:
+    min: 10M
+    max: 15M
+  test_ops:
+    test_multipart_race_complete_abort: true
+    create_bucket: true
+    create_object: true
+    upload_type: multipart
+    download_object: true
+    delete_bucket_object: false
+    sharding:
+      enable: false
+      max_shards: 0
+    compression:
+      enable: false
+      type: zlib

rgw/v2/tests/s3_swift/reusable.py

@@ -510,6 +510,7 @@ def upload_mutipart_object(
     append_data=False,
     append_msg=None,
     abort_multipart=False,
+    complete_abort_race=False
 ):
     log.info("s3 object name: %s" % s3_object_name)
     s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
@@ -606,7 +607,26 @@ def upload_mutipart_object(
     # log.info('parts_info so far: %s'% parts_info)
     if len(parts_list) == part_number:
         log.info("all parts upload completed")
-        mpu.complete(MultipartUpload=parts_info)
+        if complete_abort_race:
+            # mpu.complete(MultipartUpload=parts_info)
+            t1 = Thread(
+                target=mpu.complete,
+                kwargs={
+                    "MultipartUpload": parts_info
+                },
+            )
+            t2 = Thread(
+                target=mpu.abort,
+                kwargs={}
+            )
+
+            t1.start()
+            t2.start()
+
+            t1.join()
+            t2.join()
+        else:
+            mpu.complete(MultipartUpload=parts_info)
         log.info("multipart upload complete for key: %s" % s3_object_name)
     if config.test_ops.get("test_get_object_attributes"):
         return object_parts_info

rgw/v2/tests/s3_swift/test_Mbuckets_with_Nobjects.py

@@ -346,6 +346,16 @@ def test_exec(config, ssh_con):
                                     s3_object_name=s3_object_name,
                                     object_parts_info=object_parts_info,
                                 )
+                            elif config.test_ops.get("test_multipart_race_complete_abort"):
+                                reusable.upload_mutipart_object(
+                                    s3_object_name,
+                                    bucket,
+                                    TEST_DATA_PATH,
+                                    config,
+                                    each_user,
+                                    abort_multipart=abort_multipart,
+                                    complete_abort_race=True
+                                )
                             else:
                                 reusable.upload_mutipart_object(
                                     s3_object_name,
@@ -841,6 +851,21 @@ def test_exec(config, ssh_con):
                                     f"bucket check fix did not removed orphan objects on a bucket {bucket.name}"
                                 )
 
+                if config.test_ops.get("test_multipart_race_complete_abort"):
+                    utils.exec_shell_cmd(
+                        f"radosgw-admin gc process --include-all"
+                    )
+                    log.info("sleeping for 10 seconds before verifying objects download")
+                    time.sleep(10)
+                    log.info("downloading all objects in bucket: %s" % bucket.name)
+                    objects = s3lib.resource_op({"obj": bucket, "resource": "objects", "args": None})
+                    log.info("objects :%s" % objects)
+                    all_objects = s3lib.resource_op({"obj": objects, "resource": "all", "args": None})
+                    log.info("all objects: %s" % all_objects)
+                    for obj in all_objects:
+                        log.info("object_name: %s" % obj.key)
+                        reusable.download_object(obj.key, bucket, TEST_DATA_PATH, f"{TEST_DATA_PATH}/{obj.key}", config)
+
                 if config.test_ops.get("delete_bucket") is True:
                     reusable.delete_bucket(bucket)