Merge pull request #2161 from dhermes/rate-limit-storage-buckets

Adding 429 retry in storage for bucket create and delete.

system_tests/storage.py

@@ -30,6 +30,7 @@
 from retry import RetryErrors
 
 
+retry_429 = RetryErrors(exceptions.TooManyRequests)
 HTTP = httplib2.Http()
 _helpers.PROJECT = TESTS_PROJECT
 
@@ -49,7 +50,8 @@ def setUpModule():
     bucket_name = 'new' + unique_resource_id()
     # In the **very** rare case the bucket name is reserved, this
     # fails with a ConnectionError.
-    Config.TEST_BUCKET = Config.CLIENT.create_bucket(bucket_name)
+    Config.TEST_BUCKET = Config.CLIENT.bucket(bucket_name)
+    retry_429(Config.TEST_BUCKET.create)()
 
 
 def tearDownModule():
@@ -65,7 +67,8 @@ def setUp(self):
     def tearDown(self):
         with Config.CLIENT.batch():
             for bucket_name in self.case_buckets_to_delete:
-                Config.CLIENT.bucket(bucket_name).delete()
+                bucket = Config.CLIENT.bucket(bucket_name)
+                retry_429(bucket.delete)()
 
     def test_create_bucket(self):
         new_bucket_name = 'a-new-bucket' + unique_resource_id('-')
@@ -83,7 +86,8 @@ def test_list_buckets(self):
         ]
         created_buckets = []
         for bucket_name in buckets_to_create:
-            bucket = Config.CLIENT.create_bucket(bucket_name)
+            bucket = Config.CLIENT.bucket(bucket_name)
+            retry_429(bucket.create)()
             self.case_buckets_to_delete.append(bucket_name)
 
         # Retrieve the buckets.