testing: replace @flaky with @pytest.mark.flaky (#3496)

* testing: replace @flaky with @pytest.mark.flaky * lint * mark few tests as flaky that involves LRO polling. * lint
GoogleCloudPlatform · Apr 25, 2020 · bfcc58b · bfcc58b
1 parent b7ff1e5
commit bfcc58b
Show file tree

Hide file tree

Showing 13 changed files with 54 additions and 43 deletions.
diff --git a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py
@@ -13,13 +13,13 @@
 
 import os
 
-from flaky import flaky
+import pytest
 
 from blog import main
 
 PROJECT = os.environ['GCLOUD_PROJECT']
 
 
-@flaky
+@pytest.mark.flaky
 def test_main():
     main(PROJECT)
diff --git a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py
@@ -13,13 +13,13 @@
 
 import os
 
-from flaky import flaky
+import pytest
 
 from wiki import main
 
 PROJECT = os.environ['GCLOUD_PROJECT']
 
 
-@flaky
+@pytest.mark.flaky
 def test_main():
     main(PROJECT)
diff --git a/dns/api/main_test.py b/dns/api/main_test.py
@@ -14,7 +14,6 @@
 import os
 import uuid
 
-from flaky import flaky
 from google.cloud import dns
 from google.cloud.exceptions import NotFound
 
@@ -57,7 +56,7 @@ def zone(client):
             pass
 
 
-@flaky
+@pytest.mark.flaky
 def test_create_zone(client):
     zone = main.create_zone(
         PROJECT,
@@ -70,7 +69,7 @@ def test_create_zone(client):
     assert zone.description == TEST_ZONE_DESCRIPTION
 
 
-@flaky
+@pytest.mark.flaky
 def test_get_zone(client, zone):
     zone = main.get_zone(PROJECT, TEST_ZONE_NAME)
 
@@ -79,27 +78,27 @@ def test_get_zone(client, zone):
     assert zone.description == TEST_ZONE_DESCRIPTION
 
 
-@flaky
+@pytest.mark.flaky
 def test_list_zones(client, zone):
     zones = main.list_zones(PROJECT)
 
     assert TEST_ZONE_NAME in zones
 
 
-@flaky
+@pytest.mark.flaky
 def test_list_resource_records(client, zone):
     records = main.list_resource_records(PROJECT, TEST_ZONE_NAME)
 
     assert records
 
 
-@flaky
+@pytest.mark.flaky
 def test_list_changes(client, zone):
     changes = main.list_changes(PROJECT, TEST_ZONE_NAME)
 
     assert changes
 
 
-@flaky
+@pytest.mark.flaky
 def test_delete_zone(client, zone):
     main.delete_zone(PROJECT, TEST_ZONE_NAME)
diff --git a/iap/iap_test.py b/iap/iap_test.py
@@ -14,7 +14,7 @@
 
 """Test script for Identity-Aware Proxy code samples."""
 
-from flaky import flaky
+import pytest
 
 import make_iap_request
 import validate_jwt
@@ -34,7 +34,7 @@
 IAP_PROJECT_NUMBER = '320431926067'
 
 
-@flaky
+@pytest.mark.flaky
 def test_main(capsys):
     # JWTs are obtained by IAP-protected applications whenever an
     # end-user makes a request.  We've set up an app that echoes back

diff --git a/jobs/v3/api_client/quickstart_test.py b/jobs/v3/api_client/quickstart_test.py
@@ -11,10 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from flaky import flaky
 
+import pytest
 
-@flaky
+
+@pytest.mark.flaky
 def test_quickstart(capsys):
     import quickstart
 

diff --git a/ml_engine/online_prediction/predict_test.py b/ml_engine/online_prediction/predict_test.py
@@ -15,7 +15,6 @@
 import json
 import socket
 
-from flaky import flaky
 import pytest
 
 import predict
@@ -42,29 +41,29 @@
     BYTESTRING = f.read()
 
 
-@flaky
+@pytest.mark.flaky
 def test_predict_json():
     result = predict.predict_json(
         PROJECT, MODEL, [JSON, JSON], version=JSON_VERSION)
     assert [EXPECTED_OUTPUT, EXPECTED_OUTPUT] == result
 
 
-@flaky
+@pytest.mark.flaky
 def test_predict_json_error():
     with pytest.raises(RuntimeError):
         predict.predict_json(
             PROJECT, MODEL, [{"foo": "bar"}], version=JSON_VERSION)
 
 
-@flaky
+@pytest.mark.flaky
 def test_census_example_to_bytes():
     import tensorflow as tf
     b = predict.census_to_example_bytes(JSON)
     assert tf.train.Example.FromString(b) == tf.train.Example.FromString(
         BYTESTRING)
 
 
-@flaky(max_runs=6)
+@pytest.mark.flaky(max_runs=6)
 def test_predict_examples():
     result = predict.predict_examples(
         PROJECT, MODEL, [BYTESTRING, BYTESTRING], version=EXAMPLES_VERSION)

diff --git a/monitoring/api/v3/api-client/list_resources_test.py b/monitoring/api/v3/api-client/list_resources_test.py
@@ -23,7 +23,6 @@
 import os
 import re
 
-from flaky import flaky
 import googleapiclient.discovery
 import pytest
 
@@ -38,7 +37,7 @@ def client():
     return googleapiclient.discovery.build('monitoring', 'v3')
 
 
-@flaky
+@pytest.mark.flaky
 def test_list_monitored_resources(client, capsys):
     PROJECT_RESOURCE = "projects/{}".format(PROJECT)
     list_resources.list_monitored_resource_descriptors(
@@ -49,7 +48,7 @@ def test_list_monitored_resources(client, capsys):
     assert regex.search(stdout) is not None
 
 
-@flaky
+@pytest.mark.flaky
 def test_list_metrics(client, capsys):
     PROJECT_RESOURCE = "projects/{}".format(PROJECT)
     list_resources.list_metric_descriptors(
@@ -60,7 +59,7 @@ def test_list_metrics(client, capsys):
     assert regex.search(stdout) is not None
 
 
-@flaky
+@pytest.mark.flaky
 def test_list_timeseries(client, capsys):
     PROJECT_RESOURCE = "projects/{}".format(PROJECT)
     list_resources.list_timeseries(

diff --git a/storage/api/customer_supplied_keys_test.py b/storage/api/customer_supplied_keys_test.py
@@ -14,14 +14,14 @@
 import os
 import re
 
-from flaky import flaky
+import pytest
 
 from customer_supplied_keys import main
 
 BUCKET = os.environ['CLOUD_STORAGE_BUCKET']
 
 
-@flaky
+@pytest.mark.flaky
 def test_main(capsys):
     main(BUCKET, __file__)
     out, err = capsys.readouterr()

diff --git a/translate/cloud-client/translate_v3_batch_translate_text_test.py b/translate/cloud-client/translate_v3_batch_translate_text_test.py
@@ -33,6 +33,7 @@ def bucket():
     bucket.delete(force=True)
 
 
+@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_batch_translate_text(capsys, bucket):
     translate_v3_batch_translate_text.batch_translate_text(
         "gs://cloud-samples-data/translation/text.txt",

diff --git a/translate/cloud-client/translate_v3_batch_translate_text_with_glossary_test.py b/translate/cloud-client/translate_v3_batch_translate_text_with_glossary_test.py
@@ -52,6 +52,7 @@ def bucket():
     bucket.delete(force=True)
 
 
+@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_batch_translate_text_with_glossary(capsys, bucket, glossary):
     translate_v3_batch_translate_text_with_glossary.batch_translate_text_with_glossary(
         "gs://cloud-samples-data/translation/text_with_glossary.txt",

diff --git a/translate/cloud-client/translate_v3_create_glossary_test.py b/translate/cloud-client/translate_v3_create_glossary_test.py
@@ -13,26 +13,32 @@
 # limitations under the License.
 
 import os
+import uuid
+
+import pytest
+
 import translate_v3_create_glossary
 import translate_v3_delete_glossary
-import uuid
 
 PROJECT_ID = os.environ["GCLOUD_PROJECT"]
 GLOSSARY_INPUT_URI = "gs://cloud-samples-data/translation/glossary_ja.csv"
 
 
+@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_create_glossary(capsys):
-    glossary_id = "test-{}".format(uuid.uuid4())
-    translate_v3_create_glossary.create_glossary(
-        PROJECT_ID, GLOSSARY_INPUT_URI, glossary_id
-    )
-    out, _ = capsys.readouterr()
-    # assert
-    assert "Created:" in out
-    assert "gs://cloud-samples-data/translation/glossary_ja.csv" in out
-
-    # clean up after use
     try:
-        translate_v3_delete_glossary.delete_glossary(PROJECT_ID, glossary_id)
-    except Exception:
-        pass
+        glossary_id = "test-{}".format(uuid.uuid4())
+        translate_v3_create_glossary.create_glossary(
+            PROJECT_ID, GLOSSARY_INPUT_URI, glossary_id
+        )
+        out, _ = capsys.readouterr()
+        # assert
+        assert "Created:" in out
+        assert "gs://cloud-samples-data/translation/glossary_ja.csv" in out
+    finally:
+        # clean up after use
+        try:
+            translate_v3_delete_glossary.delete_glossary(
+                PROJECT_ID, glossary_id)
+        except Exception:
+            pass
diff --git a/translate/cloud-client/translate_v3_delete_glossary_test.py b/translate/cloud-client/translate_v3_delete_glossary_test.py
@@ -13,14 +13,18 @@
 # limitations under the License.
 
 import os
+import uuid
+
+import pytest
+
 import translate_v3_create_glossary
 import translate_v3_delete_glossary
-import uuid
 
 PROJECT_ID = os.environ["GCLOUD_PROJECT"]
 GLOSSARY_INPUT_URI = "gs://cloud-samples-data/translation/glossary_ja.csv"
 
 
+@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_delete_glossary(capsys):
     # setup
     glossary_id = "test-{}".format(uuid.uuid4())

diff --git a/vision/cloud-client/product_search/product_search_test.py b/vision/cloud-client/product_search/product_search_test.py
@@ -13,7 +13,8 @@
 # limitations under the License.
 
 import os
-from flaky import flaky
+
+import pytest
 
 from product_search import get_similar_products_file, get_similar_products_uri
 
@@ -31,7 +32,7 @@
 FILTER = 'style=womens'
 
 
-@flaky(max_runs=5, min_passes=1)
+@pytest.mark.flaky(max_runs=5, min_passes=1)
 def test_get_similar_products_file(capsys):
     get_similar_products_file(
         PROJECT_ID, LOCATION, PRODUCT_SET_ID, PRODUCT_CATEGORY, FILE_PATH_1,