From 5e9355a87881e62f306274c610b1db3557fc70fa Mon Sep 17 00:00:00 2001 From: Jeroen Overschie Date: Thu, 28 Jul 2022 15:00:00 +0000 Subject: [PATCH] Mark tests as optional --- tests/unit/callbacks/test_to_wandb.py | 9 ++++++++- tests/unit/storage/test_wandb.py | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/unit/callbacks/test_to_wandb.py b/tests/unit/callbacks/test_to_wandb.py index 5637ed6..9f51f23 100644 --- a/tests/unit/callbacks/test_to_wandb.py +++ b/tests/unit/callbacks/test_to_wandb.py @@ -113,6 +113,14 @@ def test_on_summary(wandb_callback: WandbCallback): # → assertions in `test_on_end` +@pytest.mark.xfail( + "The Weights-and-Biases platform has proven to have too large delays to be able to" + + " run integration tests on it consistently. Alhough the WandB platform does" + + " eventually upload all the data that is being sent to it, all logging is queued," + + " making it hard to rely on it in integration-/ unit test settings." + + "Therefore this test is marked as optional to prevent unnecessary build failures." + + " The test is kept in the codebase, however, for convenience in local debugging." +) @pytest.mark.dependency( depends=[ "test_on_begin", @@ -130,7 +138,6 @@ def test_on_end(wandb_callback: WandbCallback, api: Api, request: FixtureRequest # `on_end` wandb_callback.on_end() - sleep(5) # wait to give wandb run time to end # ensure finished api._runs = {} # reset wandb runs internal cache diff --git a/tests/unit/storage/test_wandb.py b/tests/unit/storage/test_wandb.py index 194eccd..86a57f6 100644 --- a/tests/unit/storage/test_wandb.py +++ b/tests/unit/storage/test_wandb.py @@ -44,7 +44,14 @@ def test_save(request: FixtureRequest): cache.set("filename", filename) -@pytest.mark.xfail(reason="Weights and Biases might not have saved the file correctly.") +@pytest.mark.xfail( + "The Weights-and-Biases platform has proven to have too large delays to be able to" + + " run integration tests on it consistently. Alhough the WandB platform does" + + " eventually upload all the data that is being sent to it, all logging is queued," + + " making it hard to rely on it in integration-/ unit test settings." + + "Therefore this test is marked as optional to prevent unnecessary build failures." + + " The test is kept in the codebase, however, for convenience in local debugging." +) @pytest.mark.dependency(depends=["test_save"]) def test_load(request: FixtureRequest): # retrieve previous run id from cache