diff --git a/docs/source/contributors_guide/contributing.rst b/docs/source/contributors_guide/contributing.rst index 5cee988..61cc227 100644 --- a/docs/source/contributors_guide/contributing.rst +++ b/docs/source/contributors_guide/contributing.rst @@ -47,24 +47,47 @@ You'll see a different output since the documentation is build with Testing ------- -If you implemented a feature or fixed a bug, please add tests for it. +.. warning:: + If you are implementing features or fixing bugs, you are expected to have + all of the three API keys (ATD, Sulu, and RapidAPI) setup and set in you + environment variables - ``JUDGE0_SULU_API_KEY``, ``JUDGE0_RAPID_API_KEY``, + and ``JUDGE0_ATD_API_KEY``. -Unfortunately, at the moment you cannot run full test suite because it requires -access to API keys for all implemented API hubs (ATD, Sulu, and RapidAPI) and -a private Judge0 instance. To partially address this situation, you can run and -test your implemented feature and tests locally and use the GitHub CI pipeline -to run the full test suite. +Every bug fix or new feature should have tests for it. The tests are located in +the ``tests`` directory and are written using `pytest `_. -To run the tests locally, you can use the following command: +While working with the tests, you should use the following fixtures: -.. code-block:: console +* ``default_ce_client`` - a client, chosen based on the environment variables set, that uses the CE flavor of the client. +* ``default_extra_ce_client`` - a client, chosen based on the environment variables set, that uses the Extra CE flavor of the client. - $ pytest -svv tests -k '' +The ``default_ce_client`` and ``default_extra_ce_client`` are fixtures that +return a client based on the environment variables set. This enables you to +run the full test suite locally, but also to run the tests on the CI pipeline +without changing the tests. -To make the test compatible with the CI pipeline, you should use one of the -client fixtures: +You can use the fixtures in your tests like this: .. code-block:: python def test_my_test(request): - client = request.getfixturevalue("judge0_ce_client") # or judge0_extra_ce_client + client = request.getfixturevalue("default_ce_client") # or default_extra_ce_client + +To run the tests locally, you can use the following command: + +.. code-block:: console + + $ pytest -svv tests -k '' + +This will enable you to run a single test, without incurring the cost of +running the full test suite. If you want to run the full test suite, you can +use the following command: + +.. code-block:: console + + $ pytest -svv tests + +or you can create a draft PR and let the CI pipeline run the tests for you. +The CI pipeline will run the tests on every PR, using a private instance +of Judge0, so you can be sure that your changes are not breaking the existing +functionality. diff --git a/tests/conftest.py b/tests/conftest.py index b1bd612..0ceddb2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,11 +13,15 @@ def judge0_ce_client(): api_key = os.getenv("JUDGE0_TEST_API_KEY") api_key_header = os.getenv("JUDGE0_TEST_API_KEY_HEADER") endpoint = os.getenv("JUDGE0_TEST_CE_ENDPOINT") - client = clients.Client( - endpoint=endpoint, - auth_headers={api_key_header: api_key}, - ) - return client + + if api_key is None or api_key_header is None or endpoint is None: + return None + else: + client = clients.Client( + endpoint=endpoint, + auth_headers={api_key_header: api_key}, + ) + return client @pytest.fixture(scope="session") @@ -25,11 +29,15 @@ def judge0_extra_ce_client(): api_key = os.getenv("JUDGE0_TEST_API_KEY") api_key_header = os.getenv("JUDGE0_TEST_API_KEY_HEADER") endpoint = os.getenv("JUDGE0_TEST_EXTRA_CE_ENDPOINT") - client = clients.Client( - endpoint=endpoint, - auth_headers={api_key_header: api_key}, - ) - return client + + if api_key is None or api_key_header is None or endpoint is None: + return None + else: + client = clients.Client( + endpoint=endpoint, + auth_headers={api_key_header: api_key}, + ) + return client @pytest.fixture(scope="session") @@ -63,6 +71,12 @@ def rapid_extra_ce_client(): @pytest.fixture(scope="session") def sulu_ce_client(): api_key = os.getenv("JUDGE0_SULU_API_KEY") + if api_key is None: + pytest.fail( + "Sulu API key is not available for testing. Make sure to have " + "JUDGE0_SULU_API_KEY in your environment variables." + ) + client = clients.SuluJudge0CE(api_key) return client @@ -70,5 +84,31 @@ def sulu_ce_client(): @pytest.fixture(scope="session") def sulu_extra_ce_client(): api_key = os.getenv("JUDGE0_SULU_API_KEY") + if api_key is None: + pytest.fail( + "Sulu API key is not available for testing. Make sure to have " + "JUDGE0_SULU_API_KEY in your environment variables." + ) + client = clients.SuluJudge0ExtraCE(api_key) return client + + +@pytest.fixture(scope="session") +def default_ce_client(judge0_ce_client, sulu_ce_client): + if judge0_ce_client is not None: + return judge0_ce_client + if sulu_ce_client is not None: + return sulu_ce_client + + pytest.fail("No default CE client available for testing.") + + +@pytest.fixture(scope="session") +def default_extra_ce_client(judge0_extra_ce_client, sulu_extra_ce_client): + if judge0_extra_ce_client is not None: + return judge0_extra_ce_client + if sulu_extra_ce_client is not None: + return sulu_extra_ce_client + + pytest.fail("No default Extra CE client available for testing.") diff --git a/tests/test_api_test_cases.py b/tests/test_api_test_cases.py index 0d08f5f..c4963e9 100644 --- a/tests/test_api_test_cases.py +++ b/tests/test_api_test_cases.py @@ -207,7 +207,7 @@ def test_single_test_case_in_iterable(self, test_cases, stdin, expected_output): def test_test_cases_from_run( source_code_or_submissions, test_cases, expected_status, request ): - client = request.getfixturevalue("judge0_ce_client") + client = request.getfixturevalue("default_ce_client") if isinstance(source_code_or_submissions, str): submissions = judge0.run( @@ -254,7 +254,7 @@ def test_test_cases_from_run( ], ) def test_no_test_cases(submissions, expected_status, request): - client = request.getfixturevalue("judge0_ce_client") + client = request.getfixturevalue("default_ce_client") submissions = judge0.run( client=client, @@ -269,7 +269,7 @@ def test_no_test_cases(submissions, expected_status, request): @pytest.mark.parametrize("n_submissions", [42, 84]) def test_batched_test_cases(n_submissions, request): - client = request.getfixturevalue("judge0_ce_client") + client = request.getfixturevalue("default_ce_client") submissions = [ Submission(source_code=f"print({i})", expected_output=f"{i}") for i in range(n_submissions) diff --git a/tests/test_submission.py b/tests/test_submission.py index ddae140..ec1885b 100644 --- a/tests/test_submission.py +++ b/tests/test_submission.py @@ -52,7 +52,7 @@ def test_from_json(): def test_status_before_and_after_submission(request): - client = request.getfixturevalue("judge0_ce_client") + client = request.getfixturevalue("default_ce_client") submission = Submission(source_code='print("Hello World!")') assert submission.status is None @@ -65,7 +65,7 @@ def test_status_before_and_after_submission(request): def test_is_done(request): - client = request.getfixturevalue("judge0_ce_client") + client = request.getfixturevalue("default_ce_client") submission = Submission(source_code='print("Hello World!")') assert submission.status is None @@ -77,7 +77,7 @@ def test_is_done(request): def test_language_before_and_after_execution(request): - client = request.getfixturevalue("judge0_ce_client") + client = request.getfixturevalue("default_ce_client") code = """\ public class Main { public static void main(String[] args) { @@ -97,7 +97,7 @@ def test_language_before_and_after_execution(request): def test_language_executable(request): - client = request.getfixturevalue("judge0_ce_client") + client = request.getfixturevalue("default_ce_client") code = b64decode( "f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAAABAAAAAAABAAAAAAAAAAEAQAAAAAAAAAAAAAEAAOAABAEAABAADAAEAAAAFAAAAABAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAJQAAAAAAAAAlAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADHAjVANsAG+GABAAInHDwUx/41HPA8FAGhlbGxvLCB3b3JsZAoALnNoc3RydGFiAC50ZXh0AC5yb2RhdGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACwAAAAEAAAAGAAAAAAAAAAAAQAAAAAAAABAAAAAAAAAXAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABEAAAABAAAAAgAAAAAAAAAYAEAAAAAAABgQAAAAAAAADQAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAABAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAlEAAAAAAAABkAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAA" # noqa: E501 )