From bea73c5c03e2980509eda5f0704547fe81176d5e Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Wed, 6 Sep 2023 15:53:00 +0300 Subject: [PATCH] re-enable c2d tests (#661) * re-enable c2d tests * fix ci * optimize tests speed * temp add ssh * cleanup runner disk space * fix cleanup * remove ssh * Delete find_format function. * Delete import. * Updated nonce for compute for tests purpose. * Fixed disabled asset error. * Fixed address for test for format timestamp. * Compare Decimals. --------- Co-authored-by: Maria Carmina --- .github/workflows/pytest.yml | 27 ++++++++++++++----- ocean_provider/run.py | 2 +- ocean_provider/utils/accounts.py | 10 ++----- ocean_provider/utils/data_nft.py | 3 ++- ocean_provider/utils/test/test_accounts.py | 9 ------- ocean_provider/utils/test/test_compute.py | 1 - .../utils/test/test_provider_fees.py | 1 - .../validation/test/test_algo_validation.py | 18 ------------- tests/helpers/compute_helpers.py | 4 +-- tests/helpers/nonce.py | 6 +++++ tests/test_compute.py | 24 ----------------- tests/test_initialize.py | 5 +--- tests/test_routes.py | 19 +++++++------ 13 files changed, 43 insertions(+), 86 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0de3935e..bae881b1 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -33,22 +33,35 @@ jobs: - name: Run Barge working-directory: ${{ github.workspace }}/barge run: | - bash -x start_ocean.sh --no-dashboard 2>&1 --with-rbac --with-provider2 --with-thegraph --skip-subgraph-deploy > start_ocean.log & + bash -x start_ocean.sh --no-dashboard 2>&1 --with-rbac --with-provider2 --with-thegraph --with-c2d --skip-subgraph-deploy > start_ocean.log & + - name: Install dependencies + working-directory: ${{ github.workspace }} + run: | + python -m pip install --upgrade pip + pip install -r requirements_dev.txt + - name: Delete default runner images + run: | + docker image rm node:14 + docker image rm node:14-alpine + docker image rm node:16 + docker image rm node:16-alpine + docker image rm node:18 + docker image rm node:18-alpine + docker image rm buildpack-deps:buster + docker image rm buildpack-deps:bullseye + docker image rm debian:10 + docker image rm debian:11 + docker image rm moby/buildkit:latest - name: Wait for contracts deployment and C2D cluster to be ready working-directory: ${{ github.workspace }}/barge run: | for i in $(seq 1 250); do sleep 10 - [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" ] && break + [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" -a -f "$HOME/.ocean/ocean-c2d/ready" ] && break done - name: Verify deployments run: | cat $HOME/.ocean/ocean-contracts/artifacts/address.json - - name: Install dependencies - working-directory: ${{ github.workspace }} - run: | - python -m pip install --upgrade pip - pip install -r requirements_dev.txt - name: Test with pytest run: | coverage run --source ocean_provider -m pytest diff --git a/ocean_provider/run.py b/ocean_provider/run.py index d5f98b05..1b83ef00 100644 --- a/ocean_provider/run.py +++ b/ocean_provider/run.py @@ -59,7 +59,7 @@ def get_services_endpoints(): ), ) ) - for (key, value) in services_endpoints.items(): + for key, value in services_endpoints.items(): services_endpoints[key] = ( list( map( diff --git a/ocean_provider/utils/accounts.py b/ocean_provider/utils/accounts.py index 7f9d8bdf..e9b00590 100644 --- a/ocean_provider/utils/accounts.py +++ b/ocean_provider/utils/accounts.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # import logging +from _decimal import Decimal from eth_keys import KeyAPI from eth_keys.backends import NativeECCBackend @@ -16,7 +17,7 @@ def verify_nonce(signer_address, nonce): db_nonce = get_nonce(signer_address) - if db_nonce and _find_nonce_format(str(nonce)) < _find_nonce_format(str(db_nonce)): + if db_nonce and Decimal(nonce) < Decimal(db_nonce): msg = ( f"Invalid signature expected nonce ({db_nonce}) > current nonce ({nonce})." ) @@ -112,10 +113,3 @@ def sign_message(message, wallet): signature = "0x" + r[2:] + s[2:] + v[2:] return signature - - -def _find_nonce_format(nonce: str): - if "." in nonce and nonce[-1] != ".": - return float(nonce) - else: - return int(nonce) diff --git a/ocean_provider/utils/data_nft.py b/ocean_provider/utils/data_nft.py index 7b1443bf..80d5432b 100644 --- a/ocean_provider/utils/data_nft.py +++ b/ocean_provider/utils/data_nft.py @@ -47,7 +47,8 @@ def get_data_nft_contract(web3: Web3, address: Optional[str] = None) -> Contract def get_metadata(web3: Web3, address: str) -> Tuple[str, str, MetadataState, bool]: """Queries the ERC721 Template smart contract getMetaData call. - Returns metaDataDecryptorUrl, metaDataDecryptorAddress, metaDataState, and hasMetaData""" + Returns metaDataDecryptorUrl, metaDataDecryptorAddress, metaDataState, and hasMetaData + """ data_nft_contract = get_data_nft_contract(web3, address) return data_nft_contract.caller.getMetaData() diff --git a/ocean_provider/utils/test/test_accounts.py b/ocean_provider/utils/test/test_accounts.py index 408d9994..06de4400 100644 --- a/ocean_provider/utils/test/test_accounts.py +++ b/ocean_provider/utils/test/test_accounts.py @@ -6,7 +6,6 @@ get_private_key, sign_message, verify_signature, - _find_nonce_format, ) from tests.helpers.nonce import build_nonce @@ -19,14 +18,6 @@ def test_get_private_key(publisher_wallet): ) -@pytest.mark.unit -def test_find_nonce_format(): - nonce = "1" - assert isinstance(_find_nonce_format(nonce), int) - nonce = "1.1" - assert isinstance(_find_nonce_format(nonce), float) - - @pytest.mark.unit def test_verify_signature(consumer_wallet, publisher_wallet): nonce = build_nonce(consumer_wallet.address) diff --git a/ocean_provider/utils/test/test_compute.py b/ocean_provider/utils/test/test_compute.py index de2c21c0..b9d247ec 100644 --- a/ocean_provider/utils/test/test_compute.py +++ b/ocean_provider/utils/test/test_compute.py @@ -14,7 +14,6 @@ @pytest.mark.unit -@pytest.mark.skip("C2D connection needs fixing.") def test_get_compute_endpoint(monkeypatch): monkeypatch.setenv("OPERATOR_SERVICE_URL", "http://with-slash.com/") assert get_compute_endpoint() == "http://with-slash.com/api/v1/operator/compute" diff --git a/ocean_provider/utils/test/test_provider_fees.py b/ocean_provider/utils/test/test_provider_fees.py index 99948afe..f4bcfa2a 100644 --- a/ocean_provider/utils/test/test_provider_fees.py +++ b/ocean_provider/utils/test/test_provider_fees.py @@ -14,7 +14,6 @@ @pytest.mark.unit -@pytest.mark.skip("C2D connection needs fixing.") @freeze_time("Feb 11th, 2012 00:00") def test_get_provider_fee_amount(web3, publisher_wallet): valid_until = get_future_valid_until() diff --git a/ocean_provider/validation/test/test_algo_validation.py b/ocean_provider/validation/test/test_algo_validation.py index 7e5df4f7..e94f9555 100644 --- a/ocean_provider/validation/test/test_algo_validation.py +++ b/ocean_provider/validation/test/test_algo_validation.py @@ -21,7 +21,6 @@ this_is_a_gist = "https://gist.githubusercontent.com/calina-c/5e8c965962bc0240eab516cb7a180670/raw/6e6cd245c039a9aac0a488857c6927d39eaafe4d/sprintf-py-conversions" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -64,7 +63,6 @@ def side_effect(*args, **kwargs): assert validator.validate() is True -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -105,7 +103,6 @@ def test_passes_raw(provider_wallet, consumer_address, web3): assert validator.validate() is True -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -152,7 +149,6 @@ def side_effect(*args, **kwargs): assert validator.message == "not_algo" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -253,7 +249,6 @@ def test_fails_meta_issues(provider_wallet, consumer_address, web3): assert validator.message == "checksum_prefix" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -386,7 +381,6 @@ def side_effect(*args, **kwargs): assert validator.message == "not_found" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -444,7 +438,6 @@ def other_service(*args, **kwargs): assert validator.message == "service_not_access_compute" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -543,7 +536,6 @@ def side_effect(*args, **kwargs): assert validator.message == "not_trusted_algo_publisher" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -568,7 +560,6 @@ def test_fails_no_asset_url(provider_wallet, consumer_address, web3): assert validator.message == "compute_services_not_in_same_provider" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch("ocean_provider.validation.algo.validate_order", side_effect=Exception("mock")) @@ -593,7 +584,6 @@ def test_fails_validate_order(provider_wallet, consumer_address, web3): assert validator.message == "order_invalid" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -621,7 +611,6 @@ def test_fails_no_service_id(provider_wallet, consumer_address, web3): assert validator.message == "missing" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -668,7 +657,6 @@ def side_effect(*args, **kwargs): assert validator.message == "did_not_found" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -722,7 +710,6 @@ def record_consume_request_side_effect(*args, **kwargs): assert validator.message == "in_use_or_not_on_chain" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -780,7 +767,6 @@ def other_service(*args, **kwargs): assert validator.message == "main_service_compute" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -832,7 +818,6 @@ def side_effect(*args, **kwargs): assert validator.message == "no_raw_algo_allowed" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -892,7 +877,6 @@ def another_side_effect(*args, **kwargs): assert validator.validate() is True -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -939,7 +923,6 @@ def another_side_effect(*args, **kwargs): assert validator.message == "missing_meta_documentId" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( @@ -985,7 +968,6 @@ def side_effect(*args, **kwargs): assert validator.message == "fees_not_paid" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit @patch("ocean_provider.validation.algo.check_asset_consumable", return_value=(True, "")) @patch( diff --git a/tests/helpers/compute_helpers.py b/tests/helpers/compute_helpers.py index a1b6c14f..119a8cab 100644 --- a/tests/helpers/compute_helpers.py +++ b/tests/helpers/compute_helpers.py @@ -8,7 +8,7 @@ from ocean_provider.utils.util import msg_hash from tests.helpers.constants import ARWEAVE_TRANSACTION_ID from tests.helpers.ddo_dict_builders import build_metadata_dict_type_algorithm -from tests.helpers.nonce import build_nonce +from tests.helpers.nonce import build_nonce, build_nonce_for_compute from tests.test_helpers import ( get_first_service_by_type, get_registered_asset, @@ -155,7 +155,7 @@ def build_and_send_ddo_with_compute_service( def get_compute_signature(client, consumer_wallet, did, job_id=None): - nonce = build_nonce(consumer_wallet.address) + nonce = build_nonce_for_compute() # prepare consumer signature on did if job_id: diff --git a/tests/helpers/nonce.py b/tests/helpers/nonce.py index 1e948f8f..a3b15c2a 100644 --- a/tests/helpers/nonce.py +++ b/tests/helpers/nonce.py @@ -1,3 +1,5 @@ +import time + from ocean_provider.user_nonce import get_nonce, update_nonce @@ -11,3 +13,7 @@ def build_nonce(address) -> int: update_nonce(address, 1) return 1 + + +def build_nonce_for_compute() -> int: + return time.time_ns() diff --git a/tests/test_compute.py b/tests/test_compute.py index 13648456..bc44d3af 100644 --- a/tests/test_compute.py +++ b/tests/test_compute.py @@ -33,18 +33,14 @@ from tests.test_helpers import get_first_service_by_type, get_ocean_token_address -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_rejected(client, monkeypatch): monkeypatch.delenv("OPERATOR_SERVICE_URL") response = post_to_compute(client, {}) assert response.status_code == 404 -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.parametrize("allow_raw_algos", [True, False]) def test_compute_raw_algo( client, @@ -120,9 +116,7 @@ def test_compute_raw_algo( assert "no_raw_algo_allowed" == response.json["error"]["dataset"] -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_specific_algo_dids( client, publisher_wallet, consumer_wallet, consumer_address, free_c2d_env ): @@ -167,9 +161,7 @@ def test_compute_specific_algo_dids( assert response.json["error"]["dataset"] == "not_trusted_algo" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute(client, publisher_wallet, consumer_wallet, free_c2d_env): valid_until = get_future_valid_until() ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( @@ -295,9 +287,7 @@ def test_compute(client, publisher_wallet, consumer_wallet, free_c2d_env): assert result_data is not None, "We should have a result" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_arweave(client, publisher_wallet, consumer_wallet, free_c2d_env): valid_until = get_future_valid_until() ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( @@ -334,9 +324,7 @@ def test_compute_arweave(client, publisher_wallet, consumer_wallet, free_c2d_env assert response.status == "200 OK", f"start compute job failed: {response.data}" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_diff_provider(client, publisher_wallet, consumer_wallet, free_c2d_env): valid_until = get_future_valid_until() ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( @@ -371,9 +359,7 @@ def test_compute_diff_provider(client, publisher_wallet, consumer_wallet, free_c assert response.status == "200 OK", f"start compute job failed: {response.data}" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_allow_all_published( client, publisher_wallet, consumer_wallet, free_c2d_env ): @@ -420,9 +406,7 @@ def test_compute_allow_all_published( assert response.status == "200 OK" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_additional_input( client, publisher_wallet, consumer_wallet, monkeypatch, free_c2d_env, web3 ): @@ -507,9 +491,7 @@ def test_compute_additional_input( assert response.status == "200 OK", f"start compute job failed: {response.data}" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_delete_job( client, publisher_wallet, consumer_wallet, consumer_address, free_c2d_env ): @@ -574,9 +556,7 @@ def test_compute_delete_job( assert response.status == "200 OK", f"delete compute job failed: {response.data}" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.unit -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_environments(): environments = get_c2d_environments() @@ -585,9 +565,7 @@ def test_compute_environments(): assert env["id"] == "ocean-compute" -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_paid_env( client, publisher_wallet, consumer_wallet, paid_c2d_env, web3 ): @@ -636,9 +614,7 @@ def test_compute_paid_env( _ = job_info.get("jobId", "") -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration -@pytest.mark.skip("C2D connection needs fixing.") def test_compute_auth_token(client, publisher_wallet, consumer_wallet, free_c2d_env): valid_until = get_future_valid_until() ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( diff --git a/tests/test_initialize.py b/tests/test_initialize.py index 322132d7..577975cc 100644 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -77,7 +77,7 @@ def test_initialize_on_disabled_asset(client, publisher_wallet, consumer_wallet, client, asset.did, service, consumer_wallet, raw_response=True ) assert "error" in response.json - assert response.json["error"] == "Asset malformed or disabled." + assert response.json["error"] == "Asset is not consumable." @pytest.mark.integration @@ -178,7 +178,6 @@ def test_can_not_initialize_compute_service_with_simple_initialize( ) -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration def test_initialize_compute_works( client, publisher_wallet, consumer_wallet, free_c2d_env @@ -232,7 +231,6 @@ def test_initialize_compute_works( assert "validOrder" not in response.json["algorithm"] -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration def test_initialize_compute_order_reused( client, publisher_wallet, consumer_wallet, free_c2d_env @@ -361,7 +359,6 @@ def test_initialize_compute_order_reused( assert "providerFee" in response.json["datasets"][0].keys() -@pytest.mark.skip("C2D connection needs fixing.") @pytest.mark.integration def test_initialize_compute_paid_env( client, publisher_wallet, consumer_wallet, paid_c2d_env diff --git a/tests/test_routes.py b/tests/test_routes.py index f7c4aaa1..955d63ef 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -103,16 +103,6 @@ def test_get_nonce(client, publisher_wallet): def test_timestamp_format_nonce(client, ganache_wallet, consumer_wallet, web3): - address = ganache_wallet.address - # Insert in db timestamp format nonce for this particular account - timestamp = str(datetime.datetime.utcnow().timestamp() * 1000) - update_nonce(address, timestamp) - nonce = get_nonce(address) - assert timestamp == nonce - - # Cast to int - nonce = int(float(nonce)) + 1 - # Send download request with the new nonce asset = get_registered_asset(ganache_wallet) service = get_first_service_by_type(asset, ServiceType.ACCESS) @@ -128,6 +118,15 @@ def test_timestamp_format_nonce(client, ganache_wallet, consumer_wallet, web3): consumer_wallet, ) + # Insert in db timestamp format nonce for this particular account + timestamp = str(datetime.datetime.utcnow().timestamp() * 1000) + update_nonce(consumer_wallet.address, timestamp) + nonce = get_nonce(consumer_wallet.address) + assert timestamp == nonce + + # Cast to int + nonce = int(float(nonce)) + 1 + payload = { "documentId": asset.did, "serviceId": service.id,