From 55ffa2ebdeb8fb538eeb847c98fd4ecb9d388bc1 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Fri, 8 Apr 2022 17:46:00 +0000 Subject: [PATCH 1/8] Adds proofs, pending tests. --- ocean_provider/routes/consume.py | 24 +++++++++++++ ocean_provider/utils/accounts.py | 4 ++- ocean_provider/utils/proof.py | 59 ++++++++++++++++++++++++++++++++ ocean_provider/utils/util.py | 29 ++++++++++++++++ tests/test_helpers.py | 27 ++------------- 5 files changed, 117 insertions(+), 26 deletions(-) create mode 100644 ocean_provider/utils/proof.py diff --git a/ocean_provider/routes/consume.py b/ocean_provider/routes/consume.py index 4c77d64a..ec600b2d 100644 --- a/ocean_provider/routes/consume.py +++ b/ocean_provider/routes/consume.py @@ -2,6 +2,7 @@ # Copyright 2021 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # +import json import logging import os @@ -18,6 +19,7 @@ validate_timestamp, ) from ocean_provider.utils.error_responses import error_response +from ocean_provider.utils.proof import send_proof from ocean_provider.utils.provider_fees import get_provider_fees, get_c2d_environments from ocean_provider.utils.services import ServiceType from ocean_provider.utils.url import append_userdata, check_url_details @@ -333,4 +335,26 @@ def download(): ) logger.info(f"download response = {response}") + provider_proof_data = json.dumps( + { + "documentId": did, + "serviceId": service_id, + "fileIndex": file_index, + "downloadedBytes": 0, # TODO + }, + separators=(",", ":"), + ) + + consumer_data = f'{did}{data.get("nonce")}' + + send_proof( + web3=get_web3(), + order_tx_id=_tx.hash, + provider_data=provider_proof_data, + consumer_data=consumer_data, + consumer_signature=data.get("signature"), + consumer_address=consumer_address, + datatoken_address=service.datatoken_address, + ) + return response diff --git a/ocean_provider/utils/accounts.py b/ocean_provider/utils/accounts.py index 93d1d4b3..6c5c04bf 100644 --- a/ocean_provider/utils/accounts.py +++ b/ocean_provider/utils/accounts.py @@ -75,9 +75,11 @@ def sign_message(message, wallet): :return: signature """ keys_pk = keys.PrivateKey(wallet.key) + hexable = Web3.toBytes(text=message) if isinstance(message, str) else message + message_hash = Web3.solidityKeccak( ["bytes"], - [Web3.toBytes(text=message)], + [Web3.toHex(hexable)], ) prefix = "\x19Ethereum Signed Message:\n32" signable_hash = Web3.solidityKeccak( diff --git a/ocean_provider/utils/proof.py b/ocean_provider/utils/proof.py new file mode 100644 index 00000000..a1ca6b9f --- /dev/null +++ b/ocean_provider/utils/proof.py @@ -0,0 +1,59 @@ +import os +import requests +from ocean_provider.utils.basics import get_provider_wallet +from ocean_provider.utils.accounts import sign_message +from ocean_provider.utils.datatoken import get_datatoken_contract +from ocean_provider.utils.util import sign_send_and_wait_for_receipt +from web3.main import Web3 + + +def send_proof( + web3, + order_tx_id, + provider_data, + consumer_data, + consumer_signature, + consumer_address, + datatoken_address, +): + if not os.getenv("USE_CHAIN_PROOF") and not os.getenv("USE_HTTP_PROOF"): + return + + provider_wallet = get_provider_wallet() + provider_signature = sign_message(provider_data, provider_wallet) + + if os.getenv("USE_HTTP_PROOF"): + payload = { + "orderTxId": order_tx_id.hex(), + "providerData": provider_data, + "providerSignature": provider_signature, + "consumerData": consumer_data, + "consumerSignature": consumer_signature, + "consumerAddress": consumer_address, + } + + try: + requests.post(os.getenv("USE_HTTP_PROOF"), payload) + except Exception: + pass + + return + + datatoken_contract = get_datatoken_contract(web3, datatoken_address) + provider_message = order_tx_id + Web3.toBytes(text=provider_data) + provider_signature = sign_message(provider_message, provider_wallet) + + consumer_message = Web3.toBytes(text=consumer_data) + + tx = datatoken_contract.functions.orderExecuted( + order_tx_id, + Web3.toBytes(text=provider_data), + provider_signature, + consumer_message, + consumer_signature, + consumer_address, + ).buildTransaction( + {"from": provider_wallet.address, "gasPrice": int(web3.eth.gas_price * 1.1)} + ) + + sign_send_and_wait_for_receipt(web3, tx, provider_wallet) diff --git a/ocean_provider/utils/util.py b/ocean_provider/utils/util.py index 0d55bcb5..e5aae76f 100644 --- a/ocean_provider/utils/util.py +++ b/ocean_provider/utils/util.py @@ -10,6 +10,7 @@ from cgi import parse_header from urllib.parse import urljoin from datetime import datetime +from typing import Tuple import requests import werkzeug @@ -18,6 +19,7 @@ from eth_account.signers.local import LocalAccount from eth_keys import KeyAPI from eth_keys.backends import NativeECCBackend +from eth_typing.encoding import HexStr from flask import Response from ocean_provider.utils.accounts import sign_message from ocean_provider.utils.basics import get_config, get_provider_wallet, get_web3 @@ -29,6 +31,7 @@ from ocean_provider.utils.services import Service from ocean_provider.utils.url import is_safe_url from web3 import Web3 +from web3.types import TxParams, TxReceipt from websockets import ConnectionClosed logger = logging.getLogger(__name__) @@ -337,3 +340,29 @@ def sign_for_compute(wallet, owner, job_id=None): signature = sign_message(msg, wallet) return nonce, signature + + +def sign_tx(web3, tx, private_key): + """ + :param web3: Web3 object instance + :param tx: transaction + :param private_key: Private key of the account + :return: rawTransaction (str) + """ + account = web3.eth.account.from_key(private_key) + nonce = web3.eth.get_transaction_count(account.address) + tx["nonce"] = nonce + signed_tx = web3.eth.account.sign_transaction(tx, private_key) + + return signed_tx.rawTransaction + + +def sign_send_and_wait_for_receipt( + web3: Web3, transaction: TxParams, from_account: LocalAccount +) -> Tuple[HexStr, TxReceipt]: + """Returns the transaction id and transaction receipt.""" + transaction_signed = sign_tx(web3, transaction, from_account.key) + transaction_hash = web3.eth.send_raw_transaction(transaction_signed) + transaction_id = Web3.toHex(transaction_hash) + + return (transaction_id, web3.eth.wait_for_transaction_receipt(transaction_hash)) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 4c71cdd9..6422cbab 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -30,6 +30,7 @@ from ocean_provider.utils.did import compute_did_from_data_nft_address_and_chain_id from ocean_provider.utils.encryption import do_encrypt from ocean_provider.utils.services import Service, ServiceType +from ocean_provider.utils.util import sign_send_and_wait_for_receipt, sign_tx from tests.helpers.ddo_dict_builders import ( build_credentials_dict, build_ddo_dict, @@ -40,7 +41,7 @@ ) from web3.logs import DISCARD from web3.main import Web3 -from web3.types import TxParams, TxReceipt +from web3.types import TxReceipt logger = logging.getLogger(__name__) BLACK_HOLE_ADDRESS = "0x0000000000000000000000000000000000000000" @@ -50,20 +51,6 @@ def get_gas_price(web3) -> int: return int(web3.eth.gas_price * 1.1) -def sign_tx(web3, tx, private_key): - """ - :param web3: Web3 object instance - :param tx: transaction - :param private_key: Private key of the account - :return: rawTransaction (str) - """ - account = web3.eth.account.from_key(private_key) - nonce = web3.eth.get_transaction_count(account.address) - tx["nonce"] = nonce - signed_tx = web3.eth.account.sign_transaction(tx, private_key) - return signed_tx.rawTransaction - - def deploy_contract(w3, _json, private_key, *args): """ :param w3: Web3 object instance @@ -94,16 +81,6 @@ def get_ocean_token_address(web3: Web3) -> HexAddress: return get_contract_address(get_config().address_file, "Ocean", 8996) -def sign_send_and_wait_for_receipt( - web3: Web3, transaction: TxParams, from_account: LocalAccount -) -> Tuple[HexStr, TxReceipt]: - """Returns the transaction id and transaction receipt.""" - transaction_signed = sign_tx(web3, transaction, from_account.key) - transaction_hash = web3.eth.send_raw_transaction(transaction_signed) - transaction_id = Web3.toHex(transaction_hash) - return (transaction_id, web3.eth.wait_for_transaction_receipt(transaction_hash)) - - def deploy_data_nft( web3: Web3, name: str, From e52326d259fdbaa8201f55ca146bdd6e22bb4802 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Mon, 18 Apr 2022 07:35:20 +0000 Subject: [PATCH 2/8] Add HTTP proof tests. --- ocean_provider/utils/proof.py | 2 ++ tests/test_proof.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/test_proof.py diff --git a/ocean_provider/utils/proof.py b/ocean_provider/utils/proof.py index a1ca6b9f..61a10098 100644 --- a/ocean_provider/utils/proof.py +++ b/ocean_provider/utils/proof.py @@ -34,6 +34,8 @@ def send_proof( try: requests.post(os.getenv("USE_HTTP_PROOF"), payload) + + return True except Exception: pass diff --git a/tests/test_proof.py b/tests/test_proof.py new file mode 100644 index 00000000..66d2b303 --- /dev/null +++ b/tests/test_proof.py @@ -0,0 +1,38 @@ +# +# Copyright 2021 Ocean Protocol Foundation +# SPDX-License-Identifier: Apache-2.0 +# +import json +import pytest +from requests.models import Response +from unittest.mock import patch, Mock + +from ocean_provider.utils.proof import send_proof + + +@pytest.mark.unit +def test_no_proof_setup(client): + assert send_proof(None, None, None, None, None, None, None) is None + + +@pytest.mark.unit +def test_http_proof(client, monkeypatch): + monkeypatch.setenv("USE_HTTP_PROOF", "http://test.com") + provider_data = json.dumps({"test_data": "test_value"}) + + with patch("requests.post") as mock: + response = Mock(spec=Response) + response.json.return_value = {"a valid response": ""} + response.status_code = 200 + mock.return_value = response + + assert send_proof(None, b'1', provider_data, None, None, None, None) is True + + mock.assert_called_once() + + with patch("requests.post") as mock: + mock.side_effect = Exception("Boom!") + + assert send_proof(None, b'1', provider_data, None, None, None, None) is None + + mock.assert_called_once() From 991efd2fcbae48574d0b3768245b7912a23de12b Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Mon, 18 Apr 2022 08:19:54 +0000 Subject: [PATCH 3/8] Adds tests. --- ocean_provider/utils/proof.py | 4 +++- tests/test_proof.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/ocean_provider/utils/proof.py b/ocean_provider/utils/proof.py index 61a10098..cbbf534f 100644 --- a/ocean_provider/utils/proof.py +++ b/ocean_provider/utils/proof.py @@ -58,4 +58,6 @@ def send_proof( {"from": provider_wallet.address, "gasPrice": int(web3.eth.gas_price * 1.1)} ) - sign_send_and_wait_for_receipt(web3, tx, provider_wallet) + transaction_id, _ = sign_send_and_wait_for_receipt(web3, tx, provider_wallet) + + return transaction_id diff --git a/tests/test_proof.py b/tests/test_proof.py index 66d2b303..fcc2fc54 100644 --- a/tests/test_proof.py +++ b/tests/test_proof.py @@ -2,12 +2,27 @@ # Copyright 2021 Ocean Protocol Foundation # SPDX-License-Identifier: Apache-2.0 # +from datetime import datetime import json import pytest from requests.models import Response from unittest.mock import patch, Mock +from ocean_provider.utils.accounts import sign_message +from ocean_provider.constants import BaseURLs from ocean_provider.utils.proof import send_proof +from ocean_provider.utils.provider_fees import get_provider_fees +from ocean_provider.utils.services import ServiceType +from tests.test_helpers import ( + get_first_service_by_type, + get_registered_asset, + mint_100_datatokens, + BLACK_HOLE_ADDRESS, + deploy_data_nft, + deploy_datatoken, + get_ocean_token_address, + start_order +) @pytest.mark.unit @@ -36,3 +51,30 @@ def test_http_proof(client, monkeypatch): assert send_proof(None, b'1', provider_data, None, None, None, None) is None mock.assert_called_once() + + +@pytest.mark.integration +def test_chain_proof(client, monkeypatch, web3, publisher_wallet, consumer_wallet): + monkeypatch.setenv("USE_CHAIN_PROOF", "1") + provider_data = json.dumps({"test_data": "test_value"}) + + asset = get_registered_asset(publisher_wallet) + service = get_first_service_by_type(asset, ServiceType.ACCESS) + mint_100_datatokens( + web3, service.datatoken_address, consumer_wallet.address, publisher_wallet + ) + tx_id, receipt = start_order( + web3, + service.datatoken_address, + consumer_wallet.address, + service.index, + get_provider_fees(asset.did, service, consumer_wallet.address, 0), + consumer_wallet, + ) + + nonce = str(datetime.utcnow().timestamp()) + + consumer_data = _msg = f"{asset.did}{nonce}" + signature = sign_message(_msg, consumer_wallet) + + assert send_proof(web3, receipt.transactionHash, provider_data, consumer_data, signature, consumer_wallet.address, service.datatoken_address) From b15ea829cc6e8fad158d060295e555e09e52a329 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Mon, 18 Apr 2022 11:58:14 +0000 Subject: [PATCH 4/8] Black formatting, click prerequisites. --- setup.py | 1 + tests/test_proof.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 86d9805c..b5d56e96 100644 --- a/setup.py +++ b/setup.py @@ -68,6 +68,7 @@ "flake8", "isort", "black==22.1.0", + "click==8.0.4", "pre-commit", "licenseheaders", "pytest-env", diff --git a/tests/test_proof.py b/tests/test_proof.py index fcc2fc54..5fd34dc3 100644 --- a/tests/test_proof.py +++ b/tests/test_proof.py @@ -21,7 +21,7 @@ deploy_data_nft, deploy_datatoken, get_ocean_token_address, - start_order + start_order, ) @@ -41,14 +41,14 @@ def test_http_proof(client, monkeypatch): response.status_code = 200 mock.return_value = response - assert send_proof(None, b'1', provider_data, None, None, None, None) is True + assert send_proof(None, b"1", provider_data, None, None, None, None) is True mock.assert_called_once() with patch("requests.post") as mock: mock.side_effect = Exception("Boom!") - assert send_proof(None, b'1', provider_data, None, None, None, None) is None + assert send_proof(None, b"1", provider_data, None, None, None, None) is None mock.assert_called_once() @@ -77,4 +77,12 @@ def test_chain_proof(client, monkeypatch, web3, publisher_wallet, consumer_walle consumer_data = _msg = f"{asset.did}{nonce}" signature = sign_message(_msg, consumer_wallet) - assert send_proof(web3, receipt.transactionHash, provider_data, consumer_data, signature, consumer_wallet.address, service.datatoken_address) + assert send_proof( + web3, + receipt.transactionHash, + provider_data, + consumer_data, + signature, + consumer_wallet.address, + service.datatoken_address, + ) From 33c8c4890f8ef8e99bf868d36b0ef331ff316321 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Mon, 18 Apr 2022 12:00:25 +0000 Subject: [PATCH 5/8] Fix pytest github action. --- .github/workflows/pytest.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4b13f4ea..2edf2b03 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -27,10 +27,7 @@ jobs: - name: Run Barge working-directory: ${{ github.workspace }}/barge env: - OPERATOR_SERVICE_VERSION: refactor_signatures CONTRACTS_VERSION: v1.0.0-alpha.28 - AQUARIUS_VERSION: refactor_signatures - PROVIDER_VERSION: refactor_signatures run: | bash -x start_ocean.sh --no-dashboard 2>&1 --with-rbac --with-provider2 --with-c2d > start_ocean.log & for i in $(seq 1 150); do From 070147704fa4facbdcb0ee583eebc0958cec0419 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Mon, 2 May 2022 06:31:30 +0000 Subject: [PATCH 6/8] Add new env vars to readme. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 336e2c0b..b8563b59 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ Refer to the [API.md](API.md) file for endpoints and payloads. * `LOG_CFG` and `LOG_LEVEL` define the location of the log file and logging leve, respectively * `IPFS_GATEWAY` defines ipfs gateway for resolving urls * `AUTHORIZED_DECRYPTERS` list of authorized addresses that are allowed to decrypt chain data. Use it to restrict access only to certain callers (e.g. custom Aquarius instance). Empty by default, meaning all decrypters are authorized. +* `USE_CHAIN_PROOF` or `USE_HTTP_PROOF` set a mechanism for saving proof-of-download information. For any present true-ish value of `USE_CHAIN_PROOF`, the proof is sent on-chain. When defining `USE_HTTP_PROOF` the env var must configure a HTTP endpoint that accepts a POST request. #### Before you commit From 825e351196457ae8ab6aca158886293a9774e363 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Mon, 2 May 2022 15:32:54 +0000 Subject: [PATCH 7/8] Send without waiting. --- ocean_provider/utils/proof.py | 4 ++-- ocean_provider/utils/util.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ocean_provider/utils/proof.py b/ocean_provider/utils/proof.py index cbbf534f..bdb85cca 100644 --- a/ocean_provider/utils/proof.py +++ b/ocean_provider/utils/proof.py @@ -3,7 +3,7 @@ from ocean_provider.utils.basics import get_provider_wallet from ocean_provider.utils.accounts import sign_message from ocean_provider.utils.datatoken import get_datatoken_contract -from ocean_provider.utils.util import sign_send_and_wait_for_receipt +from ocean_provider.utils.util import sign_and_send from web3.main import Web3 @@ -58,6 +58,6 @@ def send_proof( {"from": provider_wallet.address, "gasPrice": int(web3.eth.gas_price * 1.1)} ) - transaction_id, _ = sign_send_and_wait_for_receipt(web3, tx, provider_wallet) + _, transaction_id = sign_and_send(web3, tx, provider_wallet) return transaction_id diff --git a/ocean_provider/utils/util.py b/ocean_provider/utils/util.py index 49c3516d..cf7f8675 100644 --- a/ocean_provider/utils/util.py +++ b/ocean_provider/utils/util.py @@ -167,7 +167,7 @@ def sign_tx(web3, tx, private_key): return signed_tx.rawTransaction -def sign_send_and_wait_for_receipt( +def sign_and_send( web3: Web3, transaction: TxParams, from_account: LocalAccount ) -> Tuple[HexStr, TxReceipt]: """Returns the transaction id and transaction receipt.""" @@ -175,4 +175,13 @@ def sign_send_and_wait_for_receipt( transaction_hash = web3.eth.send_raw_transaction(transaction_signed) transaction_id = Web3.toHex(transaction_hash) + return transaction_hash, transaction_id + + +def sign_send_and_wait_for_receipt( + web3: Web3, transaction: TxParams, from_account: LocalAccount +) -> Tuple[HexStr, TxReceipt]: + """Returns the transaction id and transaction receipt.""" + transaction_id, transaction_hash = sign_and_send(web3, transaction, from_account) + return (transaction_id, web3.eth.wait_for_transaction_receipt(transaction_hash)) From 65543fd9f5addead8794196aa7f982603e53d458 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Mon, 2 May 2022 17:03:34 +0000 Subject: [PATCH 8/8] Fix typo from previous commit. --- ocean_provider/routes/consume.py | 5 ++++- ocean_provider/utils/util.py | 2 +- ocean_provider/validation/algo.py | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ocean_provider/routes/consume.py b/ocean_provider/routes/consume.py index 262bcaef..4499ef48 100644 --- a/ocean_provider/routes/consume.py +++ b/ocean_provider/routes/consume.py @@ -10,7 +10,10 @@ from ocean_provider.myapp import app from ocean_provider.requests_session import get_requests_session from ocean_provider.user_nonce import get_nonce, update_nonce -from ocean_provider.utils.asset import get_asset_from_metadatastore, check_asset_consumable +from ocean_provider.utils.asset import ( + get_asset_from_metadatastore, + check_asset_consumable, +) from ocean_provider.utils.basics import ( LocalFileAdapter, get_provider_wallet, diff --git a/ocean_provider/utils/util.py b/ocean_provider/utils/util.py index cf7f8675..17ad231a 100644 --- a/ocean_provider/utils/util.py +++ b/ocean_provider/utils/util.py @@ -182,6 +182,6 @@ def sign_send_and_wait_for_receipt( web3: Web3, transaction: TxParams, from_account: LocalAccount ) -> Tuple[HexStr, TxReceipt]: """Returns the transaction id and transaction receipt.""" - transaction_id, transaction_hash = sign_and_send(web3, transaction, from_account) + transaction_hash, transaction_id = sign_and_send(web3, transaction, from_account) return (transaction_id, web3.eth.wait_for_transaction_receipt(transaction_hash)) diff --git a/ocean_provider/validation/algo.py b/ocean_provider/validation/algo.py index 0aeaac76..faa58988 100644 --- a/ocean_provider/validation/algo.py +++ b/ocean_provider/validation/algo.py @@ -7,7 +7,10 @@ from ocean_provider.constants import BaseURLs from ocean_provider.serializers import StageAlgoSerializer -from ocean_provider.utils.asset import get_asset_from_metadatastore, check_asset_consumable +from ocean_provider.utils.asset import ( + get_asset_from_metadatastore, + check_asset_consumable, +) from ocean_provider.utils.basics import get_config, get_metadata_url from ocean_provider.utils.datatoken import ( record_consume_request,