From 179dcdb6fe315694ceb61af1d736da1ad4cc56d1 Mon Sep 17 00:00:00 2001 From: SAIKAT KARMAKAR Date: Sun, 23 Jun 2024 19:54:14 +0530 Subject: [PATCH] docs: readme updated. Examples.md page added. Index of docs webpage is updated --- README.md | 121 +++++++++++++++++++++- codecov.yaml | 10 +- docs/Examples.md | 195 ++++++++++++++++++++++++++++++++++++ docs/index.md | 83 ++++++++++++++- mkdocs.yml | 1 + src/mantaray_py/__init__.py | 3 +- 6 files changed, 405 insertions(+), 8 deletions(-) create mode 100644 docs/Examples.md diff --git a/README.md b/README.md index 07a5dac..04062e7 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,132 @@ +# Description + +With this package you can manipulate and interpret mantaray data via `MantarayNode` and `MantarayFork` abstractions. + # Installation - Install using `pip` ```py -pip install mantaray-py +pip install mantaray_py +``` + +# Usage + +### Construct Mantaray + +```py +from mantaray_py import MantarayNode, MantarayFork, init_manifest_node, gen_32_bytes + +node = init_manifest_node() +address1 = gen_32_bytes() +address2 = gen_32_bytes() +address3 = gen_32_bytes() +address4 = gen_32_bytes() +address5 = gen_32_bytes() +address6 = gen_32_bytes() + +path1 = "path1/valami/elso".encode() +path2 = "path1/valami/masodik".encode() +path3 = "path1/valami/masodik.ext".encode() +path4 = "path1/valami".encode() +path5 = "path2".encode() +path6 = "path3/haha".encode() + +node.add_fork(path1, address1, { "vmi": "elso" }) +node.add_fork(path2, address2) +node.add_fork(path3, address3) +node.add_fork(path4, address4, {"vmi": "negy"}) +node.add_fork(path5, address5) +node.add_fork(path6, address6, {"vmi": "haha"}) +node.remove_path(path3) + +print(node) +``` + +### Mantaray Storage Operations + +```py +from mantaray_py import MantarayNode + +node = MantarayNode() +""" +here `reference` parameter is a `Reference` type which can be a 32 or 64 of bytes +and `load_function` is a [load_function: (address: bytes): bytes] typed function +that returns the serialised raw data of a MantarayNode of the given reference. See tests/integration/test_int.py file for reference. +""" +node.load(load_function, reference) + +# Manipulate `node` object then save it again +# (...) + +# save into the storage with a storage handler [save_function: (data: bytes): Reference] +# See tests/integration/test_int.py file for reference. +reference = node.save(save_function) ``` + + +
+How It Works +
+ +# node binary format + +The following describes the format of a node binary format. + +``` +┌────────────────────────────────┐ +│ obfuscationKey <32 byte> │ +├────────────────────────────────┤ +│ hash("mantaray:0.1") <31 byte> │ +├────────────────────────────────┤ +│ refBytesSize <1 byte> │ +├────────────────────────────────┤ +│ entry <32/64 byte> │ +├────────────────────────────────┤ +│ forksIndexBytes <32 byte> │ +├────────────────────────────────┤ +│ ┌────────────────────────────┐ │ +│ │ Fork 1 │ │ +│ ├────────────────────────────┤ │ +│ │ ... │ │ +│ ├────────────────────────────┤ │ +│ │ Fork N │ │ +│ └────────────────────────────┘ │ +└────────────────────────────────┘ +``` + +## Fork + +``` +┌───────────────────┬───────────────────────┬──────────────────┐ +│ nodeType <1 byte> │ prefixLength <1 byte> │ prefix <30 byte> │ +├───────────────────┴───────────────────────┴──────────────────┤ +│ reference <32/64 bytes> │ +│ │ +└──────────────────────────────────────────────────────────────┘ +``` + +### Fork with metadata + +``` +┌───────────────────┬───────────────────────┬──────────────────┐ +│ nodeType <1 byte> │ prefixLength <1 byte> │ prefix <30 byte> │ +├───────────────────┴───────────────────────┴──────────────────┤ +│ reference <32/64 bytes> │ +│ │ +├─────────────────────────────┬────────────────────────────────┤ +│ metadataBytesSize <2 bytes> │ metadataBytes │ +├─────────────────────────────┘ │ +│ │ +└──────────────────────────────────────────────────────────────┘ +``` + +
+ + --- **Documentation**: https://Ankvik-Tech-Labs.github.io/mantaray-py/ diff --git a/codecov.yaml b/codecov.yaml index 058cfb7..c31e06e 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -2,8 +2,8 @@ coverage: range: 70..100 round: down precision: 1 - status: - project: - default: - target: 90% - threshold: 0.5% + # status: + # project: + # default: + # target: 90% + # threshold: 0.5% diff --git a/docs/Examples.md b/docs/Examples.md new file mode 100644 index 0000000..8e76120 --- /dev/null +++ b/docs/Examples.md @@ -0,0 +1,195 @@ +# Examples + +### Some General Examples + +```py +import pytest +from rich.console import Console + +from mantaray_py import (MantarayNode, check_for_separator, gen_32_bytes, + init_manifest_node) +from mantaray_py.node import NotFoundError + +console = Console() + + +def test_single_manaray_node_with_a_random_address(): + node = init_manifest_node() + random_address = gen_32_bytes() + node.set_entry(random_address) + serialised = node.serialise() + + new_node = MantarayNode() + new_node.deserialise(serialised) + + assert random_address == new_node.get_entry() + + +def test_get_fork_at_path_and_check_for_separator(get_sample_mantaray_node): + sample_node = get_sample_mantaray_node + node = sample_node["node"] + + # Test that get_fork_at_path throws an error for a non-existent path + with pytest.raises(Exception): + node.get_fork_at_path(b"path/not/exists") + + # Test get_fork_at_path with a path that does not contain a separator in its descendants + fork1 = node.get_fork_at_path(b"path1/valami/") + assert not check_for_separator( + fork1.node + ), "Expected no separator in the descendants" + + # Test get_fork_at_path with a path that contains a separator in its descendants + path2 = sample_node["paths"][3] + fork2 = node.get_fork_at_path(path2) + assert check_for_separator(fork2.node), "Expected a separator in the descendants" + + # Test get_fork_at_path with a path that does not contain a separator in its descendants and has no forks + path3 = sample_node["paths"][4] + fork3 = node.get_fork_at_path(path3) + assert not check_for_separator( + fork3.node + ), "Expected no separator in the descendants and no forks" + + +def test_fail_serialise_with_no_storage_saves(): + node = init_manifest_node() + rand_address = gen_32_bytes() + path = b"vmi" + node.add_fork(path, rand_address) + with pytest.raises(ValueError): + node.serialise() + + +def test_checks_the_expected_structure_of_the_sample_mantaray_node( + get_sample_mantaray_node, +): + sample_node = get_sample_mantaray_node + node = sample_node["node"] + path1 = sample_node["paths"][0] + path2 = sample_node["paths"][1] + path3 = sample_node["paths"][2] + path4 = sample_node["paths"][3] + path5 = sample_node["paths"][4] + + # * first level: 'p' + assert list(node.forks.keys()) == [path1[0]] + + second_level_fork = node.forks[path5[0]] + assert second_level_fork.prefix == b"path" + + second_level_node = second_level_fork.node + # * second level: '1', '2' + assert list(second_level_node.forks.keys()) == [path1[4], path5[4]] + + third_level_fork2 = second_level_node.forks[path5[4]] + assert third_level_fork2.prefix == bytes([path5[4]]) + + third_level_fork1 = second_level_node.forks[path1[4]] + assert third_level_fork1.prefix == b"1/valami" + + third_level_node1 = third_level_fork1.node + # * third level 1: '/' + assert list(third_level_node1.forks.keys()) == [path1[12]] + + fourth_level_fork1 = third_level_node1.forks[path1[12]] + assert fourth_level_fork1.prefix == bytes([path1[12]]) + + fourth_level_node1 = fourth_level_fork1.node + # * fourth level 1: 'e', 'm' + assert list(fourth_level_node1.forks.keys()) == [path1[13], path2[13]] + + fifth_level_fork2 = fourth_level_node1.forks[path2[13]] + assert fifth_level_fork2.prefix == b"masodik" + + fifth_level_node2 = fifth_level_fork2.node + # * fifth level 2: '.' + assert list(fifth_level_node2.forks.keys()) == [path3[20]] + + sixth_level_node1 = fifth_level_node2.forks[path3[20]] + assert sixth_level_node1.prefix == b".ext" + + +def test_remove_forks(get_sample_mantaray_node): + sample_node = get_sample_mantaray_node + node = sample_node["node"] + path1 = sample_node["paths"][0] + path2 = sample_node["paths"][1] + + # * Non-existing path check + with pytest.raises(NotFoundError): + node.remove_path(b"\x00\x01\x02") + + # * Node where the fork set will change + check_node1 = node.get_fork_at_path(b"path1/valami/").node + + # * Current forks of node + assert list(check_node1.forks.keys()) == [path1[13], path2[13]] + + node.remove_path(path2) + + # * 'm' key of prefix table disappeared + assert list(check_node1.forks.keys()) == [path1[13]] + +``` + +### Test If The Content Hash Returned Is Same As Bee Or Not + +```py +from bee_py.bee import Bee +from pathlib import Path +import asyncio + +async def test_should_generate_same_content_hash_as_bee(): + bee_class = Bee("http://localhost:1633") # localhost or other bee node url + get_debug_postage = "091a7e6472e9cb702a5b173337f0f34ea363267262b901aef47b1163729ce9cf" + PROJECT_DIR = Path("/home/path/to/project/dir") + + test_dir = PROJECT_DIR / "data" / "testpage" + upload_result = bee_class.upload_files_from_directory( + get_debug_postage, test_dir, {"pin": True, "indexDocument": "index.html"} + ) + + index_html_path = test_dir / "index.html" + image_path = test_dir / "img" / "icon.png" + text_path = test_dir / "img" / "icon.png.txt" + + index_reference, image_reference, text_reference = await asyncio.gather( + upload_file(index_html_path, get_debug_postage, bee_class), + upload_file(image_path, get_debug_postage, bee_class), + upload_file(text_path, get_debug_postage, bee_class), + ) + # console.log(index_reference, image_reference, text_reference) + + i_node = MantarayNode() + i_node.add_fork( + b"index.html", + hex_to_bytes(index_reference), + {"Content-Type": "text/html; charset=utf-8", "Filename": "index.html"}, + ) + i_node.add_fork( + b"img/icon.png.txt", + hex_to_bytes(text_reference), + {"Content-Type": "", "Filename": "icon.png.txt"}, + ) + i_node.add_fork( + b"img/icon.png", + hex_to_bytes(image_reference), + {"Content-Type": "image/png", "Filename": "icon.png"}, + ) + i_node.add_fork(b"/", bytes(32), {"website-index-document": "index.html"}) + + console.log(i_node.forks) + + save_function = create_save_function(bee_class, get_debug_postage) + i_node_ref = i_node.save(save_function) + + assert ( + str(upload_result.reference) + == "e9d46950cdb17e15d0b3712bcb325724a3107560143d65a7acd00ea781eb9cd7" + ) + + assert bytes_to_hex(i_node_ref) == upload_result.reference.value +``` + +### diff --git a/docs/index.md b/docs/index.md index 612c7a5..db5198b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1 +1,82 @@ ---8<-- "README.md" +# mantaray py + + +> Mantaray data structure in Python + + + +| Feature | Value | +| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Technology | [![Python](https://img.shields.io/badge/Python-3776AB.svg?style=flat&logo=Python&logoColor=white)](https://www.python.org/) [![Hatch project](https://img.shields.io/badge/%F0%9F%A5%9A-Hatch-4051b5.svg)](https://github.com/pypa/hatch) [![GitHub Actions](https://img.shields.io/badge/GitHub%20Actions-2088FF.svg?style=flat&logo=GitHub-Actions&logoColor=white)](https://github.com/features/actions) [![Pytest](https://img.shields.io/badge/Pytest-0A9EDC.svg?style=flat&logo=Pytest&logoColor=white)](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/tests.yml/badge.svg) | +| Type Checking | [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) | +| CI/CD | [![Release](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/release.yml/badge.svg)](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/build.yml) [![Tests](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/tests.yml/badge.svg)](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/tests.yml) [![Labeler](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/labeler.yml/badge.svg)](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/labeler.yml) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![codecov](https://codecov.io/gh/Ankvik-Tech-Labs/mantaray-py/graph/badge.svg?token=ISTIW37DO6)](https://codecov.io/gh/Ankvik-Tech-Labs/mantaray-py) | +| Docs | [![Docs](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/documentation.yml/badge.svg)](https://github.com/Ankvik-Tech-Labs/mantaray-py/actions/workflows/build.yml) | +| Package | [![PyPI - Version](https://img.shields.io/pypi/v/mantaray-py.svg)](https://pypi.org/project/mantaray-py/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mantaray-py)](https://pypi.org/project/mantaray-py/) [![PyPI - License](https://img.shields.io/pypi/l/mantaray-py)](https://pypi.org/project/mantaray-py/) | +| Meta | [![GitHub license](https://img.shields.io/github/license/Ankvik-Tech-Labs/mantaray-py?style=flat&color=1573D5)](https://github.com/Ankvik-Tech-Labs/mantaray-py/blob/main/LICENSE) [![GitHub last commit](https://img.shields.io/github/last-commit/Ankvik-Tech-Labs/mantaray-py?style=flat&color=1573D5)](https://github.com/Ankvik-Tech-Labs/mantaray-py/commits/main) [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/Ankvik-Tech-Labs/mantaray-py?style=flat&color=1573D5)](https://github.com/Ankvik-Tech-Labs/mantaray-py/graphs/commit-activity) [![GitHub top language](https://img.shields.io/github/languages/top/Ankvik-Tech-Labs/mantaray-py?style=flat&color=1573D5)](https://github.com/Ankvik-Tech-Labs/mantaray-py) | + + + +# Description + +With this package you can manipulate and interpret mantaray data via `MantarayNode` and `MantarayFork` abstractions. + +# Installation + +- Install using `pip` +```py +pip install mantaray_py +``` + +# Usage + +### Construct Mantaray + +```py +from mantaray_py import MantarayNode, MantarayFork, init_manifest_node, gen_32_bytes + +node = init_manifest_node() +address1 = gen_32_bytes() +address2 = gen_32_bytes() +address3 = gen_32_bytes() +address4 = gen_32_bytes() +address5 = gen_32_bytes() +address6 = gen_32_bytes() + +path1 = "path1/valami/elso".encode() +path2 = "path1/valami/masodik".encode() +path3 = "path1/valami/masodik.ext".encode() +path4 = "path1/valami".encode() +path5 = "path2".encode() +path6 = "path3/haha".encode() + +node.add_fork(path1, address1, { "vmi": "elso" }) +node.add_fork(path2, address2) +node.add_fork(path3, address3) +node.add_fork(path4, address4, {"vmi": "negy"}) +node.add_fork(path5, address5) +node.add_fork(path6, address6, {"vmi": "haha"}) +node.remove_path(path3) + +print(node) +``` + +### Mantaray Storage Operations + +```py +from mantaray_py import MantarayNode + +node = MantarayNode() +""" +here `reference` parameter is a `Reference` type which can be a 32 or 64 of bytes +and `load_function` is a [load_function: (address: bytes): bytes] typed function +that returns the serialised raw data of a MantarayNode of the given reference. See tests/integration/test_int.py file for reference. +""" +node.load(load_function, reference) + +# Manipulate `node` object then save it again +# (...) + +# save into the storage with a storage handler [save_function: (data: bytes): Reference] +# See tests/integration/test_int.py file for reference. +reference = node.save(save_function) +``` diff --git a/mkdocs.yml b/mkdocs.yml index 6934a59..6346b5f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -77,5 +77,6 @@ nav: - License: license.md - Authors: authors.md - Contributing: contributing.md + - Examples: Examples.md - Reference: - mantaray_py: reference/mantaray_py.md diff --git a/src/mantaray_py/__init__.py b/src/mantaray_py/__init__.py index a1bd565..bdc78b5 100644 --- a/src/mantaray_py/__init__.py +++ b/src/mantaray_py/__init__.py @@ -5,7 +5,7 @@ from rich.traceback import install -from mantaray_py.node import MantarayNode, check_for_separator, equal_nodes, load_all_nodes +from mantaray_py.node import MantarayFork, MantarayNode, check_for_separator, equal_nodes, load_all_nodes from mantaray_py.types.types import ( MetadataMapping, NodeType, @@ -26,6 +26,7 @@ ) __all__ = [ + "MantarayFork", "MantarayNode", "MetadataMapping", "NodeType",