diff --git a/.codeclimate.yml b/.codeclimate.yml index 6df7c68..b8349eb 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -14,19 +14,37 @@ engines: - 98cf09e372b83a194bdf71107a97074a - 2c1611a8d29d03a6f557fc3827aeeca0 - b084b705606f106fb116029b92e5a6cf + - 6672f12d51d66b93d2385affbf929e0a + - ebe4b23b4a9a763738af9399771ee5e9 + - df7560f7cecb58026ae7d8dcd247ffba + - 159e8adf8a3e8cd61bb5e9de1c78da33 + - 0511db0bfd510cdcfb2ecd080cbf8993 + - 0d6601bfd2a9f00e972db6f0f594fea0 + - d098e5074e270ba459e0dfb5fee17663 + - 2d21bc6adeb012585bebc59cdbf51cac + eslint: enabled: true fixme: enabled: true radon: enabled: true + gnu-complexity: + enabled: true +# nodesecurity: +# enabled: true + pep8: + enabled: true ratings: paths: - - "**.inc" - - "**.js" - - "**.jsx" - - "**.module" - - "**.php" - - "**.py" - - "**.rb" + - "**.h" + - "**.cpp" + - "**.c" + - "**.inc" + - "**.js" + - "**.jsx" + - "**.module" + - "**.php" + - "**.py" + - "**.rb" exclude_paths: [] diff --git a/.coveragerc b/.coveragerc index 5c91332..54ef579 100644 --- a/.coveragerc +++ b/.coveragerc @@ -19,4 +19,9 @@ exclude_lines = # Don't complain if non-runnable code isn't run: if 0: if False: - if __name__ == .__main__.: \ No newline at end of file + if __name__ == .__main__.: + +[paths] +source = + py_src/ + build/*/py2p/ \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..88d9e5d --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +c_src/* linguist-language=C diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4cfa655 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +./docs/py2p/.build/* +./docs/py2p/.static/* +./docs/py2p/.templates/* +./docs/py2p/py2p/* +**.pyc diff --git a/.gitmodules b/.gitmodules index f102056..e0117df 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,3 @@ -[submodule "js_src/SHA"] - path = js_src/SHA - url = https://github.com/Caligatio/jsSHA -[submodule "js_src/BigInteger"] - path = js_src/BigInteger - url = https://github.com/peterolson/BigInteger.js -[submodule "js_src/pack"] - path = js_src/pack - url = https://github.com/ryanrolds/bufferpack -[submodule "js_src/zlib"] - path = js_src/zlib - url = https://github.com/imaya/zlib.js +[submodule "cp_src/zlib"] + path = c_src/zlib + url = https://github.com/madler/zlib diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..bf69b74 --- /dev/null +++ b/.npmignore @@ -0,0 +1,14 @@ +js_src/test_doc.js +c_src/ +cp_src/ +go_src/ +jv_src/ +py_src/ +doc/ +.rst +.yml +Makefile +MANIFEST.in +.coveragerc +setup.cfg +setup.py diff --git a/.scripts/appveyor_script.bat b/.scripts/appveyor_script.bat new file mode 100644 index 0000000..62437d8 --- /dev/null +++ b/.scripts/appveyor_script.bat @@ -0,0 +1,34 @@ +IF DEFINED PIP ( + ECHO %PYTHON% %PYTHON_VERSION%%APPVEYOR_BUILD_FOLDER% + set HOME=%APPVEYOR_BUILD_FOLDER% + %PYPY% + %PIP% install --upgrade setuptools + %PIP% install pytest-coverage codecov cryptography wheel + cd %HOME% + %RUN% -m pytest -c setup.cfg --cov=./py_src/ ./py_src/ || goto :error + %RUN% setup.py sdist --universal + %PIP% install --no-index --find-links=.\\dist\\ py2p + %RUN% setup.py bdist_wheel + %RUN% setup.py build + FOR /F %%v IN ('%RUN% -c "import sys, sysconfig; print(\"{}.{}-{v[0]}.{v[1]}\".format(\"lib\", sysconfig.get_platform(), v=sys.version_info))"') DO SET BUILD_DIR=%%v + ren .coverage .covvv + %RUN% -m pytest -c setup.cfg --cov=build\\%BUILD_DIR% build\\%BUILD_DIR% || goto :error + ren .covvv .coverage.1 + ren .coverage .coverage.2 + %COV% combine + %COV% xml + %RUN% -c "import codecov; codecov.main('--token=d89f9bd9-27a3-4560-8dbb-39ee3ba020a5', '--file=coverage.xml')" +) ELSE ( + dir C:\avvm\node + powershell -Command "Install-Product node $env:NODE" + npm install . + npm install -g mocha babel-cli + mocha js_src\\test\\* || goto :error + babel js_src --out-dir build\\es5 + mocha build\\es5\\test\\* || goto :error +) +goto :EOF + +:error +ECHO Failed with error #%errorlevel%. +exit /b %errorlevel% diff --git a/.installers/installpypy2.ps1 b/.scripts/installpypy2.ps1 similarity index 100% rename from .installers/installpypy2.ps1 rename to .scripts/installpypy2.ps1 diff --git a/.installers/installpypy3.ps1 b/.scripts/installpypy3.ps1 similarity index 100% rename from .installers/installpypy3.ps1 rename to .scripts/installpypy3.ps1 diff --git a/.scripts/shippable_script.sh b/.scripts/shippable_script.sh new file mode 100644 index 0000000..3dcb15c --- /dev/null +++ b/.scripts/shippable_script.sh @@ -0,0 +1,25 @@ +set -e; +if [ $pyver ]; then + pip install codecov + make cpython pytestdeps + py.test -vv --cov=./py_src/ ./py_src/ + python setup.py sdist --universal && pip install --no-index --find-links=./dist/ py2p + mv .coverage .covvv + make cpytest cov=true + mv .covvv .coverage.1 + mv .coverage .coverage.2 + coverage combine + coverage xml + codecov --token=d89f9bd9-27a3-4560-8dbb-39ee3ba020a5 --file=coverage.xml +elif [ $jsver ]; then + sudo apt-get install build-essential libssl-dev + wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.32.0/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm + command -v nvm + nvm install $jsver + nvm use $jsver + node --version + make jstest + make ES5test +fi diff --git a/.scripts/travis_script.sh b/.scripts/travis_script.sh new file mode 100644 index 0000000..e3c6fa2 --- /dev/null +++ b/.scripts/travis_script.sh @@ -0,0 +1,32 @@ +set -e; +if [ $pyver ]; then + if [ $pyver != pypy ] && [ $pyver != pypy3 ]; then + git clone https://github.com/gappleto97/terryfy; + source terryfy/travis_tools.sh; + get_python_environment $pydist $pyver; + fi + if [ $pyver == pypy ] || [ $pyver == pypy3 ]; then + brew install $pyver; export PYTHON_EXE=$pyver; + curl $GET_PIP_URL > $DOWNLOADS_SDIR/get-pip.py; + sudo $PYTHON_EXE $DOWNLOADS_SDIR/get-pip.py --ignore-installed; + export PIP_CMD="sudo $PYTHON_EXE -m pip"; + fi + $PIP_CMD install virtualenv; + virtualenv -p $PYTHON_EXE venv; + source venv/bin/activate; + make cpython; + pip install pytest-coverage codecov wheel + py.test -vv --cov=./py_src/ ./py_src/ + python setup.py sdist --universal && pip install --no-index --find-links=./dist/ py2p + python setup.py bdist_wheel + mv .coverage .covvv + make cpytest cov=true + mv .covvv .coverage.1 + mv .coverage .coverage.2 + python -m coverage combine; + python -m coverage xml; + codecov --token=d89f9bd9-27a3-4560-8dbb-39ee3ba020a5 --file=coverage.xml +else + make jstest; + make ES5test; +fi diff --git a/.travis.yml b/.travis.yml index 26cf40d..4f457c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,13 @@ +osx_image: beta-xcode6.2 matrix: allow_failures: - env: Cov='true' pyver=pypy pydist=homebrew - env: Cov='true' pyver=pypy3 pydist=homebrew include: - - language: generic - python: 2.6 - os: osx - env: Cov='true' pyver=2.6 pydist=homebrew -# pydist=macports + # - language: generic + # python: 2.6 + # os: osx + # env: Cov='true' pyver=2.6 pydist=macports - language: generic python: 2.7 os: osx @@ -40,19 +40,20 @@ matrix: # python: pypy3 # os: osx # env: Cov='true' pyver=pypy3 pydist=homebrew + - language: node_js + node_js: "6" + os: osx + - language: node_js + node_js: "5" + os: osx + - language: node_js + node_js: "4" + os: osx -before_install: - - if [ $pyver != pypy ] && [ $pyver != pypy3 ]; then git clone https://github.com/gappleto97/terryfy; source terryfy/travis_tools.sh; get_python_environment $pydist $pyver; fi - - if [ $pyver == pypy ] || [ $pyver == pypy3 ]; then brew install $pyver; PYTHON_EXE=$pyver; curl $GET_PIP_URL > $DOWNLOADS_SDIR/get-pip.py; sudo $PYTHON_EXE $DOWNLOADS_SDIR/get-pip.py --ignore-installed; PIP_CMD="sudo $PYTHON_EXE -m pip"; export PYTHON_EXE PIP_CMD; fi - - $PIP_CMD install virtualenv - - virtualenv -p $PYTHON_EXE venv - - source venv/bin/activate -# Everything after this is user editable -install: - - if [ $Cov == 'true' ]; then pip install pytest-coverage codecov; fi - - if [ $(($RANDOM % 2)) == 0 ] || [ $pyver == pypy ] || [ $pyver == pypy3 ]; then pip install cryptography; else pip install PyOpenSSL; fi script: - - if [ $Cov == 'true' ]; then py.test -vv --cov=./py_src/ ./py_src/; fi - - if [ $Cov == 'false' ]; then py.test -vv; fi -after_success: - - if [ $Cov == 'true' ]; then python -m coverage combine; python -m coverage xml; codecov --token=d89f9bd9-27a3-4560-8dbb-39ee3ba020a5 --file=coverage.xml; fi + - sh ./.scripts/travis_script.sh + +addons: + artifacts: + paths: + - $(find . -iname *.whl -type f | tr "\n" ":") diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..805dc59 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,91 @@ +Contributing +============ + +Issues +~~~~~~ + +You may submit an issue via `this link`_ if you have a GitHub account, +or `this one`_ if you do not. + +Issues should have a description of the problem, as well as all of the +following, if possible + +#. A log from this time +#. The output from ``node.status`` + +Big Fixes +~~~~~~~~~ + +Bug fixes should be submitted as a pull request. If you are submitting a +bug fix, please title it as such, and provide a description of the bug +you’re fixing. + +Because I use waffle.io, I appreciate it if the description includes a +list like the following: + +- Connects to #71 +- Connects to #99 +- Connects to #108 + +This associates the pull request with the related issues. + +New Features +~~~~~~~~~~~~ + +New features should be submitted as a pull request. If you are +submitting a new feature, please title it as such, and provide a +description of it. Any new feature should maintain the current API where +possible, and make explicit when it does not. Your PR will not be merged +until the other implementations can be made to match it, so it helps if +you try to implement it in them as well. + +Because I use waffle.io, I appreciate it if the description includes a +list like the following: + +- Connects to #71 +- Connects to #99 +- Connects to #108 + +This associates the pull request with the related issues. + +New Network Schemas +~~~~~~~~~~~~~~~~~~~ + +If you want to add a new network schema, the following things are +required. These are written for the Python implementation, but it +applies to the others where possible. + +#. It must keep the current inheritence structure (ie, “socket”, daemon, + connection, inheriting from base\_socket, base\_daemon, + base\_connection) +#. It must use the current packet format (or any changes must be + propagated to the other schemas) +#. Where possible, it should use the same flags as the current OP-codes +#. Where possible, it should keep a similar API +#. Message handlers should use the ``register_handler`` mechanism + +New Language Implementations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you would like to write an implementation in a new language, thank +you! Following these rules will make things much faster to merge, though +exceptions can be made if necessary. + +#. The new implementation should be in a folder labelled lang\_src (ex: + py\_src) +#. Setup scripts should be in the top level folder +#. It must at minimum support the mesh network implementation +#. Where possible, it should maintain the current + inheritence/architecture scheme +#. Where possible, it should use the same flags as the current OP-codes +#. Where possible, it should keep a similar API +#. Where possible, it should have unit tests +#. Where reasonable, users should be able to register custom callbacks + +When this is ready, submit this as a pull request. If you name it as +such, it will take priority over everything but critical bugfixes. After +a review, we will discuss possible changes and plans for future +modifications, and then merge. + +.. _this link: https://github.com/gappleto97/p2p-project/issues/new +.. _this one: https://gitreports.com/issue/gappleto97/p2p-project \ No newline at end of file diff --git a/LICENSE.CITest b/LICENSE.CITest deleted file mode 100644 index d339236..0000000 --- a/LICENSE.CITest +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Gabe Appleton - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..280a6b6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,10 @@ +include LICENSE +include CONTRIBUTING.md +include README.md +include setup.py +include .coveragerc +include py_src/*.rst +include py_src/*.py +include py_src/test/*.py + +exclude MANIFEST.in \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..dd6afae --- /dev/null +++ b/Makefile @@ -0,0 +1,167 @@ +#Python setup section + +pip = -m pip install +py_deps = $(pip) cryptography +py_test_deps = $(pip) pytest-coverage +docs_deps = $(pip) sphinx sphinxcontrib-napoleon sphinx_rtd_theme + +ifeq ($(shell python -c 'import sys; print(int(hasattr(sys, "real_prefix")))'), 0) # check for virtualenv + py_deps += --user + py_test_deps += --user + docs_deps += --user +endif + +ifeq ($(shell python -c 'import sys; print((sys.version_info[0]))'), 3) + python2 = python2 + python3 = python +else + python2 = python + python3 = python3 +endif + +ifeq ($(shell python -c "import sys; print(hasattr(sys, 'pypy_version_info'))"), True) + pypy = True + ifeq ($(python2), python) + python2 = python2 + endif +else + pypy = False +endif + +pylibdir = $(shell python -c "import sys, sysconfig; print('lib.{}-{v[0]}.{v[1]}'.format(sysconfig.get_platform(), v=sys.version_info))") +py2libdir = $(shell $(python2) -c "import sys, sysconfig; print('lib.{}-{v[0]}.{v[1]}'.format(sysconfig.get_platform(), v=sys.version_info))") +py3libdir = $(shell $(python3) -c "import sys, sysconfig; print('lib.{}-{v[0]}.{v[1]}'.format(sysconfig.get_platform(), v=sys.version_info))") +ifeq ($(python2), python) + pyunvlibdir = $(pylibdir) +else + pyunvlibdir = lib +endif + +#End python setup section + +jsdeps: LICENSE + yarn || npm install + +ES5: LICENSE jsdeps + node node_modules/babel-cli/bin/babel.js --presets es2015 js_src --out-dir build/es5 + +jsdocs: + node js_src/docs_test.js + +jstest: LICENSE jsdeps + node node_modules/mocha/bin/mocha js_src/test/* + +ES5test: LICENSE ES5 + node node_modules/mocha/bin/mocha build/es5/test/* + +python: LICENSE setup.py + python $(py_deps) + python setup.py build --universal + +python3: LICENSE setup.py + $(python3) $(py_deps) + $(python3) setup.py build --universal + +python2: LICENSE setup.py + $(python2) $(py_deps) + $(python2) setup.py build --universal + +pypy: LICENSE setup.py + pypy $(py_deps) + pypy setup.py build --universal + +ifeq ($(pypy), True) +cpython: python + +else +cpython: LICENSE setup.py + python $(py_deps) +ifeq ($(debug), true) + python setup.py build --debug +else + python setup.py build +endif +endif + +cpython3: LICENSE setup.py + $(python3) $(py_deps) +ifeq ($(debug), true) + $(python3) setup.py build --debug +else + $(python3) setup.py build +endif + +cpython2: LICENSE setup.py + $(python2) $(py_deps) +ifeq ($(debug), true) + $(python2) setup.py build --debug +else + $(python2) setup.py build +endif + +pytestdeps: + python $(py_test_deps) + +py2testdeps: + $(python2) $(py_test_deps) + +py3testdeps: + $(python3) $(py_test_deps) + +pytest: LICENSE setup.py setup.cfg python pytestdeps +ifeq ($(cov), true) + python -m pytest -c ./setup.cfg --cov=build/$(pyunvlibdir) build/$(pyunvlibdir) +else + python -m pytest -c ./setup.cfg build/$(pyunvlibdir) +endif + +py2test: LICENSE setup.py setup.cfg python2 py2testdeps +ifeq ($(cov), true) + $(python2) -m pytest -c ./setup.cfg --cov=build/$(py2libdir) build/$(py2libdir) +else + $(python2) -m pytest -c ./setup.cfg build/$(py2libdir) +endif + +py3test: LICENSE setup.py setup.cfg python3 py3testdeps + @echo $(py3libdir) +ifeq ($(cov), true) + $(python3) -m pytest -c ./setup.cfg --cov=build/lib build/lib +else + $(python3) -m pytest -c ./setup.cfg build/lib +endif + +ifeq ($(pypy), True) +cpytest: pytest + +else +cpytest: LICENSE setup.py setup.cfg cpython pytestdeps +ifeq ($(cov), true) + python -m pytest -c ./setup.cfg --cov=build/$(pylibdir) build/$(pylibdir) +else + python -m pytest -c ./setup.cfg build/$(pylibdir) +endif +endif + +cpy2test: LICENSE setup.py setup.cfg cpython2 py2testdeps +ifeq ($(cov), true) + $(python2) -m pytest -c ./setup.cfg --cov=build/$(py2libdir) build/$(py2libdir) +else + $(python2) -m pytest -c ./setup.cfg build/$(py2libdir) +endif + +cpy3test: LICENSE setup.py setup.cfg cpython3 py3testdeps +ifeq ($(cov), true) + $(python3) -m pytest -c ./setup.cfg --cov=build/$(py3libdir) build/$(py3libdir) +else + $(python3) -m pytest -c ./setup.cfg build/$(py3libdir) +endif + +html: jsdocs + python $(docs_deps) + cd docs; rm -r .build; $(MAKE) html + +py_all: LICENSE setup.py setup.cfg python html cpython2 cpython3 pypy + +js_all: LICENSE ES5 html + +test_all: LICENSE jstest ES5test pytest cpy2test cpy3test diff --git a/README.md b/README.md index a1b25a0..9ed91c5 100644 --- a/README.md +++ b/README.md @@ -1,307 +1,65 @@ -Documentation for individual implementations can be found in their respective folders. +To see a better formatted, more frequently updated version of this, please visit [docs.p2p.today](https://docs.p2p.today), or for the develop branch, [dev-docs.p2p.today](https://dev-docs.p2p.today). Current build status: -[![Shippable Status for gappleto97/p2p-project](https://img.shields.io/shippable/5750887b2a8192902e225466/master.svg?maxAge=3600&label=Linux)](https://app.shippable.com/projects/5750887b2a8192902e225466) [ ![Travis-CI Status for gappleto97/p2p-project](https://img.shields.io/travis/gappleto97/p2p-project/master.svg?maxAge=3600&label=OSX)](https://travis-ci.org/gappleto97/p2p-project) [ ![Appveyor Status for gappleto97/p2p-project](https://img.shields.io/appveyor/ci/gappleto97/p2p-project/master.svg?maxAge=3600&label=Windows)](https://ci.appveyor.com/project/gappleto97/p2p-project) [ ![Code Climate Score](https://img.shields.io/codeclimate/github/gappleto97/p2p-project.svg?maxAge=3600)](https://codeclimate.com/github/gappleto97/p2p-project) [ ![Codecov Status for gappleto97/p2p-project](https://img.shields.io/codecov/c/github/gappleto97/p2p-project/master.svg?maxAge=3600)](https://codecov.io/gh/gappleto97/p2p-project) +[![shippable](https://img.shields.io/shippable/5750887b2a8192902e225466/develop.svg?maxAge=3600&label=Linux)](https://app.shippable.com/projects/5750887b2a8192902e225466) [![travis](https://img.shields.io/travis/gappleto97/p2p-project/develop.svg?maxAge=3600&label=OSX)](https://travis-ci.org/gappleto97/p2p-project) [![appveyor](https://img.shields.io/appveyor/ci/gappleto97/p2p-project/develop.svg?maxAge=3600&label=Windows)](https://ci.appveyor.com/project/gappleto97/p2p-project) [![codeclimate](https://img.shields.io/codeclimate/github/gappleto97/p2p-project.svg?maxAge=3600)](https://codeclimate.com/github/gappleto97/p2p-project) [![codecov](https://img.shields.io/codecov/c/github/gappleto97/p2p-project/develop.svg?maxAge=3600)](https://codecov.io/gh/gappleto97/p2p-project) -[![Issues in Progress](https://img.shields.io/waffle/label/gappleto97/p2p-project/backlog.svg?maxAge=3600&label=backlog)](https://waffle.io/gappleto97/p2p-project) [ ![Issues in Progress](https://img.shields.io/waffle/label/gappleto97/p2p-project/queued.svg?maxAge=3600&labal=queued)](https://waffle.io/gappleto97/p2p-project) [ ![Issues in Progress](https://img.shields.io/waffle/label/gappleto97/p2p-project/in%20progress.svg?maxAge=3600)](https://waffle.io/gappleto97/p2p-project) [ ![Issues in Progress](https://img.shields.io/waffle/label/gappleto97/p2p-project/in%20review.svg?maxAge=3600&label=in%20review)](https://waffle.io/gappleto97/p2p-project) +[![waffleio\_queued](https://img.shields.io/waffle/label/gappleto97/p2p-project/queued.svg?maxAge=3600&labal=queued)](https://waffle.io/gappleto97/p2p-project) [![waffleio\_in\_progress](https://img.shields.io/waffle/label/gappleto97/p2p-project/in%20progress.svg?maxAge=3600&labal=in%20progress)](https://waffle.io/gappleto97/p2p-project) [![waffleio\_in\_review](https://img.shields.io/waffle/label/gappleto97/p2p-project/in%20review.svg?maxAge=3600&label=in%20review)](https://waffle.io/gappleto97/p2p-project) -# Mass Broadcast Protocol +Goal +==== -1. **Abstract** +We are trying to make peer-to-peer networking easy. Right now there's very few libraries which allow multiple languages to use the same distributed network. - This project is meant to be a simple, portable peer-to-peer network. Part of its simplicity is that it will utilize no pathfinding or addressing structure outside of those provided by TCP/IP. This means that any message is either a direct transmission or a mass broadcast. This also makes it much simpler to translate the reference implementation into another language. +We're aiming to fix that. - It also is meant to be as modular as possible. If one wishes to operate on a different protocol or subnet, it takes one change to make this happen. If one wishes to use an encrypted communications channel, this is also relatively easy, so long as it inherits normal socket functions. If one wishes to compress information, this takes little change. If not, simply don't broadcast support. +What We Have +============ - This proposal is meant to formally outline the structure of such a network and its various nodes, as well as communicate this approach's disadvantages. To define the protocol, we will walk through the basic construction of a node. +There are several projects in the work right now. Several of these could be considered stable, but we're going to operate under the "beta" label for some time now. -2. **Packet Structure** +Message Serializer +================== - The first step to any of this is being able to understand messages sent through the network. To do this, you need to build a parser. Each message can be considered to have three segments: a header, metadata, and payload. The header is used to figure out the size of a message, as well as how to divide up its various packets. The metadata is used to assist routing functions. And the payload is what the user on the other end receives. +Serialization is the most important part for working with other languages. While there are several such schemes which work in most places, we made the decision to avoid these in general. We wanted something very lightweight, which could handle binary data, and operated as quickly as possible. This meant that "universal" serializers like JSON were out the window. - A more formal definitions would look like: +You can see more information about our serialization scheme in the [protocol documentation](./docs/protocol/serialization.rst). We currently have a working parser in Python, Java, Javascript, C++, and Golang. - Size of message - 4 (big-endian) bytes defining the size of the message - ------------------------All below may be compressed------------------------ - Size of packet 0 - 4 bytes defining the plaintext size of packet 0 - Size of packet 1 - 4 bytes defining the plaintext size of packet 1 - ... - Size of packet n-1 - 4 bytes defining the plaintext size of packet n-1 - Size of packet n - 4 bytes defining the plaintext size of packet n - ---------------------------------End Header-------------------------------- - Pathfinding header - [broadcast, waterfall, whisper, renegotiate] - Sender ID - A base_58 SHA384-based ID for the sender - Message ID - A base_58 SHA384-based ID for the message packets - Timestamp - A base_58 unix UTC timestamp of initial broadcast - Payload packets - Payload header - [broadcast, whisper, handshake, peers, request, response] - Payload contents +Base Network Structures +======================= - To understand this, let's work from the bottom up. When a user wants to construct a message, they feed a list of packets. For this example, let's say it's `['broadcast', 'test message']`. When this list is fed into a node, it adds the metadata section, and the list becomes: +All of our networks will be built on common base classes. Because of this, we can guarantee some network features. - `['broadcast', '6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V', '72tG7phqoAnoeWRKtWoSmseurpCtYg2wHih1y5ZX1AmUvihcH7CPZHThtm9LGvKtj7', '3EfSDb', 'broadcast', 'test message']`. +1. Networks will have as much common codebase as possible +2. Networks will have opportunistic compression across the board +3. Node IDs will be generated in a consistent manner +4. Command codes will be consistent across network types - The pathfinding header alerts nodes to how they should treat the message. If it is `broadcast` or `waterfall` they are to forward this message to their peers. If it is `whisper` they are not to do so. `renegotiate` is exclusivley uesd for connection management. +Mesh Network +============ - The sender ID is used to identify a user in your routing table. So if you go to reply to a message, it looks up this ID in your routing table. As will be discussed below, there are methods you can specifically request a user ID to connect to. +This is our unorganized network. It operates under three simple rules: - The message ID is used to filter out messages that you have seen before. If you wanted you could also use this as a checksum. +1. The first node to broadcast sends the message to all its peers +2. Each node which receives a message relays the message to each of its peers, except the node which sent it to them +3. Nodes do not relay a message they have seen before - One thing to notice is that the sender ID, message ID and timestamp are all in a strange encoding. This is base_58, borrowed from Bitcoin. It's a way to encode numbers that allows for sufficient density while still maintaining some human readability. This will get defined formally later in the paper. +Using these principles you can create a messaging network which scales linearly with the number of nodes. - All of this still leaves out the header. Constructing this goes as follows: +Currently there is an implementation in [Python](https:dev-docs.p2p.today/python/mesh) and Javascript <https:dev-docs.p2p.today/javascript/mesh>. More tractable documentation can be found in their tutorial sections. For a more in-depth explanation you can see [it's specifications](https:dev-docs.p2p.today/protocol/mesh) or [this slideshow](http://slides.p2p.today/). - For each packet, compute its length and pack this into four bytes. So a message of length 6 would look like `'\x00\x00\x00\x06'`. Take the resulting string and prepend it to your packets. In this example, you would end up with: `'\x00\x00\x00\t\x00\x00\x00B\x00\x00\x00B\x00\x00\x00\x06\x00\x00\x00\t\x00\x00\x00\x0cbroadcast6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V7iSCRDcHZwYtxGbTCz1rwDbUkt7YrbAh2VdS4A75hRuM6xan2gjmZqiVjLkMqiHE3Q3EfSDbbroadcasttest message'`. +Chord Table +=========== - After running this message through whatever compression algorith you've negotiated with your peer, compute its size, and pack it into four bytes: `'\x00\x00\x00\xc0'`. This results in a final message of: +This is a type of [distributed hash table](https://en.wikipedia.org/wiki/Distributed_hash_table) based on an [MIT paper](https://pdos.csail.mit.edu/papers/chord:sigcomm01/chord_sigcomm.pdf) which defined it. - `'\x00\x00\x00\xc0\x00\x00\x00\t\x00\x00\x00B\x00\x00\x00B\x00\x00\x00\x06\x00\x00\x00\t\x00\x00\x00\x0cbroadcast6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V7iSCRDcHZwYtxGbTCz1rwDbUkt7YrbAh2VdS4A75hRuM6xan2gjmZqiVjLkMqiHE3Q3EfSDbbroadcasttest message'` +The idea is that you can use this as a dictionary-like object. The only caveat is that all keys and values *must* be strings. It uses five separate hash tables for hash collision avoidance and data backup in case a node unexpectedly exits. -3. **Parsing a Message** +Currently there is only an implementation in Python and it is highly experimental. This section will be updated when it's ready for more general use. - Let's keep with our example above. How do we parse the resulting string? +Contributing, Credits, and Licenses +=================================== - First, check the first four bytes. Because we don't know how much data to receive through our socket, we always first check for a four byte header. Then we toss it aside and collect that much information. If we know the message will be compressed, now is when it gets decompressed. +Contributors are always welcome! Information on how you can help is located on the [Contributing page](./CONTRIBUTING.rst). - Next, we need to split up the packets. To do that, we take each four bytes and sum their values until the number of remaining bytes equals that sum. If it does not, we throw an error. An example script would look like: - - - ```python - def get_packets(string): - processed = 0 - expected = len(string) - pack_lens = [] - packets = [] - # First find each packet's length - while processed != expected: - length = struct.unpack("!L", string[processed:processed+4])[0] - processed += 4 - expected -= length - pack_lens.append(length) - # Then reconstruct the packets - for index, length in enumerate(pack_lens): - start = processed + sum(pack_lens[:index]) - end = start + length - packets.append(string[start:end]) - return packets - ``` - - From the above script we get back `['broadcast', '6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V', '72tG7phqoAnoeWRKtWoSmseurpCtYg2wHih1y5ZX1AmUvihcH7CPZHThtm9LGvKtj7', '3EfSDb', 'broadcast', 'test message']` - - A node will use the entirety of this list to decide what to do with it. In this case, it would forward it to its peers, then present to the user the payload: `['broadcast', 'test message']`. - - Now that we know how to construct a message, my examples will no longer include the headers. They will include the metadata and payload only. - -4. **IDs and Encoding** - - Knowing the overall message structure is great, but it's not very useful if you can't construct the metadata. To do this, there are three parts. - - * _base_58_ - - This encoding is taken from Bitcoin. If you've ever seen a Bitcoin address, you've seen base_58 encoding in action. The goal behind it is to provide data compression without compromising its human readability. Base_58, for the purposes of this protocol, is defined by the following python methods. - - ```python - base_58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - - def to_base_58(i): - string = "" - while i: - string = base_58[i % 58] + string - i = i // 58 # Floor division is needed to prevent floats - return string.encode() - - def from_base_58(string): - if isinstance(string, bytes): - string = string.decode() - decimal = 0 - for char in string: - decimal = decimal * 58 + base_58.index(char) - return decimal - ``` - - * _Node IDs_ - - A node ID is taken from a SHA-384 hash of three other elements. First, your outward facing address. Second, the ID of your subnet. Third, a 'user salt' generated on startup. This hash is then converted into base_58. - - That 'subnet ID' we mentioned will be explored in more detail later on, but for now consider it a constant. - - * _Message IDs_ - - A message ID is also a SHA-384 hash. In this case, it is on a message's payload and its timestamp. - - To get the hash, first join each packet together in order. Append to this the message's timestamp in base\_58. The ID you will use is the hash of this string, encoded into base\_58. - -5. **Node Construction** - - Now you're ready to parse the messages on the network, but you can't yet connect. There are important elements you need to store in order to interact with it correctly. - - 1. A daemon thread which receives messages and incoming connections - 2. A routing table of peers with the IDs and corresponding connection objects - 3. A "waterfall queue" of recently received message IDs and timestamps - 4. A user-interactable queue of recently received messages - 5. A "protocol", which contains: - 1. A sub-net flag - 2. An encryption method (or "Plaintext") - 3. A way to obtain a SHA256-based ID of this - - That last element is the 'subnet ID' we referred to in section 4. This object is used to weed out undesired connections. If someone has the wrong protocol object, then your node will reject them from connecting. A rough definition would be as follows: - - ```python - class protocol(namedtuple("protocol", ['subnet', 'encryption'])): - @property - def id(self): - info = [str(x) for x in self] + [protocol_version] - h = hashlib.sha256(''.join(info).encode()) - return to_base_58(int(h.hexdigest(), 16)) - ``` - - Or more explicitly in javascript: - - ```javascript - class protocol { - constructor(subnet, encryption) { - this.subnet = subnet; - this.encryption = encryption; - } - - get id() { - var info = [this.subnet, this.encryption, protocol_version]; - var hash = SHA256(info.join('')); - return to_base_58(BigInt(hash, 16)); - } - } - ``` - -6. **Connecting to the Network** - - This is where the protocol object becomes important. - - When you connect to a node, each of you will send a message in the following format: - - whisper - [your id] - [message id] - [timestamp] - handshake - [your id] - [your protocol id] - [your outward-facing address] - [json-ized list of supported compression methods, in order of preference] - - When you receive the corresponding message, the first thing you do is compare their protocol ID against your own. If they do not match, you shut down the connection. - - If they do match, you add them to your routing table (`{ID: connection}`), and make a note of their outward facing address and supported compression methods. Then you send a standard response: - - whisper - [your id] - [message id] - [timestamp] - peers - [json-ized copy of your routing table in format: [[addr, port], id]] - - Upon yourself receiving this message, you attempt to connect to each given address. Now you're connected to the network! But how do you process the incoming messages? - -7. **Message Propagation** - - A message is initially broadcast with the `broadcast` flag. The broadcasting node, as well as all receivers, store this message's ID and timestamp in their waterfall queue. The reciving nodes then re-broadcast this message to each of their peers, but changing the flag to `waterfall`. - - A node which receives these waterfall packets goes through the following steps: - - 1. If the message ID is not in the node's waterfall queue, continue and add it to the waterfall queue - 2. Perform cleanup on the waterfall queue - 1. Remove all possible duplicates (sending may be done in multiple threads, which may result in duplicate copies) - 2. Remove all IDs with a timestamp more than 1 minute ago - 3. Re-broadcast this message to all peers (optionally excluding the one you received it from) - - ![Figure one](./figure_one.png) - -8. **Renegotiating a Connection** - - It may be that at some point a message fails to decompress on your end. If this occurs, you have an easy solution, you can send a `renegotiate` message. This flag is used to indicate that a message should never be presented to the user, and is only used for connection management. At this time there are two possible operations. - - The `compression` subflag will allow you to renegotiate your compression methods. A message using this subflag should be constructed like so: - - renegotiate - [your id] - [message id] - [timestamp] - compression - [json-ized list of desired compression methods, in order of preference] - - Your peer will respond with the same message, excluding any methods they do not support. If this list is different than the one you sent, you reply, trimming the list of methods _you_ do not support. This process is repeated until you agree upon a list. - - You may also send a `resend` subflag, which requests your peer to resend the previous `whisper` or `broadcast`. This is structured like so: - - renegotiate - [your id] - [message id] - [timestamp] - resend - -9. **Peer Requests** - - If you want to privately reply to a message where you are not directly connected to a sender, the following method can be used: - - First, you broadcast a message to the network containing the `request` subflag. This is constructed as follows: - - broadcast - [your id] - [message id] - [timestamp] - request - [a unique, base_58 id you assign] - [the id of the desired peer] - - Then you place this in a dictionary so you can watch when this is responded to. A peer who gets this will reply: - - broadcast - [their id] - [message id] - [timestamp] - response - [the id you assigned] - [address of desired peer in format: [[addr, port], id] ] - - When this is received, you remove the request from your dictionary, make a connection to the given address, and send the message. - - Another use of this mechanism is to request a copy of your peers' routing tables. To do this, you may send a message structured like so: - - whisper - [your id] - [message id] - [timestamp] - request - * - - A node who receives this will respond exactly as they do after a successful handshake. Note that while it is technically valid to send this request as a `broadcast`, it is generally discouraged. - -10. **Potential Flaws** - - The network has a few immediately obvious shortcomings. - - First, the maximum message size is 4,294,967,299 bytes (including compression and headers). It could well be that in the future there will be more data to send in a single message. But equally so, a present-day attacker could use this to halt essentially every connection on the network. A short-term solution would be to have a soft-defined limit, but as has been shown in other protocols, this can calcify over time and do damage. - - Second, in a worst case scenario, every node will receive a given message n-1 times, and each message will generate n^2 total broadcasts, where n is the number of connected nodes. In most larger cases this will not happen, as a given node will not be connected to everyone else. But in smaller networks this will be common, and in well-connected networks this could slow things down. This calls for optimization, and will need to be explored. For instance, not propagating to a peer you receive a message from reduces the number of total broadcasts to (n-1)^2\. Limiting your number of connections can bring this down to min(max_conns, n-1) * (n-1). - - Thrid, there is quite a lot of extra data being sent. Using the default parameters, if you want to send a 4 character message it will be expanded to 175 characters. That's ~44x larger. If you want these differences to be negligble, you need to send messages on the order of 512 characters. Then there is only an increase of ~34% (0% with decent compression). This can be improved by reducing the size of the various IDs being sent, or making the packet headers shorter. Both of these have disadvantages, however. - - Results using opportunistic compression look roughly as follows (last updated in 0.2.95): - - For 4 characters… - - original 4 - plaintext 175 (4375%) - lzma 228 (5700%) - bz2 192 (4800%) - gzip 163 (4075%) - - For 498 characters… - - original 498 - plaintext 669 (134.3%) - lzma 552 (110.8%) - bz2 533 (107.0%) - gzip 471 (94.6%) - - Because the reference implementation supports all of these (except for lzma in python2), this means that the overhead will drop away after ~500 characters. Communications with other implementations may be slower than this, however. +Credits and License are located on [their own page](./docs/License.rst). diff --git a/appveyor.yml b/appveyor.yml index 880243b..d518925 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,12 +1,18 @@ environment: matrix: - - PYTHON: "C:\\Python26" - PYTHON_VERSION: "2.6" + # - PYTHON: "C:\\Python26" + # PYTHON_VERSION: "2.6" + # RUN: "%PYTHON%\\python" + # PIP: "%PYTHON%\\Scripts\\pip" + # COV: "%PYTHON%\\Scripts\\coverage" + + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7" RUN: "%PYTHON%\\python" PIP: "%PYTHON%\\Scripts\\pip" COV: "%PYTHON%\\Scripts\\coverage" - - PYTHON: "C:\\Python27" + - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7" RUN: "%PYTHON%\\python" PIP: "%PYTHON%\\Scripts\\pip" @@ -18,48 +24,62 @@ environment: PIP: "%PYTHON%\\Scripts\\pip" COV: "%PYTHON%\\Scripts\\coverage" + - PYTHON: "C:\\Python33-x64" + PYTHON_VERSION: "3.3" + RUN: "%PYTHON%\\python" + PIP: "%PYTHON%\\Scripts\\pip" + COV: "%PYTHON%\\Scripts\\coverage" + - PYTHON: "C:\\Python34" PYTHON_VERSION: "3.4" RUN: "%PYTHON%\\python" PIP: "%PYTHON%\\Scripts\\pip" COV: "%PYTHON%\\Scripts\\coverage" + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + RUN: "%PYTHON%\\python" + PIP: "%PYTHON%\\Scripts\\pip" + COV: "%PYTHON%\\Scripts\\coverage" + - PYTHON: "C:\\Python35" PYTHON_VERSION: "3.5" RUN: "%PYTHON%\\python" PIP: "%PYTHON%\\Scripts\\pip" COV: "%PYTHON%\\Scripts\\coverage" + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + RUN: "%PYTHON%\\python" + PIP: "%PYTHON%\\Scripts\\pip" + COV: "%PYTHON%\\Scripts\\coverage" + # - PYTHON_VERSION: "pypy2" # RUN: "%APPVEYOR_BUILD_FOLDER%\\pypy-4.0.1-win32\\pypy" # PIP: "%RUN% -m pip" - # PYPY: 'powershell.exe %APPVEYOR_BUILD_FOLDER%\\.installers\\installpypy2.ps1' + # PYPY: 'powershell.exe %APPVEYOR_BUILD_FOLDER%\\.scripts\\installpypy2.ps1' # COV: "%RUN% -m coverage" # - PYTHON_VERSION: "pypy3" # RUN: "%APPVEYOR_BUILD_FOLDER%\\pypy3-2.4.0-win32\\pypy" # PIP: "%RUN% -m pip" - # PYPY: "powershell.exe %APPVEYOR_BUILD_FOLDER%\\.installers\\installpypy3.ps1" + # PYPY: "powershell.exe %APPVEYOR_BUILD_FOLDER%\\.scripts\\installpypy3.ps1" # COV: "%RUN% -m coverage" + - NODE: "4" + - NODE: "5" + - NODE: "6" + matrix: allow_failures: - PYTHON_VERSION: "pypy2" - PYTHON_VERSION: "pypy3" -install: - - "ECHO %PYTHON% %PYTHON_VERSION%%APPVEYOR_BUILD_FOLDER%" - - "set HOME=%APPVEYOR_BUILD_FOLDER%" - - "%PYPY%" - - "%PIP% install pytest-coverage codecov PyOpenSSL" - - "cd %HOME%" - test: off build_script: - - "%RUN% -m pytest -vv --cov=./py_src/ ./py_src/" + - .scripts\\appveyor_script.bat -on_success: - - "%COV% combine" - - "%COV% xml" - - "%RUN% -c \"import codecov; codecov.main('--token=d89f9bd9-27a3-4560-8dbb-39ee3ba020a5', '--file=coverage.xml')\"" +artifacts: + - path: '**\*.whl' + name: Wheel diff --git a/c_src/BaseConverter.h b/c_src/BaseConverter.h new file mode 100644 index 0000000..6eda18f --- /dev/null +++ b/c_src/BaseConverter.h @@ -0,0 +1,316 @@ +/** +* Base Converter +* ============== +* +* Arbitrary precision base conversion by Daniel Gehriger +* Permission for use was given `here `_. +* This has been heavily modified since copying, has been hardcoded for a specific case, then translated to C. +*/ + +#include +#include +#include +#include + +#ifdef __cplusplus +#include +using namespace std; + +extern "C" { +#endif + +/** +* C/C++ Section +* ~~~~~~~~~~~~~ +* +* .. c:var:: const static char *base_58 +* +* This buffer contains all of the characters within the base_58 "alphabet" +* +* .. c:var:: const static char *ascii +* +* This buffer contains all of the characters within the extended ascii "alphabet" +*/ + +const static char *base_58 = (char *)"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; +const static char *ascii = (char *)"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"; + +/** +* .. c:function:: static inline size_t find_base_58(const char search) +* +* This is the equivalent of base_58.indexOf(search) +* +* :param search: The character you would like to search for +* +* :returns: The index of this character in base_58, or -1 +*/ + +static inline size_t find_base_58(const char search) { + for (size_t i = 0; i < 58; i++) { + if (base_58[i] == search) + return i; + } + return -1; +} + +/** +* .. c:function:: static inline unsigned long long from_base_58(const char *str, const size_t len) +* +* This converts a short base_58 buffer to its ascii equivalent +* +* :param str: The buffer you wish to convert +* :param len: The length of the buffer to convert +* +* :returns: The equivalent integral value +*/ + +static inline unsigned long long from_base_58(const char *str, const size_t len) { + unsigned long long ret = 0; + for (unsigned int i = 0; i < len; i++) { + ret *= (unsigned long long) 58; + ret += (unsigned long long) find_base_58(str[i]); + } + return ret; +} + +/** +* .. c:function:: static inline unsigned int base2dec(const char *value, const size_t len) +* +* Converts a small ascii buffer to its equivalent integral value +* +* :param value: The buffer you wish to convert +* :param len: The length of the buffer to convert +* +* :returns: The equivalent integral value +*/ + +static inline unsigned int base2dec(const char *value, const size_t len) { + unsigned int result = 0; + for (size_t i = 0; i < len; ++i) { + result <<= 8; + result += (unsigned char)value[i]; + } + + return result; +} + +/** +* .. c:function:: static inline void dec2base(unsigned int value, char *result, size_t *len) +* +* Converts an integral value to its equivalent binary buffer, then places this in result and updates len +* +* :param value: The value you wish to convert (as an unsigned int) +* :param result: The buffer result +* :param len: The length of the buffer result +* +* .. note:: +* +* This uses :c:func:`memmove` to transfer data, so it's helpful if you start with a larger-than-necessary buffer +*/ + +static inline void dec2base(unsigned int value, char *result, size_t *len) { + size_t pos = 4; + do { + result[--pos] = (unsigned char)value % 256; + value >>= 8; + } + while (value); + + *len = 4 - pos; + memmove(result, result + pos, *len); +} + +/** +* .. c:function:: static char *to_base_58(unsigned long long i, size_t *len) +* +* Converts an integral value to base_58, then updates len +* +* :param i: The value you want to convert +* :param len: The length of the generated buffer +* +* :returns: A buffer containing the base_58 equivalent of ``i`` +* +* .. note:: +* +* The return value needs to have :c:func:`free` called on it at some point +*/ + +static char *to_base_58(unsigned long long i, size_t *len) { + size_t pos = 0; + char *str = (char*)malloc(sizeof(char) * 4); + while (i) { + str[pos++] = base_58[i % 58]; + i /= 58; + if (pos % 4 == 0) + str = (char*)realloc(str, pos + 4); + } + if (!pos) + str[0] = base_58[0]; + else { + const size_t lim = pos - 1; + for (size_t i = 0; i < lim - i; i++) { + str[i] ^= str[lim - i]; + str[lim - i] ^= str[i]; + str[i] ^= str[lim - i]; + } + } + *len = pos; + return str; +} + +/** +* .. c:function:: static unsigned int divide_58(char *x, size_t *length) +* +* Divides an ascii buffer by 58, and returns the remainder +* +* :param x: The binary buffer you wish to divide +* :param length: The length of the buffer +* +* :returns: An unsigned int which contains the remainder of this division +*/ + +static unsigned int divide_58(char *x, size_t *length) { + const size_t const_length = *length; + size_t pos = 0; + char *quotient = (char*) malloc(sizeof(char) * const_length); + size_t len = 4; + char dec2base_str[4] = {}; + + for (size_t i = 0; i < const_length; ++i) { + const size_t j = i + 1 + (*length) - const_length; + if (*length < j) + break; + + const unsigned int value = base2dec(x, j); + + quotient[pos] = (unsigned char)(value / 58); + if (pos != 0 || quotient[pos] != ascii[0]) // Prevent leading zeros + pos++; + + dec2base(value % 58, dec2base_str, &len); + memmove(x + len, x + j, (*length) - j); + memcpy(x, dec2base_str, len); + + *length -= j; + *length += len; + } + + // calculate remainder + const unsigned int remainder = base2dec(x, *length); + + // store quotient in 'x' + memcpy(x, quotient, pos); + free(quotient); + *length = pos; + + return remainder; +} + +/** +* .. c:function:: static char *ascii_to_base_58_(const char *input, size_t length, size_t *res_len) +* +* Converts an arbitrary ascii buffer to its base_58 equivalent. The length of this buffer is placed in res_len. +* +* :param input: An input buffer +* :param length: The length of said buffer +* :param res_len: A pointer to the return buffer's length +* +* :returns: A buffer containing the base_58 equivalent of the provided buffer. +*/ + +static char *ascii_to_base_58_(const char *input, size_t length, size_t *res_len) { + char *c_input = (char*)malloc(sizeof(char) * length); + memcpy(c_input, input, length); + + const size_t res_size = ceil(length * 1.4); + size_t pos = res_size; + char *result = (char*)malloc(sizeof(char) * res_size); + + do { + result[--pos] = base_58[divide_58(c_input, &length)]; + } + while (length && !(length == 1 && c_input[0] == ascii[0])); + + free(c_input); + + *res_len = res_size - pos; + memmove(result, result + pos, *res_len); + return result; +} + +/** +* .. c:function:: static char *ascii_to_base_58(const char *input, size_t length, size_t *res_len, size_t minDigits) +* +* Converts an arbitrary ascii buffer into its base_58 equivalent. This is largely used for converting hex digests, or +* other such things which cannot conveniently be converted to an integral. +* +* :param input: An input buffer +* :param length: The length of said buffer +* :param res_len: A pointer to the return buffer's length +* :param minDigits: The minimum number of base_58 digits you would like to get back +* +* :returns: A buffer containing the base_58 equivalent of the provided buffer. +*/ + +static char *ascii_to_base_58(const char *input, size_t length, size_t *res_len, size_t minDigits) { + char *result = ascii_to_base_58_(input, length, res_len); + if (length < minDigits) { + size_t end_zeros = minDigits - *res_len; + result = (char*)realloc(result, minDigits); + memmove(result + end_zeros, result, *res_len); + memset(result, base_58[0], end_zeros); + } + return result; +} + +#ifdef __cplusplus +} + +/** +* C++ Only Section +* ~~~~~~~~~~~~~~~~ +* +* .. cpp:function:: static std::string ascii_to_base_58(std::string input) +* +* Converts an arbitrary binary buffer into its base_58 equivalent. +* +* This is a shortcut version of :c:func:`ascii_to_base_58`, with arguments: +* +* - ``input.c_str()`` +* - ``input.length()`` +* - ``&size_placeholder`` +* - ``1`` +* +* :param input: A :cpp:type:`std::string` which contains the buffer you wish to convert +* +* :returns: A :cpp:type:`std::string` which contains the resulting data +*/ + +static string ascii_to_base_58(string input) { + size_t res_size = 0; + char *c_string = ascii_to_base_58(input.c_str(), input.length(), &res_size, 1); + string result = string(c_string, res_size); + free(c_string); + return result; +} + +/** +* .. cpp:function:: static std::string to_base_58(unsigned long long i) +* +* Converts an integral value into its base_58 equivalent. This takes the data from +* :c:func:`to_base_58` and converts it to a :cpp:type:`std::string`. +* +* :param i: The integral value you wish to convert +* +* :returns: A :cpp:type:`std::string` which contains the resulting data +*/ + +static inline string to_base_58(unsigned long long i) { + size_t len = 0; + char *temp_str = to_base_58(i, &len); + string ret = string(temp_str, len); + free(temp_str); + return ret; +} + +#endif diff --git a/c_src/sha/sha2.c b/c_src/sha/sha2.c new file mode 100644 index 0000000..00441f7 --- /dev/null +++ b/c_src/sha/sha2.c @@ -0,0 +1,1082 @@ +/* + * FILE: sha2.c + * AUTHOR: Aaron D. Gifford - http://www.aarongifford.com/ + * + * Copyright (c) 2000-2001, Aaron D. Gifford + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTOR(S) ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTOR(S) BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: sha2.c,v 1.1 2001/11/08 00:01:51 adg Exp adg $ + */ + +#include /* memcpy()/memset() or bcopy()/bzero() */ +#include /* assert() */ +#include "sha2.h" + +/* + * ASSERT NOTE: + * Some sanity checking code is included using assert(). On my FreeBSD + * system, this additional code can be removed by compiling with NDEBUG + * defined. Check your own systems manpage on assert() to see how to + * compile WITHOUT the sanity checking code on your system. + * + * UNROLLED TRANSFORM LOOP NOTE: + * You can define SHA2_UNROLL_TRANSFORM to use the unrolled transform + * loop version for the hash transform rounds (defined using macros + * later in this file). Either define on the command line, for example: + * + * cc -DSHA2_UNROLL_TRANSFORM -o sha2 sha2.c sha2prog.c + * + * or define below: + * + * #define SHA2_UNROLL_TRANSFORM + * + */ + + +/*** SHA-256/384/512 Machine Architecture Definitions *****************/ +/* + * BYTE_ORDER NOTE: + * + * Please make sure that your system defines BYTE_ORDER. If your + * architecture is little-endian, make sure it also defines + * LITTLE_ENDIAN and that the two (BYTE_ORDER and LITTLE_ENDIAN) are + * equivilent. + * + * If your system does not define the above, then you can do so by + * hand like this: + * + * #define LITTLE_ENDIAN 1234 + * #define BIG_ENDIAN 4321 + * + * And for little-endian machines, add: + * + * #define BYTE_ORDER LITTLE_ENDIAN + * + * Or for big-endian machines: + * + * #define BYTE_ORDER BIG_ENDIAN + * + * The FreeBSD machine this was written on defines BYTE_ORDER + * appropriately by including (which in turn includes + * where the appropriate definitions are actually + * made). + */ +#if defined(BYTE_ORDER) && (BYTE_ORDER == LITTLE_ENDIAN || BYTE_ORDER == BIG_ENDIAN) + // no action needed +#elif defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN || \ + defined(__BIG_ENDIAN__) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(_MIBSEB) || defined(__MIBSEB) || defined(__MIBSEB__) || \ + defined(_M_PPC) + + #define BYTE_ORDER BIG_ENDIAN +#elif defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN || \ + defined(__LITTLE_ENDIAN__) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(_MIPSEL) || defined(__MIPSEL) || defined(__MIPSEL__) || \ + defined(_M_IX86) || defined(_M_X64) || defined(_M_IA64) || \ + defined(_M_ARM) + + #define BYTE_ORDER LITTLE_ENDIAN +#else + #error Define BYTE_ORDER to be equal to either LITTLE_ENDIAN or BIG_ENDIAN +#endif + +/* + * Define the followingsha2_* types to types of the correct length on + * the native archtecture. Most BSD systems and Linux define u_intXX_t + * types. Machines with very recent ANSI C headers, can use the + * uintXX_t definintions from inttypes.h by defining SHA2_USE_INTTYPES_H + * during compile or in the sha.h header file. + * + * Machines that support neither u_intXX_t nor inttypes.h's uintXX_t + * will need to define these three typedefs below (and the appropriate + * ones in sha.h too) by hand according to their system architecture. + * + * Thank you, Jun-ichiro itojun Hagino, for suggesting using u_intXX_t + * types and pointing out recent ANSI C support for uintXX_t in inttypes.h. + */ +#ifdef SHA2_USE_INTTYPES_H + +typedef uint8_t sha2_byte; /* Exactly 1 byte */ +typedef uint32_t sha2_word32; /* Exactly 4 bytes */ +typedef uint64_t sha2_word64; /* Exactly 8 bytes */ + +#else /* SHA2_USE_INTTYPES_H */ + +typedef u_int8_t sha2_byte; /* Exactly 1 byte */ +typedef u_int32_t sha2_word32; /* Exactly 4 bytes */ +typedef u_int64_t sha2_word64; /* Exactly 8 bytes */ + +#endif /* SHA2_USE_INTTYPES_H */ + + +/*** SHA-256/384/512 Various Length Definitions ***********************/ +/* NOTE: Most of these are in sha2.h */ +#define SHA256_SHORT_BLOCK_LENGTH (SHA256_BLOCK_LENGTH - 8) +#define SHA384_SHORT_BLOCK_LENGTH (SHA384_BLOCK_LENGTH - 16) +#define SHA512_SHORT_BLOCK_LENGTH (SHA512_BLOCK_LENGTH - 16) + + +/*** ENDIAN REVERSAL MACROS *******************************************/ +#if BYTE_ORDER == LITTLE_ENDIAN +#define REVERSE32(w,x) { \ + sha2_word32 tmp = (w); \ + tmp = (tmp >> 16) | (tmp << 16); \ + (x) = ((tmp & 0xff00ff00UL) >> 8) | ((tmp & 0x00ff00ffUL) << 8); \ +} +#define REVERSE64(w,x) { \ + sha2_word64 tmp = (w); \ + tmp = (tmp >> 32) | (tmp << 32); \ + tmp = ((tmp & 0xff00ff00ff00ff00ULL) >> 8) | \ + ((tmp & 0x00ff00ff00ff00ffULL) << 8); \ + (x) = ((tmp & 0xffff0000ffff0000ULL) >> 16) | \ + ((tmp & 0x0000ffff0000ffffULL) << 16); \ +} +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + +/* + * Macro for incrementally adding the unsigned 64-bit integer n to the + * unsigned 128-bit integer (represented using a two-element array of + * 64-bit words): + */ +#define ADDINC128(w,n) { \ + (w)[0] += (sha2_word64)(n); \ + if ((w)[0] < (n)) { \ + (w)[1]++; \ + } \ +} + +/* + * Macros for copying blocks of memory and for zeroing out ranges + * of memory. Using these macros makes it easy to switch from + * using memset()/memcpy() and using bzero()/bcopy(). + * + * Please define either SHA2_USE_MEMSET_MEMCPY or define + * SHA2_USE_BZERO_BCOPY depending on which function set you + * choose to use: + */ +#if !defined(SHA2_USE_MEMSET_MEMCPY) && !defined(SHA2_USE_BZERO_BCOPY) +/* Default to memset()/memcpy() if no option is specified */ +#define SHA2_USE_MEMSET_MEMCPY 1 +#endif +#if defined(SHA2_USE_MEMSET_MEMCPY) && defined(SHA2_USE_BZERO_BCOPY) +/* Abort with an error if BOTH options are defined */ +#error Define either SHA2_USE_MEMSET_MEMCPY or SHA2_USE_BZERO_BCOPY, not both! +#endif + +#ifdef SHA2_USE_MEMSET_MEMCPY +#define MEMSET_BZERO(p,l) memset((p), 0, (l)) +#define MEMCPY_BCOPY(d,s,l) memcpy((d), (s), (l)) +#endif +#ifdef SHA2_USE_BZERO_BCOPY +#define MEMSET_BZERO(p,l) bzero((p), (l)) +#define MEMCPY_BCOPY(d,s,l) bcopy((s), (d), (l)) +#endif + + +/*** THE SIX LOGICAL FUNCTIONS ****************************************/ +/* + * Bit shifting and rotation (used by the six SHA-XYZ logical functions: + * + * NOTE: The naming of R and S appears backwards here (R is a SHIFT and + * S is a ROTATION) because the SHA-256/384/512 description document + * (see http://csrc.nist.gov/cryptval/shs/sha256-384-512.pdf) uses this + * same "backwards" definition. + */ +/* Shift-right (used in SHA-256, SHA-384, and SHA-512): */ +#define R(b,x) ((x) >> (b)) +/* 32-bit Rotate-right (used in SHA-256): */ +#define S32(b,x) (((x) >> (b)) | ((x) << (32 - (b)))) +/* 64-bit Rotate-right (used in SHA-384 and SHA-512): */ +#define S64(b,x) (((x) >> (b)) | ((x) << (64 - (b)))) + +/* Two of six logical functions used in SHA-256, SHA-384, and SHA-512: */ +#define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) +#define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +/* Four of six logical functions used in SHA-256: */ +#define Sigma0_256(x) (S32(2, (x)) ^ S32(13, (x)) ^ S32(22, (x))) +#define Sigma1_256(x) (S32(6, (x)) ^ S32(11, (x)) ^ S32(25, (x))) +#define sigma0_256(x) (S32(7, (x)) ^ S32(18, (x)) ^ R(3 , (x))) +#define sigma1_256(x) (S32(17, (x)) ^ S32(19, (x)) ^ R(10, (x))) + +/* Four of six logical functions used in SHA-384 and SHA-512: */ +#define Sigma0_512(x) (S64(28, (x)) ^ S64(34, (x)) ^ S64(39, (x))) +#define Sigma1_512(x) (S64(14, (x)) ^ S64(18, (x)) ^ S64(41, (x))) +#define sigma0_512(x) (S64( 1, (x)) ^ S64( 8, (x)) ^ R( 7, (x))) +#define sigma1_512(x) (S64(19, (x)) ^ S64(61, (x)) ^ R( 6, (x))) + +/*** INTERNAL FUNCTION PROTOTYPES *************************************/ +/* NOTE: These should not be accessed directly from outside this + * library -- they are intended for private internal visibility/use + * only. + */ +void SHA512_Last(SHA512_CTX*); +void SHA256_Transform(SHA256_CTX*, const sha2_word32*); +void SHA512_Transform(SHA512_CTX*, const sha2_word64*); + + +/*** SHA-XYZ INITIAL HASH VALUES AND CONSTANTS ************************/ +/* Hash constant words K for SHA-256: */ +const static sha2_word32 K256[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, + 0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, + 0xd807aa98UL, 0x12835b01UL, 0x243185beUL, 0x550c7dc3UL, + 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, 0xc19bf174UL, + 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, + 0x983e5152UL, 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, + 0xc6e00bf3UL, 0xd5a79147UL, 0x06ca6351UL, 0x14292967UL, + 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, 0x53380d13UL, + 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, + 0xd192e819UL, 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, + 0x19a4c116UL, 0x1e376c08UL, 0x2748774cUL, 0x34b0bcb5UL, + 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, 0x682e6ff3UL, + 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; + +/* Initial hash value H for SHA-256: */ +const static sha2_word32 sha256_initial_hash_value[8] = { + 0x6a09e667UL, + 0xbb67ae85UL, + 0x3c6ef372UL, + 0xa54ff53aUL, + 0x510e527fUL, + 0x9b05688cUL, + 0x1f83d9abUL, + 0x5be0cd19UL +}; + +/* Hash constant words K for SHA-384 and SHA-512: */ +const static sha2_word64 K512[80] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL +}; + +/* Initial hash value H for SHA-384 */ +const static sha2_word64 sha384_initial_hash_value[8] = { + 0xcbbb9d5dc1059ed8ULL, + 0x629a292a367cd507ULL, + 0x9159015a3070dd17ULL, + 0x152fecd8f70e5939ULL, + 0x67332667ffc00b31ULL, + 0x8eb44a8768581511ULL, + 0xdb0c2e0d64f98fa7ULL, + 0x47b5481dbefa4fa4ULL +}; + +/* Initial hash value H for SHA-512 */ +const static sha2_word64 sha512_initial_hash_value[8] = { + 0x6a09e667f3bcc908ULL, + 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, + 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, + 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, + 0x5be0cd19137e2179ULL +}; + +/* + * Constant used by SHA256/384/512_End() functions for converting the + * digest to a readable hexadecimal character string: + */ +static const char *sha2_hex_digits = "0123456789abcdef"; + + +/*** SHA-256: *********************************************************/ +void SHA256_Init(SHA256_CTX* context) { + if (context == (SHA256_CTX*)0) { + return; + } + MEMCPY_BCOPY(context->state, sha256_initial_hash_value, SHA256_DIGEST_LENGTH); + MEMSET_BZERO(context->buffer, SHA256_BLOCK_LENGTH); + context->bitcount = 0; +} + +#ifdef SHA2_UNROLL_TRANSFORM + +/* Unrolled SHA-256 round macros: */ + +#if BYTE_ORDER == LITTLE_ENDIAN + +#define ROUND256_0_TO_15(a,b,c,d,e,f,g,h) \ + REVERSE32(*data++, W256[j]); \ + T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + \ + K256[j] + W256[j]; \ + (d) += T1; \ + (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ + j++ + + +#else /* BYTE_ORDER == LITTLE_ENDIAN */ + +#define ROUND256_0_TO_15(a,b,c,d,e,f,g,h) \ + T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + \ + K256[j] + (W256[j] = *data++); \ + (d) += T1; \ + (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ + j++ + +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + +#define ROUND256(a,b,c,d,e,f,g,h) \ + s0 = W256[(j+1)&0x0f]; \ + s0 = sigma0_256(s0); \ + s1 = W256[(j+14)&0x0f]; \ + s1 = sigma1_256(s1); \ + T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + K256[j] + \ + (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); \ + (d) += T1; \ + (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ + j++ + +void SHA256_Transform(SHA256_CTX* context, const sha2_word32* data) { + sha2_word32 a, b, c, d, e, f, g, h, s0, s1; + sha2_word32 T1, *W256; + int j; + + W256 = (sha2_word32*)context->buffer; + + /* Initialize registers with the prev. intermediate value */ + a = context->state[0]; + b = context->state[1]; + c = context->state[2]; + d = context->state[3]; + e = context->state[4]; + f = context->state[5]; + g = context->state[6]; + h = context->state[7]; + + j = 0; + do { + /* Rounds 0 to 15 (unrolled): */ + ROUND256_0_TO_15(a,b,c,d,e,f,g,h); + ROUND256_0_TO_15(h,a,b,c,d,e,f,g); + ROUND256_0_TO_15(g,h,a,b,c,d,e,f); + ROUND256_0_TO_15(f,g,h,a,b,c,d,e); + ROUND256_0_TO_15(e,f,g,h,a,b,c,d); + ROUND256_0_TO_15(d,e,f,g,h,a,b,c); + ROUND256_0_TO_15(c,d,e,f,g,h,a,b); + ROUND256_0_TO_15(b,c,d,e,f,g,h,a); + } while (j < 16); + + /* Now for the remaining rounds to 64: */ + do { + ROUND256(a,b,c,d,e,f,g,h); + ROUND256(h,a,b,c,d,e,f,g); + ROUND256(g,h,a,b,c,d,e,f); + ROUND256(f,g,h,a,b,c,d,e); + ROUND256(e,f,g,h,a,b,c,d); + ROUND256(d,e,f,g,h,a,b,c); + ROUND256(c,d,e,f,g,h,a,b); + ROUND256(b,c,d,e,f,g,h,a); + } while (j < 64); + + /* Compute the current intermediate hash value */ + context->state[0] += a; + context->state[1] += b; + context->state[2] += c; + context->state[3] += d; + context->state[4] += e; + context->state[5] += f; + context->state[6] += g; + context->state[7] += h; + + /* Clean up */ + a = b = c = d = e = f = g = h = T1 = 0; +} + +#else /* SHA2_UNROLL_TRANSFORM */ + +void SHA256_Transform(SHA256_CTX* context, const sha2_word32* data) { + sha2_word32 a, b, c, d, e, f, g, h, s0, s1; + sha2_word32 T1, T2, *W256; + int j; + + W256 = (sha2_word32*)context->buffer; + + /* Initialize registers with the prev. intermediate value */ + a = context->state[0]; + b = context->state[1]; + c = context->state[2]; + d = context->state[3]; + e = context->state[4]; + f = context->state[5]; + g = context->state[6]; + h = context->state[7]; + + j = 0; + do { +#if BYTE_ORDER == LITTLE_ENDIAN + /* Copy data while converting to host byte order */ + REVERSE32(*data++,W256[j]); + /* Apply the SHA-256 compression function to update a..h */ + T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + W256[j]; +#else /* BYTE_ORDER == LITTLE_ENDIAN */ + /* Apply the SHA-256 compression function to update a..h with copy */ + T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + (W256[j] = *data++); +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + T2 = Sigma0_256(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + + j++; + } while (j < 16); + + do { + /* Part of the message block expansion: */ + s0 = W256[(j+1)&0x0f]; + s0 = sigma0_256(s0); + s1 = W256[(j+14)&0x0f]; + s1 = sigma1_256(s1); + + /* Apply the SHA-256 compression function to update a..h */ + T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + + (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); + T2 = Sigma0_256(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + + j++; + } while (j < 64); + + /* Compute the current intermediate hash value */ + context->state[0] += a; + context->state[1] += b; + context->state[2] += c; + context->state[3] += d; + context->state[4] += e; + context->state[5] += f; + context->state[6] += g; + context->state[7] += h; + + /* Clean up */ + a = b = c = d = e = f = g = h = T1 = T2 = 0; +} + +#endif /* SHA2_UNROLL_TRANSFORM */ + +void SHA256_Update(SHA256_CTX* context, const sha2_byte *data, size_t len) { + unsigned int freespace, usedspace; + + if (len == 0) { + /* Calling with no data is valid - we do nothing */ + return; + } + + /* Sanity check: */ + assert(context != (SHA256_CTX*)0 && data != (sha2_byte*)0); + + usedspace = (context->bitcount >> 3) % SHA256_BLOCK_LENGTH; + if (usedspace > 0) { + /* Calculate how much free space is available in the buffer */ + freespace = SHA256_BLOCK_LENGTH - usedspace; + + if (len >= freespace) { + /* Fill the buffer completely and process it */ + MEMCPY_BCOPY(&context->buffer[usedspace], data, freespace); + context->bitcount += freespace << 3; + len -= freespace; + data += freespace; + SHA256_Transform(context, (sha2_word32*)context->buffer); + } else { + /* The buffer is not yet full */ + MEMCPY_BCOPY(&context->buffer[usedspace], data, len); + context->bitcount += len << 3; + /* Clean up: */ + usedspace = freespace = 0; + return; + } + } + while (len >= SHA256_BLOCK_LENGTH) { + /* Process as many complete blocks as we can */ + SHA256_Transform(context, (sha2_word32*)data); + context->bitcount += SHA256_BLOCK_LENGTH << 3; + len -= SHA256_BLOCK_LENGTH; + data += SHA256_BLOCK_LENGTH; + } + if (len > 0) { + /* There's left-overs, so save 'em */ + MEMCPY_BCOPY(context->buffer, data, len); + context->bitcount += len << 3; + } + /* Clean up: */ + usedspace = freespace = 0; +} + +void SHA256_Final(sha2_byte digest[], SHA256_CTX* context) { + sha2_word32 *d = (sha2_word32*)digest; + unsigned int usedspace; + + /* Sanity check: */ + assert(context != (SHA256_CTX*)0); + + /* If no digest buffer is passed, we don't bother doing this: */ + if (digest != (sha2_byte*)0) { + usedspace = (context->bitcount >> 3) % SHA256_BLOCK_LENGTH; +#if BYTE_ORDER == LITTLE_ENDIAN + /* Convert FROM host byte order */ + REVERSE64(context->bitcount,context->bitcount); +#endif + if (usedspace > 0) { + /* Begin padding with a 1 bit: */ + context->buffer[usedspace++] = 0x80; + + if (usedspace <= SHA256_SHORT_BLOCK_LENGTH) { + /* Set-up for the last transform: */ + MEMSET_BZERO(&context->buffer[usedspace], SHA256_SHORT_BLOCK_LENGTH - usedspace); + } else { + if (usedspace < SHA256_BLOCK_LENGTH) { + MEMSET_BZERO(&context->buffer[usedspace], SHA256_BLOCK_LENGTH - usedspace); + } + /* Do second-to-last transform: */ + SHA256_Transform(context, (sha2_word32*)context->buffer); + + /* And set-up for the last transform: */ + MEMSET_BZERO(context->buffer, SHA256_SHORT_BLOCK_LENGTH); + } + } else { + /* Set-up for the last transform: */ + MEMSET_BZERO(context->buffer, SHA256_SHORT_BLOCK_LENGTH); + + /* Begin padding with a 1 bit: */ + *context->buffer = 0x80; + } + /* Set the bit count: */ + *(sha2_word64*)&context->buffer[SHA256_SHORT_BLOCK_LENGTH] = context->bitcount; + + /* Final transform: */ + SHA256_Transform(context, (sha2_word32*)context->buffer); + +#if BYTE_ORDER == LITTLE_ENDIAN + { + /* Convert TO host byte order */ + int j; + for (j = 0; j < 8; j++) { + REVERSE32(context->state[j],context->state[j]); + *d++ = context->state[j]; + } + } +#else + MEMCPY_BCOPY(d, context->state, SHA256_DIGEST_LENGTH); +#endif + } + + /* Clean up state data: */ + MEMSET_BZERO(context, sizeof(context)); + usedspace = 0; +} + +char *SHA256_End(SHA256_CTX* context, char buffer[]) { + sha2_byte digest[SHA256_DIGEST_LENGTH], *d = digest; + int i; + + /* Sanity check: */ + assert(context != (SHA256_CTX*)0); + + if (buffer != (char*)0) { + SHA256_Final(digest, context); + + for (i = 0; i < SHA256_DIGEST_LENGTH; i++) { + *buffer++ = sha2_hex_digits[(*d & 0xf0) >> 4]; + *buffer++ = sha2_hex_digits[*d & 0x0f]; + d++; + } + *buffer = (char)0; + } else { + MEMSET_BZERO(context, sizeof(context)); + } + MEMSET_BZERO(digest, SHA256_DIGEST_LENGTH); + return buffer; +} + +char* SHA256_Data(const sha2_byte* data, size_t len, char digest[SHA256_DIGEST_STRING_LENGTH]) { + SHA256_CTX context; + + SHA256_Init(&context); + SHA256_Update(&context, data, len); + return SHA256_End(&context, digest); +} + + +/*** SHA-512: *********************************************************/ +void SHA512_Init(SHA512_CTX* context) { + if (context == (SHA512_CTX*)0) { + return; + } + MEMCPY_BCOPY(context->state, sha512_initial_hash_value, SHA512_DIGEST_LENGTH); + MEMSET_BZERO(context->buffer, SHA512_BLOCK_LENGTH); + context->bitcount[0] = context->bitcount[1] = 0; +} + +#ifdef SHA2_UNROLL_TRANSFORM + +/* Unrolled SHA-512 round macros: */ +#if BYTE_ORDER == LITTLE_ENDIAN + +#define ROUND512_0_TO_15(a,b,c,d,e,f,g,h) \ + REVERSE64(*data++, W512[j]); \ + T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + \ + K512[j] + W512[j]; \ + (d) += T1, \ + (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)), \ + j++ + + +#else /* BYTE_ORDER == LITTLE_ENDIAN */ + +#define ROUND512_0_TO_15(a,b,c,d,e,f,g,h) \ + T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + \ + K512[j] + (W512[j] = *data++); \ + (d) += T1; \ + (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)); \ + j++ + +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + +#define ROUND512(a,b,c,d,e,f,g,h) \ + s0 = W512[(j+1)&0x0f]; \ + s0 = sigma0_512(s0); \ + s1 = W512[(j+14)&0x0f]; \ + s1 = sigma1_512(s1); \ + T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + K512[j] + \ + (W512[j&0x0f] += s1 + W512[(j+9)&0x0f] + s0); \ + (d) += T1; \ + (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)); \ + j++ + +void SHA512_Transform(SHA512_CTX* context, const sha2_word64* data) { + sha2_word64 a, b, c, d, e, f, g, h, s0, s1; + sha2_word64 T1, *W512 = (sha2_word64*)context->buffer; + int j; + + /* Initialize registers with the prev. intermediate value */ + a = context->state[0]; + b = context->state[1]; + c = context->state[2]; + d = context->state[3]; + e = context->state[4]; + f = context->state[5]; + g = context->state[6]; + h = context->state[7]; + + j = 0; + do { + ROUND512_0_TO_15(a,b,c,d,e,f,g,h); + ROUND512_0_TO_15(h,a,b,c,d,e,f,g); + ROUND512_0_TO_15(g,h,a,b,c,d,e,f); + ROUND512_0_TO_15(f,g,h,a,b,c,d,e); + ROUND512_0_TO_15(e,f,g,h,a,b,c,d); + ROUND512_0_TO_15(d,e,f,g,h,a,b,c); + ROUND512_0_TO_15(c,d,e,f,g,h,a,b); + ROUND512_0_TO_15(b,c,d,e,f,g,h,a); + } while (j < 16); + + /* Now for the remaining rounds up to 79: */ + do { + ROUND512(a,b,c,d,e,f,g,h); + ROUND512(h,a,b,c,d,e,f,g); + ROUND512(g,h,a,b,c,d,e,f); + ROUND512(f,g,h,a,b,c,d,e); + ROUND512(e,f,g,h,a,b,c,d); + ROUND512(d,e,f,g,h,a,b,c); + ROUND512(c,d,e,f,g,h,a,b); + ROUND512(b,c,d,e,f,g,h,a); + } while (j < 80); + + /* Compute the current intermediate hash value */ + context->state[0] += a; + context->state[1] += b; + context->state[2] += c; + context->state[3] += d; + context->state[4] += e; + context->state[5] += f; + context->state[6] += g; + context->state[7] += h; + + /* Clean up */ + a = b = c = d = e = f = g = h = T1 = 0; +} + +#else /* SHA2_UNROLL_TRANSFORM */ + +void SHA512_Transform(SHA512_CTX* context, const sha2_word64* data) { + sha2_word64 a, b, c, d, e, f, g, h, s0, s1; + sha2_word64 T1, T2, *W512 = (sha2_word64*)context->buffer; + int j; + + /* Initialize registers with the prev. intermediate value */ + a = context->state[0]; + b = context->state[1]; + c = context->state[2]; + d = context->state[3]; + e = context->state[4]; + f = context->state[5]; + g = context->state[6]; + h = context->state[7]; + + j = 0; + do { +#if BYTE_ORDER == LITTLE_ENDIAN + /* Convert TO host byte order */ + REVERSE64(*data++, W512[j]); + /* Apply the SHA-512 compression function to update a..h */ + T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + W512[j]; +#else /* BYTE_ORDER == LITTLE_ENDIAN */ + /* Apply the SHA-512 compression function to update a..h with copy */ + T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + (W512[j] = *data++); +#endif /* BYTE_ORDER == LITTLE_ENDIAN */ + T2 = Sigma0_512(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + + j++; + } while (j < 16); + + do { + /* Part of the message block expansion: */ + s0 = W512[(j+1)&0x0f]; + s0 = sigma0_512(s0); + s1 = W512[(j+14)&0x0f]; + s1 = sigma1_512(s1); + + /* Apply the SHA-512 compression function to update a..h */ + T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + + (W512[j&0x0f] += s1 + W512[(j+9)&0x0f] + s0); + T2 = Sigma0_512(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + + j++; + } while (j < 80); + + /* Compute the current intermediate hash value */ + context->state[0] += a; + context->state[1] += b; + context->state[2] += c; + context->state[3] += d; + context->state[4] += e; + context->state[5] += f; + context->state[6] += g; + context->state[7] += h; + + /* Clean up */ + a = b = c = d = e = f = g = h = T1 = T2 = 0; +} + +#endif /* SHA2_UNROLL_TRANSFORM */ + +void SHA512_Update(SHA512_CTX* context, const sha2_byte *data, size_t len) { + unsigned int freespace, usedspace; + + if (len == 0) { + /* Calling with no data is valid - we do nothing */ + return; + } + + /* Sanity check: */ + assert(context != (SHA512_CTX*)0 && data != (sha2_byte*)0); + + usedspace = (context->bitcount[0] >> 3) % SHA512_BLOCK_LENGTH; + if (usedspace > 0) { + /* Calculate how much free space is available in the buffer */ + freespace = SHA512_BLOCK_LENGTH - usedspace; + + if (len >= freespace) { + /* Fill the buffer completely and process it */ + MEMCPY_BCOPY(&context->buffer[usedspace], data, freespace); + ADDINC128(context->bitcount, freespace << 3); + len -= freespace; + data += freespace; + SHA512_Transform(context, (sha2_word64*)context->buffer); + } else { + /* The buffer is not yet full */ + MEMCPY_BCOPY(&context->buffer[usedspace], data, len); + ADDINC128(context->bitcount, len << 3); + /* Clean up: */ + usedspace = freespace = 0; + return; + } + } + while (len >= SHA512_BLOCK_LENGTH) { + /* Process as many complete blocks as we can */ + SHA512_Transform(context, (sha2_word64*)data); + ADDINC128(context->bitcount, SHA512_BLOCK_LENGTH << 3); + len -= SHA512_BLOCK_LENGTH; + data += SHA512_BLOCK_LENGTH; + } + if (len > 0) { + /* There's left-overs, so save 'em */ + MEMCPY_BCOPY(context->buffer, data, len); + ADDINC128(context->bitcount, len << 3); + } + /* Clean up: */ + usedspace = freespace = 0; +} + +void SHA512_Last(SHA512_CTX* context) { + unsigned int usedspace; + + usedspace = (context->bitcount[0] >> 3) % SHA512_BLOCK_LENGTH; +#if BYTE_ORDER == LITTLE_ENDIAN + /* Convert FROM host byte order */ + REVERSE64(context->bitcount[0],context->bitcount[0]); + REVERSE64(context->bitcount[1],context->bitcount[1]); +#endif + if (usedspace > 0) { + /* Begin padding with a 1 bit: */ + context->buffer[usedspace++] = 0x80; + + if (usedspace <= SHA512_SHORT_BLOCK_LENGTH) { + /* Set-up for the last transform: */ + MEMSET_BZERO(&context->buffer[usedspace], SHA512_SHORT_BLOCK_LENGTH - usedspace); + } else { + if (usedspace < SHA512_BLOCK_LENGTH) { + MEMSET_BZERO(&context->buffer[usedspace], SHA512_BLOCK_LENGTH - usedspace); + } + /* Do second-to-last transform: */ + SHA512_Transform(context, (sha2_word64*)context->buffer); + + /* And set-up for the last transform: */ + MEMSET_BZERO(context->buffer, SHA512_BLOCK_LENGTH - 2); + } + } else { + /* Prepare for final transform: */ + MEMSET_BZERO(context->buffer, SHA512_SHORT_BLOCK_LENGTH); + + /* Begin padding with a 1 bit: */ + *context->buffer = 0x80; + } + /* Store the length of input data (in bits): */ + *(sha2_word64*)&context->buffer[SHA512_SHORT_BLOCK_LENGTH] = context->bitcount[1]; + *(sha2_word64*)&context->buffer[SHA512_SHORT_BLOCK_LENGTH+8] = context->bitcount[0]; + + /* Final transform: */ + SHA512_Transform(context, (sha2_word64*)context->buffer); +} + +void SHA512_Final(sha2_byte digest[], SHA512_CTX* context) { + sha2_word64 *d = (sha2_word64*)digest; + + /* Sanity check: */ + assert(context != (SHA512_CTX*)0); + + /* If no digest buffer is passed, we don't bother doing this: */ + if (digest != (sha2_byte*)0) { + SHA512_Last(context); + + /* Save the hash data for output: */ +#if BYTE_ORDER == LITTLE_ENDIAN + { + /* Convert TO host byte order */ + int j; + for (j = 0; j < 8; j++) { + REVERSE64(context->state[j],context->state[j]); + *d++ = context->state[j]; + } + } +#else + MEMCPY_BCOPY(d, context->state, SHA512_DIGEST_LENGTH); +#endif + } + + /* Zero out state data */ + MEMSET_BZERO(context, sizeof(context)); +} + +char *SHA512_End(SHA512_CTX* context, char buffer[]) { + sha2_byte digest[SHA512_DIGEST_LENGTH], *d = digest; + int i; + + /* Sanity check: */ + assert(context != (SHA512_CTX*)0); + + if (buffer != (char*)0) { + SHA512_Final(digest, context); + + for (i = 0; i < SHA512_DIGEST_LENGTH; i++) { + *buffer++ = sha2_hex_digits[(*d & 0xf0) >> 4]; + *buffer++ = sha2_hex_digits[*d & 0x0f]; + d++; + } + *buffer = (char)0; + } else { + MEMSET_BZERO(context, sizeof(context)); + } + MEMSET_BZERO(digest, SHA512_DIGEST_LENGTH); + return buffer; +} + +char* SHA512_Data(const sha2_byte* data, size_t len, char digest[SHA512_DIGEST_STRING_LENGTH]) { + SHA512_CTX context; + + SHA512_Init(&context); + SHA512_Update(&context, data, len); + return SHA512_End(&context, digest); +} + + +/*** SHA-384: *********************************************************/ +void SHA384_Init(SHA384_CTX* context) { + if (context == (SHA384_CTX*)0) { + return; + } + MEMCPY_BCOPY(context->state, sha384_initial_hash_value, SHA512_DIGEST_LENGTH); + MEMSET_BZERO(context->buffer, SHA384_BLOCK_LENGTH); + context->bitcount[0] = context->bitcount[1] = 0; +} + +void SHA384_Update(SHA384_CTX* context, const sha2_byte* data, size_t len) { + SHA512_Update((SHA512_CTX*)context, data, len); +} + +void SHA384_Final(sha2_byte digest[], SHA384_CTX* context) { + sha2_word64 *d = (sha2_word64*)digest; + + /* Sanity check: */ + assert(context != (SHA384_CTX*)0); + + /* If no digest buffer is passed, we don't bother doing this: */ + if (digest != (sha2_byte*)0) { + SHA512_Last((SHA512_CTX*)context); + + /* Save the hash data for output: */ +#if BYTE_ORDER == LITTLE_ENDIAN + { + /* Convert TO host byte order */ + int j; + for (j = 0; j < 6; j++) { + REVERSE64(context->state[j],context->state[j]); + *d++ = context->state[j]; + } + } +#else + MEMCPY_BCOPY(d, context->state, SHA384_DIGEST_LENGTH); +#endif + } + + /* Zero out state data */ + MEMSET_BZERO(context, sizeof(context)); +} + +char *SHA384_End(SHA384_CTX* context, char buffer[]) { + sha2_byte digest[SHA384_DIGEST_LENGTH], *d = digest; + int i; + + /* Sanity check: */ + assert(context != (SHA384_CTX*)0); + + if (buffer != (char*)0) { + SHA384_Final(digest, context); + + for (i = 0; i < SHA384_DIGEST_LENGTH; i++) { + *buffer++ = sha2_hex_digits[(*d & 0xf0) >> 4]; + *buffer++ = sha2_hex_digits[*d & 0x0f]; + d++; + } + *buffer = (char)0; + } else { + MEMSET_BZERO(context, sizeof(context)); + } + MEMSET_BZERO(digest, SHA384_DIGEST_LENGTH); + return buffer; +} + +char* SHA384_Data(const sha2_byte* data, size_t len, char digest[SHA384_DIGEST_STRING_LENGTH]) { + SHA384_CTX context; + + SHA384_Init(&context); + SHA384_Update(&context, data, len); + return SHA384_End(&context, digest); +} + diff --git a/c_src/sha/sha2.h b/c_src/sha/sha2.h new file mode 100644 index 0000000..f25a302 --- /dev/null +++ b/c_src/sha/sha2.h @@ -0,0 +1,203 @@ +/* + * FILE: sha2.h + * AUTHOR: Aaron D. Gifford - http://www.aarongifford.com/ + * + * Copyright (c) 2000-2001, Aaron D. Gifford + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTOR(S) ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTOR(S) BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: sha2.h,v 1.1 2001/11/08 00:02:01 adg Exp adg $ + */ + +#ifndef __SHA2_H__ +#define __SHA2_H__ + +#ifdef __cplusplus +extern "C" { +#endif + + +/* + * Import u_intXX_t size_t type definitions from system headers. You + * may need to change this, or define these things yourself in this + * file. + */ +#include + +#ifdef SHA2_USE_INTTYPES_H + +#include + +#elif defined(_WIN32) || defined(__CYGWIN32__) || defined(_MSC_VER) + +typedef unsigned char u_int8_t; /* 1-byte (8-bits) */ +typedef unsigned int u_int32_t; /* 4-bytes (32-bits) */ +typedef unsigned long long u_int64_t; /* 8-bytes (64-bits) */ + +#endif /* SHA2_USE_INTTYPES_H */ + + +/*** SHA-256/384/512 Various Length Definitions ***********************/ +#define SHA256_BLOCK_LENGTH 64 +#define SHA256_DIGEST_LENGTH 32 +#define SHA256_DIGEST_STRING_LENGTH (SHA256_DIGEST_LENGTH * 2 + 1) +#define SHA384_BLOCK_LENGTH 128 +#define SHA384_DIGEST_LENGTH 48 +#define SHA384_DIGEST_STRING_LENGTH (SHA384_DIGEST_LENGTH * 2 + 1) +#define SHA512_BLOCK_LENGTH 128 +#define SHA512_DIGEST_LENGTH 64 +#define SHA512_DIGEST_STRING_LENGTH (SHA512_DIGEST_LENGTH * 2 + 1) + + +/*** SHA-256/384/512 Context Structures *******************************/ +/* NOTE: If your architecture does not define either u_intXX_t types or + * uintXX_t (from inttypes.h), you may need to define things by hand + * for your system: + */ +#if 0 +typedef unsigned char u_int8_t; /* 1-byte (8-bits) */ +typedef unsigned int u_int32_t; /* 4-bytes (32-bits) */ +typedef unsigned long long u_int64_t; /* 8-bytes (64-bits) */ +#endif +/* + * Most BSD systems already define u_intXX_t types, as does Linux. + * Some systems, however, like Compaq's Tru64 Unix instead can use + * uintXX_t types defined by very recent ANSI C standards and included + * in the file: + * + * #include + * + * If you choose to use then please define: + * + * #define SHA2_USE_INTTYPES_H + * + * Or on the command line during compile: + * + * cc -DSHA2_USE_INTTYPES_H ... + */ +#ifdef SHA2_USE_INTTYPES_H + +typedef struct _SHA256_CTX { + uint32_t state[8]; + uint64_t bitcount; + uint8_t buffer[SHA256_BLOCK_LENGTH]; +} SHA256_CTX; +typedef struct _SHA512_CTX { + uint64_t state[8]; + uint64_t bitcount[2]; + uint8_t buffer[SHA512_BLOCK_LENGTH]; +} SHA512_CTX; + +#else /* SHA2_USE_INTTYPES_H */ + +typedef struct _SHA256_CTX { + u_int32_t state[8]; + u_int64_t bitcount; + u_int8_t buffer[SHA256_BLOCK_LENGTH]; +} SHA256_CTX; +typedef struct _SHA512_CTX { + u_int64_t state[8]; + u_int64_t bitcount[2]; + u_int8_t buffer[SHA512_BLOCK_LENGTH]; +} SHA512_CTX; + +#endif /* SHA2_USE_INTTYPES_H */ + +typedef SHA512_CTX SHA384_CTX; + + +/*** SHA-256/384/512 Function Prototypes ******************************/ +#ifndef NOPROTO +#ifdef SHA2_USE_INTTYPES_H + +void SHA256_Init(SHA256_CTX *); +void SHA256_Update(SHA256_CTX*, const uint8_t*, size_t); +void SHA256_Final(uint8_t[SHA256_DIGEST_LENGTH], SHA256_CTX*); +char* SHA256_End(SHA256_CTX*, char[SHA256_DIGEST_STRING_LENGTH]); +char* SHA256_Data(const uint8_t*, size_t, char[SHA256_DIGEST_STRING_LENGTH]); + +void SHA384_Init(SHA384_CTX*); +void SHA384_Update(SHA384_CTX*, const uint8_t*, size_t); +void SHA384_Final(uint8_t[SHA384_DIGEST_LENGTH], SHA384_CTX*); +char* SHA384_End(SHA384_CTX*, char[SHA384_DIGEST_STRING_LENGTH]); +char* SHA384_Data(const uint8_t*, size_t, char[SHA384_DIGEST_STRING_LENGTH]); + +void SHA512_Init(SHA512_CTX*); +void SHA512_Update(SHA512_CTX*, const uint8_t*, size_t); +void SHA512_Final(uint8_t[SHA512_DIGEST_LENGTH], SHA512_CTX*); +char* SHA512_End(SHA512_CTX*, char[SHA512_DIGEST_STRING_LENGTH]); +char* SHA512_Data(const uint8_t*, size_t, char[SHA512_DIGEST_STRING_LENGTH]); + +#else /* SHA2_USE_INTTYPES_H */ + +void SHA256_Init(SHA256_CTX *); +void SHA256_Update(SHA256_CTX*, const u_int8_t*, size_t); +void SHA256_Final(u_int8_t[SHA256_DIGEST_LENGTH], SHA256_CTX*); +char* SHA256_End(SHA256_CTX*, char[SHA256_DIGEST_STRING_LENGTH]); +char* SHA256_Data(const u_int8_t*, size_t, char[SHA256_DIGEST_STRING_LENGTH]); + +void SHA384_Init(SHA384_CTX*); +void SHA384_Update(SHA384_CTX*, const u_int8_t*, size_t); +void SHA384_Final(u_int8_t[SHA384_DIGEST_LENGTH], SHA384_CTX*); +char* SHA384_End(SHA384_CTX*, char[SHA384_DIGEST_STRING_LENGTH]); +char* SHA384_Data(const u_int8_t*, size_t, char[SHA384_DIGEST_STRING_LENGTH]); + +void SHA512_Init(SHA512_CTX*); +void SHA512_Update(SHA512_CTX*, const u_int8_t*, size_t); +void SHA512_Final(u_int8_t[SHA512_DIGEST_LENGTH], SHA512_CTX*); +char* SHA512_End(SHA512_CTX*, char[SHA512_DIGEST_STRING_LENGTH]); +char* SHA512_Data(const u_int8_t*, size_t, char[SHA512_DIGEST_STRING_LENGTH]); + +#endif /* SHA2_USE_INTTYPES_H */ + +#else /* NOPROTO */ + +void SHA256_Init(); +void SHA256_Update(); +void SHA256_Final(); +char* SHA256_End(); +char* SHA256_Data(); + +void SHA384_Init(); +void SHA384_Update(); +void SHA384_Final(); +char* SHA384_End(); +char* SHA384_Data(); + +void SHA512_Init(); +void SHA512_Update(); +void SHA512_Final(); +char* SHA512_End(); +char* SHA512_Data(); + +#endif /* NOPROTO */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __SHA2_H__ */ + diff --git a/c_src/zlib b/c_src/zlib new file mode 160000 index 0000000..5089329 --- /dev/null +++ b/c_src/zlib @@ -0,0 +1 @@ +Subproject commit 50893291621658f355bc5b4d450a8d06a563053d diff --git a/cp_src/base.cpp b/cp_src/base.cpp new file mode 100644 index 0000000..1a89cab --- /dev/null +++ b/cp_src/base.cpp @@ -0,0 +1,210 @@ +#include "base.h" +#include + +using namespace std; + +unsigned long getUTC() { + time_t t; + time(&t); + return mktime(gmtime(&t)); +} + +unsigned long long unpack_value(string str) { + unsigned long long val = 0; + for (unsigned int i = 0; i < str.length(); i++) { + val = val << 8; + val += (unsigned char)str[i]; + } + return val; +} + +string pack_value(size_t len, unsigned long long i) { + vector arr((size_t)len, 0); + for (size_t j = 0; j < len; j++) { + arr[len - j - 1] = i & 0xff; + i = i >> 8; + if (i == 0) + break; + } + return string(arr.begin(), arr.end()); +} + +protocol::protocol(string sub, string enc) { + CP2P_DEBUG("Defining subnet with length: %i\n", sub.length()) + subnet = sub; + CP2P_DEBUG("Defining encryption with length: %i\n", enc.length()) + encryption = enc; + CP2P_DEBUG("Done defining\n") +} + +protocol::~protocol() {} + +string protocol::id() { + if (cache.subnet == subnet && cache.encryption == encryption && cache.id != "") + return cache.id; + + char buffer[5]; + size_t buff_size = sprintf(buffer, "%llu.%llu", (unsigned long long)CP2P_PROTOCOL_MAJOR_VERSION, (unsigned long long)CP2P_PROTOCOL_MINOR_VERSION); + string info = subnet + encryption + string(buffer, buff_size); + + unsigned char digest[SHA256_DIGEST_LENGTH]; + memset(digest, 0, SHA256_DIGEST_LENGTH); + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, (unsigned char*)info.c_str(), info.length()); + SHA256_Final(digest, &ctx); + + cache.subnet = string(subnet); + cache.encryption = string(encryption); + cache.id = ascii_to_base_58(string((char*)digest, SHA256_DIGEST_LENGTH)); + return cache.id; +} + +pathfinding_message::pathfinding_message(string type, string sen, vector load) { + msg_type = type; + sender = sen; + timestamp = getUTC(); + payload = load; + compression = vector(); + compression_fail = false; +} + +pathfinding_message::pathfinding_message(string type, string sen, vector load, vector comp) { + msg_type = type; + sender = sen; + timestamp = getUTC(); + payload = load; + compression = comp; +} + +vector process_string(string str) { + unsigned long processed = 0; + unsigned long expected = str.length(); + vector pack_lens; + vector packets; + while (processed != expected) { + unsigned long tmp = unpack_value(str.substr(processed, 4)); + pack_lens.push_back(tmp); + processed += 4; + expected -= pack_lens.back(); + } + // Then reconstruct the packets + for (unsigned long i = 0; i < pack_lens.size(); i++) { + packets.push_back(str.substr(processed, pack_lens[i])); + processed += pack_lens[i]; + } + return packets; +} + +string sanitize_string(string str, bool sizeless) { + if (!sizeless) + return str.substr(4); + return str; +} + +string decompress_string(string str, vector compressions) { + return str; +} + +pathfinding_message::~pathfinding_message() {} + +string pathfinding_message::compression_used() { + if (compression.size()) + return compression[0]; + return string(""); +} + +string pathfinding_message::time_58() { + return to_base_58(timestamp); + // size_t i = 0; + // char *temp = to_base_58(timestamp, i); + // return string(temp, i); +} + +string pathfinding_message::id() { + if (cache.timestamp == timestamp && cache.payload == payload && cache.id != "") { + CP2P_DEBUG("Fetching cached ID\n") + return string(cache.id); //for copy constructor + } + + string t58 = time_58(); + size_t done = 0, expected = t58.length(); + + for (unsigned long i = 0; i < payload.size(); i++) + expected += payload[i].length(); + + unsigned char *info = new unsigned char[expected]; + + for (unsigned long i = 0; i < payload.size(); i++) { + memcpy(info + done, payload[i].c_str(), payload[i].length()); + done += payload[i].length(); + } + memcpy(info + done, t58.c_str(), t58.length()); + + unsigned char digest[SHA384_DIGEST_LENGTH]; + memset(digest, 0, SHA384_DIGEST_LENGTH); + SHA384_CTX ctx; + SHA384_Init(&ctx); + SHA384_Update(&ctx, (unsigned char*)info, expected); + SHA384_Final(digest, &ctx); + + cache.payload = vector(payload); + cache.timestamp = timestamp; + cache.id = ascii_to_base_58(string((char*)digest, SHA384_DIGEST_LENGTH)); + +#ifdef CP2P_DEBUG_FLAG + printf("ID for [\""); + for (size_t i = 0; i < expected; i++) { + printf("\\x%02x", info[i]); + } + printf("\"]:\n"); +#endif + CP2P_DEBUG("%s\n", cache.id.c_str()); + + return string(cache.id); //for copy constructor +} + +vector pathfinding_message::packets() { + vector packs; + packs.reserve(4 + payload.size()); + packs.push_back(msg_type); + packs.push_back(sender); + packs.push_back(id()); + packs.push_back(time_58()); + packs.insert(packs.end(), payload.begin(), payload.end()); + return packs; +} + +string pathfinding_message::base_string() { + if (cache.timestamp == timestamp && cache.msg_type == msg_type && cache.payload == payload) + return string(cache.base_string); //for copy constructor + + string header = ""; + string base = ""; + vector packs = packets(); + for (unsigned long i = 0; i < packs.size(); i++) { + header += pack_value(4, (unsigned long long)packs[i].size()); + base += packs[i]; + } + + //cache.timestamp = timestamp; //implied by call to packets, which calls id + //cache.payload = payload; //implied by call to packets, which calls id + cache.msg_type = msg_type; + cache.base_string = header + base; + + return string(cache.base_string); //for copy constructor +} + +string pathfinding_message::str() { + string base = base_string(); + string header = pack_value(4, (unsigned long long)base.length()); + return header + base; +} + +unsigned long long pathfinding_message::length() { + return base_string().length(); +} + +string pathfinding_message::header() { + return pack_value(4, (unsigned long long)length()); +} diff --git a/cp_src/base.h b/cp_src/base.h new file mode 100644 index 0000000..13f9f4f --- /dev/null +++ b/cp_src/base.h @@ -0,0 +1,464 @@ +/** +* Base Module +* =========== +* +* This module contains common functions and classes used throughout the rest of the library +*/ +#ifndef CP2P_PROTOCOL_MAJOR_VERSION +#define CP2P__STR( ARG ) #ARG +#define CP2P__STR__( ARG ) CP2P__STR(ARG) + +#define CP2P_PROTOCOL_MAJOR_VERSION 0 +#define CP2P_PROTOCOL_MINOR_VERSION 4 +#define CP2P_NODE_VERSION 516 +#define CP2P_VERSION CP2P__STR__(CP2P_PROTOCOL_MAJOR_VERSION) "." CP2P__STR__(CP2P_PROTOCOL_MINOR_VERSION) "." CP2P__STR__(CP2P_NODE_VERSION) +/** +* .. c:macro:: CP2P_PROTOCOL_MAJOR_VERSION +* +* This macro defines the major version number. A change here indicates a major change or release, and may be breaking. In a scheme x.y.z, it would be x +* +* .. c:macro:: CP2P_PROTOCOL_MINOR_VERSION +* +* This macro defines the minor version number. It refers specifically to minor protocol revisions, and all changes here are API compatible (after 1.0), but not compatbile with other nodes. In a scheme x.y.z, it would be y +* +* .. c:macro:: CP2P_NODE_VERSION +* +* This macro defines the patch version number. It refers specifically to node policies, and all changes here are backwards compatible. In a scheme x.y.z, it would be z +* +* .. c:macro:: CP2P_VERSION +* +* This macro is a string literal. It combines all the above macros into a single string. It will generate whatever a string literal would normally be interpreted as in that context. +* +* .. c:macro:: CP2P_DEBUG_FLAG +* +* This macro indicates whether cp2p should generate debug prints. If you define this as anything it will print +*/ + +#ifdef CP2P_DEBUG_FLAG + #define CP2P_DEBUG(...) printf(__VA_ARGS__); +#else + #define CP2P_DEBUG(...) +#endif + +//This macro was taken from: +//http://www.pixelbeat.org/programming/gcc/static_assert.html +//under the GNU All-Permissive License, which is included below: +//Copyright © Pádraig Brady 2008 +// +//Copying and distribution of this file, with or without modification, +//are permitted in any medium without royalty provided the copyright +//notice and this notice are preserved. +#define ASSERT_CONCAT_(a, b) a##b +#define ASSERT_CONCAT(a, b) ASSERT_CONCAT_(a, b) +/* These can't be used after statements in c89. */ +#ifdef __COUNTER__ + #define STATIC_ASSERT(e,m) \ + ;enum { ASSERT_CONCAT(static_assert_, __COUNTER__) = 1/(int)(!!(e)) } +#else + /* This can't be used twice on the same line so ensure if using in headers + * that the headers are not included twice (by wrapping in #ifndef...#endif) + * Note it doesn't cause an issue when used on same line of separate modules + * compiled with gcc -combine -fwhole-program. */ + #define STATIC_ASSERT(e,m) \ + ;enum { ASSERT_CONCAT(assert_line_, __LINE__) = 1/(int)(!!(e)) } +#endif +//End macro + +#include +#include +#include +#include +#include +#include +#include +#include +#include "../c_src/sha/sha2.h" +#include "../c_src/BaseConverter.h" + +using namespace std; + +STATIC_ASSERT(sizeof(size_t) >= 4, "Size of strings is too small to easily meet protocol specs"); + +namespace flags { + static const unsigned char\ + *reserved = (unsigned char*)"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F", + *implemented_compressions = (unsigned char*)""; + + static const size_t\ + reserved_len = 0x20, + compression_len = 0x00; + + /** + * .. cpp:var:: static const unsigned char *flags::reserved_cstr + * + * This binary data string contains every reserved flag. + * + * .. note:: + * + * This will be refactored later to an array of :c:type:`unsigned char *` s, but for know just know that all flags are one char long. + * + * .. cpp:var:: static const size_t flags::reserved_len + * + * The length of the above string + * + * .. cpp:var:: static const unsigned char *flags::implemented_compressions_cstr + * + * This binary data string contains the flag of every implemented compression methods. + * + * .. note:: + * + * This will be refactored later to an array of :c:type:`unsigned char *` s, but for know just know that all flags are one char long. + * + * .. cpp:var:: static const size_t flags::compression_len + * + * The length of the above string + * + * .. cpp:var:: static const unsigned char flags::other_flags + * + * These are the flags currently reserved. They are guarunteed to be the same names and values as the flags within :py:class:`py2p.base.flags`. + * + * .. note:: + * + * This will be refactored later to an array of :c:type:`unsigned char *` s, but for know just know that all flags are one char long. + */ + + static const unsigned char\ + broadcast = 0x00, // also sub-flag + waterfall = 0x01, + whisper = 0x02, // also sub-flag + renegotiate = 0x03, + ping = 0x04, // Unused, but reserved + pong = 0x05, // Unused, but reserved + + // sub-flags + //broadcast = 0x00, + compression = 0x01, + //whisper = 0x02, + handshake = 0x03, + //ping = 0x04, + //pong = 0x05, + notify = 0x06, + peers = 0x07, + request = 0x08, + resend = 0x09, + response = 0x0A, + store = 0x0B, + retrieve = 0x0C, + + // implemented compression methods + gzip = 0x11, + zlib = 0x13, + + // non-implemented compression methods (based on list from compressjs): + bwtc = 0x14, + bz2 = 0x10, + context1= 0x15, + defsum = 0x16, + dmc = 0x17, + fenwick = 0x18, + huffman = 0x19, + lzjb = 0x1A, + lzjbr = 0x1B, + lzma = 0x12, + lzp3 = 0x1C, + mtf = 0x1D, + ppmd = 0x1E, + simple = 0x1F; +} + +static string get_user_salt() { + /** + * .. cpp:function:: static std::string get_user_salt() + * + * This generates a uuid4 for use in this library + */ + srand (time(NULL)); + CP2P_DEBUG("Building user_salt\n"); + char temp_user_salt[] = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx"; + char *temp_hex_set = (char*)"0123456789abcdef"; + for (size_t i = 0; i < 36; i++) { + if (temp_user_salt[i] == 'x') + temp_user_salt[i] = temp_hex_set[(rand() % 16)]; + else if (temp_user_salt[i] == 'y') + temp_user_salt[i] = temp_hex_set[((rand() % 16) & 0x3) | 0x8]; + } + + const string user_salt = string(temp_user_salt, 36); + + return user_salt; +} + +/** +* .. cpp:var:: const static std::string user_salt +* +* A generated uuid4 for use in this library +*/ +const static string user_salt = get_user_salt(); + +unsigned long getUTC(); +/** +* .. cpp:function:: unsigned long getUTC() +* +* Returns the current UNIX second in UTC +*/ +unsigned long long unpack_value(string str); +/** +* .. cpp:function:: unsigned long long unpack_value(std::string str) +* +* Unpacks a big-endian binary value into an unsigned long long +* +* :param str: The value you'd like to unpack +* +* :returns: The value this string contained +* +* .. warning:: +* +* Integer overflow will not be accounted for +*/ +string pack_value(size_t len, unsigned long long i); +/** +* .. cpp:function:: std::string pack_value(size_t len, unsigned long long i) +* +* Packs an unsigned long long into a big-endian binary string of length len +* +* :param len: The length of the string you'd like to produce +* :param i: The value you'd like to pack +* +* :returns: A :cpp:class:`std::string` packed with the equivalent big-endian data +* +* .. warning:: +* +* Integer overflow will not be accounted for +*/ +string sanitize_string(string str, bool sizeless); +/** +* .. cpp:function:: std::string sanitize_string(std::string str, bool sizeless) +* +* This function takes in a string and removes metadata that the :cpp:class:`pathfinding_message` deserializer can't handle +* +* :param str: The string you would like to sanitize +* :param sizeless: A bool indicating if this string has a size header attached +* +* :returns: A :cpp:class:`std::string` which has the safe version of ``str`` +*/ +string decompress_string(string str, vector compressions); +/** +* .. cpp:function:: std::string decompress_string(std::string str, std::vector compressions) +* +* This function is currently an identity function which returns ``str``. In the future this function will +* decompress strings for the :cpp:class:`pathfinding_message` parser to deal with. +* +* :param str: The string you would like to decompress +* :param compressions: A :cpp:class:`std::vector\` which contains the list of possible compression methods +* +* :returns: A :cpp:class:`std::string` which has the decompressed version of ``str`` +*/ +vector process_string(string str); +/** +* .. cpp:function:: std::vector process_string(std::string str) +* +* This deserializes a :cpp:class:`pathfinding_message` string into a :cpp:class:`std::vector\` of packets +* +* :param str: The :cpp:class`std::string` you would like to parse +* +* :returns: A :cpp:class:`std::vector\` which contains the packets serialized in this string +*/ + +class protocol { + /** + * .. cpp:class:: protocol + * + * This class is used as a subnet object. Its role is to reject undesired connections. + * If you connect to someone who has a different protocol object than you, this descrepency is detected, + * and you are silently disconnected. + */ + public: + protocol(string subnet, string encryption); + /** + * .. cpp:function:: protocol::protocol(std::string, std::string encryption) + * + * :param subnet: The subnet you'd like to use + * :param encryption: The encryption method you'd like to use + */ + ~protocol(); + /** + * .. cpp:function:: protocol::~protocol() + * + * An empty deconstructor + */ + string id(); + /** + * .. cpp:function:: std::string protocol::id() + * + * :returns: A :cpp:class:`std::string` which contains the base_58 encoded, SHA256 based ID of this protocol object + */ + string subnet, encryption; + /** + * .. cpp:var:: std::string protocol::subnet + * + * .. cpp:var:: std::string protocol::encryption + */ + private: + struct { + string id, subnet, encryption; + } cache; +}; + +class pathfinding_message { + /** + * .. cpp:class:: pathfinding_message + * + * This is the message serialization/deserialization class. + */ + public: + pathfinding_message(string msg_type, string sender, vector payload); + pathfinding_message(string msg_type, string sender, vector payload, vector compressions); + /** + * .. cpp:function:: pathfinding_message::pathfinding_message(std::string msg_type, std::string sender, std::vector payload) + * + * .. cpp:function:: pathfinding_message::pathfinding_message(std::string msg_type, std::string sender, std::vector payload, std::vector compressions) + * + * + * :param msg_type: This is the main flag checked by nodes, used for routing information + * :param sender: The ID of the person sending the message + * :param payload: A :cpp:class:`std::vector\` of "packets" that you want your peers to receive + * :param compression: A :cpp:class:`std::vector\` of compression methods that the receiver supports + */ + + static pathfinding_message *feed_string(string msg) { +#ifdef CP2P_DEBUG_FLAG + printf("String fed: \""); + for (size_t i = 0; i < msg.length(); i++) { + printf("\\x%02x", msg[i]); + } + printf("\":\n"); +#endif + vector packets = process_string(msg); + pathfinding_message *pm = new pathfinding_message( + packets[0], + packets[1], + vector(packets.begin() + 4, packets.end())); + CP2P_DEBUG("Setting timestamp as %s (%i)", packets[3].c_str(), from_base_58(packets[3].c_str(), packets[3].length())) + pm->timestamp = from_base_58(packets[3].c_str(), packets[3].length()); + return pm; + } + + static pathfinding_message *feed_string(string msg, bool sizeless) { + return pathfinding_message::feed_string( + sanitize_string(msg, sizeless)); + } + + static pathfinding_message *feed_string(string msg, vector compressions) { + return pathfinding_message::feed_string( + decompress_string(msg, compressions)); + }; + + static pathfinding_message *feed_string(string msg, bool sizeless, vector compressions) { + return pathfinding_message::feed_string( + sanitize_string(msg, sizeless), + compressions); + }; + /** + * .. cpp:function:: static pathfinding_message *pathfinding_message::feed_string(std::string msg) + * + * .. cpp:function:: static pathfinding_message *pathfinding_message::feed_string(std::string msg, bool sizeless) + * + * .. cpp:function:: static pathfinding_message *pathfinding_message::feed_string(std::string msg, std::vector compressions) + * + * .. cpp:function:: static pathfinding_message *pathfinding_message::feed_string(std::string msg, bool sizeless, std::vector compressions) + * + * :param msg: A :cpp:class:`std::string` which contains the serialized message + * :param sizeless: A :c:type:`bool` which indicates if the message has a size header attached (default: it does) + * :param compressions: A :cpp:class:`std::vector\` which contains the possible compression methods this message may be using + * + * :returns: A pointer to the deserialized message + */ + ~pathfinding_message(); + /** + * .. cpp:function:: pathfinding_message::~pathfinding_message() + * + * An empty deconstructor + */ + string msg_type, sender; + unsigned long timestamp; + vector payload; + vector compression; + bool compression_fail; + /** + * .. cpp:var:: std::string pathfinding_message::msg_type + * + * .. cpp:var:: std::string pathfinding_message::sender + * + * .. cpp:var:: unsigned long pathfinding_message::timestamp + * + * .. cpp:var:: std::vector pathfinding_message::payload + * + * .. cpp:var:: std::vector pathfinding_message::compression + * + * .. cpp:var:: bool pathfinding_message::compression_fail + */ + string compression_used(); + /** + * .. cpp:function:: std::string pathfinding_message::compression_used() + * + * :returns: The compression method this message was sent under + */ + string time_58(); + /** + * .. cpp:function:: std::string pathfinding_message::time_58() + * + * :returns: :cpp:var:`pathfinding_message::timestamp` encoded in base_58 + */ + string id(); + /** + * .. cpp:function:: std::string pathfinding_message::id() + * + * :returns: A SHA384 hash of this message encoded in base_58 + */ + vector packets(); + /** + * .. cpp:function:: std::vector pathfinding_message::packets() + * + * A copy of :cpp:var:`pathfinding_message::payload` with some additional metadata appended to the front. Specifically: + * + * 0. :cpp:var:`pathfinding_message::msg_type` + * #. :cpp:var:`pathfinding_message::sender` + * #. :cpp:func:`pathfinding_message::id()` + * #. :cpp:func:`pathfinding_message::time_58()` + * #. :cpp:var:`pathfinding_message::payload` from here on out + * + * :returns: A :cpp:class:`std::vector\` in the above format + */ + string base_string(); + /** + * .. cpp:function:: std::string pathfinding_message::base_string() + * + * :returns: the serialized message, excepting the four byte size header at the beginning + */ + string str(); + /** + * .. cpp:function:: std::string pathfinding_message::str() + * + * :returns: the serialized message, including the four byte size header at the beginning + */ + unsigned long long length(); + /** + * .. cpp:function:: unsigned long long pathfinding_message::length() + * + * :returns: the length of the serialized message, excepting the four byte size header at the beginning + */ + string header(); + /** + * .. cpp:function:: std::string pathfinding_message::header() + * + * :returns: the four byte size header at the beginning of the serialized message + */ + private: + struct { + string msg_type, sender, id, base_string; + unsigned long timestamp; + vector payload; + } cache; +}; + +#endif diff --git a/cp_src/base_wrapper.cpp b/cp_src/base_wrapper.cpp new file mode 100644 index 0000000..cba9b80 --- /dev/null +++ b/cp_src/base_wrapper.cpp @@ -0,0 +1,126 @@ +/** +* Base Python Wrapper +* =================== +*/ +#include +#include +#include "structmember.h" +#include "base.h" +#include +#include "py_utils.h" +#include "protocol_wrapper.h" +#include "pathfinding_message_wrapper.h" +#include "flags_wrapper.h" + +using namespace std; + +static PyMethodDef BaseMethods[] = { + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +#if PY_MAJOR_VERSION >= 3 + + static struct PyModuleDef basemodule = { + PyModuleDef_HEAD_INIT, + "cbase", /* name of module */ + "A C++ implementation of select features from the py2p.base module",/* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + BaseMethods + }; + + PyMODINIT_FUNC PyInit_cbase() { + PyObject *cbase, *flags_wrapper; + + pmessage_wrapper_type.tp_new = PyType_GenericNew; + if (PyType_Ready(&pmessage_wrapper_type) < 0) + return NULL; + + protocol_wrapper_type.tp_new = PyType_GenericNew; + if (PyType_Ready(&protocol_wrapper_type) < 0) + return NULL; + + cbase = PyModule_Create(&basemodule); + if (cbase == NULL) + return NULL; + + flags_wrapper = PyModule_Create(&flagsmodule); + if (flags_wrapper == NULL) + return NULL; + + Py_INCREF(&protocol_wrapper_type); + PyModule_AddObject(cbase, "protocol", (PyObject *)&protocol_wrapper_type); + + Py_INCREF(&pmessage_wrapper_type); + PyModule_AddObject(cbase, "pathfinding_message", (PyObject *)&pmessage_wrapper_type); + + addConstants(cbase, flags_wrapper); + + return cbase; + } + + int main(int argc, char *argv[]) { + #if PY_MINOR_VERSION >= 5 + wchar_t *program = Py_DecodeLocale(argv[0], NULL); + #else + size_t size = strlen(argv[0]) + 1; + wchar_t* program = new wchar_t[size]; + mbstowcs(program, argv[0], size); + #endif + + if (program == NULL) { + fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); + exit(1); + } + + PyImport_AppendInittab("cbase", PyInit_cbase); + Py_SetProgramName(program); + Py_Initialize(); + PyImport_ImportModule("cbase"); + #if PY_MINOR_VERSION >= 5 + PyMem_RawFree(program); + #else + delete[] program; + #endif + return 0; + } + +#else + + + #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ + #define PyMODINIT_FUNC void + #endif + PyMODINIT_FUNC initcbase() { + PyObject *cbase, *flags_wrapper; + + pmessage_wrapper_type.tp_new = PyType_GenericNew; + if (PyType_Ready(&pmessage_wrapper_type) < 0) + return; + + protocol_wrapper_type.tp_new = PyType_GenericNew; + if (PyType_Ready(&protocol_wrapper_type) < 0) + return; + + cbase = Py_InitModule3("cbase", BaseMethods, + "C++ implementation of some base functions"); + flags_wrapper = Py_InitModule3("flags", FlagsMethods, + "Storage container for protocol level flags"); + + Py_INCREF(&pmessage_wrapper_type); + PyModule_AddObject(cbase, "pathfinding_message", (PyObject *)&pmessage_wrapper_type); + + Py_INCREF(&protocol_wrapper_type); + PyModule_AddObject(cbase, "protocol", (PyObject *)&protocol_wrapper_type); + + addConstants(cbase, flags_wrapper); + } + + int main(int argc, char *argv[]) { + Py_SetProgramName(argv[0]); + Py_Initialize(); + initcbase(); + return 0; + } + +#endif \ No newline at end of file diff --git a/cp_src/flags_wrapper.h b/cp_src/flags_wrapper.h new file mode 100644 index 0000000..db81cda --- /dev/null +++ b/cp_src/flags_wrapper.h @@ -0,0 +1,92 @@ +#ifndef CP2P_FLAGS_WRAPPER +#define CP2P_FLAGS_WRAPPER TRUE + +#include +#include +#include "structmember.h" +#include "base.h" +#include +#include "py_utils.h" + +using namespace std; + +static PyMethodDef FlagsMethods[] = { + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static void addConstants(PyObject *cbase, PyObject *flags_wrapper) { + vector compression; + for (unsigned int i = 0; i < flags::compression_len; i++) + compression.push_back(string((size_t)1, flags::implemented_compressions[i])); + PyModule_AddObject(cbase, "compression", pylist_from_vector_string(compression)); + PyModule_AddObject(cbase, "version", pybytes_from_string(string(CP2P_VERSION))); + PyModule_AddObject(cbase, "user_salt", pybytes_from_string(user_salt)); + + // Add reserved flags + vector reserved_set; + for (unsigned int i = 0; i < flags::reserved_len; i++) + reserved_set.push_back(string((size_t)1, flags::reserved[i])); + PyModule_AddObject(flags_wrapper, "reserved", pylist_from_vector_string(reserved_set)); + + // Main flags + PyModule_AddObject(flags_wrapper, "broadcast", pybytes_from_string(string((size_t) 1, flags::broadcast))); + PyModule_AddObject(flags_wrapper, "waterfall", pybytes_from_string(string((size_t) 1, flags::waterfall))); + PyModule_AddObject(flags_wrapper, "whisper", pybytes_from_string(string((size_t) 1, flags::whisper))); + PyModule_AddObject(flags_wrapper, "renegotiate", pybytes_from_string(string((size_t) 1, flags::renegotiate))); + PyModule_AddObject(flags_wrapper, "ping", pybytes_from_string(string((size_t) 1, flags::ping))); + PyModule_AddObject(flags_wrapper, "pong", pybytes_from_string(string((size_t) 1, flags::pong))); + + // Sub-flags + /*PyModule_AddObject(flags_wrapper, "broadcast", pybytes_from_string(string((size_t) 1, flags::broadcast)));*/ + PyModule_AddObject(flags_wrapper, "compression", pybytes_from_string(string((size_t) 1, flags::compression))); + /*PyModule_AddObject(flags_wrapper, "whisper", pybytes_from_string(string((size_t) 1, flags::whisper)));*/ + PyModule_AddObject(flags_wrapper, "handshake", pybytes_from_string(string((size_t) 1, flags::handshake))); + /*PyModule_AddObject(flags_wrapper, "ping", pybytes_from_string(string((size_t) 1, flags::ping)));*/ + /*PyModule_AddObject(flags_wrapper, "pong", pybytes_from_string(string((size_t) 1, flags::pong)));*/ + PyModule_AddObject(flags_wrapper, "notify", pybytes_from_string(string((size_t) 1, flags::notify))); + PyModule_AddObject(flags_wrapper, "peers", pybytes_from_string(string((size_t) 1, flags::peers))); + PyModule_AddObject(flags_wrapper, "request", pybytes_from_string(string((size_t) 1, flags::request))); + PyModule_AddObject(flags_wrapper, "resend", pybytes_from_string(string((size_t) 1, flags::resend))); + PyModule_AddObject(flags_wrapper, "response", pybytes_from_string(string((size_t) 1, flags::response))); + PyModule_AddObject(flags_wrapper, "store", pybytes_from_string(string((size_t) 1, flags::store))); + PyModule_AddObject(flags_wrapper, "retrieve", pybytes_from_string(string((size_t) 1, flags::retrieve))); + + // Implemented compression methods + PyModule_AddObject(flags_wrapper, "gzip", pybytes_from_string(string((size_t) 1, flags::gzip))); + PyModule_AddObject(flags_wrapper, "zlib", pybytes_from_string(string((size_t) 1, flags::zlib))); + + // non-implemented compression methods (based on list from compressjs): + PyModule_AddObject(flags_wrapper, "bwtc", pybytes_from_string(string((size_t) 1, flags::bwtc))); + PyModule_AddObject(flags_wrapper, "bz2", pybytes_from_string(string((size_t) 1, flags::bz2))); + PyModule_AddObject(flags_wrapper, "context1", pybytes_from_string(string((size_t) 1, flags::context1))); + PyModule_AddObject(flags_wrapper, "defsum", pybytes_from_string(string((size_t) 1, flags::defsum))); + PyModule_AddObject(flags_wrapper, "dmc", pybytes_from_string(string((size_t) 1, flags::dmc))); + PyModule_AddObject(flags_wrapper, "fenwick", pybytes_from_string(string((size_t) 1, flags::fenwick))); + PyModule_AddObject(flags_wrapper, "huffman", pybytes_from_string(string((size_t) 1, flags::huffman))); + PyModule_AddObject(flags_wrapper, "lzjb", pybytes_from_string(string((size_t) 1, flags::lzjb))); + PyModule_AddObject(flags_wrapper, "lzjbr", pybytes_from_string(string((size_t) 1, flags::lzjbr))); + PyModule_AddObject(flags_wrapper, "lzma", pybytes_from_string(string((size_t) 1, flags::lzma))); + PyModule_AddObject(flags_wrapper, "lzp3", pybytes_from_string(string((size_t) 1, flags::lzp3))); + PyModule_AddObject(flags_wrapper, "mtf", pybytes_from_string(string((size_t) 1, flags::mtf))); + PyModule_AddObject(flags_wrapper, "ppmd", pybytes_from_string(string((size_t) 1, flags::ppmd))); + PyModule_AddObject(flags_wrapper, "simple", pybytes_from_string(string((size_t) 1, flags::simple))); + + + PyObject *cbase_dict = PyModule_GetDict(cbase); + PyDict_SetItemString(cbase_dict, "flags", flags_wrapper); +} + + #if PY_MAJOR_VERSION >= 3 + +static struct PyModuleDef flagsmodule = { + PyModuleDef_HEAD_INIT, + "flags", /* name of module */ + "Storage container for protocol level flags",/* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + FlagsMethods +}; + + #endif + +#endif \ No newline at end of file diff --git a/cp_src/pathfinding_message_wrapper.h b/cp_src/pathfinding_message_wrapper.h new file mode 100644 index 0000000..0fb083b --- /dev/null +++ b/cp_src/pathfinding_message_wrapper.h @@ -0,0 +1,329 @@ +#ifndef CP2P_PMESSAGE_WRAPPER +#define CP2P_PMESSAGE_WRAPPER TRUE + +#include +#include +#include "structmember.h" +#include "base.h" +#include +#include "py_utils.h" + +using namespace std; + +typedef struct { + PyObject_HEAD + pathfinding_message *msg; + /* Type-specific fields go here. */ +} pmessage_wrapper; + +static void pmessage_wrapper_dealloc(pmessage_wrapper* self) { + delete self->msg; + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject *pmessage_wrapper_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + pmessage_wrapper *self; + + self = (pmessage_wrapper *)type->tp_alloc(type, 0); + + return (PyObject *)self; +} + +static int pmessage_wrapper_init(pmessage_wrapper *self, PyObject *args, PyObject *kwds) { + string msg_type, sender; + PyObject *py_msg=NULL, *py_sender=NULL, *payload=NULL, *compression=NULL; + + static char *kwlist[] = {(char*)"msg_type", (char*)"sender", (char*)"payload", (char*)"compressions", NULL}; + + if (! PyArg_ParseTupleAndKeywords(args, kwds, "OOO|O", kwlist, + &py_msg, &py_sender, + &payload, &compression)) + return -1; + + CP2P_DEBUG("Parsing msg_type\n") + msg_type = string_from_pybytes(py_msg); + if (PyErr_Occurred()) + return -1; + + CP2P_DEBUG("Parsing sender\n") + sender = string_from_pybytes(py_sender); + if (PyErr_Occurred()) + return -1; + + CP2P_DEBUG("Parsing payload\n") + vector load = vector_string_from_pylist(payload); + if (PyErr_Occurred()) + return -1; + + CP2P_DEBUG("Parsing compression list\n") + if (compression) { + vector comp = vector_string_from_pylist(compression); + if (PyErr_Occurred()) + return -1; + self->msg = new pathfinding_message(msg_type, sender, load, comp); + } + else { + self->msg = new pathfinding_message(msg_type, sender, load); + } + + CP2P_DEBUG("Returning\n") + return 0; +} + +static pmessage_wrapper *pmessage_feed_string(PyTypeObject *type, PyObject *args, PyObject *kwds) { + string str; + int sizeless = 0; + PyObject *py_compression=NULL, *py_str=NULL; + vector compression; + + static char *kwlist[] = {(char*)"string", (char*)"sizeless", (char*)"compressions", NULL}; + + if (! PyArg_ParseTupleAndKeywords(args, kwds, "O|pO", kwlist, + &py_str, &sizeless, &py_compression)) + return NULL; + + str = string_from_pybytes(py_str); + if (PyErr_Occurred()) + return NULL; + + pmessage_wrapper *ret = (pmessage_wrapper *)type->tp_alloc(type, 0); + + if (ret != NULL) { + if (py_compression) + compression = vector_string_from_pylist(py_compression); + if (PyErr_Occurred()) + return NULL; + + if (sizeless && py_compression) + ret->msg = pathfinding_message::feed_string(str, sizeless, compression); + else if (py_compression) + ret->msg = pathfinding_message::feed_string(str, compression); + else if (sizeless) + ret->msg = pathfinding_message::feed_string(str, sizeless); + else + ret->msg = pathfinding_message::feed_string(str.substr(4)); + } + + if (PyErr_Occurred()) + return NULL; + + return ret; +} + +static PyObject *pmessage_payload(pmessage_wrapper *self) { + PyObject *ret = pylist_from_vector_string(self->msg->payload); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_packets(pmessage_wrapper *self) { + PyObject *ret = pylist_from_vector_string(self->msg->packets()); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_str(pmessage_wrapper *self) { + string cp_str = self->msg->str(); + PyObject *ret = pybytes_from_string(cp_str); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_sender(pmessage_wrapper *self) { + string cp_str = self->msg->sender; + PyObject *ret = pybytes_from_string(cp_str); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_msg_type(pmessage_wrapper *self) { + string cp_str = self->msg->msg_type; + PyObject *ret = pybytes_from_string(cp_str); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_id(pmessage_wrapper *self) { + string cp_str = self->msg->id(); + CP2P_DEBUG("I got the id\n"); + PyObject *ret = pybytes_from_string(cp_str); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_timestamp_58(pmessage_wrapper *self) { + string cp_str = self->msg->time_58(); + PyObject *ret = pybytes_from_string(cp_str); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_timestamp(pmessage_wrapper *self) { + PyObject *ret = PyLong_FromUnsignedLong(self->msg->timestamp); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_compression_used(pmessage_wrapper *self) { + string cp_str = self->msg->compression_used(); + if (cp_str == string("")) + Py_RETURN_NONE; + + PyObject *ret = pybytes_from_string(cp_str); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyObject *pmessage_compression_get(pmessage_wrapper *self) { + PyObject *ret = pylist_from_vector_string(self->msg->compression); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static int pmessage_compression_set(pmessage_wrapper *self, PyObject *value, void *closure) { + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, "Cannot delete compression attribute"); + return -1; + } + + vector new_compression = vector_string_from_pylist(value); + if (PyErr_Occurred()) + return -1; + + self->msg->compression = new_compression; + return 0; +} + +static unsigned long long pmessage__len__(pmessage_wrapper *self) { + return self->msg->length(); +} + +static PyMemberDef pmessage_wrapper_members[] = { + {NULL} /* Sentinel */ +}; + +static PyGetSetDef pmessage_wrapper_getsets[] = { + {(char*)"payload", (getter)pmessage_payload, NULL, + (char*)"Return the payload of this message" + }, + {(char*)"packets", (getter)pmessage_packets, NULL, + (char*)"Return the packets of this message" + }, + {(char*)"string", (getter)pmessage_str, NULL, + (char*)"Return the string of this message" + }, + {(char*)"sender", (getter)pmessage_sender, NULL, + (char*)"Return the sender ID of this message" + }, + {(char*)"msg_type", (getter)pmessage_msg_type, NULL, + (char*)"Return the message type" + }, + {(char*)"time", (getter)pmessage_timestamp, NULL, + (char*)"Return the message time" + }, + {(char*)"time_58", (getter)pmessage_timestamp_58, NULL, + (char*)"Return the message encoded in base_58" + }, + {(char*)"id", (getter)pmessage_id, NULL, + (char*)"Return the message ID" + }, + {(char*)"compression_used", (getter)pmessage_compression_used, NULL, + (char*)"Return the compression method used, or None if there is none"}, + {(char*)"compression", (getter)pmessage_compression_get, (setter)pmessage_compression_set, + (char*)"A list of the compression methods available for use"}, + {NULL} /* Sentinel */ +}; + + +static PyMethodDef pmessage_wrapper_methods[] = { + {"feed_string", (PyCFunction)pmessage_feed_string, METH_CLASS | METH_KEYWORDS | METH_VARARGS, + "Constructs a pathfinding_message from a string or bytes object.\n\ +\n\ +Args:\n\ + string: The string you wish to parse\n\ + sizeless: A boolean which describes whether this string has its size header (default: it does)\n\ + compressions: A list containing the standardized compression methods this message might be under (default: [])\n\ +\n\ +Returns:\n\ + A cbase.pathfinding_message from the given string\n\ +\n\ +Raises:\n\ + TypeError: Fed a non-string, non-bytes argument\n\ + AssertionError: Initial size header is incorrect\n\ + Exception: Unrecognized compression method fed in compressions\n\ + struct.error: Packet headers are incorrect OR unrecognized compression\n\ + IndexError: See struct.error\n\ +\n\ +Note:\n\ + If you feed a unicode object, it will be decoded using utf-8. All other objects are\n\ + treated as raw_unicode_escape. If you desire a particular codec, encode it yourself\n\ + before feeding it in.\n\ +\n\ +Warning:\n\ + This part is a work in progress. Currently errors often cause segfaults, and the above Exceptions are not consistently raised.\n"}, + {NULL} /* Sentinel */ +}; + +static PySequenceMethods pmessage_as_sequence = { + (lenfunc)pmessage__len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0 /*sq_contains*/ +}; + +static PyTypeObject pmessage_wrapper_type = { + PyVarObject_HEAD_INIT(NULL, 0) + "pathfinding_message",/* tp_name */ + sizeof(pmessage_wrapper), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)pmessage_wrapper_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &pmessage_as_sequence, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,/* tp_flags */ + "C++ implementation of the pathfinding_message object",/* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + pmessage_wrapper_methods, /* tp_methods */ + pmessage_wrapper_members, /* tp_members */ + pmessage_wrapper_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)pmessage_wrapper_init,/* tp_init */ + 0, /* tp_alloc */ + pmessage_wrapper_new, /* tp_new */ +}; + +#endif \ No newline at end of file diff --git a/cp_src/protocol_wrapper.h b/cp_src/protocol_wrapper.h new file mode 100644 index 0000000..82889ed --- /dev/null +++ b/cp_src/protocol_wrapper.h @@ -0,0 +1,149 @@ +#ifndef CP2P_PROTOCOL_TYPE +#define CP2P_PROTOCOL_TYPE TRUE + +#include +#include +#include "structmember.h" +#include "base.h" +#include +#include "py_utils.h" + +using namespace std; + +typedef struct { + PyObject_HEAD + protocol *prot; + char *subnet; + char *encryption; +} protocol_wrapper; + +static void protocol_wrapper_dealloc(protocol_wrapper* self) { + delete self->prot; + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject *protocol_wrapper_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + protocol_wrapper *self; + + self = (protocol_wrapper *)type->tp_alloc(type, 0); + + return (PyObject *)self; +} + +static int protocol_wrapper_init(protocol_wrapper *self, PyObject *args, PyObject *kwds) { + const char *sub=NULL, *enc=NULL; + int sub_size = 0, enc_size = 0; + + static char *kwlist[] = {(char*)"subnet", (char*)"encryption", NULL}; + + if (! PyArg_ParseTupleAndKeywords(args, kwds, "s#s#", kwlist, + &sub, &sub_size, &enc, &enc_size)) + return -1; + + CP2P_DEBUG("Building subnet flag\n") + string sub_str = string(sub, sub_size); + + CP2P_DEBUG("Building encryption flag\n") + string enc_str = string(enc, enc_size); + + CP2P_DEBUG("Building protocol\n") + self->prot = new protocol(sub_str, enc_str); + CP2P_DEBUG("Adding subnet shortcut\n") + self->subnet = (char*) self->prot->subnet.c_str(); + CP2P_DEBUG("Adding encryption shortcut\n") + self->encryption = (char*) self->prot->encryption.c_str(); + + return 0; +} + +static PyObject *protocol_id(protocol_wrapper *self) { + string cp_str = self->prot->id(); + PyObject *ret = pybytes_from_string(cp_str); + if (PyErr_Occurred()) + return NULL; + return ret; +} + +static PyMemberDef protocol_wrapper_members[] = { + {(char*)"subnet", T_STRING, + offsetof(protocol_wrapper, subnet), + READONLY, (char*)"subnet"}, + {(char*)"encryption", T_STRING, + offsetof(protocol_wrapper, encryption), + READONLY, (char*)"encryption"}, + {NULL} /* Sentinel */ +}; + +static PyGetSetDef protocol_wrapper_getsets[] = { + {(char*)"id", (getter)protocol_id, NULL, + (char*)"Return the message ID" + }, + {NULL} /* Sentinel */ +}; + +static PyObject *protocol_getitem(protocol_wrapper *self, Py_ssize_t index) { + if (index == 0 || index == -2) + return Py_BuildValue("s#", self->subnet, self->prot->subnet.length()); + else if (index == 1 || index == -1) + return Py_BuildValue("s#", self->encryption, self->prot->encryption.length()); + + PyErr_SetString(PyExc_IndexError, "tuple index out of range"); + return NULL; +} + +static unsigned short protocol__len__(protocol_wrapper *self) { + return 2; +} + +static PySequenceMethods protocol_wrapper_sequence = { + (lenfunc)protocol__len__, /* __len__ */ + 0, /* __add__ */ + 0, /* __mul__ */ + (ssizeargfunc)protocol_getitem, /* __getitem__ */ + 0, /* __getslice__ */ + 0, /* __setitem__ */ + 0 /* __setslice__ */ +}; + +static PyTypeObject protocol_wrapper_type = { + PyVarObject_HEAD_INIT(NULL, 0) + "protocol", /* tp_name */ + sizeof(protocol_wrapper), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)protocol_wrapper_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &protocol_wrapper_sequence,/* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "C++ implementation of the protocol object",/* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + protocol_wrapper_members, /* tp_members */ + protocol_wrapper_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)protocol_wrapper_init,/* tp_init */ + 0, /* tp_alloc */ + protocol_wrapper_new, /* tp_new */ +}; + +#endif \ No newline at end of file diff --git a/cp_src/py_utils.h b/cp_src/py_utils.h new file mode 100644 index 0000000..e54ecea --- /dev/null +++ b/cp_src/py_utils.h @@ -0,0 +1,117 @@ +#ifndef CP2P_PY_UTILS +#define CP2P_PY_UTILS TRUE + +#include +#include +#include +#include + +using namespace std; + +static PyObject *pybytes_from_string(string str) { + unsigned char* c_str = (unsigned char*)str.c_str(); + Py_buffer buffer; + int res = PyBuffer_FillInfo(&buffer, 0, c_str, (Py_ssize_t)str.length(), true, PyBUF_CONTIG_RO); + if (res == -1) { + PyErr_SetString(PyExc_RuntimeError, (char*)"Could not reconvert item back to python object"); + return NULL; + } +#if PY_MAJOR_VERSION >= 3 + PyObject *memview = PyMemoryView_FromBuffer(&buffer); + PyObject *ret = PyBytes_FromObject(memview); + Py_XDECREF(memview); +#elif PY_MINOR_VERSION >= 7 + PyObject *memview = PyMemoryView_FromBuffer(&buffer); + PyObject *ret = PyObject_CallMethod(memview, (char*)"tobytes", (char*)""); + Py_XDECREF(memview); +#else + PyObject *ret = PyString_Encode((char*)c_str, (Py_ssize_t)str.length(), (char*)"raw_unicode_escape", (char*)"strict"); +#endif + return ret; +} + +static string string_from_pybytes(PyObject *bytes) { + if (PyBytes_Check(bytes)) { + CP2P_DEBUG("Decoding as bytes\n") + char *buff = NULL; + Py_ssize_t len = 0; + PyBytes_AsStringAndSize(bytes, &buff, &len); + return string(buff, len); + } +#if PY_MAJOR_VERSION >= 3 + else if (PyObject_CheckBuffer(bytes)) { + CP2P_DEBUG("Decoding as buffer\n") + PyObject *tmp = PyBytes_FromObject(bytes); + string ret = string_from_pybytes(tmp); + Py_XDECREF(tmp); + return ret; + } +#else + else if (PyByteArray_Check(bytes)) { + CP2P_DEBUG("Decoding as bytearray\n") + char *buff = PyByteArray_AS_STRING(bytes); + Py_ssize_t len = PyByteArray_GET_SIZE(bytes); + return string(buff, len); + } +#endif + else if (PyUnicode_Check(bytes)) { + CP2P_DEBUG("Decoding as unicode (incoming recursion)\n") + PyObject *tmp = PyUnicode_AsEncodedString(bytes, (char*)"utf-8", (char*)"strict"); + string ret = string_from_pybytes(tmp); + Py_XDECREF(tmp); + return ret; + } + else { + PyErr_SetObject(PyExc_TypeError, bytes); + return string(); + } +} + +static vector vector_string_from_pylist(PyObject *incoming) { + vector out; + if (PyList_Check(incoming)) { + for(Py_ssize_t i = 0; i < PyList_Size(incoming); i++) { + PyObject *value = PyList_GetItem(incoming, i); + out.push_back(string_from_pybytes(value)); + if (PyErr_Occurred()) + return out; + } + } + else if (PyTuple_Check(incoming)) { + for(Py_ssize_t i = 0; i < PyTuple_Size(incoming); i++) { + PyObject *value = PyTuple_GetItem(incoming, i); + out.push_back(string_from_pybytes(value)); + if (PyErr_Occurred()) + return out; + } + } + else { + PyObject *iter = PyObject_GetIter(incoming); + if (PyErr_Occurred()) + PyErr_SetObject(PyExc_TypeError, incoming); + else { + PyObject *item; + while ((item = PyIter_Next(iter)) != NULL) { + out.push_back(string_from_pybytes(item)); + Py_DECREF(item); + if (PyErr_Occurred()) { + Py_DECREF(iter); + return out; + } + } + Py_DECREF(iter); + } + } + return out; +} + +static PyObject *pylist_from_vector_string(vector lst) { + PyObject *listObj = PyList_New( lst.size() ); + if (!listObj) throw logic_error("Unable to allocate memory for Python list"); + for (unsigned int i = 0; i < lst.size(); i++) { + PyList_SET_ITEM(listObj, i, pybytes_from_string(lst[i])); + } + return listObj; +} + +#endif \ No newline at end of file diff --git a/docs/.static/code_wrap.css b/docs/.static/code_wrap.css new file mode 100644 index 0000000..67f59fc --- /dev/null +++ b/docs/.static/code_wrap.css @@ -0,0 +1,4 @@ +.rst-content code { + word-wrap: break-word; + white-space: pre-wrap; +} \ No newline at end of file diff --git a/docs/CONTRIBUTING_LINK.rst b/docs/CONTRIBUTING_LINK.rst new file mode 100644 index 0000000..3bdd7dc --- /dev/null +++ b/docs/CONTRIBUTING_LINK.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst \ No newline at end of file diff --git a/docs/License.rst b/docs/License.rst new file mode 100644 index 0000000..b78c457 --- /dev/null +++ b/docs/License.rst @@ -0,0 +1,46 @@ +License and Credits +=================== + +Authors +------- + +The following people have contributed to this project: + +- Gabe Appleton + +Licenses +-------- + +Overall Project +~~~~~~~~~~~~~~~ + +.. literalinclude:: ../LICENSE + :language: none + :linenos: + +C Implementation's SHA2 Header +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. literalinclude:: ../c_src/sha/sha2.h + :dedent: 2 + :lines: 2-32 + :language: none + :linenos: + +C Implementation's SHA2 Code +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. literalinclude:: ../c_src/sha/sha2.c + :dedent: 2 + :lines: 2-32 + :language: none + :linenos: + +C Literal Assert Macro +~~~~~~~~~~~~~~~~~~~~~~ + +.. literalinclude:: ../cp_src/base.h + :dedent: 2 + :lines: 43-50 + :language: none + :linenos: diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..eb2e7cc --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,225 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = .build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/py2p.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/py2p.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/py2p" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/py2p" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/c.rst b/docs/c.rst new file mode 100644 index 0000000..5375e1d --- /dev/null +++ b/docs/c.rst @@ -0,0 +1,17 @@ +C Implementation +================ + +This section contains information specific to the C implementation of p2p.today. Most users will only need to pay attention to the tutorial and last few sections (mesh, chord, kademlia). The rest is for developers who are interested in helping out. + +Contents: + +.. toctree:: + :maxdepth: 2 + + c/tutorial + c/base + c/utils + c/BaseConverter + c/mesh + c/chord + c/kademlia \ No newline at end of file diff --git a/docs/c/tutorial.rst b/docs/c/tutorial.rst new file mode 100644 index 0000000..b23b9e5 --- /dev/null +++ b/docs/c/tutorial.rst @@ -0,0 +1,2 @@ +Tutorial +======== diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..f709a69 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,397 @@ +# -*- coding: utf-8 -*- +# +# py2p documentation build configuration file, created by +# sphinx-quickstart on Tue Aug 2 13:07:12 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. + +import os +import shutil +import subprocess +import sys +import sysconfig + + +def distutils_dir_name(dname): + """Returns the name of a distutils build directory""" + f = "{dirname}.{platform}-{version[0]}.{version[1]}" + return f.format(dirname=dname, + platform=sysconfig.get_platform(), + version=sys.version_info) + +loc = os.path.dirname(os.path.abspath(__file__)) +print("Building from file %s" % loc) +bld = subprocess.call(['python', os.path.join(loc, '..', 'setup.py'), + 'build', '-b', '.py-build']) + +if os.path.isfile(os.path.join(loc, 'py2p', '__init__.py')): + shutil.rmtree(os.path.join(loc, 'py2p')) +shutil.move(os.path.join(loc, '.py-build', distutils_dir_name('lib'), 'py2p'), + os.path.join('.', 'py2p')) +shutil.rmtree(os.path.join(loc, '.py-build')) + +sys.path.insert(0, loc) + +from .py2p import version_info + + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.mathjax', + 'sphinx.ext.extlinks', + 'sphinxcontrib.napoleon', +] + +extlinks = {'issue': ('https://github.com/gappleto97/p2p-project/issues/%s', + 'issue #'), + 'commit': ('https://github.com/gappleto97/p2p-project/commit/%s', + 'commit '), + 'user': ('https://github.com/%s', + '@'), + 'gitfile': ('https://github.com/gappleto97/p2p-project/blob/%s', + 'file ')} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'p2p.today' +copyright = u'2016, Gabe Appleton' +author = u'Gabe Appleton' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'.'.join((str(x) for x in version_info[:2])) +# The full version, including alpha/beta/rc tags. +release = u'.'.join((str(x) for x in version_info)) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['.build', 'Thumbs.db', '.DS_Store'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} +html_context = { + 'cssfiles': ['.static/code_wrap.css'] +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [os.path.join('.', 'sphinx_rtd_theme')] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +# html_title = u'py2p v0.4' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +html_last_updated_fmt = ' %B %d, %Y at %H:%M:%S' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'py2pdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'p2p.today.tex', u'p2p.today Documentation', + u'Gabe Appleton', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'p2p.today', u'p2p.today Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'p2p.today', u'p2p.today Documentation', + author, 'p2p.today', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/3': None} +autodoc_member_order = 'bysource' diff --git a/docs/cpp.rst b/docs/cpp.rst new file mode 100644 index 0000000..eb08d6c --- /dev/null +++ b/docs/cpp.rst @@ -0,0 +1,21 @@ +C++ Implementation +================== + +This section contains information specific to the C++ implementation of p2p.today. Most users will only need to pay attention to the tutorial and last few sections (mesh, chord, kademlia). The rest is for developers who are interested in helping out. + +Contents: + +.. toctree:: + :maxdepth: 2 + + cpp/tutorial + cpp/base + cpp/base_wrapper + cpp/flags_wrapper + cpp/protocol_wraper + cpp/pathfinding_message_wrapper + cpp/py_utils + cpp/utils + cpp/mesh + cpp/chord + cpp/kademlia \ No newline at end of file diff --git a/docs/cpp/tutorial.rst b/docs/cpp/tutorial.rst new file mode 100644 index 0000000..b23b9e5 --- /dev/null +++ b/docs/cpp/tutorial.rst @@ -0,0 +1,2 @@ +Tutorial +======== diff --git a/docs/go.rst b/docs/go.rst new file mode 100644 index 0000000..c87aed1 --- /dev/null +++ b/docs/go.rst @@ -0,0 +1,16 @@ +Go Implementation +================= + +This section contains information specific to the Golang implementation of p2p.today. Most users will only need to pay attention to the tutorial and last few sections (mesh, chord, kademlia). The rest is for developers who are interested in helping out. + +Contents: + +.. toctree:: + :maxdepth: 2 + + go/tutorial + go/base + go/utils + go/mesh + go/chord + go/kademlia \ No newline at end of file diff --git a/docs/go/tutorial.rst b/docs/go/tutorial.rst new file mode 100644 index 0000000..b23b9e5 --- /dev/null +++ b/docs/go/tutorial.rst @@ -0,0 +1,2 @@ +Tutorial +======== diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..e40289a --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,97 @@ +.. p2p-project documentation master file, created by + sphinx-quickstart on Tue Aug 2 13:07:12 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to p2p.today's documentation! +===================================== + +Goal +~~~~ + +We are trying to make peer-to-peer networking easy. Right now there are very few libraries which allow multiple languages to use the same distributed network. + +We're aiming to fix that by providing several network models, which you can use simply by creating an object. These objects will have the simplest API possible to do the job, as well as some features under the hood which you can hook into. + +What We Have +~~~~~~~~~~~~ + +There are several projects in the work right now. Several of these could be considered stable, but we're going to operate under the "beta" label for some time now. + +Message Serializer +~~~~~~~~~~~~~~~~~~ + +Serialization is the most important part for working with other languages. While there are several such schemes which work in most places, we made the decision to avoid these in general. We wanted something very lightweight, which could handle binary data, and operated as quickly as possible. This meant that "universal" serializers like JSON were out the window. + +You can see more information about our serialization scheme in the :doc:`protocol documentation <./protocol/serialization>` . We currently have a working parser in Python, Java, Javascript, C++, and Golang. + +Base Network Structures +~~~~~~~~~~~~~~~~~~~~~~~ + +All of our networks will be built on common base classes. Because of this, we can guarantee some network features. + +#. Networks will have as much common codebase as possible +#. Networks will have opportunistic compression across the board +#. Node IDs will be generated in a consistent manner +#. Command codes will be consistent across network types + +Mesh Network +~~~~~~~~~~~~ + +This is our unorganized network. It operates under three simple rules: + +#. The first node to broadcast sends the message to all its peers +#. Each node which receives a message relays the message to each of its peers, except the node which sent it to them +#. Nodes do not relay a message they have seen before + +Using these principles you can create a messaging network which scales linearly with the number of nodes. + +Currently there is an implementation in :doc:`Python <./python/mesh>` and :doc:`Javascript <./javascript/mesh>`. More tractable documentation can be found in their tutorial sections. For a more in-depth explanation you can see :doc:`it's specifications <./protocol/mesh>` or `this slideshow `_. + +Sync Table +~~~~~~~~~~ + +This is an extension of the above network. It inherits all of the message sending properties, while also syncronizing a local dictionary-like object. + +The only limitation is that it can only have string-like keys and values. There is also an optional "leasing" system, which is enabled by default. This means that a user can own a particular key for a period of time. + +Currently there is an implementation in :doc:`Python <./python/sync>` and :doc:`Javascript <./javascript/sync>`. More tractable documentation can be found in their tutorial sections. Protocol specifications are in progress. + +Chord Table +~~~~~~~~~~~ + +This is a type of `distributed hash table `_ based on an `MIT paper `_ which defined it. + +The idea is that you can use this as a dictionary-like object. The only caveat is that all keys and values *must* be strings. It uses five separate hash tables for hash collision avoidance and data backup in case a node unexpectedly exits. + +Currently there is only an implementation in Python and it is highly experimental. This section will be updated when it's ready for more general use. + +Contributing, Credits, and Licenses +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Contributors are always welcome! Information on how you can help is located on the :doc:`Contributing page <./CONTRIBUTING_LINK>`. + +Credits and License are located on :doc:`their own page <./License>`. + +Contents +======== + +.. toctree:: + :maxdepth: 1 + + protocol + python + javascript + c + cpp + java + go + CONTRIBUTING_LINK + License + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/java.rst b/docs/java.rst new file mode 100644 index 0000000..4bb43c8 --- /dev/null +++ b/docs/java.rst @@ -0,0 +1,16 @@ +Java Implementation +=================== + +This section contains information specific to the Java implementation of p2p.today. Most users will only need to pay attention to the tutorial and last few sections (mesh, chord, kademlia). The rest is for developers who are interested in helping out. + +Contents: + +.. toctree:: + :maxdepth: 2 + + java/tutorial + java/base + java/utils + java/mesh + java/chord + java/kademlia \ No newline at end of file diff --git a/docs/java/tutorial.rst b/docs/java/tutorial.rst new file mode 100644 index 0000000..b23b9e5 --- /dev/null +++ b/docs/java/tutorial.rst @@ -0,0 +1,2 @@ +Tutorial +======== diff --git a/docs/javascript.rst b/docs/javascript.rst new file mode 100644 index 0000000..49c446d --- /dev/null +++ b/docs/javascript.rst @@ -0,0 +1,17 @@ +Javascript Implementation +========================= + +This section contains information specific to the Javascript implementation of p2p.today. Most users will only need to pay attention to the tutorial and last few sections (mesh, sync, chord, kademlia). The rest is for developers who are interested in helping out. + +Contents: + +.. toctree:: + :maxdepth: 2 + + javascript/tutorial + javascript/base + javascript/utils + javascript/mesh + javascript/sync + javascript/chord + javascript/kademlia \ No newline at end of file diff --git a/docs/javascript/sync.rst b/docs/javascript/sync.rst new file mode 100644 index 0000000..09787ba --- /dev/null +++ b/docs/javascript/sync.rst @@ -0,0 +1,93 @@ + +Sync Module +=========== + +.. js:class:: js2p.sync.metatuple(owner, timestamp) + + This class is used to store metadata for a particular key + +.. js:class:: js2p.sync.sync_socket(addr, port [, leasing [, protocol [, out_addr [, debug_level]]]]) + + This is the class for mesh network socket abstraction. It inherits from :js:class:`js2p.mesh.mesh_socket`. + Because of this inheritence, this can also be used as an alert network. + + This also implements and optional leasing system by default. This leasing system means that + if node A sets a key, node B cannot overwrite the value at that key for an hour. + + This may be turned off by setting ``leasing`` to ``false`` to the constructor. + + :param string addr: The address you'd like to bind to + :param number port: The port you'd like to bind to + :param boolean leasing: Whether this class's leasing system should be enabled (default: ``true``) + :param js2p.base.protocol protocol: The subnet you're looking to connect to + :param array out_addr: Your outward-facing address + :param number debug_level: The verbosity of debug prints + + .. js:function:: js2p.sync.sync_socket.__store(key, new_data, new_meta, error) + + Private API method for storing data + + :param key: The key you wish to store data at + :param new_data: The data you wish to store in said key + :param new_meta: The metadata associated with this storage + :param error: A boolean which says whether to raise a :py:class:`KeyError` if you can't store there + + :raises Error: If someone else has a lease at this value, and ``error`` is not ``false`` + + .. js:function:: js2p.sync.sync_socket._send_handshake_response(handler) + + Shortcut method to send a handshake response. This method is extracted from :js:func:`~js2p.mesh.mesh_socket.__handle_handshake` + in order to allow cleaner inheritence from :js:class:`js2p.sync.sync_socket` + + + .. js:function:: js2p.sync.sync_socket.__handle_store + + This callback is used to deal with data storage signals. Its two primary jobs are: + + - store data in a given key + - delete data in a given key + + :param msg: A :js:class:`~js2p.base.message` + :param handler: A :js:class:`~js2p.mesh.mesh_connection` + + :returns: Either ``true`` or ``undefined`` + + .. js:function:: js2p.sync.sync_socket.get(key [, fallback]) + + Retrieves the value at a given key + + :param key: The key you wish to look up (must be transformable into a :js:class:`Buffer` ) + :param fallback: The value it should return when the key has no data + + :returns: The value at the given key, or ``fallback``. + + :raises TypeError: If the key could not be transformed into a :js:class:`Buffer` + + .. js:function:: js2p.sync.sync_socket.set(key, value) + + Sets the value at a given key + + :param key: The key you wish to look up (must be transformable into a :js:class:`Buffer` ) + :param value: The key you wish to store (must be transformable into a :js:class:`Buffer` ) + + :raises TypeError: If a key or value could not be transformed into a :js:class:`Buffer` + :raises: See :js:func:`~js2p.sync.sync_socket.__store` + + .. js:function:: js2p.sync.sync_socket.update(update_dict) + + For each key/value pair in the given object, calls :js:func:`~js2p.sync.sync_socket.set` + + :param Object update_dict: An object with keys and values which can be transformed into a :js:class:`Buffer` + + :raises: See :js:func:`~js2p.sync.sync_socket.set` + + .. js:function:: js2p.sync.sync_socket.del(key) + + Clears the value at a given key + + :param key: The key you wish to look up (must be transformable into a :js:class:`Buffer` ) + + :raises TypeError: If a key or value could not be transformed into a :js:class:`Buffer` + :raises: See :js:func:`~js2p.sync.sync_socket.set` + + diff --git a/docs/javascript/tutorial.rst b/docs/javascript/tutorial.rst new file mode 100644 index 0000000..3fdd506 --- /dev/null +++ b/docs/javascript/tutorial.rst @@ -0,0 +1,10 @@ +Tutorial +======== + +Contents: + +.. toctree:: + :maxdepth: 2 + + tutorial/mesh + tutorial/sync \ No newline at end of file diff --git a/docs/javascript/tutorial/mesh.rst b/docs/javascript/tutorial/mesh.rst new file mode 100644 index 0000000..b0e9e73 --- /dev/null +++ b/docs/javascript/tutorial/mesh.rst @@ -0,0 +1,77 @@ +Mesh Socket +~~~~~~~~~~~ + +Basic Usage +----------- + +To connect to a mesh network, you will use the :js:class:`~js2p.mesh.mesh_socket` object. You can instantiate this as follows: + +.. code-block:: javascript + + > const mesh = require('js2p').mesh; + > sock = new mesh.mesh_socket('0.0.0.0', 4444); + +Using ``'0.0.0.0'`` will (this feature in progress) automatically grab your LAN address. If you want to use an outward-facing internet connection, there is a little more work. First you need to make sure that you have a port forward setup (NAT busting is not in the scope of this project). Then you will specify this outward address as follows: + +.. code-block:: javascript + + > const mesh = require('js2p').mesh; + > sock = new mesh.mesh_socket('0.0.0.0', 4444, null, ['35.24.77.21', 44565]); + +Specifying a different protocol object will ensure that you *only* can connect to people who share your object structure. So if someone has ``'mesh2'`` instead of ``'mesh'``, you will fail to connect. + +Unfortunately, this failure is currently silent. Because this is asynchronous in nature, raising an error is not possible. Because of this, it's good to perform the following check the truthiness of :js:attr:`.mesh_socket.routing_table`. If it is truthy, then you are connected to the network. + +To send a message, you should use the :js:func:`~js2p.mesh.mesh_socket.send` method. Each argument you supply will correspond to a packet that your peer receives. In addition, there are two keyed arguments you can use. ``flag`` will specify how other nodes relay this. These flags are defined in :js:data:`js2p.base.flags` . ``broadcast`` will indicate that other nodes are supposed to relay it. ``whisper`` will indicate that your peers are *not* supposed to relay it. There are other technically valid options, but they are not recommended. ``type`` will specify what actions other nodes are supposed to take on it. It defaults to ``broadcast``, which indicates no change from the norm. There are other valid options, but they should normally be left alone, unless you've written a handler (see below) to act on this. + +.. code-block:: javascript + + > sock.send(['this is', 'a test']); + +Receiving is a bit simpler. When you call the :js:func:`~js2p.mesh.mesh_socket.recv` method, you receive a :js:class:`~js2p.base.message` object. This has a number of methods outlined which you can find by clicking its name. Most notably, you can get the packets in a message with :js:attr:`~js2p.base.message.packets`, and reply directly with :js:func:`~js2p.base.message.reply`. + +.. code-block:: javascript + + > sock.send(['Did you get this?']); + > var msg = sock.recv(); + > console.log(msg); + message { + type: + packets: [ , ] + sender: '8vu4oLsvVBsnnH6N83z6y6RZqrMKRrVHr44xRwXCFaU9qcyYsjJDzVfKwmdGp51K4d' } + > msg.packets.forEach((packet) => { + ... var str = packet.toString() + ... console.log(util.inspect(str)); + ... }); + '\u0002' + 'yes' + 'I did' + > console.log(msg.packets); + [ , , ] + > sock.recv(10).forEach((msg) => { + ... msg.reply(["Replying to a list"]); + ... }); + +Advanced Usage +-------------- + +In addition to this, you can register a custom handler for incoming messages. This is appended to the end of the included ones. When writing your handler, you must keep in mind that you are only passed a :js:class:`~js2p.base.message` object and a :js:class:`~js2p.mesh.mesh_connection`. Fortunately you can get access to everything you need from these objects. This example is in Python, but the Javascript syntax is identical. + +.. code-block:: python + + >>> def relay_tx(msg, handler): + ... """Relays bitcoin transactions to various services""" + ... packets = msg.packets # Gives a list of the non-metadata packets + ... server = msg.server # Returns your mesh_socket object + ... if packets[0] == b'tx_relay': # It's important that this flag is bytes + ... from pycoin import tx, services + ... relay = tx.Tx.from_bin(packets[1]) + ... services.blockchain_info.send_tx(relay) + ... services.insight.InsightProvider().send_tx(relay) + ... return True # This tells the daemon to stop calling handlers + ... + >>> import py2p + >>> sock = py2p.mesh_socket('0.0.0.0', 4444) + >>> sock.register_handler(relay_tx) + +To help debug these services, you can specify a :js:attr:`~js2p.base.base_socket.debug_level` in the constructor. Using a value of 5, you can see when it enters into each handler, as well as every message which goes in or out. diff --git a/docs/javascript/tutorial/sync.rst b/docs/javascript/tutorial/sync.rst new file mode 100644 index 0000000..a7995ac --- /dev/null +++ b/docs/javascript/tutorial/sync.rst @@ -0,0 +1,79 @@ +Sync Socket +~~~~~~~~~~~ + +This is an extension of the :js:class:`~js2p.mesh.mesh_socket` which syncronizes a common :js:class:`Object`. It works by providing an extra handler to store data. This does not expose the entire :js:class:`Object` API, but it exposes a substantial subset, and we're working to expose more. + +.. note:: + + This is a fairly inefficient architecture for write intensive applications. For cases where the majority of access is reading, or for small networks, this is ideal. For larger networks where a significant portion of your operations are writing values, you should wait for the chord socket to come into beta. + +Basic Usage +----------- + +There are three limitations compared to a normal :js:class:`Object`. + +1. Keys and values must be translatable to a :js:class:`Buffer` +2. Keys and values are automatically translated to a :js:class:`Buffer` +3. By default, this implements a leasing system which prevents you from changing values set by others for a certain time + +You can override the last restriction by constructing with ``leasing`` set to ``false``, like so: + +.. code-block:: javascript + + > const sync = require('js2p').sync; + > let sock = new sync.sync_socket('0.0.0.0', 4444, false); + +The only API differences between this and :js:class:`~js2p.mesh.mesh_socket` are for access to this dictionary. They are as follows. + +:js:func:`~js2p.sync.sync_socket.get` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A value can be retrieved by using the :js:func:`~js2p.sync.sync_socket.get` method. This is reading from a local :js:class:`Object`, so speed shouldn't be a factor. + +.. code-block:: javascript + + > let foo = sock.get('test key', null) // Returns null if there is nothing at that key + > let bar = sock.get('test key') // Returns undefined if there is nothing at that key + +It is important to note that keys are all translated to a :js:class:`Buffer` before being used. + +:js:func:`~js2p.sync.sync_socket.set` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A value can be stored by using the :js:func:`~js2p.sync.sync_socket.set` method. These calls are ``O(n)``, as it has to change values on other nodes. More accurately, the delay between your node knowing of the change and the last node knowing of the change is ``O(n)``. + +.. code-block:: javascript + + > sock.set('test key', 'value'); + > sock.set('测试', 'test'); + +Like above, keys and values are all translated to :js:class:`Buffer` before being used + +This will raise an :js:class:`Error` if another node has set this value already. Their lease will expire one hour after they set it. If two leases are started at the same UTC second, the tie is settled by doing a string compare of their IDs. + +Any node which sets a value can change this value as well. Changing the value renews the lease on it. + +:js:func:`~js2p.sync.sync_socket.del` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Any node which owns a key, can clear its value. Doing this will relinquish your lease on that value. Like the above, this call is ``O(n)``. + +.. code-block:: javascript + + > sock.del('test'); + +:js:func:`~js2p.sync.sync_socket.update` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The update method is simply a wrapper which updates based on a fed :js:class:`Object`. Essentially it runs the following: + +.. code-block:: javascript + + > for (var key in update_dict) { + ... sock.set(key, update_dict[key]); + ... } + +Advanced Usage +-------------- + +Refer to :doc:`the mesh socket tutorial <./mesh>` diff --git a/docs/make.bat b/docs/make.bat new file mode 100755 index 0000000..b1a6b24 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,281 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=.build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. epub3 to make an epub3 + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + echo. dummy to check syntax errors of document sources + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\py2p.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\py2p.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "epub3" ( + %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +if "%1" == "dummy" ( + %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. Dummy builder generates no files. + goto end +) + +:end diff --git a/docs/protocol.rst b/docs/protocol.rst new file mode 100644 index 0000000..bf54baa --- /dev/null +++ b/docs/protocol.rst @@ -0,0 +1,17 @@ +Protocol Specifications +======================= + +Contents: + +.. toctree:: + :maxdepth: 1 + + protocol/goals + protocol/serialization + protocol/reconstruction + protocol/ids_and_encoding + protocol/flags + protocol/flaws + protocol/mesh + protocol/chord + protocol/kademlia \ No newline at end of file diff --git a/figure_one.png b/docs/protocol/figure_one.png similarity index 100% rename from figure_one.png rename to docs/protocol/figure_one.png diff --git a/docs/protocol/flags.rst b/docs/protocol/flags.rst new file mode 100644 index 0000000..0eb4885 --- /dev/null +++ b/docs/protocol/flags.rst @@ -0,0 +1,88 @@ +Flag Definitions +================ + +Main Flags +++++++++++ + +These flags will denote the primary purpose of a message. + +- broadcast = ``"\x00"`` +- waterfall = ``"\x01"`` +- whisper = ``"\x02"`` +- renegotiate = ``"\x03"`` +- ping = ``"\x04"`` +- pong = ``"\x05"`` + +Sub-Flags ++++++++++ + +These flags will denote the secondary purpose, or a more specific purpose, of a message. + +- broadcast = ``"\x00"`` +- compression = ``"\x01"`` +- whisper = ``"\x02"`` +- handshake = ``"\x03"`` +- ping = ``"\x04"`` +- pong = ``"\x05"`` +- notify = ``"\x06"`` +- peers = ``"\x07"`` +- request = ``"\x08"`` +- resend = ``"\x09"`` +- response = ``"\x0A"`` +- store = ``"\x0B"`` +- retrieve = ``"\x0C"`` + +Compression Flags ++++++++++++++++++ + +These flags will denote standard compression methods. + +All +~~~ + +- bwtc = ``"\x14"`` +- bz2 = ``"\x10"`` +- context1 = ``"\x15"`` +- defsum = ``"\x16"`` +- dmc = ``"\x17"`` +- fenwick = ``"\x18"`` +- gzip = ``"\x11"`` +- huffman = ``"\x19"`` +- lzjb = ``"\x1A"`` +- lzjbr = ``"\x1B"`` +- lzma = ``"\x12"`` +- lzp3 = ``"\x1C"`` +- mtf = ``"\x1D"`` +- ppmd = ``"\x1E"`` +- simple = ``"\x1F"`` +- zlib = ``"\x13"`` + +Python Implemented +~~~~~~~~~~~~~~~~~~ + +- bz2 +- gzip +- lzma +- zlib + +.. note:: + Only on systems where these modules are available + +C++ Planned +~~~~~~~~~~~ + +- gzip +- zlib + +Javascript Implemented +~~~~~~~~~~~~~~~~~~~~~~ + +- gzip +- zlib + +Reserved Flags +++++++++++++++ + +These define the flags that other applications should *not* use, as they either are (or will be) used by the standard protocol. + +Currently, this is all single byte characters from ``0x00`` to ``0x20``. This list may be expanded later. diff --git a/docs/protocol/flaws.rst b/docs/protocol/flaws.rst new file mode 100644 index 0000000..e5c983f --- /dev/null +++ b/docs/protocol/flaws.rst @@ -0,0 +1,36 @@ +Potential Serialization Flaws +============================= + +The structure has a couple immediately obvious shortcomings. + +First, the maximum message size is 4,294,967,299 bytes (including compression and headers). It could well be that in the future there will be more data to send in a single message. But equally so, a present-day attacker could use this to halt sections of a network using this structure. A short-term solution would be to have a soft-defined limit, but as has been shown in other protocols, this can calcify over time and do damage. In the end, this is more of a governance problem than a technical one. The discussion on this can be found in :issue:`84`. + +Second, there is quite a lot of extra data being sent. Using the default parameters, if you want to send a 4 character message it will be expanded to 159 characters. That's ~42x larger. If you want these differences to be negligble, you need to send messages on the order of 512 characters. Then there is only an increase of ~34% (0% with decent compression). This can be improved by reducing the size of the various IDs being sent, or making the packet headers shorter. Both of these have disadvantages, however. + +Making a shorter ID space means that you will be more likely to get a conflict. This isn't as much of a problem for node IDs as it is for message IDs, but it is certainly a problem you'd like to avoid. + +Making shorter packet headers presents few immediate problems, but it makes it more difficult for debugging, and may make it more difficult to establish standard headers in the future. + +Results using opportunistic compression look roughly as follows (last updated in 0.4.231): + +For 4 characters… + +:: + + original 4 + plaintext 167 (4175%) + lzma 220 (5500%) + bz2 189 (4725%) + gzip 156 (3900%) + +For 512 characters… + +:: + + original 512 + plaintext 677 (132.2%) + lzma 568 (110.9%) + bz2 555 (108.4%) + gzip 487 (95.1%) + +Because the reference implementations support all of these (excepting environment variations), this means that the overhead will drop away after ~500 characters. Communications with other implementations may be slower than this, however. \ No newline at end of file diff --git a/docs/protocol/goals.rst b/docs/protocol/goals.rst new file mode 100644 index 0000000..1ea304a --- /dev/null +++ b/docs/protocol/goals.rst @@ -0,0 +1,35 @@ +Protocol Goals +============== + +Currently there are very few ways to set up a peer-to-peer network between different languages. There are several proocols out there, but all of them are very specialized, complicated, or both. This is not what we want in a language-agnostic protocol. While we may go after other goals in addition to these, our explicit goals will be laid out below: + +Portable +++++++++ + +The underlying protocol should use only language agnostic features. That is to say, every message will be entirely of strings, and any formatting (like JSON) must be strictly laid out so that you can parse it without knowing the entirety of said formatting. + +Any features which are not language agnostic **must** be optional. This includes things like compression, encryption, etc. + +Fast +++++ + +Reconstructing a plaintext message with three, single-character, user-placed packets **must** take <1.5ms. (In this case, I'm judging off my laptop's time, rather than my desktop. I have a `Lenovo Carbon `_ running Kubuntu 16.04.) + +Dense ++++++ + +Where there are no disadvantages to the above, the resulting messages should be as dense as possible. + +Abstracted +++++++++++ + +The resulting protocol **must** be capturable in an object. That is to say, one should be able to call properties of an object whose underlying structures may change at any time. Parsing a message should **never** require knowledge of network state, with the sole exception of compression. + +Notes ++++++ + +These goals are subject to change in the future. If some awesome feature requires that packet reconstruction takes longer, this does not necessarily stop that feature from being implemented. The only hard rule is the portability one. + +Also worthy of note is that backwards compatability *intentionally* is not on this list. It may be that version 0.4.* and 0.5.* can understand each other, but they will *actively reject* each other. This allows for an ease of change that isn't present if you require backwards compatability. After the 1.0 release, we will try and maintain backports that you can reliably access. If you can't through a package manager, you certainly can from the git releases. + +In the next section, we will outline how you construct and serialize a message. \ No newline at end of file diff --git a/docs/protocol/ids_and_encoding.rst b/docs/protocol/ids_and_encoding.rst new file mode 100644 index 0000000..60b63b5 --- /dev/null +++ b/docs/protocol/ids_and_encoding.rst @@ -0,0 +1,86 @@ +IDs and Encoding +================ + +Knowing the overall message structure is great, but it’s not very +useful if you can’t construct the metadata. To do this, there are +four parts. + +base\_58 +++++++++ + +This encoding is taken from Bitcoin. If you’ve ever seen a Bitcoin +address, you’ve seen base\_58 encoding in action. The goal behind +it is to provide data compression without compromising its human +readability. Base\_58, for the purposes of this protocol, is +defined by the following python methods. + +.. code-block:: python + + base_58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + + def to_base_58(i): + string = "" + while i: + string = base_58[i % 58] + string + i = i // 58 # Floor division is needed to prevent floats + return string.encode() + + def from_base_58(string): + if isinstance(string, bytes): + string = string.decode() + decimal = 0 + for char in string: + decimal = decimal * 58 + base_58.index(char) + return decimal + +Subnets ++++++++ + +The last element is the ‘subnet ID’ we referred to in the previous +section. This object is used to weed out undesired connections. If +someone has the wrong protocol object, then your node will reject +them from connecting. A rough definition would be as follows: + +.. code-block:: python + + class protocol(namedtuple("protocol", ['subnet', 'encryption'])): + @property + def id(self): + info = [str(x) for x in self] + [protocol_version] + h = hashlib.sha256(''.join(info).encode()) + return to_base_58(int(h.hexdigest(), 16)) + +Or more explicitly in javascript: + +.. code-block:: javascript + + class protocol { + constructor(subnet, encryption) { + this.subnet = subnet; + this.encryption = encryption; + } + + get id() { + var info = [this.subnet, this.encryption, protocol_version]; + var hash = SHA256(info.join('')); + return to_base_58(BigInt(hash, 16)); + } + } + +Node IDs +++++++++ + +A node ID is taken from a SHA-384 hash of three other elements. +First, your outward facing address. Second, the ID of your subnet. +Third, a ‘user salt’ generated on startup. This hash is then +converted into base\_58. + +Message IDs ++++++++++++ + +A message ID is also a SHA-384 hash. In this case, it is on a +message’s payload and its timestamp. + +To get the hash, first join each packet together in order. Append +to this the message’s timestamp in base\_58. The ID you will use +is the hash of this string, encoded into base\_58. \ No newline at end of file diff --git a/docs/protocol/mesh.rst b/docs/protocol/mesh.rst new file mode 100644 index 0000000..c77e8dd --- /dev/null +++ b/docs/protocol/mesh.rst @@ -0,0 +1,238 @@ +Mesh Protocol Definition +======================== + +Problem ++++++++ + +There are very few ways to construct a peer to peer network in +dynamic languages. Everyone who wants to make such a network needs +to reinvent the wheel. For example, in Python, the only two such +libraries either never made it out of beta, or were to connect to a +cryptocurrency. There is certainly no way to communicate *between* +these languages. This section will focus on how you can make a mesh +(unorganized) network like the one in Bitcoin or Gnutella. + +Design Goals +++++++++++++ + +The network should be unorganized. This means that it’s very simple +for connections to be added, and for routes to repair themselves in +the event of obstruction. It also means that there is no central +point of failure, nor overhead to maintaining a structure. + +The network should also be as flexible as possible. It should be able +to carry binary data and should have various flags to determine how a +message is treated. + +In languages that allow it, network nodes should be able to register +custom callbacks, which can respond to incoming data in real time and +act upon it as needed. + +Most importantly, nodes should use features that are common across +multiple languages. + +And as an afterthought, nodes should be optimized for performance and +data density where possible. + +Node Construction ++++++++++++++++++ + +Now your node is ready to parse messages on the network, but it can’t +yet connect. There are important elements it needs to store in order +to interact with it correctly. + +#. A daemon thread or callback system which receives messages and incoming connections +#. A routing table of peers with the IDs and corresponding connection objects +#. A “waterfall list of recently received message IDs and timestamps +#. A user-interactable queue of recently received messages +#. A “protocol”, which contains: + + #. A sub-net flag + #. An encryption method (or “Plaintext”) + #. A way to obtain a SHA256-based ID of this + +Connecting to the Network ++++++++++++++++++++++++++ + +This is where the protocol object becomes important. + +When you connect to a node, each will send a message in the following +format: + +.. code-block:: none + + whisper + [your id] + [message id] + [timestamp] + handshake + [your id] + [your protocol id] + [your outward-facing address] + [json-ized list of supported compression methods, in order of preference] + +When your node receives the corresponding message, the first thing +your node does is compare their protocol ID against your own. If they +do not match, your node shuts down the connection. + +If they do match, your node adds them to your routing table +(``{ID: connection}``), and makes a note of their outward facing +address and supported compression methods. Then your node sends a +standard response: + +.. code-block:: none + + whisper + [your id] + [message id] + [timestamp] + peers + [json-ized copy of your routing table in format: [[addr, port], id]] + +Upon receiving this message, your node attempts to connect to each given address. Now you're connected to the network! But how do you process the incoming messages? + +Message Propagation ++++++++++++++++++++ + +A message is initially broadcast with the ``broadcast`` flag. The +broadcasting node, as well as all receivers, store this message’s ID +and timestamp in their waterfall queue. The reciving nodes then +re-broadcast this message to each of their peers, but changing the +flag to ``waterfall``. + +A node which receives these waterfall packets goes through the +following steps: + +#. If the message ID is not in the node’s waterfall queue, continue and add it to the waterfall queue +#. Perform cleanup on the waterfall queue + + a. Remove all possible duplicates (sending may be done in multiple threads, which may result in duplicate copies) + #. Remove all IDs with a timestamp more than 1 minute ago +#. Re-broadcast this message to all peers (optionally excluding the one you received it from) + +.. image:: ./figure_one.png + +Renegotiating a Connection +++++++++++++++++++++++++++ + +It may be that at some point a message fails to decompress on your +end. If this occurs, you have an easy solution, your node can send a +``renegotiate`` message. This flag is used to indicate that a message +should never be presented to the user, and is only used for +connection management. At this time there are two possible +operations. + +The ``compression`` subflag will allow your node to renegotiate your +compression methods. A message using this subflag should be +constructed like so: + +.. code-block:: none + + renegotiate + [your id] + [message id] + [timestamp] + compression + [json-ized list of desired compression methods, in order of preference] + +Your peer will respond with the same message, excluding any methods +they do not support. If this list is different than the one you sent, +you reply, trimming the list of methods *your node* does not support. +This process is repeated until the two agree upon a list. + +Your node may also send a ``resend`` subflag, which requests your +peer to resend the previous ``whisper`` or ``broadcast``. This is +structured like so: + +.. code-block:: none + + renegotiate + [your id] + [message id] + [timestamp] + resend + +Peer Requests ++++++++++++++ + +If you want to privately reply to a message where you are not +directly connected to a sender, the following method can be used: + +First, your node broadcasts a message to the network containing the +``request`` subflag. This is constructed as follows: + +.. code-block:: none + + broadcast + [your id] + [message id] + [timestamp] + request + [a unique, base_58 id you assign] + [the id of the desired peer] + +Then your node places this in a dictionary so your node can watch for +when this is responded to. A peer who gets this will reply: + +.. code-block:: none + + broadcast + [their id] + [message id] + [timestamp] + response + [the id you assigned] + [address of desired peer in format: [[addr, port], id] ] + +When this is received, your node removes the request from your +dictionary, makes a connection to the given address, and sends the +message. + +Another use of this mechanism is to request a copy of your peers’ +routing tables. To do this, your node may send a message structured +like so: + +.. code-block:: none + + whisper + [your id] + [message id] + [timestamp] + request + * + +A node who receives this will respond exactly as they do after a +successful handshake. Note that while it is technically valid to send +this request as a ``broadcast``, it is generally discouraged. + +Potential Flaws ++++++++++++++++ + +This network shcema has an immediately obvious shortcoming. + +In a worst case scenario, every node will receive a given message +:math:`n-1` times, and each message will generate :math:`n * (n-1)` total +broadcasts, where n is the number of connected nodes. This number can +be arrived at by thinking of the network serially. If you have four +nodes on a network, each connected to the other three, it will +proceed roughly as follows. + +Node A will send to B, C, and D. Node B will receive this message and +send to A, C, and D. Node C will receive the same message and send to +A, B, and D. Node D will relay to A, B, and C. This makes 12 total +messages, or :math:`n * (n-1)`. + +In most larger cases this will not happen, as a given node will not +be connected to everyone else. But in smaller networks this will be +common, and in well-connected networks this could slow things down. +This calls for optimization, and will need to be explored. + +For instance, not propagating to a peer you receive a message from +reduces the number of total broadcasts to :math:`(n-1)^2`. Using the same +example: + +Node A will send to B, C, and D. Node B will receive this message and send to C and D. +Node C will receive the same message and send to B and D. Node D will relay to B and C. +This makes 9 total messages, or :math:`(n-1)^2`. + +Limiting your number of connections can bring this down to :math:`min(MaxConns, n-1) * (n-1)`. \ No newline at end of file diff --git a/docs/protocol/reconstruction.rst b/docs/protocol/reconstruction.rst new file mode 100644 index 0000000..d60f6b8 --- /dev/null +++ b/docs/protocol/reconstruction.rst @@ -0,0 +1,38 @@ +Reconstruction +============== + +.. raw:: html + + + +Let's keep with our example from the previous section. How do we parse the resulting string? + +First, check the first four bytes. Because we don't know how much data to receive through our socket, we always first check for a four byte header. Then we can toss it aside and collect that much information. If we know the message will be compressed, this is when it gets decompressed. + +Next, we need to split up the packets. To do that, take each four bytes and sum their values until the number of remaining bytes equals that sum. If it does not, throw an error. An example script would look like: + +.. code-block:: python + + def get_packets(string): + processed = 0 + expected = len(string) + pack_lens = [] + packets = [] + # First find each packet's length + while processed != expected: + length = struct.unpack("!L", string[processed:processed+4])[0] + processed += 4 + expected -= length + pack_lens.append(length) + # Then reconstruct the packets + for index, length in enumerate(pack_lens): + start = processed + sum(pack_lens[:index]) + end = start + length + packets.append(string[start:end]) + return packets + +From the above script we get back ``['broadcast', '6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V', '72tG7phqoAnoeWRKtWoSmseurpCtYg2wHih1y5ZX1AmUvihcH7CPZHThtm9LGvKtj7', '3EfSDb', 'broadcast', 'test message']`` + +A node will use the entirety of this list to decide what to do with it. In this case, it would forward it to its peers, then present to the user the payload: ``['broadcast', 'test message']``. + +Now that we know how to construct a message, examples will no longer include the headers. They will include the metadata and payload only. \ No newline at end of file diff --git a/docs/protocol/serialization.rst b/docs/protocol/serialization.rst new file mode 100644 index 0000000..e78187b --- /dev/null +++ b/docs/protocol/serialization.rst @@ -0,0 +1,76 @@ +Serialization +============= + +.. raw:: html + + + +The first step to any of this is being able to understand messages sent +through the network. To do this, you need to build a parser. Each +message can be considered to have three segments: a header, metadata, +and payload. The header is used to figure out the size of a message, as +well as how to divide up its various packets. The metadata is used to +assist propagation functions. And the payload is what the user on the +other end receives. + +A more formal definition would look like: + +.. code-block:: none + + Size of message - 4 (big-endian) bytes defining the size of the message + ------------------------All below may be compressed------------------------ + Size of packet 0 - 4 bytes defining the plaintext size of packet 0 + Size of packet 1 - 4 bytes defining the plaintext size of packet 1 + ... + Size of packet n-1 - 4 bytes defining the plaintext size of packet n-1 + Size of packet n - 4 bytes defining the plaintext size of packet n + ---------------------------------End Header-------------------------------- + Pathfinding header - [broadcast, waterfall, whisper, renegotiate] + Sender ID - A base_58 SHA384-based ID for the sender + Message ID - A base_58 SHA384-based ID for the message packets + Timestamp - A base_58 unix UTC timestamp of initial broadcast + Payload packets + Payload header - [broadcast, whisper, handshake, peers, request, response] + Payload contents + +To understand this, let’s work from the bottom up. When a user wants to +construct a message, they feed a list of packets. For this example, +let’s say it’s ``['broadcast', 'test message']``. When this list is fed +into a node, it adds the metadata section as outlined below. + +The first element is the pathfinding header. This alerts nodes to how +they should treat the message. If it is ``broadcast`` or ``waterfall`` +they are to forward this message to their peers. If it is ``whisper`` +they are not to do so. ``renegotiate`` is exclusivley used for +connection management. + +Next is the sender ID, used to identify a user in your routing table. So +if a user wants to reply to a message, they look up this ID in their +routing table. As will be discussed below, there are methods you can +specifically request a user ID to connect to. + +After this is a message ID and timestamp, used to filter out messages +that a node has seen before. This can also be used as a checksum. + +``['broadcast', '6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V', +'72tG7phqoAnoeWRKtWoSmseurpCtYg2wHih1y5ZX1AmUvihcH7CPZHThtm9LGvKtj7', '3EfSDb', +'broadcast', 'test message']`` + +One thing to notice is that the sender ID, message ID, and timestamp are +all in a strange encoding. This is base\_58, borrowed from Bitcoin. It’s +a way to encode numbers that allows for sufficient density while still +maintaining some human readability. This will get defined formally later +in the paper. + +All of this still leaves out the header. Constructing this goes as follows: + +For each packet, compute its length and pack this into four bytes. So a +message of length 6 would look like ``'\x00\x00\x00\x06'``. Take the resulting +string and prepend it to the packets. In this example, you would end up with: +``'\x00\x00\x00\t\x00\x00\x00B\x00\x00\x00B\x00\x00\x00\x06\x00\x00\x00\t\x00\x00\x00\x0cbroadcast6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V7iSCRDcHZwYtxGbTCz1rwDbUkt7YrbAh2VdS4A75hRuM6xan2gjmZqiVjLkMqiHE3Q3EfSDbbroadcasttest message'``. + +After running this message through whatever compression algorith which has +been negotiated with the node's peer, it compute the message's size, and pack +this into four bytes: ``'\x00\x00\x00\xc0'``. This results in a final message of: + +``'\x00\x00\x00\xc0\x00\x00\x00\t\x00\x00\x00B\x00\x00\x00B\x00\x00\x00\x06\x00\x00\x00\t\x00\x00\x00\x0cbroadcast6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V7iSCRDcHZwYtxGbTCz1rwDbUkt7YrbAh2VdS4A75hRuM6xan2gjmZqiVjLkMqiHE3Q3EfSDbbroadcasttest message'`` \ No newline at end of file diff --git a/docs/python.rst b/docs/python.rst new file mode 100644 index 0000000..4138ad0 --- /dev/null +++ b/docs/python.rst @@ -0,0 +1,18 @@ +Python Implementation +===================== + +This section contains information specific to the Python implementation of p2p.today. The python version is considered the reference implementation, and is where most experimenting on new protocol ideas will come from. Most users will only need to pay attention to the tutorial and last few sections (mesh, sync, chord, kademlia). The rest is for developers who are interested in helping out. + +Contents: + +.. toctree:: + :maxdepth: 2 + + python/tutorial + python/base + python/cbase + python/utils + python/mesh + python/sync + python/chord + python/kademlia diff --git a/docs/python/base.rst b/docs/python/base.rst new file mode 100644 index 0000000..866bfa7 --- /dev/null +++ b/docs/python/base.rst @@ -0,0 +1,99 @@ +Base Module +=============== + + +.. automodule:: py2p.base + :members: + :exclude-members: flags, protocol + :special-members: __init__, __iter__ + :undoc-members: + + .. autoclass:: flags + :exclude-members: x + + .. autoattribute:: reserved + + **Main flags:** + + - .. autoattribute:: flags.broadcast + :annotation: + - .. autoattribute:: waterfall + :annotation: + - .. autoattribute:: whisper + :annotation: + - .. autoattribute:: renegotiate + :annotation: + - .. autoattribute:: ping + :annotation: + - .. autoattribute:: pong + :annotation: + + **Sub-flags:** + + - .. autoattribute:: broadcast + :annotation: + - .. autoattribute:: compression + :annotation: + - .. autoattribute:: whisper + :annotation: + - .. autoattribute:: handshake + :annotation: + - .. autoattribute:: ping + :annotation: + - .. autoattribute:: pong + :annotation: + - .. autoattribute:: notify + :annotation: + - .. autoattribute:: peers + :annotation: + - .. autoattribute:: request + :annotation: + - .. autoattribute:: resend + :annotation: + - .. autoattribute:: response + :annotation: + - .. autoattribute:: store + :annotation: + - .. autoattribute:: retrieve + :annotation: + + **Python-implemented compression methods:** + + - .. autoattribute:: bz2 + :annotation: + - .. autoattribute:: gzip + :annotation: + - .. autoattribute:: lzma + :annotation: + - .. autoattribute:: zlib + :annotation: + + **Other implementations' and/or planned compression methods:** + + - .. autoattribute:: bwtc + :annotation: + - .. autoattribute:: context1 + :annotation: + - .. autoattribute:: defsum + :annotation: + - .. autoattribute:: dmc + :annotation: + - .. autoattribute:: fenwick + :annotation: + - .. autoattribute:: huffman + :annotation: + - .. autoattribute:: lzjb + :annotation: + - .. autoattribute:: lzjbr + :annotation: + - .. autoattribute:: lzp3 + :annotation: + - .. autoattribute:: mtf + :annotation: + - .. autoattribute:: ppmd + :annotation: + - .. autoattribute:: simple + :annotation: + + .. autoclass:: protocol + :exclude-members: id \ No newline at end of file diff --git a/docs/python/cbase.rst b/docs/python/cbase.rst new file mode 100644 index 0000000..6501e27 --- /dev/null +++ b/docs/python/cbase.rst @@ -0,0 +1,104 @@ +Base Module (C++ Implementation) +================================ + + +.. automodule:: py2p.cbase + :members: + :undoc-members: + + .. autoclass:: flags + + .. note:: + This is not actually a class, it just makes it formatted much neater to treat it as such. In the C++ implementation this is a module. You should not need to import it. + + **Main flags:** + + - .. data:: broadcast + + - .. data:: waterfall + + - .. data:: whisper + + - .. data:: renegotiate + + - .. data:: ping + + - .. data:: pong + + + **Sub-flags:** + + - .. data:: broadcast + + - .. data:: compression + + - .. data:: whisper + + - .. data:: handshake + + - .. data:: ping + + - .. data:: pong + + - .. data:: notify + + - .. data:: peers + + - .. data:: request + + - .. data:: resend + + - .. data:: response + + - .. data:: store + + - .. data:: retrieve + + + **C++-planned compression methods:** + + - .. data:: gzip + + - .. data:: zlib + + + **Other implementations' and/or planned compression methods:** + + - .. data:: bwtc + + - .. data:: bz2 + + - .. data:: context1 + + - .. data:: defsum + + - .. data:: dmc + + - .. data:: fenwick + + - .. data:: huffman + + - .. data:: lzjb + + - .. data:: lzjbr + + - .. data:: lzma + + - .. data:: lzp3 + + - .. data:: mtf + + - .. data:: ppmd + + - .. data:: simple + + + .. autoclass:: py2p.cbase.pathfinding_message + :members: + :undoc-members: + :special-members: __init__, __iter__ + + .. autoclass:: py2p.cbase.protocol + :members: + :undoc-members: + :special-members: __init__, __iter__ \ No newline at end of file diff --git a/docs/python/chord.rst b/docs/python/chord.rst new file mode 100644 index 0000000..0b65da2 --- /dev/null +++ b/docs/python/chord.rst @@ -0,0 +1,27 @@ +Chord Module +============ + +.. autoclass:: py2p.chord.chord_connection + :members: + :special-members: __init__, __iter__ + :undoc-members: + :inherited-members: + +.. autoclass:: py2p.chord.chord_daemon + :members: + :special-members: __init__, __iter__ + :undoc-members: + :inherited-members: + +.. autoclass:: py2p.chord.chord_socket + :members: + :special-members: __init__, __iter__ + :private-members: + :undoc-members: + :inherited-members: + +.. automodule:: py2p.chord + :members: + :exclude-members: flags, chord_connection, chord_daemon, chord_socket + :special-members: __init__, __iter__ + :undoc-members: diff --git a/docs/python/kademlia.rst b/docs/python/kademlia.rst new file mode 100644 index 0000000..8d1c8b6 --- /dev/null +++ b/docs/python/kademlia.rst @@ -0,0 +1 @@ + diff --git a/docs/python/mesh.rst b/docs/python/mesh.rst new file mode 100644 index 0000000..e3e1372 --- /dev/null +++ b/docs/python/mesh.rst @@ -0,0 +1,28 @@ +Mesh Module +=========== + + +.. autoclass:: py2p.mesh.mesh_connection + :members: + :special-members: __init__, __iter__ + :undoc-members: + :inherited-members: + +.. autoclass:: py2p.mesh.mesh_daemon + :members: + :special-members: __init__, __iter__ + :undoc-members: + :inherited-members: + +.. autoclass:: py2p.mesh.mesh_socket + :members: + :special-members: __init__, __iter__ + :private-members: + :undoc-members: + :inherited-members: + +.. automodule:: py2p.mesh + :members: + :exclude-members: flags, mesh_connection, mesh_daemon, mesh_socket + :special-members: __init__, __iter__ + :undoc-members: diff --git a/docs/python/sync.rst b/docs/python/sync.rst new file mode 100644 index 0000000..9b48ca5 --- /dev/null +++ b/docs/python/sync.rst @@ -0,0 +1,16 @@ +Sync Module +=========== + + +.. autoclass:: py2p.sync.sync_socket + :members: + :special-members: __init__, __iter__ + :private-members: + :undoc-members: + :inherited-members: + +.. automodule:: py2p.sync + :members: + :exclude-members: flags, mesh_socket, sync_socket + :special-members: __init__, __iter__ + :undoc-members: diff --git a/docs/python/tutorial.rst b/docs/python/tutorial.rst new file mode 100644 index 0000000..ab23213 --- /dev/null +++ b/docs/python/tutorial.rst @@ -0,0 +1,11 @@ +Python Tutorials +================ + +Contents: + +.. toctree:: + :maxdepth: 2 + + tutorial/mesh + tutorial/sync + tutorial/chord diff --git a/docs/python/tutorial/chord.rst b/docs/python/tutorial/chord.rst new file mode 100644 index 0000000..45630d2 --- /dev/null +++ b/docs/python/tutorial/chord.rst @@ -0,0 +1,102 @@ +Chord Socket +~~~~~~~~~~~~ + +.. warning:: + + This module is partly unstable, and should be regarded as "pre-alpha". + + If you're considering using this, please wait until this warning is removed. Expected beta status is by end of November 2016. + +Basic Usage +----------- + +The chord schema is used as a distributed hash table. Its primary purpose is to ensure data syncronization between peers. While it's not entirely :py:class:`dict`-like, it has a substantial subset of this API. + +To connect to a chord network, use the :py:class:`~py2p.chord.chord_socket` object. this is instantiated as follows: + +.. code-block:: python + + >>> from py2p import chord + >>> sock = chord.chord_socket('0.0.0.0', 4444, k=2) + >>> sock.join() # This indicates you want to store data + +There are two arguments to explain here. + +The keyword ``k`` specifies the maximum number of seeding nodes on the network. In other words, for a given ``k``, you can have up to ``2**k`` nodes storing data, and as few as ``k``. ``k`` is also the maximum number of requests you can expect to issue for a given piece of data. So lookup time will be ``O(k)``. + +And like in :py:class:`~py2p.mesh.mesh_socket`, using ``'0.0.0.0'`` will automatically grab your LAN address. Using an outbound internet connection requires a little more work. First, ensure that you have a port forward set up (NAT busting is not in the scope of this project). Then specify your outward address as follows: + +.. code-block:: python + + >>> from py2p import chord + >>> sock = chord.chord_socket('0.0.0.0', 4444, k=2 out_addr=('35.24.77.21', 44565)) + >>> sock.join() # This indicates you want to store data + +In addition, SSL encryption can be enabled if `cryptography `_ is installed. This works by specifying a custom :py:class:`~py2p.base.protocol` object, like so: + +.. code-block:: python + + >>> from py2p import chord, base + >>> sock = chord.chord_socket('0.0.0.0', 4444, k=2, prot=base.protocol('chord', 'SSL')) + +Eventually that will be the default, but while things are being tested it will default to plaintext. If `cryptography `_ is not installed, this will generate an :py:exc:`ImportError` + +Specifying a different protocol object will ensure that the node *only* can connect to people who share its object structure. So if someone has ``'chord2'`` instead of ``'chord'``, it will fail to connect. You can see the current default by looking at :py:data:`py2p.chord.default_protocol`. + +This same check is performed for the ``k`` value provided. The full check which happens is essentially: + +.. code-block:: python + + assert your_protocol.id + to_base_58(your_k) == peer_protocol.id + to_base_58(peer_k) + +Unfortunately, this failure is currently silent. Because this is asynchronous in nature, raising an :py:exc:`Exception` is not possible. Because of this, it's good to perform the following check after connecting: + +.. code-block:: python + + >>> from py2p import chord + >>> import time + >>> sock = chord.chord_socket('0.0.0.0', 4444, k=2) + >>> sock.connect('192.168.1.14', 4567) + >>> time.sleep(1) + >>> assert sock.routing_table or sock.awaiting_ids + +Using the constructed table is very easy. Several :py:class:`dict`-like methods have been implemented. + +:py:meth:`~py2p.chord.chord_socket.get` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A value can be retrieved by using the :py:meth:`~py2p.chord.chord_socket.get` method, or alternately with :py:meth:`~py2p.chord.chord_socket.__getitem__`. + +.. code-block:: python + + >>> foo = sock.get('test key', None) # Returns None if there is nothing at that key + >>> bar = sock[b'test key'] # Raises KeyError if there is nothing at that key + +It is important to note that keys are all translated to :py:class:`bytes` before being used, so it is required that you use a :py:class:`bytes`-like object. It is also safer to manually convert :py:class:`unicode` keys to :py:class:`bytes`, as there are sometimes inconsistencies betwen the Javascript and Python implementation. If you notice one of these, please file a bug report. + +:py:meth:`~py2p.chord.chord_socket.set` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A value can be stored by using the :py:meth:`~py2p.chord.chord_socket.set` method, or alternately with :py:meth:`~py2p.chord.chord_socket.__setitem__`. + +.. code-block:: python + + >>> sock.set('test key', 'value') + >>> sock[b'test key'] = b'value' + +Like above, keys and values are all translated to :py:class:`bytes` before being used, so it is required that you use a :py:class:`bytes`-like object. + +:py:meth:`~py2p.chord.chord_socket.update` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The update method is simply a wrapper which updates based on a fed :py:class:`dict`. Essentially it runs the following: + +.. code-block:: python + + >>> for key in update_dict: + ... sock[key] = update_dict[key] + +Advanced Usage +-------------- + +Refer to :doc:`the mesh socket tutorial <./mesh>` diff --git a/docs/python/tutorial/mesh.rst b/docs/python/tutorial/mesh.rst new file mode 100644 index 0000000..597f186 --- /dev/null +++ b/docs/python/tutorial/mesh.rst @@ -0,0 +1,91 @@ +Mesh Socket +~~~~~~~~~~~ + +Basic Usage +----------- + +The mesh schema is used as an alert and messaging network. Its primary purpose is to ensure message delivery to every participant in the network. + +To connect to a mesh network, use the :py:class:`~py2p.mesh.mesh_socket` object. This is instantiated as follows: + +.. code-block:: python + + >>> from py2p import mesh + >>> sock = mesh.mesh_socket('0.0.0.0', 4444) + +Using ``'0.0.0.0'`` will automatically grab your LAN address. Using an outbound internet connection requires a little more work. First, ensure that you have a port forward set up (NAT busting is not in the scope of this project). Then specify your outward address as follows: + +.. code-block:: python + + >>> from py2p import mesh + >>> sock = mesh.mesh_socket('0.0.0.0', 4444, out_addr=('35.24.77.21', 44565)) + +In addition, SSL encryption can be enabled if `cryptography `_ is installed. This works by specifying a custom :py:class:`~py2p.base.protocol` object, like so: + +.. code-block:: python + + >>> from py2p import mesh, base + >>> sock = mesh.mesh_socket('0.0.0.0', 4444, prot=base.protocol('mesh', 'SSL')) + +Eventually that will be the default, but while things are being tested it will default to plaintext. If `cryptography `_ is not installed, this will generate an :py:exc:`ImportError` + +Specifying a different protocol object will ensure that the node *only* can connect to people who share its object structure. So if someone has ``'mesh2'`` instead of ``'mesh'``, it will fail to connect. You can see the current default by looking at :py:data:`py2p.mesh.default_protocol`. + +Unfortunately, this failure is currently silent. Because this is asynchronous in nature, raising an :py:exc:`Exception` is not possible. Because of this, it's good to perform the following check after connecting: + +.. code-block:: python + + >>> from py2p import mesh + >>> import time + >>> sock = mesh.mesh_socket('0.0.0.0', 4444) + >>> sock.connect('192.168.1.14', 4567) + >>> time.sleep(1) + >>> assert sock.routing_table + +To send a message, use the :py:meth:`~py2p.mesh.mesh_socket.send` method. Each argument supplied will correspond to a packet that the peer receives. In addition, there is a keyed argument you can use. ``flag`` will specify how other nodes relay this. These flags are defined in :py:class:`py2p.base.flags`. ``broadcast`` will indicate that other nodes are supposed to relay it. ``whisper`` will indicate that your peers are *not* supposed to relay it. + +.. code-block:: python + + >>> sock.send('this is', 'a test') + +Receiving is a bit simpler. When the :py:meth:`~py2p.mesh.mesh_socket.recv` method is called, it returns a :py:class:`~py2p.base.message` object (or ``None`` if there are no messages). This has a number of methods outlined which you can find by clicking its name. Most notably, you can get the packets in a message with :py:attr:`.message.packets`, and reply directly with :py:meth:`.message.reply`. + +.. code-block:: python + + >>> sock.send('Did you get this?') # A peer then replies + >>> msg = sock.recv() + >>> print(msg) + message(type=b'whisper', packets=[b'yes', b'I did'], sender=b'6VnYj9LjoVLTvU3uPhy4nxm6yv2wEvhaRtGHeV9wwFngWGGqKAzuZ8jK6gFuvq737V') + >>> print(msg.packets) + [b'whisper', b'yes', b'I did'] + >>> for msg in sock.recv(10): + ... msg.reply("Replying to a list") + +Advanced Usage +-------------- + +In addition to this, you can register a custom handler for incoming messages. This is appended to the end of the default handlers. These handlers are then called in a similar way to Javascripts ``Array.some()``. In other words, when a handler returns something true-like, it stops calling handlers. + +When writing your handler, keep in mind that you are only passed a :py:class:`~py2p.base.message` object and a :py:class:`~py2p.mesh.mesh_connection`. Fortunately you can get access to everything you need from these objects. + +.. code-block:: python + + >>> from py2p import mesh, base + >>> def register_1(msg, handler): # Takes in a message and mesh_connection + ... packets = msg.packets # This grabs a copy of the packets. Slightly more efficient to store this once. + ... if packets[1] == b'test': # This is the condition we want to act under + ... msg.reply(b"success") # This is the response we should give + ... return True # This tells the daemon we took an action, so it should stop calling handlers + ... + >>> def register_2(msg, handler): # This is a slightly different syntax + ... packets = msg.packets + ... if packets[1] == b'test': + ... handler.send(base.flags.whisper, base.flags.whisper, b"success") # One could instead reply to the node who relayed the message + ... return True + ... + >>> sock = mesh.mesh_socket('0.0.0.0', 4444) + >>> sock.register_handler(register_1) # The handler is now registered + +If this does not take two arguments, :py:meth:`~py2p.base.base_socket.register_handler` will raise a :py:exc:`ValueError`. + +To help debug these services, you can specify a :py:attr:`~py2p.base.base_socket.debug_level` in the constructor. Using a value of 5, you can see when it enters into each handler, as well as every message which goes in or out. diff --git a/docs/python/tutorial/sync.rst b/docs/python/tutorial/sync.rst new file mode 100644 index 0000000..a1bcacb --- /dev/null +++ b/docs/python/tutorial/sync.rst @@ -0,0 +1,80 @@ +Sync Socket +~~~~~~~~~~~ + +This is an extension of the :py:class:`~py2p.mesh.mesh_socket` which syncronizes a common :py:class:`dict`. It works by providing an extra handler to store data. This does not expose the entire :py:class:`dict` API, but it exposes a substantial subset, and we're working to expose more. + +.. note:: + + This is a fairly inefficient architecture for write intensive applications. For cases where the majority of access is reading, or for small networks, this is ideal. For larger networks where a significant portion of your operations are writing values, you should wait for the chord socket to come into beta. + +Basic Usage +----------- + +There are three limitations compared to a normal :py:class:`dict`. + +1. Keys and values can only be :py:class:`bytes`-like objects +2. Keys and values are automatically translated to :py:class:`bytes` +3. By default, this implements a leasing system which prevents you from changing values set by others for a certain time + +You can override the last restriction by constructing with ``leasing=False``, like so: + +.. code-block:: python + + >>> from py2p import sync + >>> sock = sync.sync_socket('0.0.0.0', 4444, leasing=False) + +The only API differences between this and :py:class:`~py2p.mesh.mesh_socket` are for access to this dictionary. They are as follows. + +:py:meth:`~py2p.sync.sync_socket.get` / :py:meth:`~py2p.sync.sync_socket.__getitem__` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A value can be retrieved by using the :py:meth:`~py2p.sync.sync_socket.get` method, or alternately with :py:meth:`~py2p.sync.sync_socket.__getitem__`. These calls are both ``O(1)``, as they read from a local :py:class:`dict`. + +.. code-block:: python + + >>> foo = sock.get('test key', None) # Returns None if there is nothing at that key + >>> bar = sock[b'test key'] # Raises KeyError if there is nothing at that key + >>> assert bar == foo == sock[u'test key'] # Because of the translation mentioned below, these are the same key + +It is important to note that keys are all translated to :py:class:`bytes` before being used, so it is required that you use a :py:class:`bytes`-like object. It is also safer to manually convert :py:class:`unicode` keys to :py:class:`bytes`, as there are sometimes inconsistencies betwen the Javascript and Python implementation. If you notice one of these, please file a bug report. + +:py:meth:`~py2p.sync.sync_socket.set` / :py:meth:`~py2p.sync.sync_socket.__setitem__` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A value can be stored by using the :py:meth:`~py2p.sync.sync_socket.set` method, or alternately with :py:meth:`~py2p.chord.chord_socket.__setitem__`. These calls are ``O(n)``, as it has to change values on other nodes. More accurately, the delay between your node knowing of the change and the last node knowing of the change is ``O(n)``. + +.. code-block:: python + + >>> sock.set('test key', 'value') + >>> sock[b'test key'] = b'value' + >>> sock[u'测试'] = 'test' + +Like above, keys and values are all translated to :py:class:`bytes` before being used, so it is required that you use a :py:class:`bytes`-like object. + +This will raise a :py:class:`KeyError` if another node has set this value already. Their lease will expire one hour after they set it. If two leases are started at the same UTC second, the tie is settled by doing a string compare of their IDs. + +Any node which sets a value can change this value as well. Changing the value renews the lease on it. + +:py:meth:`~py2p.sync.sync_socket.__delitem__` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Any node which owns a key, can clear its value. Doing this will relinquish your lease on that value. Like the above, this call is ``O(n)``. + +.. code-block:: python + + >>> del sock['test'] + +:py:meth:`~py2p.sync.sync_socket.update` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The update method is simply a wrapper which updates based on a fed :py:class:`dict`. Essentially it runs the following: + +.. code-block:: python + + >>> for key in update_dict: + ... sock[key] = update_dict[key] + +Advanced Usage +-------------- + +Refer to :doc:`the mesh socket tutorial <./mesh>` diff --git a/docs/python/utils.rst b/docs/python/utils.rst new file mode 100644 index 0000000..013a52b --- /dev/null +++ b/docs/python/utils.rst @@ -0,0 +1,9 @@ +Utils Module +=============== + + +.. automodule:: py2p.utils + :members: + :exclude-members: flags + :special-members: __init__, __iter__ + :undoc-members: \ No newline at end of file diff --git a/go_src/base.go b/go_src/base.go new file mode 100644 index 0000000..633ad20 --- /dev/null +++ b/go_src/base.go @@ -0,0 +1,413 @@ +/** +* Base Module +* =========== +* +* This module contains common functions and types used in the rest of the library. +*/ +package main + +import ( + "bytes" + "compress/gzip" + "compress/zlib" + "crypto/rand" + "crypto/sha256" + "crypto/sha512" + "fmt" + "io/ioutil" + "reflect" + "strings" + "time" +) + +func pseudo_uuid() (string) { + b := make([]byte, 16) + _, err := rand.Read(b) + if err != nil { + panic(err) + } + + b[8] = (b[8] | 0x80) & 0xBF + b[6] = (b[6] | 0x40) & 0x4F + + return fmt.Sprintf("%X-%X-%X-%X-%X", b[:4], b[4:6], b[6:8], b[8:10], b[10:]) +} + +func getUTC() (int64) { + return time.Now().UTC().Unix() +} + +func get_ulong(str interface{}) (int64) { + switch str.(type) { + case []byte: + return get_ulong_from_bytes(str.([]byte)) + case string: + b := []byte(str.(string)) + return get_ulong_from_bytes(b) + default: + panic("Invalid type") + } +} + +func get_ulong_from_bytes(arr []byte) (int64) { + if len(arr) != 4 { + panic("not size of long") + } + val := int64(0) + for i := 0; i < len(arr); i++ { + val *= 256 + val += int64(arr[i]) + } + return val +} + +func pack_ulong(i int64) ([]byte) { + arr := []byte("\x00\x00\x00\x00") + for c := 3; c >= 0; c-- { + arr[c] = byte(i % int64(256)) + i /= 256 + } + return arr +} + +var compression []string = []string{"gzip", "zlib"} + +func compress(data []byte, method string) ([]byte) { + switch method { + case "gzip": + var b bytes.Buffer + gz := gzip.NewWriter(&b) + if _, err := gz.Write(data); err != nil { + panic(err) + } + if err := gz.Flush(); err != nil { + panic(err) + } + if err := gz.Close(); err != nil { + panic(err) + } + return b.Bytes() + case "zlib": + var b bytes.Buffer + gz := zlib.NewWriter(&b) + if _, err := gz.Write(data); err != nil { + panic(err) + } + if err := gz.Flush(); err != nil { + panic(err) + } + if err := gz.Close(); err != nil { + panic(err) + } + return b.Bytes() + default: + panic("Unknown compression method") + } +} + +func decompress(data []byte, method string) ([]byte) { + switch method { + case "gzip": + b := bytes.NewBuffer(data) + gz, err := gzip.NewReader(b) + if err != nil { + panic(err) + } + var ret []byte + if ret, err = ioutil.ReadAll(gz); err != nil { + panic(err) + } + return ret + case "zlib": + b := bytes.NewBuffer(data) + gz, err := zlib.NewReader(b) + if err != nil { + panic(err) + } + var ret []byte + if ret, err = ioutil.ReadAll(gz); err != nil { + panic(err) + } + return ret + default: + panic("Unknown compression method") + } +} + +const base_58 string = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + +func to_base_58(i interface{}) (string) { + switch i.(type) { + case int32: + return to_base_58_from_int64(int64(i.(int32))) + case int: + return to_base_58_from_int64(int64(i.(int))) + case int64: + return to_base_58_from_int64(i.(int64)) + case []byte: + return to_base_58_from_bytes(i.([]byte)) + case string: + return to_base_58_from_bytes([]byte(i.(string))) + default: + panic(reflect.TypeOf(i)) + } +} + +func to_base_58_from_int64(i int64) (string) { + str := "" + for i != 0 { + str += string(base_58[i % int64(58)]) + i /= int64(58) + } + if str == "" { + str = string(base_58[0]) + } + return str +} + +func divide_byte_by_58(b []byte, remainder int) ([]byte, int) { + answer := []byte("") + for i := 0; i < len(b); i++ { + char := int(b[i]) + c := remainder * 256 + char + d := c / 58 + remainder = c % 58 + if len(answer) != 0 || d != 0 { + answer = append(answer, byte(d)) + } + } + return answer, remainder +} + +func to_base_58_from_bytes(b []byte) (string) { + answer := []byte("") + var char int + for len(b) != 0 { + b, char = divide_byte_by_58(b, 0) + answer = append([]byte{base_58[char]}, answer...) + } + return string(answer) +} + +func from_base_58(str string) (int64) { + decimal := int64(0) + for i := len(str) - 1; i >= 0; i-- { + decimal *= 58 + decimal += int64(strings.Index(base_58, string(str[i]))) + } + return decimal +} + +type protocol struct { + subnet string + encryption string +} + +func (p protocol) id() string { + info := []byte(p.subnet + p.encryption) + hash := sha256.New() + hash.Write(info) + return to_base_58(hash.Sum([]byte(""))) +} + +type pathfinding_message struct { + protocol protocol + msg_type string + sender string + payload [][]byte + time int64 + compression []string + compression_fail bool +} + +func (msg pathfinding_message) time_58() string { + return to_base_58_from_int64(msg.time) +} + +func (msg pathfinding_message) id() string { + payload_bytes := make([]byte, 0) + for i := 0; i < len(msg.payload); i++ { + payload_bytes = append(payload_bytes, msg.payload[i]...) + } + hash := sha512.New384() + hash.Write(payload_bytes) + return to_base_58_from_bytes(hash.Sum([]byte(""))) +} + +func (msg pathfinding_message) packets() [][]byte { + meta := [][]byte{ + []byte(msg.msg_type), + []byte(msg.sender), + []byte(msg.id()), + []byte(msg.time_58())} + return append(meta, msg.payload...) +} + +func (msg pathfinding_message) compression_used() string { + for i := 0; i < len(msg.compression); i++ { + for j := 0; j < len(compression); j++ { + if msg.compression[i] == compression[j] { + return msg.compression[i] + } + } + } + return "" +} + +func (msg pathfinding_message) base_bytes() []byte { + header := make([]byte, 0) + payload := make([]byte, 0) + packets := msg.packets() + for i := 0; i < len(packets); i++ { + header = append(header, pack_ulong(int64(len(packets[i])))...) + payload = append(payload, packets[i]...) + } + if comp := msg.compression_used(); comp != "" { + return compress(append(header, payload...), comp) + } + return append(header, payload...) +} + +func (msg pathfinding_message) bytes() []byte { + payload := msg.base_bytes() + header := pack_ulong(int64(len(payload))) + return append(header, payload...) +} + +func new_pathfinding_message(prot protocol, msg_type string, sender string, payload interface{}, compressions interface{}) (pathfinding_message) { + fmtd_payload := make([][]byte, 0) + fmtd_compression := make([]string, 0) + switch payload.(type) { + case [][]byte: + fmtd_payload = payload.([][]byte) + case []string: + payload := payload.([]string) + for i := 0; i < len(payload); i++ { + fmtd_payload = append(fmtd_payload, []byte(payload[i])) + } + default: + panic("payload is wrong type") + } + switch compressions.(type) { + case [][]byte: + compressions := compressions.([][]byte) + for i := 0; i < len(compressions); i++ { + fmtd_compression = append(fmtd_compression, string(compressions[i])) + } + case []string: + fmtd_compression = compressions.([]string) + default: + panic("compressions are wrong type") + } + return pathfinding_message{ + protocol: prot, + msg_type: msg_type, + sender: sender, + payload: fmtd_payload, + compression: fmtd_compression, + compression_fail: false, + time: getUTC()} +} + +func sanitize_string(b []byte, sizeless bool) []byte { + if sizeless { + return b + } + if get_ulong(string(b[:4])) != int64(len(b[4:])) { + panic("Size header is incorrect") + } + return b[4:] +} + +func decompress_string(b []byte, compressions []string) []byte { + compression_used := "" + for i := 0; i < len(compressions); i++ { + for j := 0; j < len(compression); j++ { + if compressions[i] == compression[j] { + compression_used = compressions[i] + break + } + } + } + if compression_used != "" { + return decompress(b, compression_used) + } + return b +} + +func process_string(b []byte) [][]byte { + processed := 0 + expected := len(b) + pack_lens := make([]int, 0) + packets := make([][]byte, 0) + for processed != expected { + pack_lens = append(pack_lens, int(get_ulong(b[processed:processed+4]))) + processed += 4 + expected -= pack_lens[len(pack_lens)-1] + } + for _, val := range pack_lens { + start := processed + processed += val + end := processed + packets = append(packets, b[start:end]) + } + return packets +} + +func feed_string(prot protocol, b []byte, sizeless bool, compressions []string) pathfinding_message { + b = sanitize_string(b, sizeless) + b = decompress_string(b, compressions) + packets := process_string(b) + return pathfinding_message{ + protocol: prot, + msg_type: string(packets[0]), + sender: string(packets[1]), + payload: packets[4:], + compression: compressions, + compression_fail: false, + time: from_base_58(string(packets[3]))} +} + +func main() { + fmt.Println(get_ulong("\xFF\x00\x00\x04")) + fmt.Println(to_base_58(4)) + fmt.Println(to_base_58(256)) + fmt.Println(from_base_58(to_base_58(4))) + fmt.Println(from_base_58(to_base_58(256))) + fmt.Println(from_base_58(to_base_58([]byte("\xFF\x00\x00\x04")))) + fmt.Println(protocol{ + subnet: "hi", + encryption: "Plaintext"}.id()) + fmt.Println(pseudo_uuid()) + test_msg := new_pathfinding_message( + protocol{ + "hi", + "Plaintext"}, + "test", + "test sender", + []string{"test1", "test2"}, + []string{} ) + fmt.Println(test_msg.bytes()) + fmt.Println(string(test_msg.bytes())) + test_msg.compression = []string{"gzip"} + fmt.Println("With gzip") + fmt.Println(test_msg.bytes()) + fmt.Println(string(test_msg.bytes())) + test_msg.compression = []string{"zlib"} + fmt.Println("With zlib") + fmt.Println(test_msg.bytes()) + fmt.Println(string(test_msg.bytes())) + fmt.Println(compress([]byte("test"), "gzip")) + fmt.Println(compress([]byte("test"), "zlib")) + fmt.Println(string(decompress(compress([]byte("test"), "gzip"), "gzip"))) + fmt.Println(string(decompress(compress([]byte("test"), "zlib"), "zlib"))) + fmt.Println(test_msg.bytes()) + from_feed := feed_string(protocol{ + "hi", + "Plaintext"}, + test_msg.bytes(), + false, + []string{"zlib"}) + fmt.Println(from_feed.bytes()) + fmt.Println(string(test_msg.bytes()) == string(from_feed.bytes())) +} \ No newline at end of file diff --git a/js_src/BigInteger b/js_src/BigInteger deleted file mode 160000 index cda5bcc..0000000 --- a/js_src/BigInteger +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cda5bcce74c3a4eb34951201d50c1b8776a56eca diff --git a/js_src/SHA b/js_src/SHA deleted file mode 160000 index 28fe708..0000000 --- a/js_src/SHA +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 28fe708c1913b7902876bb6228d6d9827dab38c7 diff --git a/js_src/base.js b/js_src/base.js new file mode 100644 index 0000000..e02e935 --- /dev/null +++ b/js_src/base.js @@ -0,0 +1,977 @@ +/** +* Base Module +* =========== +* +* This module contains common classes and functions which are used throughout the rest of the js2p library. +*/ + +"use strict"; + +var buffer = require('buffer'); // These ensure parser compatability with browserify +var Buffer = buffer.Buffer; +var BigInt = require('big-integer'); +var SHA = require('jssha'); +var zlib = require('zlibjs'); +var assert = require('assert'); +var net = require('net'); +var util = require('util'); + +/** +* .. note:: +* +* This library will likely issue a warning over the size of data that :js:class:`Buffer` can hold. On most implementations +* of Javascript this is 2GiB, but the maximum size message that can be transmitted in our serialization scheme is 4GiB. +* This shouldn't be a problem for most applications, but you can discuss it in :issue:`83`. +*/ +if (buffer.kMaxLength < 4294967299) { + console.log(`WARNING: This implementation of javascript does not support the maximum protocol length. The largest message you may receive is 4294967299 bytes, but you can only allocate ${buffer.kMaxLength}, or ${(buffer.kMaxLength / 4294967299 * 100).toFixed(2)}% of that.`); +} + +var base; + +if( typeof exports !== 'undefined' ) { + if( typeof module !== 'undefined' && module.exports ) { + base = exports = module.exports; + } + base = exports; +} +else { + root.base = {}; + base = root; +} + +/** +* .. js:data:: js2p.base.version_info +* +* A list containing the version numbers in the format ``[major, minor, patch]``. +* +* The first two numbers refer specifically to the protocol version. The last number increments with each build. +* +* .. js:data:: js2p.base.node_policy_version +* +* This is the last number in :js:data:`~js2p.base.version_info` +* +* .. js:data:: js2p.base.protocol_version +* +* This is the first two numbers of :js:data:`~js2p.base.version_info` joined in the format ``'a.b'`` +* +* .. warning:: +* +* Nodes with different versions of this variable will actively reject connections with each other +* +* .. js:data:: js2p.base.version +* +* This is :js:data:`~js2p.base.version_info` joined in the format ``'a.b.c'`` +*/ + +base.version_info = [0, 4, 516]; +base.node_policy_version = base.version_info[2].toString(); +base.protocol_version = base.version_info.slice(0, 2).join("."); +base.version = base.version_info.join('.'); + +base.flags = { + /** + * .. js:data:: js2p.base.flags + * + * A "namespace" which defines protocol reserved flags + */ + reserved: ['\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', + '\x08', '\x09', '\x0A', '\x0B', '\x0C', '\x0D', '\x0E', '\x0F', + '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', + '\x18', '\x19', '\x1A', '\x1B', '\x1C', '\x1D', '\x1E', '\x1F'], + + //main flags + broadcast: '\x00', + waterfall: '\x01', + whisper: '\x02', + renegotiate: '\x03', + ping: '\x04', + pong: '\x05', + + //sub-flags + //broadcast: '\x00', + compression: '\x01', + //whisper: '\x02', + handshake: '\x03', + //ping: '\x04', + //pong: '\x05', + notify: '\x06', + peers: '\x07', + request: '\x08', + resend: '\x09', + response: '\x0A', + store: '\x0B', + retrieve: '\x0C', + + //compression methods + bz2: '\x10', + gzip: '\x11', + lzma: '\x12', + zlib: '\x13', + bwtc: '\x14', + context1: '\x15', + defsum: '\x16', + dmc: '\x17', + fenwick: '\x18', + huffman: '\x19', + lzjb: '\x1A', + lzjbr: '\x1B', + lzp3: '\x1C', + mtf: '\x1D', + ppmd: '\x1E', + simple: '\x1F' +}; + +base.compression = [base.flags.zlib, base.flags.gzip]; +base.json_compressions = JSON.stringify(base.compression); + +// User salt generation pulled from: http://stackoverflow.com/a/2117523 +base.user_salt = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + var r = Math.random()*16|0; + var v = c === 'x' ? r : (r&0x3|0x8); + return v.toString(16); +}); + +base.intersect = function intersect() { + /** + * .. js:function:: js2p.base.intersect(array1, array2, [array3, [...]]) + * + * This function returns the intersection of two or more arrays. + * That is, it returns an array of the elements present in all arrays, + * in the order that they were present in the first array. + * + * :param arrayn: Any array-like object + * + * :returns: An array + */ + var last = arguments.length - 1; + var seen={}; + var result=[]; + for (var i = 1; i <= last; i++) { + for (var j = 0; j < arguments[i].length; j++) { + if (seen[arguments[i][j]]) { + seen[arguments[i][j]] += 1; + } + else if (i === 1) { + seen[arguments[i][j]] = 1; + } + } + } + for (var i = 0; i < arguments[0].length; i++) { + if ( seen[arguments[0][i]] === last) + result.push(arguments[0][i]); + } + return result; +} + +base.unpack_value = function unpack_value(str) { + /** + * .. js:function:: js2p.base.unpack_value(str) + * + * This function unpacks a string into its corresponding big endian value + * + * :param str: The string you want to unpack + * + * :returns: A big-integer + */ + str = new Buffer(str, 'ascii'); + var val = BigInt.zero; + for (var i = 0; i < str.length; i++) { + val = val.shiftLeft(8); + val = val.add(str[i]); + } + return val; +} + +base.pack_value = function pack_value(len, i) { + /** + * .. js:function:: js2p.base.pack_value(len, i) + * + * This function packs an integer i into a buffer of length len + * + * :param len: An integral value + * :param i: An integeral value + * + * :returns: A big endian buffer of length len + */ + var arr = new Buffer(new Array(len)); + for (var j = 0; j < len && i != 0; j++) { + arr[len - j - 1] = i & 0xff; + i = i >> 8; + } + return arr; +} + +base.base_58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'; + +base.to_base_58 = function to_base_58(i) { + /** + * .. js:function:: js2p.base.to_base_58(i) + * + * Takes an integer and returns its corresponding base_58 string + * + * :param i: An integral value + * + * :returns: the corresponding base_58 string + */ + var string = ""; + if (!BigInt.isInstance(i)) { + i = new BigInt(i); + } + while (i.notEquals(0)) { + string = base.base_58[i.mod(58)] + string; + i = i.divide(58); + } + if (!string) { + string = "1"; + } + return string; +}; + + +base.from_base_58 = function from_base_58(string) { + /** + * .. js:function:: js2p.base.from_base_58(string) + * + * Takes a base_58 string and returns its corresponding integer + * + * :param string: A base_58 string or string-like object + * + * :returns: A big-integer + */ + try { + string = string.toString() + } + finally { + var decimal = new BigInt(0); + //for char in string { + for (var i = 0; i < string.length; i++) { + decimal = decimal.times(58).plus(base.base_58.indexOf(string[i])); + } + return decimal; + } +}; + + +base.getUTC = function getUTC() { + /** + * .. js:function:: js2p.base.getUTC() + * + * :returns: An integral value containing the unix timestamp in seconds UTC + */ + return Math.floor(Date.now() / 1000); +}; + + +base.SHA384 = function SHA384(text) { + /** + * .. js:function:: js2p.base.SHA384(text) + * + * This function returns the hex digest of the SHA384 hash of the input text + * + * :param string text: A string you wish to hash + * + * :returns: the hex SHA384 hash + */ + var hash = new SHA("SHA-384", "TEXT"); + hash.update(text); + return hash.getHash("HEX"); +}; + + +base.SHA256 = function SHA256(text) { + /** + * .. js:function:: js2p.base.SHA256(text) + * + * This function returns the hex digest of the SHA256 hash of the input text + * + * :param string text: A string you wish to hash + * + * :returns: the hex SHA256 hash + */ + var hash = new SHA("SHA-256", "TEXT"); + hash.update(text); + return hash.getHash("HEX"); +}; + + +base.compress = function compress(text, method) { + /** + * .. js:function:: js2p.base.compress(text, method) + * + * This function is a shortcut for compressing data using a predefined method + * + * :param text: The string or Buffer-like object you wish to compress + * :param method: A compression method as defined in :js:data:`~js2p.base.flags` + * + * :returns: A variabley typed object containing a compressed version of text + */ + if (method === base.flags.zlib) { + return zlib.deflateSync(new Buffer(text)); + } + else if (method === base.flags.gzip) { + return zlib.gzipSync(new Buffer(text)); + } + else { + throw new Error("Unknown compression method"); + } +}; + + +base.decompress = function decompress(text, method) { + /** + * .. js:function:: js2p.base.decompress(text, method) + * + * This function is a shortcut for decompressing data using a predefined method + * + * :param text: The string or Buffer-like object you wish to decompress + * :param method: A compression method as defined in :js:data:`~js2p.base.flags` + * + * :returns: A variabley typed object containing a decompressed version of text + */ + if (method === base.flags.zlib) { + return zlib.inflateSync(new Buffer(text)); + } + else if (method === base.flags.gzip) { + return zlib.gunzipSync(new Buffer(text)); + } + else { + throw new Error("Unknown compression method"); + } +}; + + +base.protocol = class protocol { + /** + * .. js:class:: js2p.base.protocol(subnet, encryption) + * + * This class is used as a subnet object. Its role is to reject undesired connections. + * If you connect to someone who has a different protocol object than you, this descrepency is detected, + * and you are silently disconnected. + * + * :param string subnet: The subnet ID you wish to connect to. Ex: ``'mesh'`` + * :param string encryption: The encryption method you wish to use. Ex: ``'Plaintext'`` + */ + constructor(subnet, encryption) { + this.subnet = subnet; + this.encryption = encryption; + } + + get id() { + /** + * .. js:attribute:: js2p.base.protocol.id + * + * The ID of your desired network + */ + var protocol_hash = base.SHA256([this.subnet, this.encryption, base.protocol_version].join('')); + return base.to_base_58(new BigInt(protocol_hash, 16)); + } +}; + +base.default_protocol = new base.protocol('', 'Plaintext'); + +base.pathfinding_message = class pathfinding_message { + /** + * .. js:class:: js2p.base.pathfinding_message(msg_type, sender, payload, compression, timestamp) + * + * This is the message serialization/deserialization class. + * + * :param msg_type: This is the main flag checked by nodes, used for routing information + * :param sender: The ID of the person sending the message + * :param payload: A list of "packets" that you want your peers to receive + * :param compression: A list of compression methods that the receiver supports + * :param number timestamp: The time at which this message will be sent in seconds UTC + */ + constructor(msg_type, sender, payload, compression, timestamp) { + this.msg_type = new Buffer(msg_type); + this.sender = new Buffer(sender); + this.payload = payload || []; + for (var i = 0; i < this.payload.length; i++) { + this.payload[i] = new Buffer(this.payload[i]); + } + this.time = timestamp || base.getUTC(); + this.compression = compression || []; + this.compression_fail = false + } + + static feed_string(string, sizeless, compressions) { + /** + * .. js:function:: js2p.base.pathfinding_message.feed_string(string, sizeless, compressions) + * + * This method deserializes a message + * + * :param string: The message you would like to deserialize + * :param sizeless: A bool-like object describing whether the size header is present + * :param compressions: A list of possible compression methods this message may be under + * + * :returns: A :js:class:`~js2p.base.pathfinding_message` object containing the deserialized message + */ + string = base.pathfinding_message.sanitize_string(string, sizeless) + var compression_return = base.pathfinding_message.decompress_string(string, compressions) + var compression_fail = compression_return[1] + string = compression_return[0] + var packets = base.pathfinding_message.process_string(string) + var msg = new base.pathfinding_message(packets[0], packets[1], packets.slice(4), compressions) + msg.time = base.from_base_58(packets[3]) + msg.compression_fail = compression_fail + assert (msg.id === packets[2].toString(), `ID check failed. ${msg.id} !== ${packets[2].toString()}`) + return msg + } + + static sanitize_string(string, sizeless) { + try { + string = new Buffer(string) + } + finally { + if (!sizeless) { + if (base.unpack_value(string.slice(0,4)) + 4 !== string.length) { + //console.log(`slice given: ${string.slice(0, 4).inspect()}. Value expected: ${string.length - 4}. Value derived: ${base.unpack_value(string.slice(0, 4))}`) + throw "The following expression must be true: unpack_value(string.slice(0,4)) === string.length - 4" + } + string = string.slice(4) + } + return string + } + } + + static decompress_string(string, compressions) { + var compression_fail = false + compressions = compressions || [] + for (var i = 0; i < compressions.length; i++) { + //console.log(`Checking ${compressions[i]} compression`) + if (base.compression.indexOf(compressions[i]) > -1) { // module scope compression + //console.log(`Trying ${compressions[i]} compression`) + try { + string = base.decompress(string, compressions[i]) + compression_fail = false + //console.log(`Compression ${compressions[i]} succeeded`) + break + } + catch(err) { + compression_fail = true + //console.log(`compresion ${compressions[i]} failed: ${err}`) + continue + } + } + } + return [string, compression_fail] + } + + static process_string(string) { + var processed = 0; + var expected = string.length; + var pack_lens = []; + var packets = []; + while (processed < expected) { + pack_lens = pack_lens.concat(base.unpack_value(new Buffer(string.slice(processed, processed+4)))); + processed += 4; + expected -= pack_lens[pack_lens.length - 1]; + } + if (processed > expected) { + throw `Could not parse correctly processed=${processed}, expected=${expected}, pack_lens=${pack_lens}`; + } + // Then revarruct the packets + for (var i=0; i < pack_lens.length; i++) { + var end = processed + pack_lens[i]; + packets = packets.concat([string.slice(processed, end)]); + processed = end; + } + return packets; + } + + get compression_used() { + /** + * .. js:attribute:: js2p.base.pathfinding_message.compression_used + * + * Returns the compression method used in this message, as defined in :js:data:`~js2p.base.flags`, or ``undefined`` if none + */ + for (var i = 0; i < base.compression.length; i++) { + for (var j = 0; j < this.compression.length; j++) { + if (base.compression[i] === this.compression[j]) { + return base.compression[i]; + } + } + } + } + + get time_58() { + /** + * .. js:attribute:: js2p.base.pathfinding_message.time + * + * Returns the timestamp of this message + * + * + * .. js:attribute:: js2p.base.pathfinding_message.time_58 + * + * Returns the timestamp encoded in base_58 + */ + return base.to_base_58(this.time); + } + + get id() { + /** + * .. js:attribute:: js2p.base.pathfinding_message.id + * + * Returns the ID/checksum associated with this message + */ + try { + var payload_string = this.payload.join('') + var payload_hash = base.SHA384(payload_string + this.time_58) + return base.to_base_58(new BigInt(payload_hash, 16)) + } + catch (err) { + console.log(err); + console.log(this.payload); + } + } + + get packets() { + /** + * .. js:attribute:: js2p.base.pathfinding_message.payload + * + * Returns the payload "packets" associated with this message + * + * + * .. js:attribute:: js2p.base.pathfinding_message.packets + * + * Returns the total "packets" associated with this message + */ + var meta = [this.msg_type, this.sender, this.id, this.time_58] + return meta.concat(this.payload) + } + + get __non_len_string() { + var buf_array = []; + var packets = this.packets; + for (var i = 0; i < packets.length; i++) { + buf_array.push(new Buffer(packets[i])); + } + var total = Buffer.concat(buf_array); + var headers = []; + for (var i = 0; i < buf_array.length; i++) { + headers = headers.concat(base.pack_value(4, buf_array[i].length)); + } + total = Buffer.concat(headers.concat(total)); + if (this.compression_used) { + total = base.compress(total, this.compression_used); + } + return total; + } + + get string() { + /** + * .. js:attribute:: js2p.base.pathfinding_message.string + * + * Returns a Buffer containing the serialized version of this message + */ + var string = this.__non_len_string + return Buffer.concat([base.pack_value(4, string.length), string]); + } + + get length() { + /** + * .. js:attribute:: js2p.base.pathfinding_message.length + * + * Returns the length of this message when serialized + */ + return this.__non_len_string.length + } + + len() { + return pack_vlaue(4, this.length) + } +}; + +base.message = class message { + /** + * .. js:class:: js2p.base.message(msg, server) + * + * This is the message class we present to the user. + * + * :param js2p.base.pathfinding_message msg: This is the serialization object you received + * :param js2p.base.base_socket sender: This is the "socket" object that received it + */ + constructor(msg, server) { + this.msg = msg + this.server = server + } + + inspect() { + var packets = this.packets; + var type = packets[0]; + var payload = packets.slice(1); + var text = "message {\n"; + text += ` type: ${util.inspect(type)}\n`; + text += ` packets: ${util.inspect(payload)}\n`; + text += ` sender: ${util.inspect(this.sender.toString())} }`; + return text; + } + + get time() { + /** + * .. js:attribute:: js2p.base.message.time + * + * Returns the time (in seconds UTC) this message was sent at + */ + return this.msg.time + } + + get time_58() { + /** + * .. js:attribute:: js2p.base.message.time_58 + * + * Returns the time (in seconds UTC) this message was sent at, encoded in base_58 + */ + return this.msg.time_58 + } + + get sender() { + /** + * .. js:attribute:: js2p.base.message.sender + * + * Returns the ID of this message's sender + */ + return this.msg.sender + } + + get id() { + /** + * .. js:attribute:: js2p.base.message.id + * + * Returns the ID/checksum associated with this message + */ + return this.msg.id + } + + get packets() { + /** + * .. js:attribute:: js2p.base.message.packets + * + * Returns the packets the sender wished you to have, sans metadata + */ + return this.msg.payload + } + + get length() { + /** + * .. js:attribute:: js2p.base.message.length + * + * Returns the serialized length of this message + */ + return this.msg.length + } + + get protocol() { + /** + * .. js:attribute:: js2p.base.message.protocol + * + * Returns the :js:class:`~js2p.base.protocol` associated with this message + */ + return this.server.protocol + } + + reply(packs) { + /** + * .. js:function:: js2p.base.message.reply(packs) + * + * Replies privately to this message. + * + * .. warning:: + * + * Using this method has potential effects on the network composition. + * If you are not connected to the sender, we cannot garuntee + * the message will get through. If successful, you will experience + * higher network load on average. + * + * :param packs: A list of packets you want the other user to receive + */ + if (this.server.routing_table[this.sender]) { + this.server.routing_table[this.sender].send(base.flags.whisper, [base.flags.whisper].concat(packs)); + } + else { + var request_hash = base.SHA384(this.sender + base.to_base_58(base.getUTC())); + var request_id = base.to_base_58(new BigInt(request_hash, 16)); + this.server.requests[request_id] = [packs, base.flags.whisper, base.flags.whisper]; + this.server.send([request_id, this.sender], base.flags.broadcast, base.flags.request); + console.log("You aren't connected to the original sender. This reply is not guarunteed, but we're trying to make a connection and put the message through."); + } + } +}; + +base.base_connection = class base_connection { + /** + * .. js:class:: js2p.base.base_connection(sock, server, outgoing) + * + * This is the template class for connection abstracters. + * + * :param sock: This is the raw socket object + * :param js2p.base.base_socket server: This is a link to the :js:class:`~js2p.base.base_socket` parent + * :param outgoing: This bool describes whether ``server`` initiated the connection + */ + constructor(sock, server, outgoing) { + this.sock = sock; + this.server = server; + this.outgoing = outgoing | false; + this.buffer = new Buffer(0); + this.id = null; + this.time = base.getUTC(); + this.addr = null; + this.compression = []; + this.last_sent = []; + this.expected = 4; + this.active = false; + var self = this; + + this.sock.on('data', function(data) { + self.collect_incoming_data(self, data); + }); + this.sock.on('end', function() { + self.onEnd(); + }); + this.sock.on('error', function(err) { + self.onError(err); + }); + this.sock.on('close', function() { + self.onClose(); + }); + } + + onEnd() { + /** + * .. js:function:: js2p.base.base_connection.onEnd() + * + * This function is run when a connection is ended + */ + console.log(`Connection to ${this.id || this} ended. This is the template function`); + } + + onError(err) { + /** + * .. js:function:: js2p.base.base_connection.onError() + * + * This function is run when a connection experiences an error + */ + console.log(`Error: ${err}`); + this.sock.end(); + this.sock.destroy(); + } + + onClose() { + /** + * .. js:function:: js2p.base.base_connection.onClose() + * + * This function is run when a connection is closed + */ + console.log(`Connection to ${this.id || this} closed. This is the template function`); + } + + send(msg_type, packs, id, time) { + /** + * .. js:function:: js2p.base.base_connection.send(msg_type, packs, id, time) + * + * Sends a message through its connection. + * + * :param msg_type: Message type, corresponds to the header in a :js:class:`~js2p.base.pathfinding_message` object + * :param packs: A list of Buffer-like objects, which correspond to the packets to send to you + * :param id: The ID this message should appear to be sent from (default: your ID) + * :param number time: The time this message should appear to be sent from (default: now in UTC) + * + * :returns: the :js:class:`~js2p.base.pathfinding_message` object you just sent, or ``undefined`` if the sending was unsuccessful + */ + + //This section handles waterfall-specific flags + // console.log(packs); + id = id || this.server.id; //Latter is returned if key not found + time = time || base.getUTC(); + //Begin real method + var msg = new base.pathfinding_message(msg_type, id, packs, this.compression, time); + // console.log(msg.payload); + if (msg_type === base.flags.whisper || msg_type === base.flags.broadcast) { + this.last_sent = [msg_type].concat(packs); + } + // this.__print__(`Sending ${[msg.len()].concat(msg.packets)} to ${this}`, 4); + if (msg.compression_used) { + //console.log(`Compressing with ${JSON.stringify(msg.compression_used)}`); + // self.__print__(`Compressing with ${msg.compression_used}`, level=4) + } + // try { + //console.log(`Sending message ${JSON.stringify(msg.string.toString())} to ${this.id}`); + this.sock.write(msg.string, 'ascii') + return msg + // } + // catch(e) { + // self.server.daemon.exceptions.append((e, traceback.format_exc())) + // self.server.disconnect(self) + // } + } + + get protocol() { + return this.server.protocol; + } + + collect_incoming_data(self, data) { + /** + * .. js:function:: js2p.base.base_connection.collect_incoming_data(self, data) + * + * Collects and processes data which just came in on the socket + * + * :param self: A reference to this connection. Will be refactored out. + * :param Buffer data: The data which was just received + */ + self.buffer = Buffer.concat([self.buffer, data]); + //console.log(self.buffer); + self.time = base.getUTC(); + while (self.buffer.length >= self.expected) { + if (!self.active) { + // this.__print__(this.buffer, this.expected, this.find_terminator(), level=4) + self.expected = base.unpack_value(self.buffer.slice(0, 4)).add(4); + self.active = true; + // this.found_terminator(); + } + if (self.active && self.buffer.length >= self.expected) { //gets checked again because the answer may have changed + self.found_terminator(); + } + } + return true; + } + + found_terminator() { + /** + * .. js:function:: js2p.base.base_connection.found_terminator() + * + * This method is called when the expected amount of data is received + * + * :returns: The deserialized message received + */ + //console.log("I got called"); + var msg = base.pathfinding_message.feed_string(this.buffer.slice(0, this.expected), false, this.compression); + this.buffer = this.buffer.slice(this.expected); + this.expected = 4; + this.active = false; + return msg; + } + + handle_renegotiate(packets) { + /** + * .. js:function:: js2p.base.base_connection.handle_renegotiate(packets) + * + * This function handles connection renegotiations. This is used when compression methods + * fail, or when a node needs a message resent. + * + * :param packs: The array of packets which were received to initiate the renegotiation + * + * :returns: ``true`` if action was taken, ``undefined`` if not + */ + if (packets[0] == base.flags.renegotiate) { + if (packets[4] == base.flags.compression) { + var encoded_methods = JSON.parse(packets[5]); + var respond = (this.compression != encoded_methods); + this.compression = encoded_methods; + // self.__print__("Compression methods changed to: %s" % repr(self.compression), level=2) + if (respond) { + var decoded_methods = base.intersect(base.compression, this.compression); + self.send(base.flags.renegotiate, base.flags.compression, JSON.stringify(decoded_methods)) + } + return true; + } + else if (packets[4] == base.flags.resend) { + var type = self.last_sent[0]; + var packs = self.last_sent.slice(1); + self.send(type, packs); + return true; + } + } + } + + __print__() { + + } +}; + +base.base_socket = class base_socket { + /** + * .. js:class:: js2p.base.base_socket(addr, port [, protocol [, out_addr [, debug_level]]]) + * + * This is the template class for socket abstracters. + * + * :param string addr: The address you'd like to bind to + * :param number port: The port you'd like to bind to + * :param js2p.base.protocol protocol: The subnet you're looking to connect to + * :param array out_addr: Your outward-facing address + * :param number debug_level: The verbosity of debug prints + * + * .. js:attribute:: js2p.base.base_socket.routing_table + * + * An object which contains :js:class:`~js2p.base.base_connection` s keyed by their IDs + * + * .. js:attribute:: js2p.base.base_socket.awaiting_ids + * + * An array which contains :js:class:`~js2p.base.base_connection` s that are awaiting handshake information + */ + constructor(addr, port, protocol, out_addr, debug_level) { + var self = this; + this.addr = [addr, port]; + this.incoming = new net.Server(); + this.incoming.listen(port, addr); + this.protocol = protocol || base.default_protocol; + this.out_addr = out_addr || this.addr; + this.debug_level = debug_level || 0; + + this.awaiting_ids = []; + this.routing_table = {}; + this.id = base.to_base_58(BigInt(base.SHA384(`(${addr}, ${port})${this.protocol.id}${base.user_salt}`), 16)); + this.__handlers = []; + this.exceptions = []; + } + + get status() { + /** + * .. js:attribute:: js2p.base.base_socket.status + * + * This attribute describes whether the socket is operating as expected. + * + * It will either return a string ``"Nominal"`` or a list of Error/Traceback pairs + */ + if (this.exceptions.length) { + return this.exceptions; + } + return "Nominal"; + } + + register_handler(callback) { + /** + * .. js:function:: js2p.base.base_socket.register_handler(callback) + * + * This registers a message callback. Each is run through until one returns ``true``, + * rather like :js:func:`Array.some()`. The callback is expected to be of the form: + * + * .. code-block:: javascript + * + * function callback(msg, conn) { + * var packets = msg.packets; + * if (packets[0] === some_expected_value) { + * some_action(msg, conn); + * return true; + * } + * } + * + * :param function callback: A function formatted like the above + */ + this.__handlers = this.__handlers.concat(callback); + } + + handle_msg(msg, conn) { + var ret = false; + this.__handlers.some(function(handler) { + // self.__print__("Checking handler: %s" % handler.__name__, level=4) + // console.log(`Entering handler ${handler.name}`); + if (handler(msg, conn)) { + // self.__print__("Breaking from handler: %s" % handler.__name__, level=4) + // console.log(`breaking from ${handler.name}`); + ret = true; + return true + } + }); + return ret; + } +}; diff --git a/js_src/docs_test.js b/js_src/docs_test.js new file mode 100644 index 0000000..5ccb273 --- /dev/null +++ b/js_src/docs_test.js @@ -0,0 +1,96 @@ +var fs = require('fs'); +var path = require('path'); + +function walk(dir, done) { + var results = []; + fs.readdir(dir, function(err, list) { + if (err) return done(err); + var pending = list.length; + if (!pending) return done(null, results); + list.forEach(function(file) { + file = path.resolve(dir, file); + file = path.relative(dir, file); + fs.stat(file, function(err, stat) { + if (stat && stat.isDirectory()) { + walk(file, function(err, res) { + results = results.concat(res); + if (!--pending) done(null, results); + }); + } else { + results.push(file); + if (!--pending) done(null, results); + } + }); + }); + }); +}; + + +function execAll(re, string) { + var match = null; + var matches = []; + // For each match in the string: + while (match = re.exec(string)) { + matches.push(match.slice()); // Push a copy of it onto the return array + } + return matches; +} + +var folder_map = { + js_src: "javascript", + jv_src: "java", + go_src: "go", + cp_src: "cpp", + c_src: "c" +}; +var find_relevant_comments = /\/\*\*\r?\n(?:(?:(?![\n\r])\s)*\*([^\r\n]*)\r?\n)*\s*\*\//g; // http://regexr.com/3eb1d +var extract_lines = /(?:(?![\n\r])\s)*\*[ ]?([^\r\n\*\/][^\r\n]*)?\r?\n/g; // http://regexr.com/3ecnv + +function gen_walker(folder) { + return function done(err, res) { + if (err) throw err; + var docs_folder = path.resolve('docs', folder_map[folder]); + for (var i = 0; i < res.length; i++) { + var file = path.resolve('.', folder, res[i]); + if (fs.lstatSync(file).isDirectory()) { + continue; + } + var ending = res[i].substr(res[i].lastIndexOf(".") && res[i].indexOf("wrapper") < 0); + if (ending === ".cpp" || ending === ".c") + continue; + var file_dest_name = res[i].substr(0, res[i].lastIndexOf(".")) + ".rst"; + var file_dest = path.resolve(docs_folder, file_dest_name); + var string = fs.readFileSync(file); + var comments = execAll(find_relevant_comments, string); + var content = ""; + for (var j = 0; j < comments.length; j++) { + var lines = execAll(extract_lines, comments[j]); + for (var h = 0; h < lines.length; h++) { + if (lines[h][1] === undefined) { + content += "\n"; + } + else { + content += lines[h][1] + "\n"; + } + } + } + content += "\n\n"; + var options = { flag : 'w' }; + fs.writeFile(file_dest, content, options, (err) => { + if (err && err.code !== 'ENOENT') + throw err; + }); + } + } +} + +walk('js_src', gen_walker('js_src')); +walk('jv_src', gen_walker('jv_src')); +walk('go_src', gen_walker('go_src')); +walk('cp_src', gen_walker('cp_src')); +walk('c_src', gen_walker('c_src')); + +// walk(dir, function(err, results) { +// if (err) throw err; +// console.log(results); +// }); \ No newline at end of file diff --git a/js_src/js2p.js b/js_src/js2p.js new file mode 100644 index 0000000..5ebf8b7 --- /dev/null +++ b/js_src/js2p.js @@ -0,0 +1,24 @@ +"use strict"; + +var m; + +if( typeof exports !== 'undefined' ) { + if( typeof module !== 'undefined' && module.exports ) { + m = exports = module.exports; + } + m = exports; +} +else { + root.js2p = {}; + m = root; +} + +m.base = require('./base.js'); +m.mesh = require('./mesh.js'); +m.sync = require('./sync.js'); +m.version = m.base.version; +m.version_info = m.base.version_info; + +m.bootstrap = function bootstrap() { + throw "Not Implemented"; +} diff --git a/js_src/mesh.js b/js_src/mesh.js new file mode 100644 index 0000000..803643f --- /dev/null +++ b/js_src/mesh.js @@ -0,0 +1,567 @@ +/** +* Mesh Module +* =========== +*/ + +"use strict"; + +const BigInt = require('big-integer'); +const base = require('./base.js'); +const net = require('net'); +const util = require('util'); + +var m; + +if( typeof exports !== 'undefined' ) { + if( typeof module !== 'undefined' && module.exports ) { + m = exports = module.exports; + } + m = exports; +} +else { + root.mesh = {}; + m = root; +} + +m.max_outgoing = 4; + +m.default_protocol = new base.protocol('mesh', "Plaintext"); +/** +* .. js:data:: js2p.mesh.default_protocol +* +* A :js:class:`~js2p.base.protocol` object which is used by default in the mesh module +*/ + +m.mesh_connection = class mesh_connection extends base.base_connection { + /** + * .. js:class:: js2p.mesh.mesh_connection(sock, server, outgoing) + * + * This is the class for mesh connection abstractraction. It inherits from :js:class:`js2p.base.base_connection` + * + * :param sock: This is the raw socket object + * :param js2p.mesh.mesh_socket server: This is a link to the :js:class:`~js2p.mesh.mesh_socket` parent + * :param outgoing: This bool describes whether ``server`` initiated the connection + */ + constructor(sock, server, outgoing) { + super(sock, server, outgoing); + } + + send(msg_type, packs, id, time) { + /** + * .. js:function:: js2p.mesh.mesh_connection.send(msg_type, packs, id, time) + * + * Sends a message through its connection. + * + * :param msg_type: Message type, corresponds to the header in a :js:class:`~js2p.base.pathfinding_message` object + * :param packs: A list of Buffer-like objects, which correspond to the packets to send to you + * :param id: The ID this message should appear to be sent from (default: your ID) + * :param number time: The time this message should appear to be sent from (default: now in UTC) + * + * :returns: ``undefined`` + */ + // console.log(msg_type); + // console.log(packs); + try { + var msg = super.send(msg_type, packs, id, time); + //add msg to waterfall + var contained = false; + const mid = msg.id; + this.server.waterfalls.some(function(entry) { + if (!BigInt.isInstance(entry[1])) { + entry[1] = new BigInt(entry[1]); + } + if (entry[0] === mid && entry[1].equals(msg.time)) { + contained = true; + return true; + } + }); + if (!contained) { + this.server.waterfalls.unshift([mid, msg.time]); + } + } + catch(err) { + console.log(`There was an unhandled exception with peer id ${this.id}. This peer is being disconnected, and the relevant exception is added to the debug queue. If you'd like to report this, please post a copy of your mesh_socket.status to http://git.p2p.today/issues`); + this.server.exceptions.push(err); + this.sock.emit('error'); + } + } + + found_terminator() { + /** + * .. js:function:: js2p.mesh.mesh_connection.found_terminator() + * + * This method is called when the expected amount of data is received + * + * :returns: ``undefined`` + */ + try { + var msg = super.found_terminator(); + //console.log(msg.packets); + if (this.handle_waterfall(msg, msg.packets)) { + return true; + } + else if (this.handle_renegotiate(msg.packets)) { + return true; + } + this.server.handle_msg(new base.message(msg, this.server), this); + } + catch(err) { + console.log(`There was an unhandled exception with peer id ${this.id}. This peer is being disconnected, and the relevant exception is added to the debug queue. If you'd like to report this, please post a copy of your mesh_socket.status to http://git.p2p.today/issues`); + this.server.exceptions.push(err); + this.sock.emit('error'); + } + } + + handle_waterfall(msg, packets) { + /** + * .. js:function:: js2p.mesh.mesh_connection.handle_waterfall(msg, packets) + * + * This method determines whether this message has been previously received or not. + * If it has been previously received, this method returns ``true``. + * If it is older than a preset limit, this method returns ``true``. + * Otherwise this method returns ``undefined``, and forwards the message appropriately. + * + * :param js2p.base.pathfinding_message msg: The message in question + * :param packets: The message's packets + * + * :returns: ``true`` or ``undefined`` + */ + if (packets[0] == base.flags.waterfall || packets[0] == base.flags.broadcast) { + if (base.from_base_58(packets[3]) < base.getUTC() - 60) { + // this.__print__("Waterfall expired", level=2); + return true; + } + else if (!this.server.waterfall(new base.message(msg, this.server))) { + // this.__print__("Waterfall already captured", level=2); + return true; + } + // this.__print__("New waterfall received. Proceeding as normal", level=2) + } + } + + onClose() { + /** + * .. js:function:: js2p.mesh.mesh_connection.onClose() + * + * This function is run when a connection is closed + */ + if (this.server.routing_table[this.id]) { + delete this.server.routing_table[this.id]; + } + } + + onEnd() { + /** + * .. js:function:: js2p.mesh.mesh_connection.onEnd() + * + * This function is run when a connection is ended + */ + this.sock.end(); + this.sock.destroy(); + } +} + +m.mesh_socket = class mesh_socket extends base.base_socket { + /** + * .. js:class:: js2p.mesh.mesh_socket(addr, port [, protocol [, out_addr [, debug_level]]]) + * + * This is the class for mesh network socket abstraction. It inherits from :js:class:`js2p.base.base_socket` + * + * :param string addr: The address you'd like to bind to + * :param number port: The port you'd like to bind to + * :param js2p.base.protocol protocol: The subnet you're looking to connect to + * :param array out_addr: Your outward-facing address + * :param number debug_level: The verbosity of debug prints + * + * .. js:attribute:: js2p.mesh.mesh_socket.routing_table + * + * An object which contains :js:class:`~js2p.mesh.mesh_connection` s keyed by their IDs + * + * .. js:attribute:: js2p.mesh.mesh_socket.awaiting_ids + * + * An array which contains :js:class:`~js2p.mesh.mesh_connection` s that are awaiting handshake information + */ + constructor(addr, port, protocol, out_addr, debug_level) { + super(addr, port, protocol || m.default_protocol, out_addr, debug_level); + var self = this; + this.waterfalls = []; + this.requests = {}; + this.queue = []; + this.register_handler(function handle_handshake(msg, conn) {return self.__handle_handshake(msg, conn);}); + this.register_handler(function handle_peers(msg, conn) {return self.__handle_peers(msg, conn);}); + this.register_handler(function handle_response(msg, conn) {return self.__handle_response(msg, conn);}); + this.register_handler(function handle_request(msg, conn) {return self.__handle_request(msg, conn);}); + + this.incoming.on('connection', function onConnection(sock) { + var conn = new m.mesh_connection(sock, self, false); + self._send_handshake_response(conn); + self.awaiting_ids = self.awaiting_ids.concat(conn); + }); + } + + get outgoing() { + /** + * .. js:attribute:: js2p.mesh.mesh_socket.outgoing + * + * This is an array of all outgoing connections. The length of this array is used to determine + * whether the "socket" should automatically initiate connections + */ + var outs = []; + var self = this; + Object.keys(this.routing_table).forEach(function(key) { + if (self.routing_table[key].outgoing) { + outs.push(self.routing_table[key]); + } + }); + return outs; + } + + recv(num) { + /** + * .. js:function:: js2p.mesh.mesh_socket.recv([num]) + * + * This function has two behaviors depending on whether num is truthy. + * + * If num is truthy, it will return a list of :js:class:`~js2p.base.message` objects up to length len. + * + * If num is not truthy, it will return either a single :js:class:`~js2p.base.message` object, or ``undefined`` + * + * :param number num: The maximum number of :js:class:`~js2p.base.message` s you would like to pull + * + * :returns: A list of :js:class:`~js2p.base.message` s, an empty list, a single :js:class:`~js2p.base.message` , or ``undefined`` + */ + var ret; + if (num) { + ret = this.queue.slice(0, num); + this.queue = this.queue.slice(num); + } + else { + ret = this.queue[0]; + this.queue = this.queue.slice(1); + } + return ret; + } + + _send_handshake_response(handler) { + /** + * .. js:function:: js2p.mesh.mesh_socket._send_handshake_response(handler) + * + * Shortcut method to send a handshake response. This method is extracted from + * :js:meth:`~js2p.mesh.mesh_socket.__handle_handshake` in order to allow cleaner + * inheritence from :js:class:`js2p.sync.sync_socket` + */ + handler.send(base.flags.whisper, [base.flags.handshake, this.id, this.protocol.id, `["${this.out_addr[0]}", ${this.out_addr[1]}]`, base.json_compressions]); + } + + connect(addr, port, id) { + /** + * .. js:function:: js2p.mesh.mesh_socket.connect(addr, port [, id]) + * + * This function connects you to a specific node in the overall network. + * Connecting to one node *should* connect you to the rest of the network, + * however if you connect to the wrong subnet, the handshake failure involved + * is silent. You can check this by looking at the truthiness of this objects + * routing table. Example: + * + * .. code-block:: javascript + * + * > var conn = new mesh.mesh_socket('localhost', 4444); + * > conn.connect('localhost', 5555); + * > //do some other setup for your program + * > if (!conn.routing_table) { + * ... conn.connect('localhost', 6666); // any fallback address + * ... } + * + * :param string addr: A string address + * :param number port: A positive, integral port + * :param id: A string-like object which represents the expected ID of this node + * + * .. note:: + * + * While in the Python version there are more thorough checks on this, the Javascript + * implementation *can* connect to itself. There are checks to keep this from happening + * automatically, but it's still trivial to override this via human intervention. Please + * do not try to connect to yourself. + */ + // self.__print__("Attempting connection to %s:%s with id %s" % (addr, port, repr(id)), level=1) + // if socket.getaddrinfo(addr, port)[0] == socket.getaddrinfo(*self.out_addr)[0] or \ + // id in self.routing_table: + // self.__print__("Connection already established", level=1) + // return false + var shouldBreak = (id == this.id || [addr, port] == this.out_addr || [addr, port] == this.addr); + var self = this; + Object.keys(this.routing_table).some(function(key) { + if (key == id || self.routing_table[key].addr == [addr, port]) { + shouldBreak = true; + } + if (shouldBreak) { + return true; + } + }); + if (shouldBreak) { + return false; + } + var conn = new net.Socket(); + // conn.settimeout(1) + conn.connect(port, addr); + var handler = new m.mesh_connection(conn, this, true); + handler.id = id; + this._send_handshake_response(handler); + if (id) { + this.routing_table[id] = handler; + } + else { + this.awaiting_ids = this.awaiting_ids.concat(handler); + } + } + + disconnect(handler) { + /** + * .. js:function:: js2p.mesh.mesh_socket.disconnect(handler) + * + * Closes a given connection, and removes it from your routing tables + * + * :param js2p.mesh.mesh_connection handler: The connection you wish to close + */ + handler.sock.end(); + handler.sock.destroy(); //These implicitly remove from routing table + } + + handle_msg(msg, conn) { + if (!super.handle_msg(msg, conn)) { + var packs = msg.packets; + if (packs[0] == base.flags.whisper || packs[0] == base.flags.broadcast) { + this.queue = this.queue.concat(msg); + } + // else { + // this.__print__("Ignoring message with invalid subflag", level=4); + // } + } + } + + __get_peer_list() { + /** + * .. js:function:: js2p.mesh.mesh_socket.__get_peer_list() + * + * This function is used to generate a list-formatted group of your peers. It goes in format ``[ [[addr, port], ID], ...]`` + * + * :returns: An array in the above format + */ + var ret = []; + var self = this; + Object.keys(this.routing_table).forEach(function(key) { + ret = ret.concat([[self.routing_table[key].addr, key]]); + }); + return ret; + } + + __handle_handshake(msg, conn) { + /** + * .. js:function:: js2p.mesh.mesh_socket.__handle_handshake(msg, conn) + * + * This callback is used to deal with handshake signals. Its three primary jobs are: + * + * - reject connections seeking a different network + * - set connection state + * - deal with connection conflicts + * + * :param js2p.base.message msg: + * :param js2p.mesh.mesh_connection conn: + * + * :returns: Either ``true`` or ``undefined`` + */ + var packets = msg.packets; + if (packets[0].toString() == base.flags.handshake) { + if (packets[2] != msg.protocol.id) { + this.disconnect(conn); + return true; + } + // else if (handler is not this.routing_table.get(packets[1], handler)) { + // this.__resolve_connection_conflict(handler, packets[1]); + // } + conn.id = packets[1]; + conn.addr = JSON.parse(packets[3]); + //console.log(`changed compression methods to: ${packets[4]}`); + conn.compression = JSON.parse(packets[4]); + // self.__print__("Compression methods changed to %s" % repr(handler.compression), level=4) + if (this.awaiting_ids.indexOf(conn) > -1) { // handler in this.awaiting_ids + this.awaiting_ids.splice(this.awaiting_ids.indexOf(conn), 1); + } + this.routing_table[packets[1]] = conn; + conn.send(base.flags.whisper, [base.flags.peers, JSON.stringify(this.__get_peer_list())]); + return true; + } + } + + __handle_peers(msg, conn) { + /** + * .. js:function:: js2p.mesh.mesh_socket.__handle_peers(msg, conn) + * + * This callback is used to deal with peer signals. Its primary jobs is to connect to the given peers, if this does not exceed :js:data:`js2p.mesh.max_outgoing` + * + * :param js2p.base.message msg: + * :param js2p.mesh.mesh_connection conn: + * + * :returns: Either ``true`` or ``undefined`` + */ + var packets = msg.packets; + if (packets[0].toString() == base.flags.peers) { + var new_peers = JSON.parse(packets[1]); + var self = this; + new_peers.forEach(function(peer_array) { + if (self.outgoing.length < m.max_outgoing) { + // try: + var addr = peer_array[0]; + var id = peer_array[1]; + self.connect(addr[0], addr[1], id); + } + // except: # pragma: no cover + // self.__print__("Could not connect to %s:%s because\n%s" % (addr[0], addr[1], traceback.format_exc()), level=1) + // continue + }); + return true; + } + } + + __handle_response(msg, conn) { + /** + * .. js:function:: js2p.mesh.mesh_socket.__handle_response(msg, conn) + * + * This callback is used to deal with response signals. Its two primary jobs are: + * + * - if it was your request, send the deferred message + * - if it was someone else's request, relay the information + * + * :param js2p.base.message msg: + * :param js2p.mesh.mesh_connection conn: + * + * :returns: Either ``true`` or ``undefined`` + */ + var packets = msg.packets; + if (packets[0].toString() == base.flags.response) { + // self.__print__("Response received for request id %s" % packets[1], level=1) + if (this.requests[packets[1]]) { + var addr = JSON.parse(packets[2]); + if (addr) { + var msg = this.requests[packets[1]]; + // console.log(msg); + this.connect(addr[0][0], addr[0][1], addr[1]); + this.routing_table[addr[1]].send(msg[1], [msg[2]].concat(msg[0])); + delete this.requests[packets[1]]; + } + } + return true; + } + } + + __handle_request(msg, conn) { + /** + * .. js:function:: js2p.mesh.mesh_socket.__handle_request(msg, conn) + * + * This callback is used to deal with request signals. Its three primary jobs are: + * + * - respond with a peers signal if packets[1] is ``'*'`` + * - if you know the ID requested, respond to it + * - if you don't, make a request with your peers + * + * :param js2p.base.message msg: + * :param js2p.mesh.mesh_connection conn: + * + * :returns: Either ``true`` or ``undefined`` + */ + var packets = msg.packets; + //console.log(packets[0].toString()); + //console.log(packets[1].toString()); + if (packets[0].toString() == base.flags.request) { + if (packets[1].toString() == '*') { + conn.send(base.flags.whisper, [base.flags.peers, JSON.stringify(this.__get_peer_list())]); + } + else if (this.routing_table[packets[2]]) { + conn.send(base.flags.broadcast, [base.flags.response, packets[1], JSON.stringify([this.routing_table[packets[2]].addr, packets[2]])]); + } + return true; + } + } + + send(packets, flag, type) { + /** + * .. js:function:: js2p.mesh.mesh_socket.send(packets [, flag [, type]]) + * + * This sends a message to all of your peers. If you use default values it will send it to everyone on the network + * + * :param packets: A list of strings or Buffer-like objects you want your peers to receive + * :param flag: A string or Buffer-like object which defines your flag. In other words, this defines packet 0. + * :param type: A string or Buffer-like object which defines your message type. Changing this from default can have adverse effects. + * + * .. warning:: + * + * If you change the type attribute from default values, bad things could happen. It **MUST** be a value from :js:data:`js2p.base.flags` , + * and more specifically, it **MUST** be either ``broadcast`` or ``whisper``. The only other valid flags are ``waterfall`` and ``renegotiate``, + * but these are **RESERVED** and must **NOT** be used. + */ + var send_type = type || base.flags.broadcast; + var main_flag = flag || base.flags.broadcast; + var self = this; + Object.keys(this.routing_table).forEach(function(key) { + self.routing_table[key].send(main_flag, [send_type].concat(packets)); + }); + } + + __clean_waterfalls() { + /** + * .. js:function:: js2p.mesh.mesh_socket.__clean_waterfalls() + * + * This function cleans the list of recently relayed messages based on + * the following heurisitics: + * + * - Delete all duplicates + * - Delete all older than 60 seconds + */ + this.waterfalls = Array.from(new Set(this.waterfalls)); + var new_waterfalls = []; + var filter_time = base.getUTC() - 60; + for (var i in this.waterfalls) { + if (this.waterfalls[i][1] > filter_time) { + new_waterfalls.push(this.waterfalls[i]); + } + } + this.waterfalls = new_waterfalls; + } + + waterfall(msg) { + /** + * .. js:function:: js2p.mesh.mesh_socket.waterfall(msg) + * + * This function handles message relays. Its return value is based on + * whether it took an action or not. + * + * :param js2p.base.message msg: The message in question + * + * :returns: ``true`` if the message was then forwarded. ``false`` if not. + */ + var contained = false; + const id = msg.id; + this.waterfalls.some(function(entry) { + if (entry[0] === id && entry[1].equals(msg.time)) { + contained = true; + return true; + } + }); + if (!contained) { + this.waterfalls.unshift([id, msg.time]); + var self = this; + Object.keys(this.routing_table).forEach(function(key) { + var handler = self.routing_table[key]; + if (handler.id.toString() !== msg.sender.toString()) { + handler.send(base.flags.waterfall, msg.packets, msg.sender, msg.time); + } + }); + this.__clean_waterfalls() + return true + } + else { + // this.__print__("Not rebroadcasting", level=3) + return false + } + } +}; diff --git a/js_src/p2p.js b/js_src/p2p.js deleted file mode 100644 index bb21ff0..0000000 --- a/js_src/p2p.js +++ /dev/null @@ -1,290 +0,0 @@ -var BigInt = require('./BigInteger/BigInteger.js'); -var struct = require('./pack/bufferpack.js'); -var SHA = require('./SHA/src/sha.js'); -var zlib = require('./zlib/bin/node-zlib.js'); - -function p2p() { - "use strict"; - var m = this; - - if (!Array.prototype.last) { - Array.prototype.last = function() { - return this[this.length - 1]; - }; - } - - m.protocol_version = "0.2"; - m.node_policy_version = "136"; - - m.version = [m.protocol_version, m.node_policy_version].join('.'); - m.compression = ['gzip']; - - // User salt generation pulled from: http://stackoverflow.com/a/2117523 - m.user_salt = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { - var r = Math.random()*16|0, v = c === 'x' ? r : (r&0x3|0x8); - return v.toString(16); - }); - - m.base_58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'; - - m.to_base_58 = function(i) { - //Takes an integer and returns its corresponding base_58 string - var string = ""; - if (!BigInt.isInstance(i)) { - i = BigInt(i); - } - while (i.notEquals(0)) { - string = m.base_58[i.mod(58)] + string; - i = i.divide(58); - } - return string; - }; - - - m.from_base_58 = function(string) { - //Takes a base_58 string and returns its corresponding integer - var decimal = BigInt(0); - //for char in string { - for (var i = 0; i < string.length; i++) { - decimal = decimal.times(58).plus(m.base_58.indexOf(string[i])); - } - return decimal; - }; - - - m.getUTC = function() { - return Math.floor(Date.now() / 1000); - }; - - - m.SHA384 = function(text) { - var hash = new SHA("SHA-384", "TEXT"); - hash.update(text); - return hash.getHash("HEX"); - }; - - - m.SHA256 = function(text) { - var hash = new SHA("SHA-256", "TEXT"); - hash.update(text); - return hash.getHash("HEX"); - }; - - - m.compress = function(text, method) { - if (method === "gzip") { - return zlib.deflateSync(Buffer(text)); - } - else { - throw "Unknown compression method"; - } - }; - - - m.decompress = function(text, method) { - if (method === "gzip") { - return zlib.inflateSync(Buffer(text)); - } - else { - throw "Unknown compression method"; - } - }; - - - m.protocol = class protocol { - constructor(subnet, encryption) { - this.subnet = subnet; - this.encryption = encryption; - } - - id() { - var protocol_hash = m.SHA256([this.subnet, this.encryption, m.protocol_version].join('')); - return m.to_base_58(BigInt(protocol_hash, 16)); - } - }; - - m.default_protocol = new m.protocol('', 'Plaintext'); - - m.pathfinding_message = class pathfinding_message { - constructor(protocol, msg_type, sender, payload, compression) { - this.protocol = protocol - this.msg_type = msg_type - this.sender = sender - this.payload = payload - this.time = m.getUTC() - if (compression) { - this.compression = compression - } - else { - this.compression = [] - } - this.compression_fail = false - } - - static feed_string(protocol, string, sizeless, compressions) { - string = m.pathfinding_message.sanitize_string(string, sizeless) - var compression_return = m.pathfinding_message.decompress_string(string, compressions) - var compression_fail = compression_return[1] - string = compression_return[0] - var packets = m.pathfinding_message.process_string(string) - var msg = new m.pathfinding_message(protocol, packets[0], packets[1], packets.slice(4), compressions) - msg.time = m.from_base_58(packets[3]) - msg.compression_fail = compression_fail - return msg - } - - static sanitize_string(string, sizeless) { - try { - string = string.toString() - } - finally { - if (!sizeless) { - if (struct.unpack("!L", Buffer(string.substring(0,4)))[0] !== string.substring(4).length) { - throw "The following expression must be true: struct.unpack(\"!L\", Buffer(string.substring(0,4)))[0] === string.substring(4).length" - } - string = string.substring(4) - } - return string - } - } - - static decompress_string(string, compressions) { - var compression_fail = false - compressions = compressions || [] - for (var i = 0; i < compressions.length; i++) { - if (compressions[i] in m.compression) { // module scope compression - console.log("Trying %s compression" % method) - try { - string = m.decompress(string, method) - compression_fail = false - break - } - catch(err) { - compression_fail = true - continue - } - } - } - return [string, compression_fail] - } - - static process_string(string) { - var processed = 0 - var expected = string.length - var pack_lens = [] - var packets = [] - function add(a, b) { - return a + b - } - while (processed !== expected) { - pack_lens = pack_lens.concat(struct.unpack("!L", Buffer(string.substring(processed, processed+4)))) - processed += 4 - expected -= pack_lens.last() - } - // Then reconstruct the packets - for (var i=0; i < pack_lens.length; i++) { - var start = processed + pack_lens.slice(0, i).reduce(add, 0) - var end = start + pack_lens[i] - packets = packets.concat([string.substring(start, end)]) - } - return packets - } - - get compression_used() { - for (var i = 0; i < m.compression.length; i++) { - for (var j = 0; j < this.compression.length; j++) { - if (m.compression[i] === this.compression[j]) { - return m.compression[i] - } - } - } - return null - } - - get time_58() { - return m.to_base_58(this.time) - } - - get id() { - var payload_string = this.payload.join('') - var payload_hash = m.SHA384(payload_string + this.time_58) - return m.to_base_58(BigInt(payload_hash, 16)) - } - - get packets() { - var meta = [this.msg_type, this.sender, this.id, this.time_58] - return meta.concat(this.payload) - } - - get __non_len_string() { - var string = this.packets.join('') - var headers = [] - for (var i = 0; i < this.packets.length; i++) { - headers = headers.concat(struct.pack("!L", [this.packets[i].length])) - } - string = headers.join('') + string - if (this.compression_used) { - string = m.compress(string, this.compression_used) - } - return string - } - - get string() { - var string = this.__non_len_string - return struct.pack("!L", [string.length]) + string - } - - get length() { - return this.__non_len_string.length - } - - len() { - return struct.pack("!L", [this.length]) - } - }; - - m.message = class message { - constructor(msg, server) { - this.msg = msg - this.server = server - } - - get time() { - return this.msg.time - } - - get sender() { - return this.msg.sender - } - - get protocol() { - return this.msg.protocol - } - - get id() { - return this.msg.id - } - - get packets() { - return this.msg.payload - } - - get length() { - return this.msg.length - } - - reply(args) { - throw "Not implemented" - } - }; -} - -if( typeof exports !== 'undefined' ) { - if( typeof module !== 'undefined' && module.exports ) { - exports = module.exports = new p2p() - } - exports.p2p = new p2p() -} -else { - root.p2p = new p2p() -} \ No newline at end of file diff --git a/js_src/pack b/js_src/pack deleted file mode 160000 index 575b314..0000000 --- a/js_src/pack +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 575b31464f966c6796511c369ab90756336a42d7 diff --git a/js_src/sync.js b/js_src/sync.js new file mode 100644 index 0000000..0a5c60b --- /dev/null +++ b/js_src/sync.js @@ -0,0 +1,208 @@ +/** +* Sync Module +* =========== +*/ + +"use strict"; + +const base = require('./base.js'); +const mesh = require('./mesh.js'); + +var m; + +if( typeof exports !== 'undefined' ) { + if( typeof module !== 'undefined' && module.exports ) { + m = exports = module.exports; + } + m = exports; +} +else { + root.sync = {}; + m = root; +} + +m.default_protocol = new base.protocol('sync', 'Plaintext'); + +m.metatuple = class metatuple { + /** + * .. js:class:: js2p.sync.metatuple(owner, timestamp) + * + * This class is used to store metadata for a particular key + */ + constructor(owner, timestamp) { + this.owner = owner; + this.timestamp = timestamp; + } +} + +m.sync_socket = class sync_socket extends mesh.mesh_socket { + /** + * .. js:class:: js2p.sync.sync_socket(addr, port [, leasing [, protocol [, out_addr [, debug_level]]]]) + * + * This is the class for mesh network socket abstraction. It inherits from :js:class:`js2p.mesh.mesh_socket`. + * Because of this inheritence, this can also be used as an alert network. + * + * This also implements and optional leasing system by default. This leasing system means that + * if node A sets a key, node B cannot overwrite the value at that key for an hour. + * + * This may be turned off by setting ``leasing`` to ``false`` to the constructor. + * + * :param string addr: The address you'd like to bind to + * :param number port: The port you'd like to bind to + * :param boolean leasing: Whether this class's leasing system should be enabled (default: ``true``) + * :param js2p.base.protocol protocol: The subnet you're looking to connect to + * :param array out_addr: Your outward-facing address + * :param number debug_level: The verbosity of debug prints + */ + constructor(addr, port, leasing, protocol, out_addr, debug_level) { + if (!protocol) { + protocol = m.default_protocol; + } + let lease_descriptor = (leasing !== false) ? '1' : '0'; + let protocol_used = new base.protocol(protocol.subnet + lease_descriptor, protocol.encryption); + super(addr, port, protocol_used, out_addr, debug_level); + this.data = {}; + this.metadata = {}; + const self = this; + this.register_handler(function handle_store(msg, conn) {return self.__handle_store(msg, conn);}); + } + + __store(key, new_data, new_meta, error) { + /** + * .. js:function:: js2p.sync.sync_socket.__store(key, new_data, new_meta, error) + * + * Private API method for storing data + * + * :param key: The key you wish to store data at + * :param new_data: The data you wish to store in said key + * :param new_meta: The metadata associated with this storage + * :param error: A boolean which says whether to raise a :py:class:`KeyError` if you can't store there + * + * :raises Error: If someone else has a lease at this value, and ``error`` is not ``false`` + */ + let meta = this.metadata[key]; + if ( (!meta) || (!this.__leasing) || (meta.owner == new_meta.owner) || + (meta.timestamp > new_meta.timestamp) || (meta.timestamp < base.getUTC() - 3600) || + (meta.timestamp == new_meta.timestamp && meta.owner > new_meta.owner) ) { + if (new_data !== new Buffer('')) { + this.metadata[key] = new_meta; + this.data[key] = new_data; + } + else { + delete this.data[key]; + delete this.metadata[key]; + } + } + else if (error !== false) { + throw new Error("You don't have permission to change this yet"); + } + } + + _send_handshake_response(handler) { + /** + * .. js:function:: js2p.sync.sync_socket._send_handshake_response(handler) + * + * Shortcut method to send a handshake response. This method is extracted from :js:func:`~js2p.mesh.mesh_socket.__handle_handshake` + * in order to allow cleaner inheritence from :js:class:`js2p.sync.sync_socket` + * + */ + super._send_handshake_response(handler) + for (var key in this.data) { + let meta = this.metadata[key]; + handler.send(base.flags.whisper, [base.flags.store, key, this.data[key], meta.owner, base.to_base_58(meta.timestamp)]); + } + } + + __handle_store(msg, handler) { + /** + * .. js:function:: js2p.sync.sync_socket.__handle_store + * + * This callback is used to deal with data storage signals. Its two primary jobs are: + * + * - store data in a given key + * - delete data in a given key + * + * :param msg: A :js:class:`~js2p.base.message` + * :param handler: A :js:class:`~js2p.mesh.mesh_connection` + * + * :returns: Either ``true`` or ``undefined`` + */ + const packets = msg.packets; + if (packets[0].toString() === base.flags.store) { + let meta = new m.metatuple(msg.sender, msg.time); + if (packets.length === 5) { + if (this.data[packets[1]]) { + return; + } + meta = new m.metatuple(packets[3], base.from_base_58(packets[4])); + } + this.__store(packets[1], packets[2], meta, false); + return true; + } + } + + get(key, fallback) { + /** + * .. js:function:: js2p.sync.sync_socket.get(key [, fallback]) + * + * Retrieves the value at a given key + * + * :param key: The key you wish to look up (must be transformable into a :js:class:`Buffer` ) + * :param fallback: The value it should return when the key has no data + * + * :returns: The value at the given key, or ``fallback``. + * + * :raises TypeError: If the key could not be transformed into a :js:class:`Buffer` + */ + let l_key = new Buffer(key); + return this.data[l_key] || fallback; + } + + set(key, data) { + /** + * .. js:function:: js2p.sync.sync_socket.set(key, value) + * + * Sets the value at a given key + * + * :param key: The key you wish to look up (must be transformable into a :js:class:`Buffer` ) + * :param value: The key you wish to store (must be transformable into a :js:class:`Buffer` ) + * + * :raises TypeError: If a key or value could not be transformed into a :js:class:`Buffer` + * :raises: See :js:func:`~js2p.sync.sync_socket.__store` + */ + let new_meta = new m.metatuple(this.id, base.getUTC()); + let s_key = new Buffer(key); + let s_data = (data) ? new Buffer(data) : new Buffer('') + this.__store(s_key, s_data, new_meta); + this.send([s_key, s_data], undefined, base.flags.store); + } + + update(update_dict) { + /** + * .. js:function:: js2p.sync.sync_socket.update(update_dict) + * + * For each key/value pair in the given object, calls :js:func:`~js2p.sync.sync_socket.set` + * + * :param Object update_dict: An object with keys and values which can be transformed into a :js:class:`Buffer` + * + * :raises: See :js:func:`~js2p.sync.sync_socket.set` + */ + for (var key in update_dict) { + this.set(key, update_dict[key]); + } + } + + del(key) { + /** + * .. js:function:: js2p.sync.sync_socket.del(key) + * + * Clears the value at a given key + * + * :param key: The key you wish to look up (must be transformable into a :js:class:`Buffer` ) + * + * :raises TypeError: If a key or value could not be transformed into a :js:class:`Buffer` + * :raises: See :js:func:`~js2p.sync.sync_socket.set` + */ + this.set(key); + } +} diff --git a/js_src/test/test_base.js b/js_src/test/test_base.js new file mode 100644 index 0000000..736d778 --- /dev/null +++ b/js_src/test/test_base.js @@ -0,0 +1,174 @@ +"use strict"; + +var assert = require('assert'); +var base = require('../base.js'); +var BigInt = require('big-integer'); + +function get_random_buffer(len) { + var pre_buffer = []; + for (var j = 0; j < len; j++) { + pre_buffer.push(Math.floor(Math.random() * 256)); + } + return new Buffer(pre_buffer); +} + +function get_random_array(len) { + var ret = []; + for (var i = 0; i < len; i++) { + ret.push(get_random_buffer(Math.floor(Math.random() * 20))); + } + return ret; +} + +function test_pathfinding_message(payload, instance) { + if (!instance) { + var msg = new base.pathfinding_message(base.flags.broadcast, new Buffer('\u00ff', 'ascii'), payload); + } + else { + var msg = instance; + } + var expected_packets = [new Buffer(base.flags.broadcast), new Buffer('\u00ff', 'ascii'), msg.id, msg.time_58].concat(payload); + var packets = msg.packets; + for (var j = 0; j < packets.length; j++) { + assert.equal(packets[j].toString(), expected_packets[j].toString(), `At position ${j}: ${packets[j]} != ${expected_packets[j]}`); + } + var p_hash_info = payload.concat(msg.time_58).join(''); + var p_hash = base.SHA384(p_hash_info); + assert.equal(msg.id, base.to_base_58(new BigInt(p_hash, 16))); +} + +describe('base', function() { + + describe('compress/decompress', function() { + + it('should always be reversable', function() { + this.timeout(1500); + for (var i = 0; i < 500; i++) { + // Step one: generate a random buffer up to size 40 + var len = Math.floor(Math.random() * 40); + var test_string = get_random_buffer(len); + // Then: For each flag in compression, assert that the compressed then decompressed version equals the original + for (var j = 0; j < base.compression.length; j++) { + assert(test_string.equals( base.decompress( base.compress(test_string, base.compression[j]), base.compression[j] ) )); + } + } + }); + + it('should raise an exception when given an unkown method', function() { + function compress_err() { + base.compress('', get_random_buffer(4).toString()); + } + function decompress_err() { + base.decompress('', get_random_buffer(4).toString()); + } + for (var i = 0; i < 100; i++) { + assert.throws(compress_err, Error); + assert.throws(decompress_err, Error); + } + }); + }); + + describe('to_base_58/from_base_58', function() { + it('should return "1" if fed 0', function() { + this.timeout(25); + assert.equal("1", base.to_base_58(0)); + }); + + it('should always be reversable', function() { + this.timeout(125); + for (var i = 0; i < 500; i++) { + var test = Math.floor(Math.random() * 1000000000); + var shouldEqual = base.from_base_58(base.to_base_58(test)); + assert(shouldEqual.equals(test), `${test} != ${shouldEqual}`); + } + }); + }); + + describe('protocol', function() { + it('should have information assigned to the correct getters', function() { + for (var i = 0; i < 250; i++) { + var sub = get_random_buffer(4); + var enc = get_random_buffer(4); + // Make sure it's normal ascii. utf-8 intentionally not supported here + for (var j = 0; j < 4; j++) { + sub[j] = sub[j] % 128; + enc[j] = enc[j] % 128; + } + var test = new base.protocol(sub, enc); + assert.equal(test.subnet, sub); + assert.equal(test.encryption, enc); + var p_hash_info = [sub, enc, base.protocol_version].join(''); + var p_hash = base.SHA256(p_hash_info); + assert.equal(test.id, base.to_base_58(new BigInt(p_hash, 16))); + } + }); + }); + + describe('pathfinding_message', function() { + it('should pass some short, statically defined tests', function() { + var string = '\x00\x00\x00{\x00\x00\x00\t\x00\x00\x00\x0b\x00\x00\x00B\x00\x00\x00\x06\x00\x00\x00\x0bbroadcasttest sender2ypz9RTBAFbw75WSJTNwaXZ6zSVLG8wvqbQDNRtoh74Hkxg3JAozHAZtCfwg1PEmpe3EdmDctest packet'; + var zlib = new Buffer("\x00\x00\x00{x\x9cc``\xe0d``\xe0\x06b' f\x03\xb1\x93\x8a\xf2\x13S\x92\x13\x8bKJR\x8bK\x14\x8aS\xf3RR\x8b\x8c*\x0b\xaa,\x83B\x9c\x1c\xdd\x92\xca\xcdM\xc3\x83\xbdB\xfc\xca\x13#\xa2\xcc\xaa\x82\xc3|\xdc-\xca\xcb\n\x93\x02]\xfc\x82J\xf23\xccM<\xb2+\xd2\x8d\xbd\x1c\xf3\xab<\x1c\xa3J\x9c\xd3\xca\xd3\r\x03\\s\x0bR\x8d]Sr]\x92\xc1F\x16$&g\xa7\x96\x00\x00\xbfC%T", "ascii"); + var gzip = new Buffer("\x00\x00\x00\x87\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03c``\xe0d``\xe0\x06b' f\x03\xb1\x93\x8a\xf2\x13S\x92\x13\x8bKJR\x8bK\x14\x8aS\xf3RR\x8b\x8c*\x0b\xaa,\x83B\x9c\x1c\xdd\x92\xca\xcdM\xc3\x83\xbdB\xfc\xca\x13#\xa2\xcc\xaa\x82\xc3|\xdc-\xca\xcb\n\x93\x02]\xfc\x82J\xf23\xccM<\xb2+\xd2\x8d\xbd\x1c\xf3\xab<\x1c\xa3J\x9c\xd3\xca\xd3\r\x03\\s\x0bR\x8d]Sr]\x92\xc1F\x16$&g\xa7\x96\x00\x00m\xbeb\xef{\x00\x00\x00", "ascii"); + + var pm = base.pathfinding_message.feed_string(string); + var msg = new base.message(pm); + + var zlib_pm = base.pathfinding_message.feed_string(zlib, false, [base.flags.zlib]); + + var gzip_pm = base.pathfinding_message.feed_string(gzip, false, [base.flags.gzip]); + + var expected = [ new Buffer('broadcast'), new Buffer('test sender'), '2ypz9RTBAFbw75WSJTNwaXZ6zSVLG8wvqbQDNRtoh74Hkxg3JAozHAZtCfwg1PEmpe', '3EdmDc', new Buffer('test packet') ]; + + assert (JSON.stringify(pm.packets) == JSON.stringify(expected), "pathfinding_message is not extracting packets correctly"); + assert (JSON.stringify(msg.packets) == JSON.stringify(expected.slice(4)), "message is not extracting from pathfinding_message correctly"); + + assert (JSON.stringify(zlib_pm.packets) == JSON.stringify(expected), "pathfinding_message is not extracting zlib packets correctly"); + + assert (JSON.stringify(gzip_pm.packets) == JSON.stringify(expected), "pathfinding_message is not extracting gzip packets correctly"); + }); + + it('should serialize and deserialize', function() { + this.timeout(0); + for (var i = 0; i < 250; i++) { + for (var j = 0; j <= base.compression.length; j++) { + var compressions = []; + if (j < base.compression.length) { + compressions.push(base.compression[j]); + } + var payload = get_random_array(Math.floor(Math.random() * 16)); + var msg = new base.pathfinding_message(base.flags.broadcast, new Buffer('\u00ff', 'ascii'), payload, compressions); + var deserialized = base.pathfinding_message.feed_string(msg.string, false, compressions); + test_pathfinding_message(payload, deserialized); + } + } + }); + + it('should have information assigned to the correct getters', function() { + for (var i = 0; i < 250; i++) { + var payload = get_random_array(Math.floor(Math.random() * 16)); + test_pathfinding_message(payload); + } + }); + }); + + describe('message', function() { + it('should have information assigned to the correct getters', function() { + for (var i = 0; i < 150; i++) { + var sen = get_random_buffer(4); + for (var j = 0; j < 4; j++) { // utf-8 is supported here, but it doesn't make much sense to have that as a sender id + sen[j] = sen[j] % 128; + } + var pac = get_random_array(36); + var base_msg = new base.pathfinding_message(base.flags.broadcast, sen, pac); + var test = new base.message(base_msg, null); + assert.equal(test.packets, pac); + assert.equal(test.msg, base_msg); + assert.equal(test.sender.toString(), sen); + assert.equal(test.id, base_msg.id); + assert.equal(test.time, base_msg.time); + assert.equal(test.time_58, base_msg.time_58); + assert(base.from_base_58(test.time_58).equals(test.time)); + } + }) + }); +}); diff --git a/js_src/test/test_mesh.js b/js_src/test/test_mesh.js new file mode 100644 index 0000000..b644e5e --- /dev/null +++ b/js_src/test/test_mesh.js @@ -0,0 +1,95 @@ +"use strict"; + +var assert = require('assert'); +var base = require('../base.js'); +var mesh = require('../mesh.js'); +var start_port = 44565; + +describe('mesh', function() { + + describe('mesh_socket', function() { + + it('should propagate messages to everyone in the network (over plaintext)', function(done) { + this.timeout(6000); + var count = 3; + + var nodes = [new mesh.mesh_socket('localhost', start_port++)]; + for (var j = 1; j < count; j++) { + var node = new mesh.mesh_socket('localhost', start_port++); + var addr = nodes[nodes.length - 1].addr; + node.connect(addr[0], addr[1]); + nodes.push(node); + } + setTimeout(function() { + nodes[0].send(['hello']); + setTimeout(function() { + for (var h = 1; h < count; h++) { + var msg = nodes[h].recv(); + assert.ok(msg); + } + done(); + }, count * 500); + }, count * 250); + }); + + it('should reject connections with a different protocol object (over plaintext)', function(done) { + var node1 = new mesh.mesh_socket('localhost', start_port++, new base.protocol('mesh1', 'Plaintext')); + var node2 = new mesh.mesh_socket('localhost', start_port++, new base.protocol('mesh2', 'Plaintext')); + + node1.connect(node2.addr[0], node2.addr[1]); + setTimeout(function() { + assert.deepEqual(node1.routing_table, {}); + assert.deepEqual(node2.routing_table, {}); + done(); + }, 500); + }); + + function register_1(msg, handler) { + var packets = msg.packets; + if (packets[1].toString() === 'test') { + handler.send(base.flags.whisper, [base.flags.whisper, 'success']); + return true; + } + } + + function register_2(msg, handler) { + var packets = msg.packets; + if (packets[1].toString() === 'test') { + msg.reply(['success']); + return true; + } + } + + function test_callback(callback, done) { + var node1 = new mesh.mesh_socket('localhost', start_port++); + var node2 = new mesh.mesh_socket('localhost', start_port++); + + node2.register_handler(callback); + node1.connect(node2.addr[0], node2.addr[1]); + + setTimeout(function() { + node1.send(['test']); + setTimeout(function() { + assert.ok(node1.recv()); + assert.ok(!node2.recv()); + node1.send(['not test']); + setTimeout(function() { + assert.ok(!node1.recv()); + assert.ok(node2.recv()); + done(); + }, 500); + }, 500); + }, 250); + } + + it('should be able to register and use message callbacks (over plaintext)', function(done) { + test_callback(register_1, done); + }); + + it('should let you reply to messages via the message object (over plaintext)', function(done) { + test_callback(register_2, done); + }); + + }); + +}); diff --git a/js_src/test/test_sync.js b/js_src/test/test_sync.js new file mode 100644 index 0000000..0f0b455 --- /dev/null +++ b/js_src/test/test_sync.js @@ -0,0 +1,49 @@ +"use strict"; + +const assert = require('assert'); +const sync = require('../sync.js'); +var start_port = 44665; + +describe('sync', function() { + + describe('sync_socket', function() { + + function test_storage(leasing, done) { + var node1 = new sync.sync_socket('localhost', start_port++, leasing); + var node2 = new sync.sync_socket('localhost', start_port++, leasing); + + node1.connect(node2.addr[0], node2.addr[1]); + + setTimeout(function() { + node1.set('test', 'value'); + setTimeout(function() { + assert.ok(node1.get('test')); + assert.ok(node2.get('test')); + node2.update({ + '测试': '成功', + 'store': 'store' + }); + setTimeout(function() { + assert.equal(node1.get('测试').toString(), '成功'); + assert.equal(node2.get('测试').toString(), '成功'); + assert.equal(node1.get('store').toString(), 'store'); + assert.equal(node2.get('store').toString(), 'store'); + done(); + }, 500); + }, 500); + }, 250); + }; + + it('should store values correctly when leasing', function(done) { + this.timeout(2500); + test_storage(true, done); + }); + + it('should store values correctly when not leasing', function(done) { + this.timeout(2500); + test_storage(false, done); + }); + + }); + +}); \ No newline at end of file diff --git a/js_src/test_protocol.js b/js_src/test_protocol.js deleted file mode 100644 index 5c071da..0000000 --- a/js_src/test_protocol.js +++ /dev/null @@ -1,11 +0,0 @@ -var base = require('./p2p.js'); -const assert = require('assert'); - -var string = '\u0000\u0000\u0000{\u0000\u0000\u0000\t\u0000\u0000\u0000\u000b\u0000\u0000\u0000B\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u000bbroadcasttest sender2ypz9RTBAFbw75WSJTNwaXZ6zSVLG8wvqbQDNRtoh74Hkxg3JAozHAZtCfwg1PEmpe3EdmDctest packet' -var pm = base.pathfinding_message.feed_string(base.default_protocol, string) -var msg = new base.message(pm) - -var expected = [ 'broadcast', 'test sender', '2ypz9RTBAFbw75WSJTNwaXZ6zSVLG8wvqbQDNRtoh74Hkxg3JAozHAZtCfwg1PEmpe', '3EdmDc', 'test packet' ] - -assert (JSON.stringify(pm.packets) == JSON.stringify(expected), "pathfinding_message is not extracting packets correctly") -assert (JSON.stringify(msg.packets) == JSON.stringify(expected.slice(4)), "message is not extracting from pathfinding_message correctly") \ No newline at end of file diff --git a/js_src/zlib b/js_src/zlib deleted file mode 160000 index b99bd33..0000000 --- a/js_src/zlib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b99bd33d485d63e93310b911a1f8dbf9ceb265d5 diff --git a/jv_src/base.java b/jv_src/base.java new file mode 100644 index 0000000..1cc7ce4 --- /dev/null +++ b/jv_src/base.java @@ -0,0 +1,267 @@ +/** +* Base Module +* =========== +* +* This module contains common functions and classes used in the rest of the library. +*/ +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.math.BigInteger; +import java.lang.Character; +import java.lang.StringBuilder; +import java.time.Instant; +import java.util.Arrays; +import java.util.ArrayList; + +public class base { + + public int[] version_info = {0, 4, 319}; + public String protocol_version = String.valueOf(version_info[0]) + "." + String.valueOf(version_info[1]); + + public class protocol { + String subnet; + String encryption; + + public protocol(String subnet, String encryption) { + this.subnet = subnet; + this.encryption = encryption; + } + + String id() throws java.security.NoSuchAlgorithmException { + String info = this.subnet + this.encryption + protocol_version; + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(info.getBytes()); + BigInteger i = new BigInteger(1, digest); + return to_base_58(i); + } + } + + public class pathfinding_message { + String msg_type; + String sender; + String[] payload; + long time; + byte[][] compression; + + public pathfinding_message(String msg_type, String sender, String[] payload, byte[][] compression, long timestamp) { + this.msg_type = msg_type; + this.sender = sender; + this.payload = payload; + this.compression = compression; + this.time = timestamp; + } + + public pathfinding_message(String msg_type, String sender, String[] payload, byte[][] compression) { + this(msg_type, sender, payload, compression, getUTC()); + } + + public pathfinding_message(String msg_type, String sender, String[] payload, long timestamp) { + this(msg_type, sender, payload, new byte[0][0], timestamp); + } + + public pathfinding_message(String msg_type, String sender, String[] payload) { + this(msg_type, sender, payload, new byte[0][0], getUTC()); + } + + private static byte[] sanitize_string(byte[] fed_string, boolean sizeless) throws Exception { + if (!sizeless) { + if (unpack_value(fed_string, 4) != fed_string.length - 4) { + throw new Exception("Size header inaccurate " + Arrays.toString(Arrays.copyOfRange(fed_string, 0, 4)) + ", " + String.valueOf(fed_string.length - 4)); + } + return Arrays.copyOfRange(fed_string, 4, fed_string.length); + } + return fed_string; + } + + private static byte[] decompress_string(byte[] fed_string, byte[][] compressions) { + return fed_string; //Eventually this will do decompression + } + + private static String[] process_string(byte[] fed_string) { + int processed = 0; + int expected = fed_string.length; + ArrayList pack_lens = new ArrayList(); + + while (processed != expected) { + byte[] pack_len = Arrays.copyOfRange(fed_string, processed, processed + 4); + int new_len = (int)unpack_value(pack_len, 4); + processed += 4; + expected -= new_len; + pack_lens.add(pack_lens.size(), new_len); + } + + String[] packets = new String[pack_lens.size()]; + + for (int i = 0; i < packets.length; i++) { + int end = processed + pack_lens.get(i); + packets[i] = new String(Arrays.copyOfRange(fed_string, processed, end)); + processed = end; + } + + return packets; + } + + public static pathfinding_message feed_string(byte[] fed_string, boolean sizeless, byte[][] compressions) throws Exception { + byte[] sanitized_string = pathfinding_message.sanitize_string(fed_string, sizeless); + sanitized_string = pathfinding_message.decompress_string(sanitized_string, compressions); + String[] packets = pathfinding_message.process_string(sanitized_string); + + String[] payload = new String[packets.length-4]; + for (int i = 0; i < payload.length; i++) { + payload[i] = packets[4 + i]; + } + + pathfinding_message msg = new pathfinding_message(packets[0], packets[1], payload); + msg.time = from_base_58(packets[3]); + + if (!packets[2].equals(msg.id())) { + throw new Exception("Checksum match failed"); + } + + return msg; + } + + public static pathfinding_message feed_string(byte[] fed_string, boolean sizeless) throws Exception { + return pathfinding_message.feed_string(fed_string, sizeless, new byte[0][0]); + } + + public static pathfinding_message feed_string(byte[] fed_string, byte[][] compressions) throws Exception { + return pathfinding_message.feed_string(fed_string, false, compressions); + } + + public static pathfinding_message feed_string(byte[] fed_string) throws Exception { + return pathfinding_message.feed_string(fed_string, false, new byte[0][0]); + } + + public String time_58() { + return to_base_58(this.time); + } + + public String id() throws java.security.NoSuchAlgorithmException { + StringBuilder builder = new StringBuilder(); + for(String s : this.payload) { + builder.append(s); + } + String info = builder.toString() + this.time_58(); + MessageDigest md = MessageDigest.getInstance("SHA-384"); + byte[] digest = md.digest(info.getBytes()); + BigInteger i = new BigInteger(1, digest); + return to_base_58(i); + } + + public String[] getPackets() throws java.security.NoSuchAlgorithmException { + String[] packets = new String[4 + this.payload.length]; + packets[0] = this.msg_type; + packets[1] = this.sender; + packets[2] = this.id(); + packets[3] = this.time_58(); + for (int i = 0; i < this.payload.length; i++) { + packets[4 + i] = this.payload[i]; + } + return packets; + } + + public byte[] non_len_string() throws java.security.NoSuchAlgorithmException { + String id = this.id(); + String[] packets = this.getPackets(); + byte[][] encoded_packets = new byte[packets.length][]; + byte[] encoded_lengths = new byte[4 * packets.length]; + int total_length = 4 * packets.length; + + for (int i = 0; i < packets.length; i++) { + encoded_packets[i] = packets[i].getBytes(); + total_length += encoded_packets[i].length; + } + + byte[] ret = new byte[total_length]; + + for (int i = 0; i < encoded_packets.length; i++) { + byte[] len = pack_value(encoded_packets[i].length, 4); + ret[(4 * i) + 0] = len[0]; + ret[(4 * i) + 1] = len[1]; + ret[(4 * i) + 2] = len[2]; + ret[(4 * i) + 3] = len[3]; + } + + int index = encoded_packets.length * 4; + for (int i = 0; i < encoded_packets.length; i++) { + for (int j = 0; j < encoded_packets[i].length; j++) { + ret[index++] = encoded_packets[i][j]; + } + } + + return ret; + } + + public byte[] serialize() throws java.security.NoSuchAlgorithmException { + byte[] non_len_string = this.non_len_string(); + + // This is where compression should happen in the future + + byte[] ret = new byte[4 + non_len_string.length]; + byte[] len = pack_value(non_len_string.length, 4); + + ret[0] = len[0]; + ret[1] = len[1]; + ret[2] = len[2]; + ret[3] = len[3]; + + for (int i = 0; i < non_len_string.length; i++) { + ret[4 + i] = non_len_string[i]; + } + + return ret; + } + } + + public long unpack_value(byte[] arr, int len) { + long ret = 0; + for (int i = 0; i < len; i++) { + ret *= 256; + ret += arr[i]; + } + return ret; + } + + public byte[] pack_value(long value, int length) { + byte[] ret = new byte[length]; + Arrays.fill(ret, (byte) 0); + for (int i = length - 1; i != 0 && value != 0; i++) { + ret[i] = (byte) (value % 256); + value /= 256; + } + return ret; + } + + public String base_58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + + public String to_base_58(long i) { + return to_base_58(BigInteger.valueOf(i)); + } + + public String to_base_58(BigInteger i) { + String ret = ""; + BigInteger fifty_eight = new BigInteger("58"); + BigInteger working_value = i; + while (working_value.toString() != "0") { + BigInteger[] divAndRem = working_value.divideAndRemainder(fifty_eight); + int index = divAndRem[1].intValue(); + ret = Character.toString(base_58.charAt(index)) + ret; + working_value = divAndRem[0]; + } + return ret; + } + + public long from_base_58(String str) { + long ret = 0; + for (int i = 0; i < str.length(); i++) { + ret *= 58; + ret += base_58.indexOf(str.charAt(i)); + } + return ret; + } + + public long getUTC() { + return Instant.now().getEpochSecond(); + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..136a627 --- /dev/null +++ b/package.json @@ -0,0 +1,24 @@ +{ + "name": "js2p", + "main": "js_src/js2p.js", + "version": "0.4.470", + "license": "LGPL-3.0", + "repository": { + "type": "git", + "url": "https://github.com/gappleto97/p2p-project.git" + }, + "bugs": { + "url": "https://github.com/gappleto97/p2p-project/issues" + }, + "dependencies": { + "big-integer": "^1.6.16", + "buffer": "^4.9.1", + "jssha": "^2.2.0", + "zlibjs": "^0.2.0" + }, + "devDependencies": { + "babel-cli": "^6.16.0", + "babel-preset-es2015": "^6.16.0", + "mocha": "^3.1.0" + } +} diff --git a/py_src/API.rst b/py_src/API.rst deleted file mode 100644 index 775baea..0000000 --- a/py_src/API.rst +++ /dev/null @@ -1,546 +0,0 @@ -`Skip to file-wise API <#file-wise-api>`__ - -Public API -========== - -Constants ---------- - -- ``__version__``: A string containing the major, minor, and patch - release number. -- ``version_info``: A ``tuple`` version of the above -- ``protocol_version``: A string containing the major and minor release - number. This refers to the underlying protocol -- ``node_policy_version``: A string containing the build number - associated with this version. This refers to the node and its - policies. - -Classes -------- - -- `mesh_socket <#mesh_socket>`__ -- `protocol <#protocol>`__ - -File-wise API -============= - -base.py -======= - -This is used mostly for inheriting common functions with -`mesh.py <#meshpy>`__ and the planned `chord.py <#chordpy>`__ - -Constants ---------- - -- ``version``: A string containing the major, minor, and patch release - number. This version refers to the underlying protocol. -- ``protocol_version``: A string containing the major and minor release - number. This refers to the underlying protocol -- ``node_policy_version``: A string containing the build number - associated with this version. This refers to the node and its - policies. -- ``user_salt``: A ``uuid4`` which is generated uniquely in each - running instance -- ``compression``: A ``list`` of the compression methods your instance - supports -- ``default_protocol``: The default `protocol <#protocol>`__ - definition. This uses an empty string as the subnet and - ``SSL`` encryption, as supplied by `ssl\_wrapper.py <#ssl_wrapperpy>`__ (in - alpha releases this will use ``Plaintext``) -- ``base_58``: The characterspace of base\_58, ordered from least to - greatest value - -Methods -------- - -- ``to_base_58(i)``: Takes an ``int`` (or ``long``) and returns its corresponding base\_58 string (type: ``bytes``) -- ``from_base_58(string)``: Takes a base\_58 string (or ``bytes``) and returns its corresponding integer (type: ``int``, ``long``) -- ``getUTC()``: Returns the current unix time in UTC (type: ``int``) -- ``compress(msg, method)``: Shortcut method for compression (type: ``bytes``) -- ``decompress(msg, method)``: Shortcut method for decompression (type: ``bytes``) -- ``get_lan_ip()``: Returns either your current local IP, or ``"127.0.0.1"`` -- ``get_socket(protocol, serverside)``: Shortcut method to generate an appropriate socket object - -Classes -------- - -flags -~~~~~ - -This class is used as a namespace to store the various protocol defined -flags. - -- ``broadcast`` -- ``bz2`` -- ``compression`` -- ``gzip`` -- ``handshake`` -- ``lzma`` -- ``peers`` -- ``waterfall`` -- ``resend`` -- ``response`` -- ``renegotiate`` -- ``request`` -- ``whisper`` - -pathfinding\_message -~~~~~~~~~~~~~~~~~~~~ - -This class is used internally to deal with packet parsing from a socket -level. If you find yourself calling this as a user, something's gone -wrong. - -Constructor -^^^^^^^^^^^ - -``pathfinding_message(protocol, msg_type, sender, payload, compressions=None)`` -``pathfinding_message.feed_string(protocol, string, sizeless=False, compressions=None)`` - -- ``protocol``: The `protocol <#protocol>`__ this message uses -- ``msg_type``: The chief `flag <#flags>`__ this message uses, to broadcast intent -- ``sender``: The SHA384-based sender ID -- ``payload``: A ``list`` of additional packets to send -- ``compressions``: A ``list`` of possible compression methods used/to use -- ``string``: The raw message to parse -- ``sizeless``: An indicator as to whether this message contains the length header - -Constants -^^^^^^^^^ - -- ``protocol``: The protocol this message is sent under -- ``msg_type``: The main `flag <#flags>`__ of the message (ie: ``['broadcast', 'waterfall', 'whisper', 'renegotiate']``) -- ``sender``: The sender id of this message -- ``time``: An ``int`` of the message's timestamp -- ``compression``: The ``list`` of compression methods this message may be under -- ``compression_fail``: A debug property which is triggered if you give - compression methods, but the message fed from ``feed_string`` is - actually in plaintext - -Properties -^^^^^^^^^^ - -- ``payload``: Returns the message's payload -- ``compression_used``: Returns the compression method used -- ``time_58``: Returns the timestamp in base\_58 -- ``id``: Returns the message's id -- ``len``: Returns the messages length header -- ``packets``: Returns a ``list`` of the packets in this message, excluding the length header -- ``string``: Returns a string version of the message, including the length header -- ``__non_len_string``: Returns the string of this message without the size header - -Methods -^^^^^^^ - -- ``__len__()``: Returns the length of this message excluding the length header - -Class Methods -^^^^^^^^^^^^^ - -- ``feed_string(ptorocol, string, sizeless=False, compressions=None)``: - Given a `protocol <#protocol>`__, a string or ``bytes``, process - this into a ``pathfinding_message``. If compressions are enabled, you - must provide a ``list`` of possible methods. If the size header is - not included, you must specify this with ``sizeless=True``. Possible - errors: - - - ``AttributeError``: Fed a non-string, non-\ ``bytes`` argument - - ``AssertionError``: Initial size header is incorrect - - ``Exception``: Unrecognized compression method fed in - ``compressions`` - - ``struct.error``: Packet headers are incorrect OR unrecognized - compression - - ``IndexError``: See ``struct.error`` - -- ``sanitize_string(string, sizeless=False)``: Given an ``str`` or - ``bytes``, returns a ``bytes`` object with no size header. Possible - errors: - - - ``AttributeError``: Fed a non-string, non-\ ``bytes`` argument - - ``AssertionError``: Initial size header is incorrect - -- ``decompress_string(string, compressions=None)``: Given a ``bytes`` - object and list of possible compression methods, returns a - decompressed version and a ``bool`` indicating if decompression - failed. If decompression occurs, this will always return ``bytes``. - If not, it will return whatever you pass in. Decompression failure is - defined as it being unable to decompress despite a list of possible - methods being provided. Possible errors: - - - ``Exception``: Unrecognized compression method fed in - ``compressions`` - -- ``process_string(string)``: Given a ``bytes``, return a ``list`` of - its contained packets. Possible errors: - - - ``IndexError``: Packet headers are incorrect OR not fed plaintext - - ``struct.error``: See ``IndexError`` OR fed non-\ ``bytes`` object - -message -~~~~~~~ - -This class is returned to the user when a non-automated message is -received. It contains sufficient information to parse a message or reply -to it. - -Constructor -^^^^^^^^^^^ - -``message(msg, server)`` - -- ``msg``: This contains the - `pathfinding_message <#pathfinding_message>`__ you received -- ``server``: The `base_socket <#base_socket>`__ which received the - message - -Constants -^^^^^^^^^ - -- ``msg``: This contains the - `pathfinding_message <#pathfinding_message>`__ you received -- ``server``: The `base_socket <#base_socket>`__ which received the - message - -Properties -^^^^^^^^^^ - -- ``time``: The UTC Unix time at which the message was sent -- ``sender``: The original sender's ID -- ``protocol``: The `protocol <#protocol>`__ you received this - under -- ``packets``: Returns a ``list`` of the packets received, with the - first item being the subflag -- ``id``: Returns the SHA384-based message id - -Methods -^^^^^^^ - -- ``reply(*args)``: Sends a `whisper <#flags>`__ to the original - sender with the arguments being each packet after that. If you are - not connected, it uses the `request/response <#flags>`__ - mechanism to try making a connection - -protocol -~~~~~~~~ - -This class inherits most of its methods from a ``namedtuple``. This -means that each of the properties in the constructor can be accessed by -name or index. Mostly you'll be doing this by name. - -Constructor -^^^^^^^^^^^ - -``protocol(subnet, encryption)`` - -Constants -^^^^^^^^^ - -- ``subnet``: A flag to allow people with the same package version to - operate different networks -- ``encryption``: Defines the encryption standard used on the socket - -Properties -^^^^^^^^^^ - -- ``id``: Returns the SHA256-based protocol id - -base\_socket -~~~~~~~~~~~~ - -Variables -^^^^^^^^^ - -- ``debug_level``: The verbosity of the socket with debug prints -- ``routing_table``: The current ``dict`` of peers in format - ``{id: connection}`` -- ``awaiting_ids``: A ``list`` of connections awaiting a handshake -- ``queue``: A ``deque`` of recently received - `message <#message>`__\ s -- ``daemon``: This node's `base_daemon <#base_daemon>`__ object - -Properties -^^^^^^^^^^ - -- ``outgoing``: A ``list`` of ids for outgoing connections -- ``incoming``: A ``list`` of ids for incoming connections -- ``status``: Returns ``"Nominal"`` or - ``base_socket.daemon.exceptions`` if there are ``Exceptions`` - collected - -Methods -^^^^^^^ - -- ``recv(quantity=1)``: Receive `message <#message>`__\ s; If - ``quantity != 1``, returns a ``list`` of - `message <#message>`__\ s, otherwise returns one -- ``__print__(*args, level=None)``: Prints debug information if - ``level >= debug_level`` - -base\_daemon -~~~~~~~~~~~~ - -Constructor -^^^^^^^^^^^ - -``base_daemon(addr, port, server, prot=default_protocol)`` - -- ``addr``: The address it should bind its incoming connection to -- ``port``: The port it should bind its incoming connection to -- ``server``: This daemon's `base_socket <#base_socket>`__ -- ``prot``: This daemon's `protocol <#protocol>`__ - -Variables -^^^^^^^^^ - -- ``protocol``: This daemon's `protocol <#protocol>`__ object -- ``server``: A pointer to this daemon's - `base_socket <#base_socket>`__ -- ``sock``: This daemon's ``socket`` object -- ``alive``: A checker to shutdown the daemon. If ``False``, its thread - will stop running eventually. -- ``exceptions``: A ``list`` of unhandled ``Exception``\ s raised in - ``mainloop`` -- ``daemon``: A ``Thread`` which runs through ``mainloop`` - -Methods -^^^^^^^ - -- ``__print__(*args, level=None)``: Prints debug information if - ``level >= server.debug_level`` - -base\_connection -~~~~~~~~~~~~~~~~ - -Constructor -^^^^^^^^^^^ - -``base_connection(sock, server, prot=default_protocol, outgoing=False)`` - -- ``sock``: A ``socket.socket`` -- ``server``: This node's `base_socket <#base_socket>`__ -- ``prot``: This node's `protocol <#protocol>`__ -- ``outgoing``: Whether or not this node is an outgoing connection - -Variables -^^^^^^^^^ - -- ``sock``: This connection's ``socket`` object -- ``server``: A pointer to this connection's - `base_socket <#base_socket>`__ object -- ``protocol``: This connection's `protocol <#protocol>`__ object -- ``outgoing``: A ``bool`` that states whether this connection is - outgoing -- ``buffer``: A ``list`` of recently received characters -- ``id``: This node's SHA384-based id -- ``time``: The time at which this node last received data -- ``addr``: This node's outward-facing address -- ``compression``: A ``list`` of this node's supported compression - methods -- ``last_sent``: A copy of the most recently sent ``whisper`` or - ``broadcast`` -- ``expected``: The number of bytes expected in the next message -- ``active``: A ``bool`` which says whether the next message is a size - header, or a message (``True`` if message) - -Methods -^^^^^^^ - -- ``fileno()``: Returns ``sock``'s file number -- ``collect_incoming_data(data)``: Adds new data to the buffer -- ``find_terminator()``: Determines if a message has been fully - received (name is a relic of when this had an ``end_of_tx`` flag) -- ``__print__(*args, level=None)``: Prints debug information if - ``level >= server.debug_level`` - -mesh.py -======= - -Note: This inherits a *lot* from `base.py <#basepy>`__, and imported -values will *not* be listed here, for brevity's sake. - -Constants ---------- - -- ``json_compression``: A json dump of the ``list`` of the compression methods your instance supports -- ``max_outgoing``: The (rough) maximum number of outgoing connections your node will maintain -- ``default_protocol``: The default `protocol <#protocol>`__ definition. This uses ``'mesh'`` as the subnet and ``SSL`` encryption, as supplied by `ssl\_wrapper.py <#ssl_wrapperpy>`__ (in alpha releases this will use ``Plaintext``) - -Classes -------- - -mesh\_socket -~~~~~~~~~~~~ - -This peer-to-peer socket is the main purpose behind this library. It -maintains a connection to a mesh network. Details on how it works -specifically are outlined `here <../README.md>`__, but the basics are -outlined below. - -It also inherits all the attributes of -`base_socket <#base_socket>`__, though they are also outlined here - -Constructor -^^^^^^^^^^^ - -``mesh_socket(addr, port, prot=default_protocol, out_addr=None, debug_level=0)`` - -- ``addr``: The address you'd like to bind to -- ``port``: The port you'd like to bind to -- ``prot``: The `protocol <#protocol>`__ you'd like to use -- ``out_addr``: Your outward-facing address, if that is different from ``(addr, port)`` -- ``debug_level``: The verbosity at which this and its associated `mesh_daemon <#mesh_daemon>`__ prints debug information - -Variables -^^^^^^^^^ - -- ``protocol``: A `protocol <#protocol>`__ object which contains the subnet flag and the encryption method -- ``debug_level``: The verbosity of the socket with debug prints -- ``routing_table``: The current ``dict`` of peers in format ``{id: connection}`` -- ``awaiting_ids``: A ``list`` of connections awaiting a handshake -- ``outgoing``: A ``list`` of ids for outgoing connections -- ``incoming``: A ``list`` of ids for incoming connections -- ``requests``: A ``dict`` of the requests this node has made in format ``{request_id: delayed_message_contents}`` -- ``waterfalls``: A ``deque`` of metadata for recently received `message <#message>`__\ s -- ``queue``: A ``deque`` of recently received `message <#message>`__\ s -- ``out_addr``: A ``tuple`` which contains the outward facing address and port -- ``id``: This node's SHA384-based id -- ``daemon``: This node's `mesh_daemon <#mesh_daemon>`__ object - -Methods -^^^^^^^ - -- ``connect(addr, port, id=None)``: Connect to another ``mesh_socket`` (and assigns id if specified) -- ``send(*args, flag=flags.broadcast, type=flags.broadcast)``: Send a message to your peers with each argument as a packet Type specifies the subflag (packet 4), flag specifies the flag (packet 0). -- ``recv(quantity=1)``: Receive `message <#message>`__\ s; If ``quantity != 1``, returns a ``list`` of - `message <#message>`__\ s, otherwise returns one -- ``handle_msg(msg, conn)``: Allows the daemon to parse subflag-level actions -- ``waterfall(msg)``: Waterfalls a `message <#message>`__ to your peers. -- ``disconnect(handler)``: Closes a given `mesh_connection <#mesh_connection>`__ and removes its information from the various routing tables -- ``register_handler(method)``: Registers a callback method for certain types of messages. This is appended after the default callbacks and should take the format: - - .. code-block:: python - - >>> def relay_tx(msg, handler): - ... """Relays bitcoin transactions to various services""" - ... packets = msg.packets # Gives a list of the non-metadata packets - ... server = msg.server # Returns your mesh_socket object - ... if packets[0] == b'tx_relay': # It's important that this flag is bytes - ... from pycoin import tx, services - ... relay = tx.Tx.from_bin(packets[1]) - ... services.blockchain_info.send_tx(relay) - ... services.insight.InsightProvider().send_tx(relay) - ... return True # This tells the daemon to stop calling handlers - -mesh\_daemon -~~~~~~~~~~~~ - -This inherits all the attributes of `base_daemon <#base_daemon>`__, -though they are also outlined here - -Constructor -^^^^^^^^^^^ - -``mesh_daemon(addr, port, server, prot=default_protocol)`` - -- ``addr``: The address it should bind its incoming connection to -- ``port``: The port it should bind its incoming connection to -- ``server``: This daemon's `mesh_socket <#mesh_socket>`__ -- ``prot``: This daemon's `protocol <#protocol>`__ - -Variables -^^^^^^^^^ - -- ``protocol``: This daemon's `protocol <#protocol>`__ object -- ``server``: A pointer to this daemon's - `mesh_socket <#mesh_socket>`__ -- ``sock``: This daemon's ``socket`` object -- ``alive``: A checker to shutdown the daemon. If ``False``, its thread - will stop running eventually. -- ``exceptions``: A ``list`` of unhandled ``Exception``\ s raised in - ``mainloop`` -- ``daemon``: A ``Thread`` which runs through ``mainloop`` - -Methods -^^^^^^^ - -- ``mainloop()``: The method through which ``daemon`` parses. This runs - as long as ``alive`` is ``True``, and alternately calls the - ``collect_incoming_data`` methods of - `mesh_connection <#mesh_connection>`__\ s and ``handle_accept``. -- ``process_data()``: The portion of ``mainloop`` which handles received data -- ``handle_accept()``: Deals with incoming connections -- ``kill_old_nodes(handler)``: If a node hasn't completed their message within 60 seconds, disconnects it -- ``__print__(*args, level=None)``: Prints debug information if ``level >= server.debug_level`` - -mesh\_connection -~~~~~~~~~~~~~~~~ - -This inherits all the attributes of -`base_connection <#base_connection>`__, though they are also -outlined here - -Constructor -^^^^^^^^^^^ - -``base_connection(sock, server, prot=default_protocol, outgoing=False)`` - -- ``sock``: A ``socket.socket`` -- ``server``: This node's `mesh_socket <#mesh_socket>`__ -- ``prot``: This node's `protocol <#protocol>`__ -- ``outgoing``: Whether or not this node is an outgoing connection - -Variables -^^^^^^^^^ - -- ``sock``: This connection's ``socket`` object -- ``server``: A pointer to this connection's - `mesh_socket <#mesh_socket>`__ object -- ``protocol``: This connection's `protocol <#protocol>`__ object -- ``outgoing``: A ``bool`` that states whether this connection is - outgoing -- ``buffer``: A ``list`` of recently received characters -- ``id``: This node's SHA384-based id -- ``time``: The time at which this node last received data -- ``addr``: This node's outward-facing address -- ``compression``: A ``list`` of this node's supported compression - methods -- ``last_sent``: A copy of the most recently sent - `whisper <#flags>`__ or `broadcast <#flags>`__ -- ``expected``: The number of bytes expected in the next message -- ``active``: A ``bool`` which says whether the next message is a size - header, or a message (``True`` if message) - -Methods -^^^^^^^ - -- ``fileno()``: Returns ``sock``'s file number -- ``collect_incoming_data(data)``: Adds new data to the buffer -- ``find_terminator()``: Determines if a message has been fully - received (name is a relic of when this had an ``end_of_tx`` flag) -- ``found_terminator()``: Deals with any data received when - ``find_terminator`` returns ``True`` -- ``send(msg_type, *args, id=server.id, time=base.getUTC())``: Sends a - message via ``sock`` -- ``__print__(*args, level=None)``: Prints debug information if - ``level >= server.debug_level`` - -net.py -====== - -Deprecated. Scheduled to be removed in the next release. - -ssl\_wrapper.py -=============== - -Variables ---------- - -- ``cleanup_files``: Only present in python2; A list of files to clean up using the ``atexit`` module. Because of this setup, sudden crashes of Python will not clean up keys or certs. - -Methods -------- - -- ``generate_self_signed_cert(cert_file, key_file)``: Given two file-like objects, generate an SSL certificate and key file -- ``get_socket(server_side)``: Returns an ``ssl.SSLSocket`` for use in other parts of this library -- ``cleanup()``: Only present in python2; Calls ``os.remove`` on all files in ``cleanup_files``. diff --git a/py_src/__init__.py b/py_src/__init__.py index f61e300..b84bf81 100644 --- a/py_src/__init__.py +++ b/py_src/__init__.py @@ -4,35 +4,64 @@ Constants - * __version__: A string containing the major, minor, and patch release number. + * __version__: A string containing the major, minor, and patch release + number. * version_info: A tuple version of the above - * protocol_version: A string containing the major and minor release number. This refers to the underlying protocol - * node_policy_version: A string containing the build number associated with this version. This refers to the node and its policies. + * protocol_version: A string containing the major and minor release number. + This refers to the underlying protocol + * node_policy_version: A string containing the build number associated + with this version. This refers to the node + and its policies. Classes - * mesh_socket(addr, port, out_addr=None, prot=py2p.mesh.default_protocol, debug_level=0): + * mesh_socket(addr, port, out_addr=None, debug_level=0, + prot=py2p.mesh.default_protocol): - addr: The address you'd like to bind to - port: The port you'd like to bind to - - out_addr: Your outward-facing address, if that is different from (addr, port) + - out_addr: Your outward-facing address, if that is different + from (addr, port) - prot: The py2p.base.protocol object you'd like to use - debug_level: The verbosity at which this and its associated py2p.mesh.mesh_daemon prints debug information Submodules: - * base: A library of common functions and classes to enable mesh and the planned chord + * base: A library of common functions and classes to enable mesh + and the planned chord * mesh: A library to deal with mesh networking * chord: A planned library to deal with distributed hash tables * ssl_wrapper: A shortcut library to generate peer-to-peer ssl.SSLSockets * test: Unit tests for this library """ +from .base import protocol, version from .mesh import mesh_socket # from .chord import chord_socket -from .base import version as __version__ -from .base import protocol +# from .kademlia import kademlia_socket +# dht_socket = kademlia_socket + +__version__ = version version_info = tuple(map(int, __version__.split("."))) -__all__ = ["mesh", "chord", "base", "ssl_wrapper"] + +def bootstrap(socket_type, proto, addr, port, *args, **kargs): + raise NotImplementedError + # global seed + # seed = dht_socket(addr, port, out_addr = kargs.get('out_addr')) + # seed.connect(standard_starting_conn) + # time.sleep(1) + # conn_list = json.loads(seed.get(proto.id)) + # ret = socket_type(addr, port, *args, prot=proto, **kargs) + # for addr, port in conn_list: + # ret.connect(addr, port) + # return ret + +__all__ = ["mesh", "chord", "kademlia", "base", "ssl_wrapper"] + +try: + import cbase + __all__.append("cbase") +except ImportError: + pass diff --git a/py_src/base.py b/py_src/base.py index c3ef2fa..43fce73 100644 --- a/py_src/base.py +++ b/py_src/base.py @@ -1,92 +1,265 @@ """A library to store common functions and protocol definitions""" +from __future__ import absolute_import from __future__ import print_function -import hashlib, json, select, socket, struct, time, threading, traceback, uuid, warnings -from collections import namedtuple, deque +from __future__ import with_statement -protocol_version = "0.3" -node_policy_version = "213" +import hashlib +import inspect +import json +import socket +import struct +import sys +import threading +import traceback +import uuid + +from collections import namedtuple +from .utils import (getUTC, intersect, get_lan_ip, get_socket, sanitize_packet) + +protocol_version = "0.4" +node_policy_version = "516" version = '.'.join([protocol_version, node_policy_version]) +plock = threading.Lock() + +class brepr(bytearray): + """An extension of the bytearray object which prints a different value than it stores. This is mostly used for debugging purposes.""" + def __init__(self, value, rep=None): + """Initializes a brepr object + + Args: + value: The value you want this bytearray to store + rep: The value you want this bytearray to print + """ + super(brepr, self).__init__(value) + self.__rep = (rep or value) + + def __repr__(self): + return self.__rep + class flags(): """A namespace to hold protocol-defined flags""" + # Reserved set of bytes + reserved = [struct.pack('!B', x) for x in range(0x20)] + # main flags - broadcast = b'broadcast' # also sub-flag - waterfall = b'waterfall' - whisper = b'whisper' # also sub-flag - renegotiate = b'renegotiate' + broadcast = brepr(b'\x00', rep='broadcast') # also sub-flag + waterfall = brepr(b'\x01', rep='waterfall') + whisper = brepr(b'\x02', rep='whisper') # also sub-flag + renegotiate = brepr(b'\x03', rep='renegotiate') + ping = brepr(b'\x04', rep='ping') # Unused, but reserved + pong = brepr(b'\x05', rep='pong') # Unused, but reserved # sub-flags - handshake = b'handshake' - request = b'request' - response = b'response' - resend = b'resend' - peers = b'peers' - compression = b'compression' - - # compression methods - gzip = b'gzip' - bz2 = b'bz2' - lzma = b'lzma' - -user_salt = str(uuid.uuid4()).encode() + # broadcast = brepr(b'\x00', rep='broadcast') + compression = brepr(b'\x01', rep='compression') + # whisper = brepr(b'\x02', rep='whisper') + handshake = brepr(b'\x03', rep='handshake') + # ping = brepr(b'\x04', rep='ping') + # pong = brepr(b'\x05', rep='pong') + notify = brepr(b'\x06', rep='notify') + peers = brepr(b'\x07', rep='peers') + request = brepr(b'\x08', rep='request') + resend = brepr(b'\x09', rep='resend') + response = brepr(b'\x0A', rep='response') + store = brepr(b'\x0B', rep='store') + retrieve = brepr(b'\x0C', rep='retrieve') + + # implemented compression methods + bz2 = brepr(b'\x10', rep='bz2') + gzip = brepr(b'\x11', rep='gzip') + lzma = brepr(b'\x12', rep='lzma') + zlib = brepr(b'\x13', rep='zlib') + + # non-implemented compression methods (based on list from compressjs): + bwtc = brepr(b'\x14', rep='bwtc') + context1 = brepr(b'\x15', rep='context1') + defsum = brepr(b'\x16', rep='defsum') + dmc = brepr(b'\x17', rep='dmc') + fenwick = brepr(b'\x18', rep='fenwick') + huffman = brepr(b'\x19', rep='huffman') + lzjb = brepr(b'\x1A', rep='lzjb') + lzjbr = brepr(b'\x1B', rep='lzjbr') + lzp3 = brepr(b'\x1C', rep='lzp3') + mtf = brepr(b'\x1D', rep='mtf') + ppmd = brepr(b'\x1E', rep='ppmd') + simple = brepr(b'\x1F', rep='simple') + + +user_salt = str(uuid.uuid4()).encode() compression = [] # This should be in order of preference, with None being implied as last # Compression testing section try: import zlib - compression.append(flags.gzip) -except: # pragma: no cover + compression.extend((flags.zlib, flags.gzip)) +except ImportError: # pragma: no cover pass try: import bz2 compression.append(flags.bz2) -except: # pragma: no cover +except ImportError: # pragma: no cover pass try: import lzma compression.append(flags.lzma) -except: # pragma: no cover +except ImportError: # pragma: no cover pass -# Utility method/class section; feel free to mostly ignore +json_compressions = json.dumps([method.decode() for method in compression]) + + +if sys.version_info < (3, ): + def pack_value(l, i): + """For value i, pack it into bytes of size length + + Args: + length: A positive, integral value describing how long to make the packed array + i: A positive, integral value to pack into said array + + Returns: + A bytes object containing the given value + + Raises: + ValueError: If length is not large enough to contain the value provided + """ + ret = b"" + for x in range(l): + ret = chr(i & 0xFF) + ret + i = i >> 8 + if i == 0: + break + if i: + raise ValueError("Value not allocatable in size given") + return ("\x00" * (l - len(ret))) + ret + + def unpack_value(string): + """For a string, return the packed value inside of it + + Args: + string: A string or bytes-like object + + Returns: + An integral value interpreted from this, as if it were a big-endian, unsigned integral + """ + val = 0 + for char in string: + val = val << 8 + val += ord(char) + return val + +else: + def pack_value(l, i): + """For value i, pack it into bytes of size length + + Args: + length: A positive, integral value describing how long to make the packed array + i: A positive, integral value to pack into said array + + Returns: + A :py:class:`bytes` object containing the given value + + Raises: + ValueError: If length is not large enough to contain the value provided + """ + ret = b"" + for x in range(l): + ret = bytes([i & 0xFF]) + ret + i = i >> 8 + if i == 0: + break + if i: + raise ValueError("Value not allocatable in size given") + return (b"\x00" * (l - len(ret))) + ret + + def unpack_value(string): + """For a string, return the packed value inside of it + + Args: + string: A string or bytes-like object + + Returns: + An integral value interpreted from this, as if it were a big-endian, unsigned integral + """ + val = 0 + if not isinstance(string, (bytes, bytearray)): + string = bytes(string, 'raw_unicode_escape') + val = 0 + for char in string: + val = val << 8 + val += char + return val + base_58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' -def to_base_58(i): # returns bytes - """Takes an integer and returns its corresponding base_58 string""" +def to_base_58(i): + """Takes an integer and returns its corresponding base_58 string + + Args: + i: The integral value you wish to encode + + Returns: + A :py:class:`bytes` object which contains the base_58 string + + Raises: + TypeError: If you feed a non-integral value + """ string = "" while i: string = base_58[i % 58] + string i = i // 58 + if not string: + string = base_58[0] return string.encode() -def from_base_58(string): # returns int (or long) - """Takes a base_58 string and returns its corresponding integer""" +def from_base_58(string): + """Takes a base_58 string and returns its corresponding integer + + Args: + string: The base_58 value you wish to decode (string, bytes, or bytearray) + + Returns: + Returns integral value which corresponds to the fed string + """ decimal = 0 - if isinstance(string, bytes): + if isinstance(string, (bytes, bytearray)): string = string.decode() for char in string: decimal = decimal * 58 + base_58.index(char) return decimal -def getUTC(): # returns int - """Returns the current unix time in UTC""" - import calendar, time - return calendar.timegm(time.gmtime()) +def compress(msg, method): + """Shortcut method for compression + Args: + msg: The message you wish to compress, the type required is defined by the requested method + method: The compression method you wish to use. Supported (assuming installed): + :py:class:`~base.flags.gzip`, + :py:class:`~base.flags.zlib`, + :py:class:`~base.flags.bz2`, + :py:class:`~base.flags.lzma` -def compress(msg, method): # takes bytes, returns bytes - """Shortcut method for compression""" + Returns: + Defined by the compression method, but typically the bytes of the compressed message + + Warning: + The types fed are dependent on which compression method you use. Best to assume most values are :py:class:`bytes` or :py:class:`bytearray` + """ if method == flags.gzip: - return zlib.compress(msg) + compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, 31) + return compressor.compress(msg) + compressor.flush() + elif method == flags.zlib: + compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, 15) + return compressor.compress(msg) + compressor.flush() elif method == flags.bz2: return bz2.compress(msg) elif method == flags.lzma: @@ -95,9 +268,24 @@ def compress(msg, method): # takes bytes, returns bytes raise Exception('Unknown compression method') -def decompress(msg, method): # takes bytes, returns bytes - """Shortcut method for decompression""" - if method == flags.gzip: +def decompress(msg, method): + """Shortcut method for decompression + + Args: + msg: The message you wish to decompress, the type required is defined by the requested method + method: The decompression method you wish to use. Supported (assuming installed): + :py:class:`~base.flags.gzip`, + :py:class:`~base.flags.zlib`, + :py:class:`~base.flags.bz2`, + :py:class:`~base.flags.lzma` + + Returns: + Defined by the decompression method, but typically the bytes of the compressed message + + Warning: + The types fed are dependent on which decompression method you use. Best to assume most values are :py:class:`bytes` or :py:class:`bytearray` + """ + if method in (flags.gzip, flags.zlib): return zlib.decompress(msg, zlib.MAX_WBITS | 32) elif method == flags.bz2: return bz2.decompress(msg) @@ -107,205 +295,65 @@ def decompress(msg, method): # takes bytes, returns bytes raise Exception('Unknown decompression method') -def intersect(*args): # returns list - """Returns the ordered intersection of all given iterables, where the order is defined by the first iterable""" - if not all(args): - return [] - intersection = args[0] - for l in args[1:]: - intersection = [item for item in intersection if item in l] - return intersection - - -def get_lan_ip(): - """Retrieves the LAN ip. Expanded from http://stackoverflow.com/a/28950776""" - import socket - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # doesn't even have to be reachable - s.connect(('8.8.8.8', 23)) - IP = s.getsockname()[0] - except: - IP = '127.0.0.1' - finally: - s.shutdown(socket.SHUT_RDWR) - return IP - - class protocol(namedtuple("protocol", ['subnet', 'encryption'])): - """Defines service variables so that you can reject connections looking for a different service""" + """Defines service variables so that you can reject connections looking for a different service + + Attributes: + subnet: The subnet flag this protocol uses + encryption: The encryption method this protocol uses + id: The SHA-256 based ID of this protocol + """ @property def id(self): + """The SHA-256-based ID of the protocol""" h = hashlib.sha256(''.join([str(x) for x in self] + [protocol_version]).encode()) return to_base_58(int(h.hexdigest(), 16)) -default_protocol = protocol('', "Plaintext") # PKCS1_v1.5") - - -def get_socket(protocol, serverside=False): - if protocol.encryption == "Plaintext": - return socket.socket() - elif protocol.encryption == "SSL": - from . import ssl_wrapper - return ssl_wrapper.get_socket(serverside) - else: # pragma: no cover - raise ValueError("Unkown encryption method") - - -class base_connection(object): - """The base class for a connection""" - def __init__(self, sock, server, prot=default_protocol, outgoing=False): - self.sock = sock - self.server = server - self.protocol = prot - self.outgoing = outgoing - self.buffer = [] - self.id = None - self.time = getUTC() - self.addr = None - self.compression = [] - self.last_sent = [] - self.expected = 4 - self.active = False - - def collect_incoming_data(self, data): - """Collects incoming data""" - if not bool(data): - self.__print__(data, time.time(), level=5) - try: - self.sock.shutdown(socket.SHUT_RDWR) - except: - pass - return False - self.buffer.append(data) - self.time = getUTC() - if not self.active and self.find_terminator(): - self.__print__(self.buffer, self.expected, self.find_terminator(), level=4) - self.expected = struct.unpack("!L", ''.encode().join(self.buffer))[0] + 4 - self.active = True - return True - - def find_terminator(self): - """Returns whether the definied return sequences is found""" - return len(''.encode().join(self.buffer)) == self.expected - - def fileno(self): - return self.sock.fileno() - - def __print__(self, *args, **kargs): - """Private method to print if level is <= self.server.debug_level""" - self.server.__print__(*args, **kargs) - - -class base_daemon(object): - """The base class for a daemon""" - def __init__(self, addr, port, server, prot=default_protocol): - self.protocol = prot - self.server = server - self.sock = get_socket(self.protocol, True) - self.sock.bind((addr, port)) - self.sock.listen(5) - self.sock.settimeout(0.1) - self.exceptions = [] - self.alive = True - self.daemon = threading.Thread(target=self.mainloop) - self.daemon.daemon = True - self.daemon.start() - - def __del__(self): - self.alive = False - try: - self.sock.shutdown(socket.SHUT_RDWR) - except: # pragma: no cover - pass - - def __print__(self, *args, **kargs): - """Private method to print if level is <= self.server.debug_level""" - self.server.__print__(*args, **kargs) - - -class base_socket(object): - """The base class for a peer-to-peer socket""" - def recv(self, quantity=1): - """Receive 1 or several message objects. Returns none if none are present. Non-blocking.""" - if quantity != 1: - ret_list = [] - while len(self.queue) and quantity > 0: - ret_list.append(self.queue.pop()) - quantity -= 1 - return ret_list - elif len(self.queue): - return self.queue.pop() - else: - return None - - @property - def status(self): - """Returns "Nominal" if all is going well, or a list of unexpected (Excpetion, traceback) tuples if not""" - return self.daemon.exceptions or "Nominal" - - @property - def outgoing(self): - """IDs of outgoing connections""" - return [handler.id for handler in self.routing_table.values() if handler.outgoing] - - @property - def incoming(self): - """IDs of incoming connections""" - return [handler.id for handler in self.routing_table.values() if not handler.outgoing] - - def __print__(self, *args, **kargs): - """Private method to print if level is <= self.__debug_level""" - if kargs.get('level') <= self.debug_level: - print(self.out_addr[1], *args) - - def __del__(self): - handlers = list(self.routing_table.values()) + self.awaiting_ids - for handler in handlers: - self.disconnect(handler) +default_protocol = protocol('', "Plaintext") # SSL") class pathfinding_message(object): """An object used to build and parse protocol-defined message structures""" @classmethod - def feed_string(cls, protocol, string, sizeless=False, compressions=None): - """Constructs a pathfinding_message from a string or bytes object. - Possible errors: - AttributeError: Fed a non-string, non-bytes argument - AssertionError: Initial size header is incorrect - Exception: Unrecognized compression method fed in compressions - struct.error: Packet headers are incorrect OR unrecognized compression - IndexError: See struct.error""" - # First section checks size header - string = cls.sanitize_string(string, sizeless) - # Then we attempt to decompress - string, compression_fail = cls.decompress_string(string, compressions) - # After this, we process the packet size headers - packets = cls.process_string(string) - msg = cls(protocol, packets[0], packets[1], packets[4:], compression=compressions) - msg.time = from_base_58(packets[3]) - msg.compression_fail = compression_fail - return msg - - @classmethod - def sanitize_string(cls, string, sizeless=False): + def __sanitize_string(cls, string, sizeless=False): """Removes the size header for further processing. Also checks if the header is valid. - Possible errors: - AttributeError: Fed a non-string, non-bytes argument - AssertionError: Initial size header is incorrect""" - if not isinstance(string, bytes): + + Args: + string: The string you wish to sanitize + sizeless: Whether this string is missing a size header (default: ``False``) + + Returns: + The fed string without the size header + + Raises: + AttributeError: Fed a non-string, non-bytes argument + AssertionError: Initial size header is incorrect + """ + if not isinstance(string, (bytes, bytearray)): string = string.encode() if not sizeless: - assert struct.unpack('!L', string[:4])[0] == len(string[4:]), \ - "Must assert struct.unpack('!L', string[:4])[0] == len(string[4:])" + assert unpack_value(string[:4]) == len(string[4:]), \ + "Must assert base.unpack_value(string[:4]) == len(string[4:])" string = string[4:] return string @classmethod - def decompress_string(cls, string, compressions=None): + def __decompress_string(cls, string, compressions=None): """Returns a tuple containing the decompressed bytes and a boolean as to whether decompression failed or not - Possible errors: - Exception: Unrecognized compression method fed in compressions""" + + Args: + string: The possibly-compressed message you wish to parse + compressions: A list of the standard compression methods this message may be under (defualt: []) + + Returns: + A decompressed version of the message + + Raises: + Exception: Unrecognized compression method fed in compressions + + Warning: + Do not feed it with the size header, it will throw errors + """ compression_fail = False for method in intersect(compressions, compression): # second is module scope compression try: @@ -318,11 +366,22 @@ def decompress_string(cls, string, compressions=None): return (string, compression_fail) @classmethod - def process_string(cls, string): + def __process_string(cls, string): """Given a sanitized, plaintext string, returns a list of its packets - Possible errors: - struct.error: Packet headers are incorrect OR not fed plaintext - IndexError: See struct.error""" + + Args: + string: The message you wish to parse + + Returns: + A list containing the message's packets + + Raises: + struct.error: Packet headers are incorrect OR not fed plaintext + IndexError: See case of :py:class:`struct.error` + + Warning: + Do not feed a message with the size header. Do not feed a compressed message. + """ processed, expected = 0, len(string) pack_lens, packets = [], [] while processed != expected: @@ -330,30 +389,75 @@ def process_string(cls, string): processed += 4 expected -= pack_lens[-1] # Then reconstruct the packets - for index, length in enumerate(pack_lens): - start = processed + sum(pack_lens[:index]) - end = start + length - packets.append(string[start:end]) + for length in pack_lens: + end = processed + length + packets.append(string[processed:end]) + processed = end return packets - def __init__(self, protocol, msg_type, sender, payload, compression=None): - self.protocol = protocol - self.msg_type = msg_type - self.sender = sender - self.__payload = payload - self.time = getUTC() + @classmethod + def feed_string(cls, string, sizeless=False, compressions=None): + """Constructs a pathfinding_message from a string or bytes object. + + Args: + string: The string you wish to parse + sizeless: A boolean which describes whether this string has its size header (default: it does) + compressions: A list containing the standardized compression methods this message might be under (default: []) + + Returns: + A base.pathfinding_message from the given string + + Raises: + AttributeError: Fed a non-string, non-bytes argument + AssertionError: Initial size header is incorrect + Exception: Unrecognized compression method fed in compressions + struct.error: Packet headers are incorrect OR unrecognized compression + IndexError: See case of :py:class:`struct.error` + """ + # First section checks size header + string = cls.__sanitize_string(string, sizeless) + # Then we attempt to decompress + string, compression_fail = cls.__decompress_string(string, compressions) + # After this, we process the packet size headers + packets = cls.__process_string(string) + msg = cls(packets[0], packets[1], packets[4:], compression=compressions) + msg.time = from_base_58(packets[3]) + msg.compression_fail = compression_fail + assert packets[2] == msg.id, "Checksum failed" + return msg + + def __init__(self, msg_type, sender, payload, compression=None, timestamp=None): + """Initializes a pathfinding_message instance + + Args: + msg_type: A bytes-like header for the message you wish to send + sender: A bytes-like sender ID the message is using + payload: A list of bytes-like objects containing the payload of the message + compression: A list of the compression methods this message may use (default: []) + timestamp: The current UTC timestamp (as an integer) (default: result of utils.getUTC()) + + Raises: + TypeError: If you feed an object which cannot convert to bytes + + Warning: + If you feed a unicode object, it will be decoded using utf-8. All other objects are + treated as raw bytes. If you desire a particular codec, encode it yourself + before feeding it in. + """ + self.msg_type = sanitize_packet(msg_type) + self.sender = sanitize_packet(sender) + self.__payload = [sanitize_packet(packet) for packet in payload] + self.time = timestamp or getUTC() + self.compression_fail = False + if compression: self.compression = compression else: self.compression = [] - self.compression_fail = False @property def payload(self): """Returns a list containing the message payload encoded as bytes""" - for i, val in enumerate(self.__payload): - if not isinstance(val, bytes): - self.__payload[i] = val.encode() return self.__payload @property @@ -371,35 +475,30 @@ def time_58(self): @property def id(self): """Returns the message id""" - payload_string = b''.join(self.payload) + payload_string = b''.join((bytes(pac) for pac in self.payload)) payload_hash = hashlib.sha384(payload_string + self.time_58) return to_base_58(int(payload_hash.hexdigest(), 16)) @property def packets(self): """Returns the full list of packets in this message encoded as bytes, excluding the header""" - meta = [self.msg_type, self.sender, self.id, self.time_58] - for i, val in enumerate(meta): - if not isinstance(val, bytes): - meta[i] = val.encode() - return meta + self.payload + return [self.msg_type, self.sender, self.id, self.time_58] + self.payload @property def __non_len_string(self): """Returns a bytes object containing the entire message, excepting the total length header""" packets = self.packets - header = struct.pack("!" + str(len(packets)) + "L", - *[len(x) for x in packets]) - string = header + b''.join(packets) + header = [pack_value(4, len(x)) for x in packets] + string = b''.join((bytes(pac) for pac in header + packets)) if self.compression_used: string = compress(string, self.compression_used) return string - + @property def string(self): """Returns a string representation of the message""" string = self.__non_len_string - return struct.pack("!L", len(string)) + string + return pack_value(4, len(string)) + string def __len__(self): return len(self.__non_len_string) @@ -407,14 +506,345 @@ def __len__(self): @property def len(self): """Return the struct-encoded length header""" - return struct.pack("!L", self.__len__()) + return pack_value(4, self.__len__()) + + +class base_connection(object): + """The base class for a connection""" + def __init__(self, sock, server, outgoing=False): + """Sets up a connection to another peer-to-peer socket + + Args: + sock: The connected socket object + server: A reference to your peer-to-peer socket + outgoing: Whether this connection is outgoing (default: False) + """ + self.sock = sock + self.server = server + self.outgoing = outgoing + self.buffer = [] + self.id = None + self.time = getUTC() + self.addr = None + self.compression = [] + self.last_sent = [] + self.expected = 4 + self.active = False + + def send(self, msg_type, *args, **kargs): + """Sends a message through its connection. + + Args: + msg_type: Message type, corresponds to the header in a :py:class:`py2p.base.pathfinding_message` object + *args: A list of bytes-like objects, which correspond to the packets to send to you + **kargs: There are two available keywords: + id: The ID this message should appear to be sent from (default: your ID) + time: The time this message should appear to be sent from (default: now in UTC) + + Returns: + the pathfinding_message object you just sent, or None if the sending was unsuccessful + """ + # This section handles waterfall-specific flags + id = kargs.get('id', self.server.id) # Latter is returned if key not found + time = kargs.get('time') or getUTC() + # Begin real method + msg = pathfinding_message(msg_type, id, list(args), self.compression, timestamp=time) + if msg_type in [flags.whisper, flags.broadcast]: + self.last_sent = [msg_type] + list(args) + self.__print__("Sending %s to %s" % ([msg.len] + msg.packets, self), level=4) + if msg.compression_used: self.__print__("Compressing with %s" % repr(msg.compression_used), level=4) + try: + self.sock.send(msg.string) + return msg + except (IOError, socket.error) as e: # pragma: no cover + self.server.daemon.exceptions.append((e, traceback.format_exc())) + self.server.disconnect(self) + + @property + def protocol(self): + """Returns server.protocol""" + return self.server.protocol + + def collect_incoming_data(self, data): + """Collects incoming data + + Args: + data: The most recently received byte + + Returns: + ``True`` if the data collection was successful, ``False`` if the connection was closed + """ + if not bool(data): + try: + self.sock.shutdown(socket.SHUT_RDWR) + except: + pass + return False + self.buffer.append(data) + self.time = getUTC() + if not self.active and self.find_terminator(): + self.__print__(self.buffer, self.expected, self.find_terminator(), level=4) + self.expected = struct.unpack("!L", ''.encode().join(self.buffer))[0] + 4 + self.active = True + return True + + def find_terminator(self): + """Returns whether the definied return sequences is found""" + return len(''.encode().join(self.buffer)) == self.expected + + def found_terminator(self): + """Processes received messages""" + raw_msg = ''.encode().join(self.buffer) + self.__print__("Received: %s" % repr(raw_msg), level=6) + self.expected = 4 + self.buffer = [] + self.active = False + msg = pathfinding_message.feed_string(raw_msg, False, self.compression) + return msg + + def handle_renegotiate(self, packets): + """The handler for connection renegotiations + + This is to deal with connection maintenence. For instance, it could + be that a compression method fails to decode on the other end, and a + node will need to renegotiate which methods it is using. Hence the + name of the flag associated with it, "renegotiate". + + Args: + packets: A list containing the packets received in this message + + Returns: + ``True`` if an action was taken, ``None`` if not + """ + if packets[0] == flags.renegotiate: + if packets[4] == flags.compression: + encoded_methods = [algo.encode() for algo in json.loads(packets[5].decode())] + respond = (self.compression != encoded_methods) + self.compression = encoded_methods + self.__print__("Compression methods changed to: %s" % repr(self.compression), level=2) + if respond: + decoded_methods = [algo.decode() for algo in intersect(compression, self.compression)] + self.send(flags.renegotiate, flags.compression, json.dumps(decoded_methods)) + return True + elif packets[4] == flags.resend: + self.send(*self.last_sent) + return True + + def fileno(self): + """Mirror for the fileno() method of the connection's underlying socket""" + return self.sock.fileno() + + def __print__(self, *args, **kargs): + """Private method to print if level is <= self.server.debug_level + + Args: + *args: Each argument you wish to feed to the print method + **kargs: One keyword is used here: level, which defines the + lowest value of self.server.debug_level at which the message + will be printed + """ + self.server.__print__(*args, **kargs) + + +class base_daemon(object): + """The base class for a daemon""" + def __init__(self, addr, port, server): + """Sets up a daemon process for your peer-to-peer socket + + Args: + addr: The address you wish to bind to + port: The port you wish to bind to + server: A reference to the peer-to-peer socket + + Raises: + socket.error: The address you wanted is already in use + ValueError: If your peer-to-peer socket is set up with an unknown encryption method + """ + self.server = server + self.sock = get_socket(self.protocol, True) + self.sock.bind((addr, port)) + self.sock.listen(5) + self.sock.settimeout(0.1) + self.exceptions = [] + self.alive = True + self.main_thread = threading.current_thread() + self.daemon = threading.Thread(target=self.mainloop) + self.daemon.start() + + @property + def protocol(self): + """Returns server.protocol""" + return self.server.protocol + + def kill_old_nodes(self, handler): + """Cleans out connections which never finish a message""" + if handler.active and handler.time < getUTC() - 60: + self.server.disconnect(handler) + + def __del__(self): + self.alive = False + try: + self.sock.shutdown(socket.SHUT_RDWR) + except: # pragma: no cover + pass + + def __print__(self, *args, **kargs): + """Private method to print if level is <= self.server.debug_level + + Args: + *args: Each argument you wish to feed to the print method + **kargs: One keyword is used here: level, which defines the + lowest value of self.server.debug_level at which the message + will be printed + """ + self.server.__print__(*args, **kargs) + + +class base_socket(object): + """The base class for a peer-to-peer socket abstractor""" + def __init__(self, addr, port, prot=default_protocol, out_addr=None, debug_level=0): + """Initializes a peer to peer socket + + Args: + addr: The address you wish to bind to (ie: "192.168.1.1") + port: The port you wish to bind to (ie: 44565) + prot: The protocol you wish to operate over, defined by a :py:class:`py2p.base.protocol` object + out_addr: Your outward facing address. Only needed if you're connecting + over the internet. If you use '0.0.0.0' for the addr argument, this will + automatically be set to your LAN address. + debug_level: The verbosity you want this socket to use when printing event data + + Raises: + socket.error: The address you wanted could not be bound, or is otherwise used + """ + self.protocol = prot + self.debug_level = debug_level + self.routing_table = {} # In format {ID: handler} + self.awaiting_ids = [] # Connected, but not handshook yet + if out_addr: # Outward facing address, if you're port forwarding + self.out_addr = out_addr + elif addr == '0.0.0.0': + self.out_addr = get_lan_ip(), port + else: + self.out_addr = addr, port + info = [str(self.out_addr).encode(), prot.id, user_salt] + h = hashlib.sha384(b''.join(info)) + self.id = to_base_58(int(h.hexdigest(), 16)) + self.__handlers = [] + self.__closed = False + + def close(self): + """If the socket is not closed, close the socket + + Raises: + RuntimeError: The socket was already closed + """ + if self.__closed: + raise RuntimeError("Already closed") + else: + self.daemon.alive = False + self.daemon.daemon.join() + self.debug_level = 0 + try: + self.daemon.sock.shutdown(socket.SHUT_RDWR) + except: + pass + conns = list(self.routing_table.values()) + self.awaiting_ids + for conn in conns: + self.disconnect(conn) + self.__closed = True + + if sys.version_info >= (3, ): + def register_handler(self, method): + """Register a handler for incoming method. + + Args: + method: A function with two given arguments. Its signature + should be of the form ``handler(msg, handler)``, where msg + is a :py:class:`py2p.base.message` object, and handler is + a :py:class:`py2p.base.base_connection` object. It should + return ``True`` if it performed an action, to reduce the + number of handlers checked. + + Raises: + ValueError: If the method signature doesn't parse correctly + """ + args = inspect.signature(method) + if len(args.parameters) != (3 if args.parameters.get('self') else 2): + raise ValueError("This method must contain exactly two arguments (or three if first is self)") + self.__handlers.append(method) + + else: + def register_handler(self, method): + """Register a handler for incoming method. + + Args: + method: A function with two given arguments. Its signature + should be of the form ``handler(msg, handler)``, where msg + is a :py:class:`py2p.base.message` object, and handler is + a :py:class:`py2p.base.base_connection` object. It should + return ``True`` if it performed an action, to reduce the + number of handlers checked. + + Raises: + ValueError: If the method signature doesn't parse correctly + """ + args = inspect.getargspec(method) + if args[1:] != (None, None, None) or len(args[0]) != (3 if args[0][0] == 'self' else 2): + raise ValueError("This method must contain exactly two arguments (or three if first is self)") + self.__handlers.append(method) + + def handle_msg(self, msg, conn): + """Decides how to handle various message types, allowing some to be handled automatically + + Args: + msg: A :py:class:`py2p.base.message` object + conn: A :py:class:`py2p.base.base_connection` object + + Returns: + True if an action was taken, None if not. + """ + for handler in self.__handlers: + self.__print__("Checking handler: %s" % handler.__name__, level=4) + if handler(msg, conn): + self.__print__("Breaking from handler: %s" % handler.__name__, level=4) + return True + + @property + def status(self): + """The status of the socket. + + Returns: + ``"Nominal"`` if all is going well, or a list of unexpected (Excpetion, traceback) tuples if not""" + return self.daemon.exceptions or "Nominal" + + def __print__(self, *args, **kargs): + """Private method to print if level is <= self.debug_level + + Args: + *args: Each argument you wish to feed to the print method + **kargs: One keyword is used here: level, which defines the + lowest value of self.debug_level at which the message will + be printed + """ + if kargs.get('level', 0) <= self.debug_level: + with plock: + print(self.out_addr[1], *args) + + def __del__(self): + if not self.__closed: + self.close() class message(object): """An object which gets returned to a user, containing all necessary information to parse and reply to a message""" def __init__(self, msg, server): - if not isinstance(msg, pathfinding_message): # pragma: no cover - raise TypeError("message must be passed a pathfinding_message") + """Initializes a message object + + Args: + msg: A :py:class:`py2p.base.pathfinding_message` object + server: A :py:class:`py2p.base.base_socket` object + """ self.msg = msg self.server = server @@ -433,11 +863,6 @@ def sender(self): """The ID of this message's sender""" return self.msg.sender - @property - def protocol(self): - """The protocol this message was sent under""" - return self.msg.protocol - @property def id(self): """This message's ID""" @@ -460,7 +885,13 @@ def __repr__(self): return string + repr(self.sender) + ")" def reply(self, *args): - """Replies to the sender if you're directly connected. Tries to make a connection otherwise""" + """Replies to the sender if you're directly connected. Tries to make a connection otherwise + + Args: + *args: Each argument given is a packet you wish to send. This is + prefixed with base.flags.whisper, so the other end will receive + ``[base.flags.whisper, *args]`` + """ if self.server.routing_table.get(self.sender): self.server.routing_table.get(self.sender).send(flags.whisper, flags.whisper, *args) else: diff --git a/py_src/chord.py b/py_src/chord.py index e69de29..7223934 100644 --- a/py_src/chord.py +++ b/py_src/chord.py @@ -0,0 +1,616 @@ +from __future__ import print_function +from __future__ import absolute_import + +import hashlib +import json +import random +import select +import socket +import struct +import sys +import time +import traceback +import warnings + +try: + from .cbase import protocol +except: + from .base import protocol + +from .base import (flags, compression, to_base_58, from_base_58, + base_connection, message, base_daemon, base_socket, + pathfinding_message, json_compressions) +from .utils import (getUTC, get_socket, intersect, awaiting_value, most_common) + +default_protocol = protocol('chord', "Plaintext") # SSL") +hashes = ['sha1', 'sha224', 'sha256', 'sha384', 'sha512'] + +if sys.version_info >= (3,): + xrange = range + + +def distance(a, b, limit): + """This is a clockwise ring distance function. + It depends on a globally defined k, the key size. + The largest possible node id is limit (or 2**k).""" + return (b - a) % limit + + +class chord_connection(base_connection): + """The class for chord connection abstraction. This inherits from :py:class:`py2p.base.base_connection`""" + def found_terminator(self): + """This method is called when the expected amount of data is received + + Returns: + ``None`` + """ + try: + msg = super(chord_connection, self).found_terminator() + except (IndexError, struct.error): + self.__print__("Failed to decode message: %s. Expected compression: %s." % \ + (raw_msg, intersect(compression, self.compression)[0]), level=1) + self.send(flags.renegotiate, flags.compression, json.dumps([])) + self.send(flags.renegotiate, flags.resend) + return + packets = msg.packets + self.__print__("Message received: %s" % packets, level=1) + if self.handle_renegotiate(packets): + return + self.server.handle_msg(message(msg, self.server), self) + + @property + def id_10(self): + """Returns the nodes ID as an integer""" + return from_base_58(self.id) + + def __hash__(self): + return self.id_10 or id(self) + + +class chord_daemon(base_daemon): + """The class for chord daemon. This inherits from :py:class:`py2p.base.base_daemon`""" + def mainloop(self): + """Daemon thread which handles all incoming data and connections""" + while self.main_thread.is_alive() and self.alive: + conns = list(self.server.routing_table.values()) + self.server.awaiting_ids + for handler in select.select(conns + [self.sock], [], [], 0.01)[0]: + if handler == self.sock: + self.handle_accept() + else: + self.process_data(handler) + for handler in conns: + self.kill_old_nodes(handler) + self.server.update_fingers() + + def handle_accept(self): + """Handle an incoming connection""" + if sys.version_info >= (3, 3): + exceptions = (socket.error, ConnectionError) + else: + exceptions = (socket.error, ) + try: + conn, addr = self.sock.accept() + self.__print__('Incoming connection from %s' % repr(addr), level=1) + handler = chord_connection(conn, self.server) + handler.sock.settimeout(1) + self.server.awaiting_ids.append(handler) + except exceptions: + pass + + def process_data(self, handler): + """Collects incoming data from nodes""" + try: + while not handler.find_terminator(): + if not handler.collect_incoming_data(handler.sock.recv(1)): + self.__print__("disconnecting node %s while in loop" % handler.id, level=6) + self.server.disconnect(handler) + return + handler.found_terminator() + except socket.timeout: # pragma: no cover + return # Shouldn't happen with select, but if it does... + except Exception as e: + if isinstance(e, socket.error) and e.args[0] in (9, 104, 10053, 10054, 10058): + node_id = handler.id + if not node_id: + node_id = repr(handler) + self.__print__("Node %s has disconnected from the network" % node_id, level=1) + else: + self.__print__("There was an unhandled exception with peer id %s. This peer is being disconnected, and the relevant exception is added to the debug queue. If you'd like to report this, please post a copy of your chord_socket.status to github.com/gappleto97/p2p-project/issues." % handler.id, level=0) + self.exceptions.append((e, traceback.format_exc())) + self.server.disconnect(handler) + + +class chord_socket(base_socket): + """The class for chord socket abstraction. This inherits from :py:class:`py2p.base.base_socket`""" + def __init__(self, addr, port, k=6, prot=default_protocol, out_addr=None, debug_level=0): + """Initializes a chord socket + + Args: + addr: The address you wish to bind to (ie: "192.168.1.1") + port: The port you wish to bind to (ie: 44565) + k: This number indicates the node counts the network can support. You must have > (k+1) nodes. + You may only have up to 2**k nodes, but at that count you will likely get ID conficts. + prot: The protocol you wish to operate over, defined by a :py:class:`py2p.base.protocol` object + out_addr: Your outward facing address. Only needed if you're connecting over the internet. If you + use '0.0.0.0' for the addr argument, this will automatically be set to your LAN address. + debug_level: The verbosity you want this socket to use when printing event data + + Raises: + socket.error: The address you wanted could not be bound, or is otherwise used + """ + super(chord_socket, self).__init__(addr, port, prot, out_addr, debug_level) + self.k = k # 160 # SHA-1 namespace + self.limit = 2**k + self.id_10 = from_base_58(self.id) % self.limit + self.id = to_base_58(self.id_10) + self.data = dict(((method, dict()) for method in hashes)) + self.daemon = chord_daemon(addr, port, self) + self.requests = {} + self.predecessors = [] + self.register_handler(self.__handle_handshake) + self.register_handler(self.__handle_peers) + self.register_handler(self.__handle_response) + self.register_handler(self.__handle_request) + self.register_handler(self.__handle_retrieve) + self.register_handler(self.__handle_store) + self.next = self + self.prev = self + self.leeching = True + warnings.warn("This network configuration supports %s total nodes and requires a theoretical minimum of %s nodes" % (min(self.limit, 2**160), self.k), RuntimeWarning, stacklevel=2) + + @property + def addr(self): + """An alternate binding for ``self.out_addr``, in order to better handle self-references in the daemon thread""" + return self.out_addr + + def __findFinger__(self, key): + current=self + for x in xrange(self.k): + if distance(current.id_10, key, self.limit) > \ + distance(self.routing_table.get(x, self).id_10, key, self.limit): + current=self.routing_table.get(x, self) + return current + + def __get_fingers(self): + """Returns a finger table for your peer""" + peer_list = [] + peer_list = list(set(((tuple(node.addr), node.id.decode()) for node in list(self.routing_table.values()) + self.awaiting_ids if node.addr))) + if self.next is not self: + peer_list.append((self.next.addr, self.next.id.decode())) + if self.prev is not self: + peer_list.append((self.prev.addr, self.prev.id.decode())) + return peer_list + + def set_fingers(self, handler): + """Given a handler, check to see if it's the closest connection to an ideal slot. + + In other words, if it's the closest ID you know of to a power of two distance from you, + add it to your connection table. + + Args: + handler: A :py:class:`~py2p.chord.chord_connection` + """ + for x in xrange(self.k): + goal = self.id_10 + 2**x + if distance(self.__findFinger__(goal).id_10, goal, self.limit) \ + > distance(handler.id_10, goal, self.limit): + former = self.__findFinger__(goal) + self.routing_table[x] = handler + if former not in self.routing_table.values(): + self.disconnect(former) + + def is_saturated(self): + """Returns whether all ideal connection slots are filled""" + for x in xrange(self.k): + node = self.__findFinger__(self.id_10 + 2**x % self.limit) + if distance(node.id_10, self.id_10 + 2**x, self.limit) != 0: + return False + return True + + def update_fingers(self): + """Updates your connection table, and sends a request for more peers whenever ``getUTC() % 5 == 0 and not self.is_saturated()`` + + Is this efficient? No. + + Will it be fixed? Yes. See the warning up top. + """ + should_request = (not self.leeching) and (not (getUTC() % 5)) and (not self.is_saturated()) + for handler in list(self.routing_table.values()) + self.awaiting_ids + self.predecessors: + if handler.id: + self.set_fingers(handler) + if should_request: + handler.send(flags.whisper, flags.request, b'*') + + def handle_msg(self, msg, conn): + """Decides how to handle various message types, allowing some to be handled automatically""" + if not super(chord_socket, self).handle_msg(msg, conn): + self.__print__("Ignoring message with invalid subflag", level=4) + + def __handle_handshake(self, msg, handler): + """This callback is used to deal with handshake signals. Its two primary jobs are: + + - reject connections seeking a different network + - set connection state + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.chord.chord_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.handshake: + if packets[2] != self.protocol.id + to_base_58(self.k): + self.disconnect(handler) + return True + if not handler.id: + handler.id = packets[1] + self._send_handshake(handler) + handler.addr = json.loads(packets[3].decode()) + handler.compression = json.loads(packets[4].decode()) + handler.compression = [algo.encode() for algo in handler.compression] + self.__print__("Compression methods changed to %s" % repr(handler.compression), level=4) + self.set_fingers(handler) + handler.send(flags.whisper, flags.peers, json.dumps(self.__get_fingers())) + if distance(self.id_10, self.next.id_10-1, self.limit) \ + > distance(self.id_10, handler.id_10, self.limit): + self.next = handler + if distance(self.prev.id_10+1, self.id_10, self.limit) \ + > distance(handler.id_10, self.id_10, self.limit): + self.prev = handler + return True + + def __handle_peers(self, msg, handler): + """This callback is used to deal with peer signals. Its primary jobs is to connect to the given peers, if they are a better connection given the chord schema + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.chord.chord_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.peers: + new_peers = json.loads(packets[1].decode()) + for addr, key in new_peers: + key = from_base_58(key) + for index in xrange(self.k): + goal = self.id_10 + 2**index + self.__print__("%s : %s" % (distance(self.__findFinger__(goal).id_10, goal, self.limit), + distance(key, goal, self.limit)), level=5) + if distance(self.__findFinger__(goal).id_10, goal, self.limit) \ + > distance(key, goal, self.limit): + self.__connect(*addr) + if distance(self.id_10, self.next.id_10-1, self.limit) \ + > distance(self.id_10, key, self.limit): + self.__connect(*addr) + if distance(self.prev.id_10+1, self.id_10, self.limit) \ + > distance(key, self.id_10, self.limit): + self.__connect(*addr) + return True + + def __handle_response(self, msg, handler): + """This callback is used to deal with response signals. Its two primary jobs are: + + - if it was your request, send the deferred message + - if it was someone else's request, relay the information + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.chord.chord_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.response: + self.__print__("Response received for request id %s" % packets[1], level=1) + if self.requests.get((packets[1], packets[2])): + value = self.requests.get((packets[1], packets[2])) + value.value = packets[3] + if value.callback: + value.callback_method(packets[1], packets[2]) + return True + + def __handle_request(self, msg, handler): + """This callback is used to deal with request signals. Its three primary jobs are: + + - respond with a peers signal if packets[1] is ``'*'`` + - if you know the ID requested, respond to it + - if you don't, make a request with your peers + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.chord.chord_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.request: + if packets[1] == b'*': + handler.send(flags.whisper, flags.peers, json.dumps(self.__get_fingers())) + else: + goal = from_base_58(packets[1]) + node = self.__findFinger__(goal) + if node is not self: + node.send(flags.whisper, flags.request, packets[1], msg.id) + ret = awaiting_value() + ret.callback = handler + self.requests.update({(packets[1], msg.id): ret}) + else: + handler.send(flags.whisper, flags.response, packets[1], packets[2], self.out_addr) + return True + + def __handle_retrieve(self, msg, handler): + """This callback is used to deal with data retrieval signals. Its two primary jobs are: + + - respond with data you possess + - if you don't possess it, make a request with your closest peer to that key + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.chord.chord_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.retrieve: + if packets[1] in hashes: + val = self.__lookup(packets[1], from_base_58(packets[2]), handler) + if isinstance(val.value, str): + self.__print__(val.value) + handler.send(flags.whisper, flags.response, packets[1], packets[2], val.value) + return True + + def __handle_store(self, msg, handler): + """This callback is used to deal with data storage signals. Its two primary jobs are: + + - store data in keys you're responsible for + - if you aren't responsible, make a request with your closest peer to that key + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.chord.chord_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.store: + method = packets[1] + key = from_base_58(packets[2]) + self.__store(method, key, packets[3]) + return True + + def dump_data(self, start, end=None): + """Args: + start: An :py:class:`int` which indicates the start of the desired key range. + ``0`` will get all data. + end: An :py:class:`int` which indicates the end of the desired key range. + ``None`` will get all data. (default: ``None``) + + Returns: + A nested :py:class:`dict` containing your data from start to end + """ + i = start + ret = dict(((method, {}) for method in hashes)) + for method in self.data: + for key in self.data[method]: + if key >= start % self.limit and (not end or key < end % self.limit): + print(method, key, self.data) + ret[method].update({key: self.data[method][key]}) + return ret + + def connect(self, addr, port): + """This function connects you to a specific node in the overall network. + Connecting to one node *should* connect you to the rest of the network, + however if you connect to the wrong subnet, the handshake failure involved + is silent. You can check this by looking at the truthiness of this objects + routing table. Example: + + .. code:: python + + >>> conn = chord.chord_socket('localhost', 4444) + >>> conn.connect('localhost', 5555) + >>> conn.join() + >>> # do some other setup for your program + >>> if (!conn.routing_table): + ... conn.connect('localhost', 6666) # any fallback address + ... conn.join() + + Args: + addr: A string address + port: A positive, integral port + id: A string-like object which represents the expected ID of this node + """ + self.__print__("Attempting connection to %s:%s" % (addr, port), level=1) + if socket.getaddrinfo(addr, port)[0] == socket.getaddrinfo(*self.out_addr)[0]: + self.__print__("Connection already established", level=1) + return False + conn = get_socket(self.protocol, False) + conn.settimeout(1) + conn.connect((addr, port)) + handler = chord_connection(conn, self, outgoing=True) + self.awaiting_ids.append(handler) + return handler + + def _send_handshake(self, handler): + """Shortcut method for sending a handshake to a given handler + + Args: + handler: A :py:class:`~py2p.chord.chord_connection` + """ + json_out_addr = '["{}", {}]'.format(*self.out_addr) + handler.send(flags.whisper, flags.handshake, self.id, \ + self.protocol.id + to_base_58(self.k), \ + json_out_addr, json_compressions) + + def __connect(self, addr, port): + """Private API method for connecting and handshaking + + Args: + addr: the address you want to connect to/handshake + port: the port you want to connect to/handshake + """ + try: + handler = self.connect(addr, port) + if handler and not self.leeching: + self._send_handshake(handler) + except: + pass + + def join(self): + """Tells the node to start seeding the chord table""" + # for handler in self.awaiting_ids: + self.leeching = False + handler = random.choice(self.awaiting_ids or list(self.routing_table.values())) + self._send_handshake(handler) + + def unjoin(self): + """Tells the node to stop seeding the chord table, and dumps the data to the proper nodes""" + self.leeching = True + temp_data = self.data + self.data = dict(((method, dict()) for method in hashes)) + + peers = self.awaiting_ids + list(self.routing_table.values()) + addrs = set([tuple(node.addr) for node in peers]) + + for node in peers: + self.disconnect(node) + + for addr in addrs: + self.connect(*addr) + + for algo in temp_data: + for key in temp_data[algo]: + self.__store(algo, key, temp_data[algo][key]) + + def __lookup(self, method, key, handler=None): + if self.routing_table: + node = self.__findFinger__(key) + else: + node = random.choice(self.awaiting_ids) + if node in (self, None): + return awaiting_value(self.data[method].get(key, '')) + else: + node.send(flags.whisper, flags.retrieve, method, to_base_58(key)) + ret = awaiting_value() + if handler: + ret.callback = handler + self.requests.update({(method, to_base_58(key)): ret}) + return ret + + def lookup(self, key): + """Looks up the value at a given key. + + Under the covers, this actually checks five different hash tables, and + returns the most common value given. + + Args: + key: The key that you wish to check. Must be a :py:class:`str` or + :py:class:`bytes`-like object + + Returns: + The value at said key + + Raises: + socket.timeout: If the request goes partly-unanswered for >=10 seconds + KeyError: If the request is made for a key with no agreed-upon value + """ + if not isinstance(key, (bytes, bytearray)): + key = str(key).encode() + keys = [int(hashlib.new(algo, key).hexdigest(), 16) for algo in hashes] + vals = [self.__lookup(method, x) for method, x in zip(hashes, keys)] + common, count = most_common(vals) + iters = 0 + limit = 100 + while common == -1 and iters < limit: + time.sleep(0.1) + iters += 1 + common, count = most_common(vals) + if common not in (None, '') and count > len(hashes) // 2: + return common + elif iters == limit: + raise socket.timeout() + raise KeyError("This key does not have an agreed-upon value", vals) + + def __getitem__(self, key): + return self.lookup(key) + + def get(self, key): + return self.__getitem__(key) + + def __store(self, method, key, value): + node = self.__findFinger__(key) + if self.leeching and node is self: + node = random.choice(self.awaiting_ids) + if node in (self, None): + self.data[method].update({key: value}) + else: + node.send(flags.whisper, flags.store, method, to_base_58(key), value) + + def store(self, key, value): + """Updates the value at a given key. + + Under the covers, this actually uses five different hash tables, and + updates the value in all of them. + + Args: + key: The key that you wish to update. Must be a :py:class:`str` or + :py:class:`bytes`-like object + value: The value you wish to put at this key. Must be a :py:class:`str` + or :py:class:`bytes`-like object + """ + if not isinstance(key, (bytes, bytearray)): + key = str(key).encode() + keys = [int(hashlib.new(algo, key).hexdigest(), 16) for algo in hashes] + for method, x in zip(hashes, keys): + self.__store(method, x, value) + + def __setitem__(self, key, value): + return self.store(key, value) + + def set(self, key, value): + return self.__setitem__(key, value) + + def update(self, update_dict): + """Equivalent to :py:meth:`dict.update` + + This calls :py:meth:`.chord_socket.store` for each key/value pair in the + given dictionary. + + Args: + update_dict: A :py:class:`dict`-like object to extract key/value pairs from. + Key and value be a :py:class:`str` or :py:class:`bytes`-like + object + """ + for key in update_dict: + value = update_dict[key] + self.__setitem__(key, value) + + def disconnect(self, handler): + """Closes a given connection, and removes it from your routing tables + + Args: + handler: the connection you would like to close + """ + node_id = handler.id + if not node_id: + node_id = repr(handler) + self.__print__("Connection to node %s has been closed" % node_id, level=1) + if handler in self.awaiting_ids: + self.awaiting_ids.remove(handler) + elif handler in self.routing_table.values(): + for key in list(self.routing_table.keys()): + if self.routing_table[key] is handler: + self.routing_table.pop(key) + elif handler in self.predecessors: + self.predecessors.remove(handler) + try: + handler.sock.shutdown(socket.SHUT_RDWR) + except: + pass diff --git a/py_src/kademlia.py b/py_src/kademlia.py new file mode 100644 index 0000000..8c25360 --- /dev/null +++ b/py_src/kademlia.py @@ -0,0 +1,34 @@ +from __future__ import print_function + +import hashlib +import json +import select +import socket +import struct +import sys +import traceback +import warnings + +from .base import (flags, compression, to_base_58, from_base_58, protocol, + base_connection, message, base_daemon, base_socket, + pathfinding_message, json_compressions) +from .utils import (getUTC, get_socket, intersect, file_dict, + awaiting_value, most_common) + +default_protocol = protocol('chord', "Plaintext") # SSL") +hashes = ['sha1', 'sha224', 'sha256', 'sha384', 'sha512'] + +if sys.version_info >= (3,): + xrange = range + +def distance(a, b): + raise NotImplementedError + + +class kademlia_connection(base_connection): pass + + +class kademlia_daemon(base_daemon): pass + + +class kademlia_socket(base_socket): pass \ No newline at end of file diff --git a/py_src/mesh.py b/py_src/mesh.py index 57c7698..8c88c9b 100644 --- a/py_src/mesh.py +++ b/py_src/mesh.py @@ -1,328 +1,479 @@ -from __future__ import print_function -import hashlib, inspect, json, random, select, socket, struct, sys, traceback, warnings -from collections import namedtuple, deque -from .base import flags, user_salt, compression, to_base_58, from_base_58, \ - getUTC, compress, decompress, intersect, get_lan_ip, protocol, get_socket, \ - base_connection, base_daemon, base_socket, message, pathfinding_message - -max_outgoing = 4 -default_protocol = protocol('mesh', "Plaintext") # SSL") -json_compressions = json.dumps([method.decode() for method in compression]) - -class mesh_connection(base_connection): - def found_terminator(self): - """Processes received messages""" - raw_msg = ''.encode().join(self.buffer) - self.expected = 4 - self.buffer = [] - self.active = False - try: - msg = pathfinding_message.feed_string(self.protocol, raw_msg, False, self.compression) - except (IndexError, struct.error): - self.__print__("Failed to decode message: %s. Expected compression: %s." % \ - (raw_msg, intersect(compression, self.compression)[0]), level=1) - self.send(flags.renegotiate, flags.compression, json.dumps([])) - self.send(flags.renegotiate, flags.resend) - return - packets = msg.packets - self.__print__("Message received: %s" % packets, level=1) - if self.__handle_waterfall(msg, packets): - return - elif self.__handle_renegotiate(packets): - return - self.server.handle_msg(message(msg, self.server), self) - - def __handle_waterfall(self, msg, packets): - if packets[0] in [flags.waterfall, flags.broadcast]: - if from_base_58(packets[3]) < getUTC() - 60: - self.__print__("Waterfall expired", level=2) - return True - elif not self.server.waterfall(message(msg, self.server)): - self.__print__("Waterfall already captured", level=2) - return True - self.__print__("New waterfall received. Proceeding as normal", level=2) - - def __handle_renegotiate(self, packets): - if packets[0] == flags.renegotiate: - if packets[4] == flags.compression: - encoded_methods = [algo.encode() for algo in json.loads(packets[5].decode())] - respond = (self.compression != encoded_methods) - self.compression = encoded_methods - self.__print__("Compression methods changed to: %s" % repr(self.compression), level=2) - if respond: - decoded_methods = [algo.decode() for algo in intersect(compression, self.compression)] - self.send(flags.renegotiate, flags.compression, json.dumps(decoded_methods)) - return True - elif packets[4] == flags.resend: - self.send(*self.last_sent) - return True - - def send(self, msg_type, *args, **kargs): - """Sends a message through its connection. The first argument is message type. All after that are content packets""" - # This section handles waterfall-specific flags - id = kargs.get('id', self.server.id) # Latter is returned if key not found - time = kargs.get('time', getUTC()) - # Begin real method - msg = pathfinding_message(self.protocol, msg_type, id, list(args), self.compression) - if (msg.id, msg.time) not in self.server.waterfalls: - self.server.waterfalls.appendleft((msg.id, msg.time)) - if msg_type in [flags.whisper, flags.broadcast]: - self.last_sent = [msg_type] + list(args) - self.__print__("Sending %s to %s" % ([msg.len] + msg.packets, self), level=4) - if msg.compression_used: self.__print__("Compressing with %s" % msg.compression_used, level=4) - try: - self.sock.send(msg.string) - except (IOError, socket.error) as e: - self.server.daemon.exceptions.append((e, traceback.format_exc())) - self.server.disconnect(self) - - -class mesh_daemon(base_daemon): - def mainloop(self): - """Daemon thread which handles all incoming data and connections""" - while self.alive: - conns = list(self.server.routing_table.values()) + self.server.awaiting_ids - if conns: - for handler in select.select(conns, [], [], 0.01)[0]: - self.process_data(handler) - for handler in conns: - self.kill_old_nodes(handler) - self.handle_accept() - - def handle_accept(self): - """Handle an incoming connection""" - if sys.version_info >= (3, 3): - exceptions = (socket.error, ConnectionError) - else: - exceptions = (socket.error, ) - try: - conn, addr = self.sock.accept() - self.__print__('Incoming connection from %s' % repr(addr), level=1) - handler = mesh_connection(conn, self.server, self.protocol) - handler.send(flags.whisper, flags.handshake, self.server.id, self.protocol.id, \ - json.dumps(self.server.out_addr), json_compressions) - handler.sock.settimeout(1) - self.server.awaiting_ids.append(handler) - except exceptions: - pass - - def process_data(self, handler): - """Collects incoming data from nodes""" - try: - while not handler.find_terminator(): - if not handler.collect_incoming_data(handler.sock.recv(1)): - self.__print__("disconnecting node %s while in loop" % handler.id, level=6) - self.server.disconnect(handler) - self.server.request_peers() - return - handler.found_terminator() - except socket.timeout: # pragma: no cover - return # Shouldn't happen with select, but if it does... - except Exception as e: - if isinstance(e, socket.error) and e.args[0] in (9, 104, 10053, 10054, 10058): - node_id = handler.id - if not node_id: - node_id = repr(handler) - self.__print__("Node %s has disconnected from the network" % node_id, level=1) - else: - self.__print__("There was an unhandled exception with peer id %s. This peer is being disconnected, and the relevant exception is added to the debug queue. If you'd like to report this, please post a copy of your mesh_socket.status to github.com/gappleto97/p2p-project/issues." % handler.id, level=0) - self.exceptions.append((e, traceback.format_exc())) - self.server.disconnect(handler) - self.server.request_peers() - - def kill_old_nodes(self, handler): - """Cleans out connections which never finish a message""" - if handler.active and handler.time < getUTC() - 60: - self.server.disconnect(handler) - - -class mesh_socket(base_socket): - def __init__(self, addr, port, prot=default_protocol, out_addr=None, debug_level=0): - self.protocol = prot - self.debug_level = debug_level - self.routing_table = {} # In format {ID: handler} - self.awaiting_ids = [] # Connected, but not handshook yet - self.requests = {} # Metadata about message replies where you aren't connected to the sender - self.waterfalls = deque() # Metadata of messages to waterfall - self.queue = deque() # Queue of received messages. Access through recv() - if out_addr: # Outward facing address, if you're port forwarding - self.out_addr = out_addr - elif addr == '0.0.0.0': - self.out_addr = get_lan_ip(), port - else: - self.out_addr = addr, port - info = [str(self.out_addr).encode(), prot.id, user_salt] - h = hashlib.sha384(b''.join(info)) - self.id = to_base_58(int(h.hexdigest(), 16)) - self.daemon = mesh_daemon(addr, port, self, prot) - self.__handlers = [self.__handle_handshake, self.__handle_peers, - self.__handle_response, self.__handle_request] - - def handle_msg(self, msg, conn): - """Decides how to handle various message types, allowing some to be handled automatically""" - for handler in self.__handlers: - self.__print__("Checking handler: %s" % handler.__name__, level=4) - if handler(msg, conn): - self.__print__("Breaking from handler: %s" % handler.__name__, level=4) - break - else: # misnomer: more accurately "if not break" - if msg.packets[0] in [flags.whisper, flags.broadcast]: - self.queue.appendleft(msg) - else: - self.__print__("Ignoring message with invalid subflag", level=4) - - def __get_peer_list(self): - peer_list = [(self.routing_table[key].addr, key.decode()) for key in self.routing_table] - random.shuffle(peer_list) - return peer_list - - def __resolve_connection_conflict(self, handler, h_id): - self.__print__("Resolving peer conflict on id %s" % repr(h_id), level=1) - to_keep, to_kill = None, None - if bool(from_base_58(self.id) > from_base_58(h_id)) ^ bool(handler.outgoing): # logical xor - self.__print__("Closing outgoing connection", level=1) - to_keep, to_kill = self.routing_table[h_id], handler - self.__print__(to_keep.outgoing, level=1) - else: - self.__print__("Closing incoming connection", level=1) - to_keep, to_kill = handler, self.routing_table[h_id] - self.__print__(not to_keep.outgoing, level=1) - self.disconnect(to_kill) - self.routing_table.update({h_id: to_keep}) - - def __handle_handshake(self, msg, handler): - packets = msg.packets - if packets[0] == flags.handshake: - if packets[2] != self.protocol.id: - self.disconnect(handler) - return True - elif handler is not self.routing_table.get(packets[1], handler): - self.__resolve_connection_conflict(handler, packets[1]) - handler.id = packets[1] - handler.addr = json.loads(packets[3].decode()) - handler.compression = json.loads(packets[4].decode()) - handler.compression = [algo.encode() for algo in handler.compression] - self.__print__("Compression methods changed to %s" % repr(handler.compression), level=4) - if handler in self.awaiting_ids: - self.awaiting_ids.remove(handler) - self.routing_table.update({packets[1]: handler}) - handler.send(flags.whisper, flags.peers, json.dumps(self.__get_peer_list())) - return True - - def __handle_peers(self, msg, handler): - packets = msg.packets - if packets[0] == flags.peers: - new_peers = json.loads(packets[1].decode()) - for addr, id in new_peers: - if len(self.outgoing) < max_outgoing: - try: - self.connect(addr[0], addr[1], id.encode()) - except: # pragma: no cover - self.__print__("Could not connect to %s:%s because\n%s" % (addr[0], addr[1], traceback.format_exc()), level=1) - continue - return True - - def __handle_response(self, msg, handler): - packets = msg.packets - if packets[0] == flags.response: - self.__print__("Response received for request id %s" % packets[1], level=1) - if self.requests.get(packets[1]): - addr = json.loads(packets[2].decode()) - if addr: - msg = self.requests.get(packets[1]) - self.requests.pop(packets[1]) - self.connect(addr[0][0], addr[0][1], addr[1]) - self.routing_table[addr[1]].send(*msg) - return True - - def __handle_request(self, msg, handler): - packets = msg.packets - if packets[0] == flags.request: - if packets[1] == b'*': - handler.send(flags.whisper, flags.peers, json.dumps(self.__get_peer_list())) - elif self.routing_table.get(packets[2]): - handler.send(flags.broadcast, flags.response, packets[1], json.dumps([self.routing_table.get(packets[2]).addr, packets[2].decode()])) - return True - - def send(self, *args, **kargs): - """Sends data to all peers. type flag will override normal subflag. Defaults to 'broadcast'""" - send_type = kargs.pop('type', flags.broadcast) - main_flag = kargs.pop('flag', flags.broadcast) - # map(methodcaller('send', 'broadcast', 'broadcast', *args), self.routing_table.values()) - handlers = list(self.routing_table.values()) - for handler in handlers: - handler.send(main_flag, send_type, *args) - - def __clean_waterfalls(self): - """Cleans up the waterfall deque""" - self.waterfalls = deque(set(self.waterfalls)) - self.waterfalls = deque((i for i in self.waterfalls if i[1] > getUTC() - 60)) - - def waterfall(self, msg): - """Handles the waterfalling of received messages""" - if msg.id not in (i for i, t in self.waterfalls): - self.waterfalls.appendleft((msg.id, msg.time)) - for handler in self.routing_table.values(): - if handler.id != msg.sender: - handler.send(flags.waterfall, *msg.packets, time=msg.time_58, id=msg.sender) - self.__clean_waterfalls() - return True - else: - self.__print__("Not rebroadcasting", level=3) - return False - - def connect(self, addr, port, id=None): - """Connects to a specified node. Specifying ID will immediately add to routing table. Blocking""" - self.__print__("Attempting connection to %s:%s with id %s" % (addr, port, repr(id)), level=1) - if socket.getaddrinfo(addr, port)[0] == socket.getaddrinfo(*self.out_addr)[0] or \ - id in self.routing_table: - self.__print__("Connection already established", level=1) - return False - conn = get_socket(self.protocol, False) - conn.settimeout(1) - conn.connect((addr, port)) - handler = mesh_connection(conn, self, self.protocol, outgoing=True) - handler.id = id - handler.send(flags.whisper, flags.handshake, self.id, self.protocol.id, \ - json.dumps(self.out_addr), json_compressions) - if id: - self.routing_table.update({id: handler}) - else: - self.awaiting_ids.append(handler) - - def disconnect(self, handler): - """Disconnects a node""" - node_id = handler.id - if not node_id: - node_id = repr(handler) - self.__print__("Connection to node %s has been closed" % node_id, level=1) - if handler in self.awaiting_ids: - self.awaiting_ids.remove(handler) - elif self.routing_table.get(handler.id) is handler: - self.routing_table.pop(handler.id) - try: - handler.sock.shutdown(socket.SHUT_RDWR) - except: - pass - - def request_peers(self): - """Requests your peers' routing tables""" - self.send('*', type=flags.request, flag=flags.whisper) - - def register_handler(self, method): - """Register a handler for incoming method. Should be roughly of the form: - def handler(msg, handler): - packets = msg.packets - if packets[0] == expected_value: - action() - return True - """ - if sys.version_info >= (3, 0): - args = inspect.signature(method) - if len(args.parameters) != 2: - raise ValueError("This method must contain exactly two arguments") - else: - args = inspect.getargspec(method) - if args[1:] != (None, None, None) or len(args[0]) != 2: - raise ValueError("This method must contain exactly two arguments") - self.__handlers.append(method) \ No newline at end of file +from __future__ import print_function +from __future__ import absolute_import + +import inspect +import json +import random +import select +import socket +import struct +import sys +import traceback + +from collections import deque + +try: + from .cbase import protocol +except: + from .base import protocol +from .base import (flags, compression, to_base_58, from_base_58, + base_connection, message, base_daemon, base_socket, + pathfinding_message, json_compressions) +from .utils import getUTC, get_socket, intersect + +max_outgoing = 4 +default_protocol = protocol('mesh', "Plaintext") # SSL") + +class mesh_connection(base_connection): + """The class for mesh connection abstraction. This inherits from :py:class:`py2p.base.base_connection`""" + def send(self, msg_type, *args, **kargs): + """Sends a message through its connection. + + Args: + msg_type: Message type, corresponds to the header in a :py:class:`py2p.base.pathfinding_message` object + *args: A list of bytes-like objects, which correspond to the packets to send to you + **kargs: There are two available keywords: + id: The ID this message should appear to be sent from (default: your ID) + time: The time this message should appear to be sent from (default: now in UTC) + + Returns: + the :py:class:`~py2p.base.pathfinding_message` object you just sent, or None if the sending was unsuccessful + """ + msg = super(mesh_connection, self).send(msg_type, *args, **kargs) + if msg and (msg.id, msg.time) not in self.server.waterfalls: + self.server.waterfalls.appendleft((msg.id, msg.time)) + + def found_terminator(self): + """This method is called when the expected amount of data is received + + Returns: + ``None`` + """ + try: + msg = super(mesh_connection, self).found_terminator() + except (IndexError, struct.error): + self.__print__("Failed to decode message. Expected first compression of: %s." % \ + intersect(compression, self.compression), level=1) + self.send(flags.renegotiate, flags.compression, json.dumps([])) + self.send(flags.renegotiate, flags.resend) + return + packets = msg.packets + self.__print__("Message received: %s" % packets, level=1) + if self.handle_waterfall(msg, packets): + return + elif self.handle_renegotiate(packets): + return + self.server.handle_msg(message(msg, self.server), self) + + def handle_waterfall(self, msg, packets): + """This method determines whether this message has been previously received or not. + + If it has been previously received, this method returns ``True``. + + If it is older than a preset limit, this method returns ``True``. + + Otherwise this method returns ``None``, and forwards the message appropriately. + + Args: + msg: The message in question + packets: The message's packets + + Returns: + Either ``True`` or ``None`` + """ + if packets[0] in [flags.waterfall, flags.broadcast]: + if from_base_58(packets[3]) < getUTC() - 60: + self.__print__("Waterfall expired", level=2) + return True + elif not self.server.waterfall(message(msg, self.server)): + self.__print__("Waterfall already captured", level=2) + return True + self.__print__("New waterfall received. Proceeding as normal", level=2) + + +class mesh_daemon(base_daemon): + """The class for mesh daemon. This inherits from :py:class:`py2p.base.base_daemon`""" + def mainloop(self): + """Daemon thread which handles all incoming data and connections""" + while self.main_thread.is_alive() and self.alive: + conns = list(self.server.routing_table.values()) + self.server.awaiting_ids + for handler in select.select(conns + [self.sock], [], [], 0.01)[0]: + if handler == self.sock: + self.handle_accept() + else: + self.process_data(handler) + for handler in conns: + self.kill_old_nodes(handler) + + def handle_accept(self): + """Handle an incoming connection""" + if sys.version_info >= (3, 3): + exceptions = (socket.error, ConnectionError) + else: + exceptions = (socket.error, ) + try: + conn, addr = self.sock.accept() + self.__print__('Incoming connection from %s' % repr(addr), level=1) + handler = mesh_connection(conn, self.server) + self.server._send_handshake(handler) + handler.sock.settimeout(1) + self.server.awaiting_ids.append(handler) + except exceptions: + pass + + def process_data(self, handler): + """Collects incoming data from nodes""" + try: + while not handler.find_terminator(): + if not handler.collect_incoming_data(handler.sock.recv(1)): + self.__print__("disconnecting node %s while in loop" % handler.id, level=6) + self.server.disconnect(handler) + self.server.request_peers() + return + handler.found_terminator() + except socket.timeout: # pragma: no cover + return # Shouldn't happen with select, but if it does... + except Exception as e: + if isinstance(e, socket.error) and e.args[0] in (9, 104, 10053, 10054, 10058): + node_id = handler.id + if not node_id: + node_id = repr(handler) + self.__print__("Node %s has disconnected from the network" % node_id, level=1) + else: + self.__print__("There was an unhandled exception with peer id %s. This peer is being disconnected, and the relevant exception is added to the debug queue. If you'd like to report this, please post a copy of your mesh_socket.status to github.com/gappleto97/p2p-project/issues." % handler.id, level=0) + self.exceptions.append((e, traceback.format_exc())) + self.server.disconnect(handler) + self.server.request_peers() + + +class mesh_socket(base_socket): + """The class for mesh socket abstraction. This inherits from :py:class:`py2p.base.base_socket`""" + def __init__(self, addr, port, prot=default_protocol, out_addr=None, debug_level=0): + """Initializes a mesh socket + + Args: + addr: The address you wish to bind to (ie: "192.168.1.1") + port: The port you wish to bind to (ie: 44565) + prot: The protocol you wish to operate over, defined by a :py:class:`py2p.base.protocol` object + out_addr: Your outward facing address. Only needed if you're connecting + over the internet. If you use '0.0.0.0' for the addr argument, this will + automatically be set to your LAN address. + debug_level: The verbosity you want this socket to use when printing event data + + Raises: + socket.error: The address you wanted could not be bound, or is otherwise used + """ + super(mesh_socket, self).__init__(addr, port, prot, out_addr, debug_level) + self.requests = {} # Metadata about message replies where you aren't connected to the sender + self.waterfalls = deque() # Metadata of messages to waterfall + self.queue = deque() # Queue of received messages. Access through recv() + self.daemon = mesh_daemon(addr, port, self) + self.register_handler(self.__handle_handshake) + self.register_handler(self.__handle_peers) + self.register_handler(self.__handle_response) + self.register_handler(self.__handle_request) + + @property + def outgoing(self): + """IDs of outgoing connections""" + return [handler.id for handler in self.routing_table.values() if handler.outgoing] + + @property + def incoming(self): + """IDs of incoming connections""" + return [handler.id for handler in self.routing_table.values() if not handler.outgoing] + + def handle_msg(self, msg, conn): + """Decides how to handle various message types, allowing some to be handled automatically""" + if not super(mesh_socket, self).handle_msg(msg, conn): + if msg.packets[0] in [flags.whisper, flags.broadcast]: + self.queue.appendleft(msg) + else: + self.__print__("Ignoring message with invalid subflag", level=4) + + def __get_peer_list(self): + """This function is used to generate a list-formatted group of your peers. It goes in format ``[ ((addr, port), ID), ...]``""" + peer_list = [(self.routing_table[key].addr, key.decode()) for key in self.routing_table] + random.shuffle(peer_list) + return peer_list + + def _send_handshake(self, handler): + """Shortcut method for sending a handshake to a given handler + + Args: + handler: A :py:class:`~py2p.mesh.mesh_connection` + """ + json_out_addr = '["{}", {}]'.format(*self.out_addr) + handler.send(flags.whisper, flags.handshake, self.id, self.protocol.id, \ + json_out_addr, json_compressions) + + def __resolve_connection_conflict(self, handler, h_id): + """Sometimes in trying to recover a network a race condition is created. + This function applies a heuristic to try and organize the fallout from + that race condition. While it isn't perfect, it seems to have increased + connection recovery rate from ~20% to ~75%. This statistic is from memory + on past tests. Much improvement can be made here, but this statistic can + likely never be brought to 100%. + + In the failure condition, the overall network is unaffacted *for large + networks*. In small networks this failure condition causes a fork, usually + where an individual node is kicked out. + + Args: + handler: The handler with whom you have a connection conflict + h_id: The id of this handler + """ + self.__print__("Resolving peer conflict on id %s" % repr(h_id), level=1) + to_keep, to_kill = None, None + if bool(from_base_58(self.id) > from_base_58(h_id)) ^ bool(handler.outgoing): # logical xor + self.__print__("Closing outgoing connection", level=1) + to_keep, to_kill = self.routing_table[h_id], handler + self.__print__(to_keep.outgoing, level=1) + else: + self.__print__("Closing incoming connection", level=1) + to_keep, to_kill = handler, self.routing_table[h_id] + self.__print__(not to_keep.outgoing, level=1) + self.disconnect(to_kill) + self.routing_table.update({h_id: to_keep}) + + def _send_handshake_response(self, handler): + """Shortcut method to send a handshake response. This method is extracted from :py:meth:`.__handle_handshake` + in order to allow cleaner inheritence from :py:class:`py2p.sync.sync_socket`""" + handler.send(flags.whisper, flags.peers, json.dumps(self.__get_peer_list())) + + def __handle_handshake(self, msg, handler): + """This callback is used to deal with handshake signals. Its three primary jobs are: + + - reject connections seeking a different network + - set connection state + - deal with connection conflicts + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.mesh.mesh_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.handshake: + if packets[2] != self.protocol.id: + self.__print__("Connected to peer on wrong subnet. ID: %s" % packets[2], level=2) + self.disconnect(handler) + return True + elif handler is not self.routing_table.get(packets[1], handler): + self.__print__("Connection conflict detected. Trying to resolve", level=2) + self.__resolve_connection_conflict(handler, packets[1]) + handler.id = packets[1] + handler.addr = json.loads(packets[3].decode()) + handler.compression = json.loads(packets[4].decode()) + handler.compression = [algo.encode() for algo in handler.compression] + self.__print__("Compression methods changed to %s" % repr(handler.compression), level=4) + if handler in self.awaiting_ids: + self.awaiting_ids.remove(handler) + self.routing_table.update({packets[1]: handler}) + self._send_handshake_response(handler) + return True + + def __handle_peers(self, msg, handler): + """This callback is used to deal with peer signals. Its primary jobs is to connect to the given peers, if this does not exceed :py:const:`py2p.mesh.max_outgoing` + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.mesh.mesh_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.peers: + new_peers = json.loads(packets[1].decode()) + for addr, id in new_peers: + if len(self.outgoing) < max_outgoing: + try: + self.connect(addr[0], addr[1], id.encode()) + except: # pragma: no cover + self.__print__("Could not connect to %s because\n%s" % (addr, traceback.format_exc()), level=1) + continue + return True + + def __handle_response(self, msg, handler): + """This callback is used to deal with response signals. Its two primary jobs are: + + - if it was your request, send the deferred message + - if it was someone else's request, relay the information + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.mesh.mesh_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.response: + self.__print__("Response received for request id %s" % packets[1], level=1) + if self.requests.get(packets[1]): + addr = json.loads(packets[2].decode()) + if addr: + msg = self.requests.get(packets[1]) + self.requests.pop(packets[1]) + self.connect(addr[0][0], addr[0][1], addr[1]) + self.routing_table[addr[1]].send(*msg) + return True + + def __handle_request(self, msg, handler): + """This callback is used to deal with request signals. Its three primary jobs are: + + - respond with a peers signal if packets[1] is ``'*'`` + - if you know the ID requested, respond to it + - if you don't, make a request with your peers + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.mesh.mesh_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.request: + if packets[1] == b'*': + handler.send(flags.whisper, flags.peers, json.dumps(self.__get_peer_list())) + elif self.routing_table.get(packets[2]): + handler.send(flags.broadcast, flags.response, packets[1], json.dumps([self.routing_table.get(packets[2]).addr, packets[2].decode()])) + return True + + def send(self, *args, **kargs): + """This sends a message to all of your peers. If you use default values it will send it to everyone on the network + + Args: + *args: A list of strings or bytes-like objects you want your peers to receive + **kargs: There are two keywords available: + flag: A string or bytes-like object which defines your flag. In other words, this defines packet 0. + type: A string or bytes-like object which defines your message type. Changing this from default can have adverse effects. + + Warning: + + If you change the type attribute from default values, bad things could happen. It **MUST** be a value from :py:data:`py2p.base.flags` , + and more specifically, it **MUST** be either ``broadcast`` or ``whisper``. The only other valid flags are ``waterfall`` and ``renegotiate``, + but these are **RESERVED** and must **NOT** be used. + """ + send_type = kargs.pop('type', flags.broadcast) + main_flag = kargs.pop('flag', flags.broadcast) + # map(methodcaller('send', 'broadcast', 'broadcast', *args), self.routing_table.values()) + handlers = list(self.routing_table.values()) + for handler in handlers: + handler.send(main_flag, send_type, *args) + + def __clean_waterfalls(self): + """This function cleans the list of recently relayed messages based on the following heurisitics: + + * Delete all duplicates + * Delete all older than 60 seconds + """ + self.waterfalls = deque(set(self.waterfalls)) + self.waterfalls = deque((i for i in self.waterfalls if i[1] > getUTC() - 60)) + + def waterfall(self, msg): + """This function handles message relays. Its return value is based on + whether it took an action or not. + + Args: + msg: The :py:class:`~py2p.base.message` in question + + Returns: + ``True`` if the message was then forwarded. ``False`` if not. + """ + if msg.id not in (i for i, t in self.waterfalls): + self.waterfalls.appendleft((msg.id, msg.time)) + for handler in self.routing_table.values(): + if handler.id != msg.sender: + handler.send(flags.waterfall, *msg.packets, time=msg.time, id=msg.sender) + self.__clean_waterfalls() + return True + else: + self.__print__("Not rebroadcasting", level=3) + return False + + def connect(self, addr, port, id=None): + """This function connects you to a specific node in the overall network. + Connecting to one node *should* connect you to the rest of the network, + however if you connect to the wrong subnet, the handshake failure involved + is silent. You can check this by looking at the truthiness of this objects + routing table. Example: + + .. code:: python + + >>> conn = mesh.mesh_socket('localhost', 4444) + >>> conn.connect('localhost', 5555) + >>> # do some other setup for your program + >>> if (!conn.routing_table): + ... conn.connect('localhost', 6666) # any fallback address + + Args: + addr: A string address + port: A positive, integral port + id: A string-like object which represents the expected ID of this node + """ + self.__print__("Attempting connection to %s:%s with id %s" % (addr, port, repr(id)), level=1) + if socket.getaddrinfo(addr, port)[0] == socket.getaddrinfo(*self.out_addr)[0] or \ + id in self.routing_table: + self.__print__("Connection already established", level=1) + return False + conn = get_socket(self.protocol, False) + conn.settimeout(1) + conn.connect((addr, port)) + handler = mesh_connection(conn, self, outgoing=True) + self._send_handshake(handler) + if id: + self.routing_table.update({id: handler}) + else: + self.awaiting_ids.append(handler) + + def disconnect(self, handler): + """Closes a given connection, and removes it from your routing tables + + Args: + handler: the connection you would like to close + """ + node_id = handler.id + if not node_id: + node_id = repr(handler) + self.__print__("Connection to node %s has been closed" % node_id, level=1) + if handler in self.awaiting_ids: + self.awaiting_ids.remove(handler) + elif self.routing_table.get(handler.id) is handler: + self.routing_table.pop(handler.id) + try: + handler.sock.shutdown(socket.SHUT_RDWR) + except: + pass + + def request_peers(self): + """Requests your peers' routing tables""" + self.send('*', type=flags.request, flag=flags.whisper) + + def recv(self, quantity=1): + """This function has two behaviors depending on whether quantity is truthy. + + If truthy is truthy, it will return a list of :py:class:`~py2p.base.message` objects up to length len. + + If truthy is not truthy, it will return either a single :py:class:`~py2p.base.message` object, or ``None`` + + Args: + quantity: The maximum number of :py:class:`~py2p.base.message` s you would like to pull + + Returns: + A list of :py:class:`~py2p.base.message` s, an empty list, a single :py:class:`~py2p.base.message` , or ``None`` + """ + if quantity != 1: + ret_list = [] + while len(self.queue) and quantity > 0: + ret_list.append(self.queue.pop()) + quantity -= 1 + return ret_list + elif len(self.queue): + return self.queue.pop() + else: + return None diff --git a/py_src/ssl_wrapper.py b/py_src/ssl_wrapper.py index c6e5006..34e48db 100644 --- a/py_src/ssl_wrapper.py +++ b/py_src/ssl_wrapper.py @@ -1,88 +1,94 @@ -import os, socket, ssl, sys, tempfile +from __future__ import with_statement -try: - from OpenSSL import crypto +import datetime +import os +import socket +import ssl +import sys +import uuid - def generate_self_signed_cert(cert_file, key_file): - """Given two file-like objects, generate an SSL key and certificate.""" - # create a key pair - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, 2048) +from tempfile import NamedTemporaryFile - # create a self-signed cert - cert = crypto.X509() - cert.get_subject().C = 'PY' - cert.get_subject().ST = 'py2p generated cert' - cert.get_subject().L = 'py2p generated cert' - cert.get_subject().O = 'py2p generated cert' - cert.get_subject().OU = 'py2p generated cert' - cert.get_subject().CN = socket.gethostname() - cert.set_serial_number(1000) - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) - cert.set_issuer(cert.get_subject()) - cert.set_pubkey(key) - cert.sign(key, 'sha1') +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption +from cryptography.x509.oid import NameOID - cert_file.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert)) - key_file.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, key)) -except ImportError: - try: - from cryptography import x509 - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.asymmetric import rsa - from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption - from cryptography.x509.oid import NameOID - import datetime, uuid +if sys.version_info < (3, ): + import atexit + cleanup_files = [] - def generate_self_signed_cert(cert_file, key_file): - """Given two file-like objects, generate an SSL key and certificate.""" - one_day = datetime.timedelta(1, 0, 0) - private_key = rsa.generate_private_key( - public_exponent=65537, - key_size=2048, - backend=default_backend() - ) - public_key = private_key.public_key() - builder = x509.CertificateBuilder() - builder = builder.subject_name(x509.Name([ - x509.NameAttribute(NameOID.COMMON_NAME, u'cryptography.io'), - ])) - builder = builder.issuer_name(x509.Name([ - x509.NameAttribute(NameOID.COMMON_NAME, u'cryptography.io'), - ])) - builder = builder.not_valid_before(datetime.datetime.today() - one_day) - builder = builder.not_valid_after(datetime.datetime.today() + datetime.timedelta(365*10)) - builder = builder.serial_number(int(uuid.uuid4())) - builder = builder.public_key(public_key) - builder = builder.add_extension( - x509.BasicConstraints(ca=False, path_length=None), critical=True, - ) - certificate = builder.sign( - private_key=private_key, algorithm=hashes.SHA256(), - backend=default_backend() - ) + def cleanup(): # pragma: no cover + """Cleans SSL certificate and key files""" + for f in cleanup_files: + os.remove(f) - key_file.write(private_key.private_bytes( - Encoding.PEM, - PrivateFormat.TraditionalOpenSSL, - NoEncryption() - )) - cert_file.write(certificate.public_bytes(Encoding.PEM)) + atexit.register(cleanup) + + +def generate_self_signed_cert(cert_file, key_file): + """Given two file-like objects, generate an SSL key and certificate + + Args: + cert_file: The certificate file you wish to write to + key_file: The key file you wish to write to + """ + one_day = datetime.timedelta(1, 0, 0) + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend() + ) + public_key = private_key.public_key() + builder = x509.CertificateBuilder() + builder = builder.subject_name(x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, u'cryptography.io'), + ])) + builder = builder.issuer_name(x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, u'cryptography.io'), + ])) + builder = builder.not_valid_before(datetime.datetime.today() - one_day) + builder = builder.not_valid_after(datetime.datetime.today() + + datetime.timedelta(365*10)) + builder = builder.serial_number(int(uuid.uuid4())) + builder = builder.public_key(public_key) + builder = builder.add_extension( + x509.BasicConstraints(ca=False, path_length=None), critical=True, + ) + certificate = builder.sign( + private_key=private_key, algorithm=hashes.SHA256(), + backend=default_backend() + ) + + key_file.write(private_key.private_bytes( + Encoding.PEM, + PrivateFormat.TraditionalOpenSSL, + NoEncryption() + )) + cert_file.write(certificate.public_bytes(Encoding.PEM)) - except ImportError: # pragma: no cover - raise def get_socket(server_side): + """Returns a socket set up as server or client side + + Args: + server_side: Whether the socket should be server side or not + + Returns: + An SSL socket object + """ if server_side: names = (None, None) - with tempfile.NamedTemporaryFile(delete=False, suffix=".cert") as cert_file: - with tempfile.NamedTemporaryFile(delete=False, suffix=".key") as key_file: + with NamedTemporaryFile(delete=False, suffix=".cert") as cert_file: + with NamedTemporaryFile(delete=False, suffix=".key") as key_file: generate_self_signed_cert(cert_file, key_file) names = (cert_file.name, key_file.name) - sock = ssl.wrap_socket(socket.socket(), suppress_ragged_eofs=True, server_side=True, keyfile=names[1], certfile=names[0]) + sock = ssl.wrap_socket(socket.socket(), suppress_ragged_eofs=True, + server_side=True, keyfile=names[1], + certfile=names[0]) if sys.version_info >= (3, ): os.remove(names[0]) os.remove(names[1]) @@ -90,14 +96,5 @@ def get_socket(server_side): cleanup_files.extend(names) return sock else: - return ssl.wrap_socket(socket.socket(), server_side=False, suppress_ragged_eofs=True) - -if sys.version_info < (3, ): - import atexit - cleanup_files = [] - - def cleanup(): # pragma: no cover - for f in cleanup_files: - os.remove(f) - - atexit.register(cleanup) \ No newline at end of file + return ssl.wrap_socket(socket.socket(), server_side=False, + suppress_ragged_eofs=True) diff --git a/py_src/sync.py b/py_src/sync.py new file mode 100644 index 0000000..5b38634 --- /dev/null +++ b/py_src/sync.py @@ -0,0 +1,160 @@ +from . import mesh +from .utils import (getUTC, sanitize_packet) +from .base import (flags, to_base_58, from_base_58) + +try: + from .cbase import protocol +except: + from .base import protocol + +from collections import namedtuple + +default_protocol = protocol('sync', "Plaintext") # SSL") + +class metatuple(namedtuple('meta', ['owner', 'timestamp'])): + """This class is used to store metadata for a particular key""" + pass + +class sync_socket(mesh.mesh_socket): + """This class is used to sync dictionaries between programs. It extends :py:class:`py2p.mesh.mesh_socket` + + Because of this inheritence, this can also be used as an alert network + + This also implements and optional leasing system by default. This leasing system means that + if node A sets a key, node B cannot overwrite the value at that key for an hour. + + This may be turned off by adding ``leasing=False`` to the constructor.""" + def __init__(self, addr, port, prot=default_protocol, out_addr=None, debug_level=0, leasing=True): + protocol_used = protocol(prot[0] + str(int(leasing)), prot[1]) + self.__leasing = leasing + super(sync_socket, self).__init__(addr, port, protocol_used, out_addr, debug_level) + self.data = {} + self.metadata = {} + self.register_handler(self.__handle_store) + + def __store(self, key, new_data, new_meta, error=True): + """Private API method for storing data. You have permission to store something if: + + - The network is not enforcing leases, or + - There is no value at that key, or + - The lease on that key has lapsed (not been set in the last hour), or + - You are the owner of that key + + Args: + key: The key you wish to store data at + new_data: The data you wish to store in said key + new_meta: The metadata associated with this storage + error: A boolean which says whether to raise a :py:class:`KeyError` if you can't store there + + Raises: + KeyError: If someone else has a lease at this value, and ``error`` is ``True`` + """ + meta = self.metadata.get(key, None) + if (not meta) or (not self.__leasing) or (meta.owner == new_meta.owner) or \ + (meta.timestamp > new_meta.timestamp) or (meta.timestamp < getUTC() - 3600) or \ + (meta.timestamp == new_meta.timestamp and meta.owner > new_meta.owner): + if new_data not in ('', b''): + self.metadata[key] = new_meta + self.data[key] = new_data + else: + del self.data[key] + del self.metadata[key] + elif error: + raise KeyError("You don't have permission to change this yet") + + def _send_handshake_response(self, handler): + """Shortcut method to send a handshake response. This method is extracted from :py:meth:`.__handle_handshake` + in order to allow cleaner inheritence from :py:class:`py2p.sync.sync_socket`""" + super(sync_socket, self)._send_handshake_response(handler) + for key in self: + meta = self.metadata[key] + handler.send(flags.whisper, flags.store, key, self[key], meta.owner, to_base_58(meta.timestamp)) + + def __handle_store(self, msg, handler): + """This callback is used to deal with data storage signals. Its two primary jobs are: + + - store data in a given key + - delete data in a given key + + Args: + msg: A :py:class:`~py2p.base.message` + handler: A :py:class:`~py2p.mesh.mesh_connection` + + Returns: + Either ``True`` or ``None`` + """ + packets = msg.packets + if packets[0] == flags.store: + meta = metatuple(msg.sender, msg.time) + if len(packets) == 5: + if self.data.get(packets[1]): + return + meta = metatuple(packets[3], from_base_58(packets[4])) + self.__store(packets[1], packets[2], meta, error=False) + return True + + def __setitem__(self, key, data): + new_meta = metatuple(self.id, getUTC()) + key = sanitize_packet(key) + data = sanitize_packet(data) + self.__store(key, data, new_meta) + if data is None: + self.send(key, '', type=flags.store) + else: + self.send(key, data, type=flags.store) + + def set(self, key, data): + """Updates the value at a given key. + + Args: + key: The key that you wish to update. Must be a :py:class:`str` or + :py:class:`bytes`-like object + value: The value you wish to put at this key. Must be a :py:class:`str` + or :py:class:`bytes`-like object + + Raises: + KeyError: If you do not have the lease for this slot. Lease is given + automatically for one hour if the slot is open. + """ + self.__setitem__(key, data) + + def update(self, update_dict): + """Equivalent to :py:meth:`dict.update` + + This calls :py:meth:`.sync_socket.__setitem__` for each key/value pair in the + given dictionary. + + Args: + update_dict: A :py:class:`dict`-like object to extract key/value pairs from. + Key and value be a :py:class:`str` or :py:class:`bytes`-like + object + """ + for key in update_dict: + value = update_dict[key] + self.__setitem__(key, value) + + def __getitem__(self, key): + key = sanitize_packet(key) + return self.data[key] + + def get(self, key, ret=None): + """Retrieves the value at a given key. + + Args: + key: The key that you wish to update. Must be a :py:class:`str` or + :py:class:`bytes`-like object + + Returns: + The value at this key, or ``ret`` if there is none. + """ + key = sanitize_packet(key) + return self.data.get(key, ret) + + def __len__(self): + return len(self.data) + + def __delitem__(self, key): + self[key] = None + + def __iter__(self): + return iter(self.data) diff --git a/py_src/test/__init__.py b/py_src/test/__init__.py index be167e1..6f00a37 100644 --- a/py_src/test/__init__.py +++ b/py_src/test/__init__.py @@ -1,3 +1,5 @@ -from . import test_base, test_mesh, test_chord +from . import (test_base, test_cbase, test_utils, + test_mesh, test_chord, test_kademlia) -__all__ = ['test_base', 'test_mesh', 'test_chord'] \ No newline at end of file +__all__ = ['test_base', 'test_cbase', 'test_utils', + 'test_mesh', 'test_chord', 'test_kademlia'] diff --git a/py_src/test/test_base.py b/py_src/test/test_base.py index 67427f5..e03aea6 100644 --- a/py_src/test/test_base.py +++ b/py_src/test/test_base.py @@ -1,74 +1,37 @@ -import datetime, hashlib, os, random, struct, sys, time, uuid +from __future__ import print_function +from __future__ import absolute_import + +import hashlib +import os +import random +import struct +import sys +import uuid + +import pytest + from functools import partial from .. import base -if sys.version_info[0] > 2: +if sys.version_info >= (3, ): xrange = range + def try_identity(in_func, out_func, data_gen, iters): - for i in xrange(iters): + for _ in xrange(iters): test = data_gen() assert test == out_func(in_func(test)) + def gen_random_list(item_size, list_size): - return [os.urandom(item_size) for i in xrange(list_size)] + return [os.urandom(item_size) for _ in xrange(list_size)] + def test_base_58(iters=1000): max_val = 2**32 - 1 data_gen = partial(random.randint, 0, max_val) try_identity(base.to_base_58, base.from_base_58, data_gen, iters) -def test_intersect(iters=200): - max_val = 2**12 - 1 - for i in xrange(iters): - pair1 = sorted([random.randint(0, max_val), random.randint(0, max_val)]) - pair2 = sorted([random.randint(0, max_val), random.randint(0, max_val)]) - cross1 = [pair1[0], pair2[0]] - cross2 = [pair1[1], pair2[1]] - if max(cross1) < min(cross2): - assert base.intersect(range(*pair1), range(*pair2)) == \ - list(range(max(cross1), min(cross2))) - else: - assert base.intersect(range(*pair1), range(*pair2)) == [] - -def test_getUTC(iters=20): - while iters: - nowa, nowb = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1), base.getUTC() - assert nowa.days * 86400 + nowa.seconds in xrange(nowb-1, nowb+2) # 1 second error margin - time.sleep(random.random()) - iters -= 1 - -def test_lan_ip(): - if sys.platform[:5] in ('linux', 'darwi'): - lan_ip_validation_linux() - elif sys.platform[:3] in ('win', 'cyg'): - lan_ip_validation_windows() - else: # pragma: no cover - raise Exception("Unrecognized patform; don't know what command to test against") - -def lan_ip_validation_linux(): - import subprocess - # command pulled from http://stackoverflow.com/a/13322549 - command = """ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'""" - if sys.version_info >= (2, 7): - output = subprocess.check_output(command, universal_newlines=True, shell=True) - else: # fix taken from http://stackoverflow.com/a/4814985 - output = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).communicate()[0] - assert base.get_lan_ip() in output - -def lan_ip_validation_windows(): - import subprocess - # command pulled from http://stackoverflow.com/a/17634009 - command = """for /f "delims=[] tokens=2" %%a in ('ping %computername% -4 -n 1 ^| findstr "["') do (echo %%a)""" - test_file = open('test.bat', 'w') - test_file.write(command) - test_file.close() - if sys.version_info >= (2, 7): - output = subprocess.check_output(['test.bat']) - else: # fix taken from http://stackoverflow.com/a/4814985 - output = subprocess.Popen(['test.bat'], stdout=subprocess.PIPE).communicate()[0] - assert base.get_lan_ip().encode() in output - os.remove('test.bat') def test_compression(iters=100): for method in base.compression: @@ -77,88 +40,81 @@ def test_compression(iters=100): data_gen = partial(os.urandom, 36) try_identity(compress, decompress, data_gen, iters) + def test_compression_exceptions(iters=100): - for i in xrange(iters): + for _ in xrange(iters): test = os.urandom(36) - try: + with pytest.raises(Exception): base.compress(test, os.urandom(4)) - except: - pass - else: # pragma: no cover - raise Exception("Unknown compression method should raise error") - try: + + with pytest.raises(Exception): base.decompress(test, os.urandom(4)) - except: - pass - else: # pragma: no cover - raise Exception("Unknown compression method should raise error") -def test_pathfinding_message(iters=500): + +def test_pathfinding_message(iters=500, impl=base): max_val = 2**8 - for i in xrange(iters): + for _ in xrange(iters): length = random.randint(0, max_val) array = gen_random_list(36, length) - pathfinding_message_constructor_validation(array) - pathfinding_message_exceptions_validiation(array) + pathfinding_message_constructor_validation(array, impl) + pathfinding_message_exceptions_validiation(array, impl) + -def pathfinding_message_constructor_validation(array): - msg = base.pathfinding_message(base.default_protocol, base.flags.broadcast, 'TEST SENDER', array) +def pathfinding_message_constructor_validation(array, impl): + msg = impl.pathfinding_message(base.flags.broadcast, u'\xff', array) assert array == msg.payload - assert msg.packets == [base.flags.broadcast, 'TEST SENDER'.encode(), msg.id, msg.time_58] + array - for method in base.compression: + assert msg.packets == [base.flags.broadcast, u'\xff'.encode('utf-8'), msg.id, msg.time_58] + array + p_hash = hashlib.sha384(b''.join(array + [msg.time_58])) + assert base.to_base_58(int(p_hash.hexdigest(), 16)) == msg.id + assert impl.pathfinding_message.feed_string(msg.string).id == msg.id + for method in impl.compression: msg.compression = [] string = base.compress(msg.string[4:], method) string = struct.pack('!L', len(string)) + string msg.compression = [method] - comp = base.pathfinding_message.feed_string(base.default_protocol, string, False, [method]) - assert msg.string == string == comp.string + comp1 = impl.pathfinding_message.feed_string(string, False, [method]) + comp2 = base.pathfinding_message.feed_string(string, False, [method]) + assert msg.string == string == comp1.string == comp2.string -def pathfinding_message_exceptions_validiation(array): - msg = base.pathfinding_message(base.default_protocol, base.flags.broadcast, 'TEST SENDER', array) - for method in base.compression: + +def pathfinding_message_exceptions_validiation(array, impl): + msg = impl.pathfinding_message(base.flags.broadcast, 'TEST SENDER', array) + for method in impl.compression: msg.compression = [method] - try: - base.pathfinding_message.feed_string(base.default_protocol, msg.string, True, [method]) - except: - pass - else: # pragma: no cover - raise Exception("Erroneously parses sized message with sizeless: %s" % string) - try: - base.pathfinding_message.feed_string(base.default_protocol, msg.string[4:], False, [method]) - except: - pass - else: # pragma: no cover - raise Exception("Erroneously parses sizeless message with size %s" % string) - try: - base.pathfinding_message.feed_string(base.default_protocol, msg.string) - except: - pass - else: # pragma: no cover - raise Exception("Erroneously parses compressed message as plaintext %s" % string) - -def test_protocol(iters=200): - for i in range(iters): + with pytest.raises(Exception): + impl.pathfinding_message.feed_string(msg.string, True, [method]) + + with pytest.raises(Exception): + impl.pathfinding_message.feed_string(msg.string[4:], False, [method]) + + with pytest.raises(Exception): + impl.pathfinding_message.feed_string(msg.string) + + +def test_protocol(iters=200, impl=base): + for _ in range(iters): sub = str(uuid.uuid4()) enc = str(uuid.uuid4()) - test = base.protocol(sub, enc) + print("constructing") + test = impl.protocol(sub, enc) + print("testing subnet equality") assert test.subnet == test[0] == sub + print("testing encryption equality") assert test.encryption == test[1] == enc p_hash = hashlib.sha256(''.join([sub, enc, base.protocol_version]).encode()) - assert int(p_hash.hexdigest(), 16) == base.from_base_58(test.id) + print("testing ID equality") + assert base.to_base_58(int(p_hash.hexdigest(), 16)) == test.id + def test_message_sans_network(iters=1000): - for i in range(iters): - sub = str(uuid.uuid4()) - enc = str(uuid.uuid4()) + for _ in range(iters): sen = str(uuid.uuid4()) pac = gen_random_list(36, 10) - prot = base.protocol(sub, enc) - base_msg = base.pathfinding_message(prot, base.flags.broadcast, sen, pac) + base_msg = base.pathfinding_message(base.flags.broadcast, sen, pac) test = base.message(base_msg, None) assert test.packets == pac assert test.msg == base_msg - assert test.sender == sen - assert test.protocol == prot + assert test.sender == sen.encode() assert test.id == base_msg.id assert test.time == base_msg.time == base.from_base_58(test.time_58) == base.from_base_58(base_msg.time_58) - assert sen in repr(test) \ No newline at end of file + assert sen in repr(test) diff --git a/py_src/test/test_cbase.py b/py_src/test/test_cbase.py new file mode 100644 index 0000000..9ce1a83 --- /dev/null +++ b/py_src/test/test_cbase.py @@ -0,0 +1,51 @@ +from __future__ import print_function +from __future__ import absolute_import + +try: + import hashlib + import os + import random + import struct + import sys + import uuid + + from functools import partial + from .. import base, cbase + + from . import test_base + + if sys.version_info >= (3, ): + xrange = range + + def test_flags(): + bf = base.flags + cf = cbase.flags + assert bf.reserved == cf.reserved + + #main flags + bf_main = (bf.broadcast, bf.waterfall, bf.whisper, + bf.renegotiate, bf.ping, bf.pong) + cf_main = (cf.broadcast, cf.waterfall, cf.whisper, + cf.renegotiate, cf.ping, cf.pong) + assert bf_main == cf_main + + #sub-flags + bf_sub = (bf.broadcast, bf.compression, bf.whisper, bf.handshake, + bf.ping, bf.pong, bf.notify, bf.peers, bf.request, + bf.resend, bf.response, bf.store, bf.retrieve) + cf_sub = (cf.broadcast, cf.compression, cf.whisper, cf.handshake, + cf.ping, cf.pong, cf.notify, cf.peers, cf.request, + cf.resend, cf.response, cf.store, cf.retrieve) + assert bf_sub == cf_sub + + #common compression methods + assert (bf.zlib, bf.gzip) == (cf.zlib, cf.gzip) + + def test_protocol(): + test_base.test_protocol(impl=cbase) + + def test_pathfinding_message(): + test_base.test_pathfinding_message(impl=cbase) + +except ImportError: + pass \ No newline at end of file diff --git a/py_src/test/test_chord.py b/py_src/test/test_chord.py index 09a0aa2..74ae2b1 100644 --- a/py_src/test/test_chord.py +++ b/py_src/test/test_chord.py @@ -1,5 +1,138 @@ -import sys -from .. import chord - -if sys.version_info[0] > 2: - xrange = range \ No newline at end of file +from __future__ import print_function +from __future__ import absolute_import + +import random +import sys +import time +import uuid + +import pytest + +from .. import chord +from .test_mesh import close_all_nodes + +if sys.version_info >= (3, ): + xrange = range + + +# def protocol_rejection_validation(iters, start_port, encryption, k=4, name='test'): +# for i in xrange(iters): +# print("----------------------Test start----------------------") +# f = chord.chord_socket('localhost', start_port + i*2, k=4, prot=chord.protocol('test', encryption), debug_level=5) +# g = chord.chord_socket('localhost', start_port + i*2 + 1, k=k, prot=chord.protocol(name, encryption), debug_level=5) +# print("----------------------Test event----------------------") +# g.connect('localhost', start_port + i*2) +# g.join() +# time.sleep(1) +# print("----------------------Test ended----------------------") +# assert len(f.routing_table) == len(f.awaiting_ids) == len(g.routing_table) == len(g.awaiting_ids) == 0 +# print(f.status) +# print(g.status) +# close_all_nodes([f, g]) + + +# def test_protocol_rejection_Plaintext(iters=3): +# protocol_rejection_validation(iters, 6000, 'Plaintext', name='test2') + + +# def test_protocol_rejection_SSL(iters=3): +# protocol_rejection_validation(iters, 6100, 'SSL', name='test2') + + +# def test_size_rejection_Plaintext(iters=3): +# protocol_rejection_validation(iters, 6200, 'Plaintext', k=5) + + +# def test_size_rejection_SSL(iters=3): +# protocol_rejection_validation(iters, 6300, 'SSL', k=3) + + +# def gen_connected_list(start_port, encryption, k=2): +# nodes = [chord.chord_socket('localhost', start_port + x, k=k, debug_level=0) for x in xrange(2**k)] + +# for index, node in enumerate(nodes): +# node.id_10 = index +# node.id = chord.to_base_58(index) +# node.connect(*nodes[(index - 1) % len(nodes)].addr) +# time.sleep(0.1) + +# for node in nodes: +# node.join() + +# time.sleep(3 * k) + +# for node in nodes: +# print("%s:" % node.id) +# print(node.status) +# for key in node.routing_table: +# print("entry %i: %s" % (key, node.routing_table[key].id)) + +# return nodes + + +# def routing_validation(iters, start_port, encryption, k=3): +# for i in xrange(iters): +# nodes = gen_connected_list(start_port + i * 2**k, encryption, k) + +# assertion_list = list(map(len, (node.routing_table for node in nodes) )) +# print(assertion_list) +# close_all_nodes(nodes) +# assert min(assertion_list) >= 1 + + +# def test_routing_Plaintext(iters=3): +# routing_validation(iters, 6400, 'Plaintext', k=3) + + +# def test_routing_SSL(iters=1): +# routing_validation(iters, 6500, 'SSL', k=3) + + +# def storage_validation(iters, start_port, encryption, k=2): +# for i in xrange(iters): +# nodes = gen_connected_list(start_port + i * 2**k, encryption, k) + +# test_key = str(uuid.uuid4()) +# test_data = str(uuid.uuid4()) + +# nodes[0][test_key] = test_data + +# time.sleep(2*k) + +# for meth in chord.hashes: +# assert any((bool(node.data[meth]) for node in nodes)) + +# close_all_nodes(nodes) + + +# def test_storage_Plaintext(iters=1): +# storage_validation(iters, 6600, 'Plaintext') + + +# def test_storage_SSL(iters=1): +# storage_validation(iters, 6700, 'SSL') + + +# def retrieval_validation(iters, start_port, encryption, k=2): +# for i in xrange(iters): +# nodes = gen_connected_list(start_port + i * 2**k, encryption, k) + +# test_key = str(uuid.uuid4()) +# test_data = str(uuid.uuid4()) + +# nodes[0][test_key] = test_data + +# time.sleep(2*k) + +# for node in nodes: +# assert node[test_key] == test_data +# with pytest.raises(KeyError): +# node[test_data] + + +# def test_retrieval_Plaintext(iters=1): +# retrieval_validation(iters, 6800, 'Plaintext') + + +# def test_retrieval_SSL(iters=1): +# retrieval_validation(iters, 6900, 'SSL') diff --git a/py_src/test/test_kademlia.py b/py_src/test/test_kademlia.py new file mode 100644 index 0000000..e69de29 diff --git a/py_src/test/test_mesh.py b/py_src/test/test_mesh.py index f589e26..a714562 100644 --- a/py_src/test/test_mesh.py +++ b/py_src/test/test_mesh.py @@ -1,16 +1,30 @@ -import socket, sys, time +from __future__ import print_function +from __future__ import absolute_import + +import socket +import sys +import time + from .. import mesh from ..base import flags -if sys.version_info[0] > 2: +if sys.version_info >= (3, ): xrange = range + +def close_all_nodes(nodes): + for node in nodes: + node.close() + + def propagation_validation(iters, start_port, num_nodes, encryption): for i in xrange(iters): print("----------------------Test start----------------------") - nodes = [mesh.mesh_socket('localhost', start_port + i*num_nodes, prot=mesh.protocol('', encryption), debug_level=5)] + nodes = [mesh.mesh_socket('localhost', start_port + i*num_nodes, + prot=mesh.protocol('', encryption), debug_level=5)] for j in xrange(1, num_nodes): - new_node = mesh.mesh_socket('localhost', start_port + i*num_nodes + j, prot=mesh.protocol('', encryption), debug_level=5) + new_node = mesh.mesh_socket('localhost', start_port + i*num_nodes + j, + prot=mesh.protocol('', encryption), debug_level=5) nodes[-1].connect('localhost', start_port + i*num_nodes + j) nodes.append(new_node) time.sleep(0.5) @@ -25,60 +39,94 @@ def propagation_validation(iters, start_port, num_nodes, encryption): assert b"hello" == node.recv().packets[1] # Failure is either no message received: AttributeError # message doesn't match: AssertionError - del nodes[:] + close_all_nodes(nodes) + def test_propagation_Plaintext(iters=3): propagation_validation(iters, 5100, 3, 'Plaintext') + def test_propagation_SSL(iters=3): propagation_validation(iters, 5200, 3, 'SSL') + def protocol_rejection_validation(iters, start_port, encryption): for i in xrange(iters): print("----------------------Test start----------------------") - f = mesh.mesh_socket('localhost', start_port + i*2, prot=mesh.protocol('test', encryption), debug_level=5) - g = mesh.mesh_socket('localhost', start_port + i*2 + 1, prot=mesh.protocol('test2', encryption), debug_level=5) + f = mesh.mesh_socket('localhost', start_port + i*2, + prot=mesh.protocol('test', encryption), debug_level=5) + g = mesh.mesh_socket('localhost', start_port + i*2 + 1, + prot=mesh.protocol('test2', encryption), debug_level=5) print("----------------------Test event----------------------") g.connect('localhost', start_port + i*2) time.sleep(1) print("----------------------Test ended----------------------") assert len(f.routing_table) == len(f.awaiting_ids) == len(g.routing_table) == len(g.awaiting_ids) == 0 - del f, g + close_all_nodes([f, g]) + def test_protocol_rejection_Plaintext(iters=3): protocol_rejection_validation(iters, 5300, 'Plaintext') + def test_protocol_rejection_SSL(iters=3): protocol_rejection_validation(iters, 5400, 'SSL') -def handler_registry_validation(iters, start_port, encryption): + +def register_1(msg, handler): + packets = msg.packets + if packets[1] == b'test': + handler.send(flags.whisper, flags.whisper, b"success") + return True + + +def register_2(msg, handler): + packets = msg.packets + if packets[1] == b'test': + msg.reply(b"success") + return True + + +def handler_registry_validation(iters, start_port, encryption, reg): for i in xrange(iters): print("----------------------Test start----------------------") - f = mesh.mesh_socket('localhost', start_port + i*2, prot=mesh.protocol('', encryption), debug_level=5) - g = mesh.mesh_socket('localhost', start_port + i*2 + 1, prot=mesh.protocol('', encryption), debug_level=5) - - def register(msg, handler): - packets = msg.packets - if packets[1] == b'test': - handler.send(flags.whisper, flags.whisper, b"success") - return True + f = mesh.mesh_socket('localhost', start_port + i*2, + prot=mesh.protocol('', encryption), debug_level=5) + g = mesh.mesh_socket('localhost', start_port + i*2 + 1, + prot=mesh.protocol('', encryption), debug_level=5) - f.register_handler(register) + f.register_handler(reg) g.connect('localhost', start_port + i*2) time.sleep(1) - print("----------------------Test event----------------------") + print("----------------------1st event----------------------") g.send('test') time.sleep(1) - print("----------------------Test ended----------------------") - assert not f.recv() - assert g.recv() - del f, g + print("----------------------1st ended----------------------") + assert all((not f.recv(), g.recv())) + time.sleep(1) + print("----------------------2nd event----------------------") + g.send('not test') + time.sleep(1) + print("----------------------2nd ended----------------------") + assert all((f.recv(), not g.recv())) + close_all_nodes([f, g]) + def test_hanlder_registry_Plaintext(iters=3): - handler_registry_validation(iters, 5500, 'Plaintext') + handler_registry_validation(iters, 5500, 'Plaintext', register_1) + def test_hanlder_registry_SSL(iters=3): - handler_registry_validation(iters, 5600, 'SSL') + handler_registry_validation(iters, 5600, 'SSL', register_1) + + +def test_reply_Plaintext(iters=3): + handler_registry_validation(iters, 5700, 'Plaintext', register_2) + + +def test_reply_SSL(iters=3): + handler_registry_validation(iters, 5800, 'SSL', register_2) + # def disconnect(node, method): # if method == 'crash': @@ -89,6 +137,7 @@ def test_hanlder_registry_SSL(iters=3): # else: # pragma: no cover # raise ValueError() + # def connection_recovery_validation(iters, start_port, encryption, method): # for i in xrange(iters): # print("----------------------Test start----------------------") @@ -113,16 +162,20 @@ def test_hanlder_registry_SSL(iters=3): # print("f.status: %s\n" % repr(f.status)) # print("g.status: %s\n" % repr(g.status)) # print("h.status: %s\n" % repr(h.status)) -# del f, g, h +# close_all_nodes([f, g, h]) + # def test_disconnect_recovery_Plaintext(iters=1): # connection_recovery_validation(iters, 5500, 'Plaintext', 'disconnect') + # def test_disconnect_recovery_SSL(iters=3): # connection_recovery_validation(iters, 5600, 'SSL', 'disconnect') + # def test_conn_error_recovery_Plaintext(iters=1): # connection_recovery_validation(iters, 5600, 'Plaintext', 'crash') + # def test_conn_error_recovery_SSL(iters=3): -# connection_recovery_validation(iters, 5800, 'SSL', 'crash') \ No newline at end of file +# connection_recovery_validation(iters, 5800, 'SSL', 'crash') diff --git a/py_src/test/test_sync.py b/py_src/test/test_sync.py new file mode 100644 index 0000000..c708a90 --- /dev/null +++ b/py_src/test/test_sync.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +from __future__ import print_function +from __future__ import absolute_import + +import sys +import time + +import pytest + +from .. import sync +from ..base import flags + +from .test_mesh import close_all_nodes + +if sys.version_info >= (3, ): + xrange = range + + +def storage_validation(iters, start_port, num_nodes, encryption, leasing): + for i in xrange(iters): + print("----------------------Test start----------------------") + nodes = [sync.sync_socket('localhost', start_port + i*num_nodes, + prot=sync.protocol('', encryption), + debug_level=5, leasing=leasing)] + nodes[0].set('store', b"store") + for j in xrange(1, num_nodes): + new_node = sync.sync_socket('localhost', start_port + i*num_nodes + j, + prot=sync.protocol('', encryption), + debug_level=5, leasing=leasing) + nodes[-1].connect('localhost', start_port + i*num_nodes + j) + nodes.append(new_node) + time.sleep(0.5) + print("----------------------Test event----------------------") + nodes[0]['test'] = b"hello" + nodes[1][u'测试'] = u'成功' + time.sleep(num_nodes) + print("----------------------Test ended----------------------") + print(nodes[0].id) + print([len(n.routing_table) for n in nodes]) + for node in nodes[1:]: + print(node.status, len(node.routing_table)) + assert b"store" == node['store'] + assert b"hello" == node['test'] + assert b'\xe6\x88\x90\xe5\x8a\x9f' == node[u'测试'] + if leasing: + with pytest.raises(KeyError): + node['test'] = b"This shouldn't work" + with pytest.raises(KeyError): + node['test2'] + + close_all_nodes(nodes) + + +def test_storage_leasing_Plaintext(iters=2): + storage_validation(iters, 7100, 3, 'Plaintext', True) + + +def test_storage_leasing_SSL(iters=2): + storage_validation(iters, 7200, 3, 'SSL', True) + + +def test_storage_Plaintext(iters=2): + storage_validation(iters, 7300, 3, 'Plaintext', True) + + +def test_storage_SSL(iters=2): + storage_validation(iters, 7400, 3, 'SSL', True) diff --git a/py_src/test/test_utils.py b/py_src/test/test_utils.py new file mode 100644 index 0000000..ed8671c --- /dev/null +++ b/py_src/test/test_utils.py @@ -0,0 +1,58 @@ +from __future__ import print_function +from __future__ import absolute_import + +import datetime +import os +import random +import subprocess +import sys +import time + +from .. import utils + +if sys.version_info >= (3, ): + xrange = range + +def test_intersect(iters=200): + max_val = 2**12 - 1 + for _ in xrange(iters): + pair1 = sorted([random.randint(0, max_val), random.randint(0, max_val)]) + pair2 = sorted([random.randint(0, max_val), random.randint(0, max_val)]) + cross1 = [pair1[0], pair2[0]] + cross2 = [pair1[1], pair2[1]] + if max(cross1) < min(cross2): + assert utils.intersect(range(*pair1), range(*pair2)) == \ + list(range(max(cross1), min(cross2))) + else: + assert utils.intersect(range(*pair1), range(*pair2)) == [] + +def test_getUTC(iters=20): + while iters: + nowa, nowb = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1), utils.getUTC() + assert nowa.days * 86400 + nowa.seconds in xrange(nowb-1, nowb+2) # 1 second error margin + time.sleep(random.random()) + iters -= 1 + +def test_lan_ip(): + if sys.platform[:5] in ('linux', 'darwi'): + lan_ip_validation_linux() + elif sys.platform[:3] in ('win', 'cyg'): + lan_ip_validation_windows() + else: # pragma: no cover + raise Exception("Unrecognized patform; don't know what command to test against") + +def lan_ip_validation_linux(): + # command pulled from http://stackoverflow.com/a/13322549 + command = """ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'""" + output = subprocess.check_output(command, universal_newlines=True, shell=True) + assert utils.get_lan_ip() in output + +def lan_ip_validation_windows(): + # command pulled from http://stackoverflow.com/a/17634009 + command = """for /f "delims=[] tokens=2" %%a in ('ping %computername% -4 -n 1 ^| findstr "["') do (echo %%a)""" + test_file = open('test.bat', 'w') + test_file.write(command) + test_file.close() + output = subprocess.check_output(['test.bat']) + assert utils.get_lan_ip().encode() in output + os.remove('test.bat') diff --git a/py_src/utils.py b/py_src/utils.py new file mode 100644 index 0000000..8010071 --- /dev/null +++ b/py_src/utils.py @@ -0,0 +1,126 @@ +from __future__ import print_function +from __future__ import with_statement + +import base64 +import calendar +import os +import shutil +import socket +import tempfile +import time + +try: + import cPickle as pickle +except ImportError: + import pickle + +def sanitize_packet(packet): + """Function to sanitize a packet for pathfinding_message serialization, or dict keying""" + if isinstance(packet, type(u'')): + return packet.encode('utf-8') + elif not isinstance(packet, (bytes, bytearray)): + return packet.encode('raw_unicode_escape') + return packet + +def intersect(*args): # returns list + """Finds the intersection of several iterables + + Args: + *args: Several iterables + + Returns: + A list containing the ordered intersection of all given iterables, + where the order is defined by the first iterable + """ + if not all(args): + return [] + intersection = args[0] + for l in args[1:]: + intersection = [item for item in intersection if item in l] + return intersection + + +def get_lan_ip(): + """Retrieves the LAN ip. Expanded from http://stackoverflow.com/a/28950776 + + Note: This will return '127.0.0.1' if it is not connected to a network + """ + import socket + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + # doesn't even have to be reachable + s.connect(('8.8.8.8', 23)) + IP = s.getsockname()[0] + except: + IP = '127.0.0.1' + finally: + s.shutdown(socket.SHUT_RDWR) + return IP + + +def getUTC(): + """Returns the current unix time in UTC + + Note: This will always return an integral value + """ + return calendar.timegm(time.gmtime()) + + +def get_socket(protocol, serverside=False): + """Given a protocol object, return the appropriate socket + + Args: + protocol: A py2p.base.protocol object + serverside: Whether you are the server end of a connection (default: False) + + Raises: + ValueError: If your protocol object has an unknown encryption method + + Returns: + A socket-like object + """ + if protocol.encryption == "Plaintext": + return socket.socket() + elif protocol.encryption == "SSL": + from . import ssl_wrapper # This is inline to prevent dependency issues + return ssl_wrapper.get_socket(serverside) + else: # pragma: no cover + raise ValueError("Unkown encryption method") + + +class awaiting_value(object): + """Proxy object for an asynchronously retrieved item""" + def __init__(self, value=-1): + self.value = value + self.callback = False + + def callback_method(self, method, key): + from .base import flags + self.callback.send(flags.whisper, flags.response, method, key, self.value) + + def __repr__(self): + return "<" + repr(self.value) + ">" + + +def most_common(tmp): + """Returns the most common element in a list + + Args: + tmp: A non-string iterable + + Returns: + The most common element in the iterable + + Warning: + If there are multiple elements which share the same count, it will return a random one. + """ + lst = [] + for item in tmp: + if isinstance(item, awaiting_value): + lst.append(item.value) + else: + lst.append(item) + ret = max(set(lst), key=lst.count) + if lst.count(ret) == lst.count(-1): + return -1, lst.count(ret) + return ret, lst.count(ret) diff --git a/setup.cfg b/setup.cfg index 81a93c5..9438b9f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,8 @@ -[metadata] -description-file = py_src/README.rst - [bdist_wheel] universal=1 + +[tool:pytest] +addopts = -vv + +[pytest] +addopts = -vv \ No newline at end of file diff --git a/setup.py b/setup.py index 0ce2b1d..b881579 100644 --- a/setup.py +++ b/setup.py @@ -1,28 +1,117 @@ -# from distutils.core import setup -from setuptools import setup +from __future__ import with_statement + +import os +import sys + +try: + import setuptools + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension + from py_src import __version__ -setup(name='py2p', - description='A python library for peer-to-peer networking', - version=__version__, - author='Gabe Appleton', - author_email='gappleto97+development@gmail.com', - url='https://github.com/gappleto97/p2p-project', - license='LGPLv3', - packages=['py2p', 'py2p.test'], - package_dir={'py2p': 'py_src'}, - classifiers=['Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python', - 'Programming Language :: JavaScript', - 'Operating System :: OS Independent', - 'Topic :: Communications', - 'Topic :: Internet']) \ No newline at end of file +# Set up the long_description + +loc = os.path.dirname(os.path.realpath(__file__)) +with open(os.path.join(loc, 'py_src', 'README.rst'), 'r') as fd: + long_description = fd.read() + +# Determine whether to build C binaries +# The exception is made for bdist_wheel because it genuinely uses the --universal flag + +__USE_C__ = '--universal' not in sys.argv and os.path.isfile(os.path.join(loc, 'cp_src', 'base.cpp')) +if '--universal' in sys.argv and 'bdist_wheel' not in sys.argv: + sys.argv.remove('--universal') + +__DEBUG__ = [("CP2P_DEBUG_FLAG", "a")] if ('--debug' in sys.argv and __USE_C__) else [] + +# This sets up the program's classifiers + +classifiers = ['Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', + 'Operating System :: OS Independent', + 'Topic :: Communications', + 'Topic :: Internet', + 'Programming Language :: C++', + 'Programming Language :: JavaScript', + 'Programming Language :: Other', + 'Programming Language :: Python'] + +classifiers.extend(( + ('Programming Language :: Python :: %s' % x) for x in + '2 3 2.7 3.3 3.4 3.5'.split())) + + + +def has_environment_marker_support(): + """ + Tests that setuptools has support for PEP-426 environment marker support. + The first known release to support it is 0.7 (and the earliest on PyPI seems to be 0.7.2 + so we're using that), see: http://pythonhosted.org/setuptools/history.html#id142 + References: + * https://wheel.readthedocs.io/en/latest/index.html#defining-conditional-dependencies + * https://www.python.org/dev/peps/pep-0426/#environment-markers + Method extended from pytest. Credit goes to developers there. + """ + try: + from pkg_resources import parse_version + return parse_version(setuptools.__version__) >= parse_version('0.7.2') + except Exception as exc: + sys.stderr.write("Could not test setuptool's version: %s\n" % exc) + return False + +def main(): + ext_modules = [] + install_requires = [] + extras_require = {'SSL': ['cryptography']} + if has_environment_marker_support(): + pass + else: + pass + + if __USE_C__: + ext_modules.append( + Extension( + 'py2p.cbase', + sources=[os.path.join(loc, 'cp_src', 'base_wrapper.cpp'), + os.path.join(loc, 'cp_src', 'base.cpp'), + os.path.join(loc, 'c_src', 'sha', 'sha2.c')], + define_macros=__DEBUG__)) + + try: + setup(name='py2p', + description='A python library for peer-to-peer networking', + long_description=long_description, + version=__version__, + author='Gabe Appleton', + author_email='gappleto97+development@gmail.com', + url='https://github.com/gappleto97/p2p-project', + license='LGPLv3', + packages=['py2p', 'py2p.test'], + package_dir={'py2p': os.path.join(loc, 'py_src')}, + ext_modules=ext_modules, + classifiers=classifiers, + install_requires=install_requires, + extras_require=extras_require + ) + except: + print("Not building C++ code due to errors") + setup(name='py2p', + description='A python library for peer-to-peer networking', + long_description=long_description, + version=__version__, + author='Gabe Appleton', + author_email='gappleto97+development@gmail.com', + url='https://github.com/gappleto97/p2p-project', + license='LGPLv3', + packages=['py2p', 'py2p.test'], + package_dir={'py2p': 'py_src'}, + classifiers=classifiers, + install_requires=install_requires, + extras_require=extras_require + ) + +if __name__ == "__main__": + main() diff --git a/shippable.yml b/shippable.yml index ec90388..8ec1989 100644 --- a/shippable.yml +++ b/shippable.yml @@ -1,25 +1,38 @@ language: python python: - - 2.6 - - 2.7 - - 3.3 - - 3.4 - 3.5 - - pypy -# - pypy3 +env: pyver=3.5 matrix: + include: + # - python: 2.6 + # env: pyver=2.6 + - python: 2.7 + env: pyver=2.7 + - python: 3.3 + env: pyver=3.3 + - python: 3.4 + env: pyver=3.4 + # - python: 3.5 + # env: pyver=3.5 + - python: pypy + env: pyver=pypy + # - python: pypy3 + # env: pyver=pypy3 + - language: node_js + node_js: "4" + env: jsver=4 + - language: node_js + node_js: "5" + env: jsver=5 + - language: node_js + node_js: "6" + env: jsver=6 allow_failures: - python: pypy - python: pypy3 build: ci: - - pip install pytest-coverage codecov - - if [ $(($RANDOM % 2)) == 0 ] || [ $SHIPPABLE_PYTHON_VERSION == pypy ] || [ $SHIPPABLE_PYTHON_VERSION == pypy3 ]; then pip install cryptography; else pip install PyOpenSSL; fi - - py.test -vv --cov=./py_src/ ./py_src/ - on_success: - - coverage combine - - coverage xml - - codecov --token=d89f9bd9-27a3-4560-8dbb-39ee3ba020a5 --file=coverage.xml + - sh ./.scripts/shippable_script.sh diff --git a/sm_src/SM2P-GVA.1.mcz b/sm_src/SM2P-GVA.1.mcz new file mode 100644 index 0000000..3ddf28f Binary files /dev/null and b/sm_src/SM2P-GVA.1.mcz differ