From ceaebc221bc342e7e5a3cacd978bcb82a954e972 Mon Sep 17 00:00:00 2001 From: emrgnt-cmplxty Date: Tue, 20 Aug 2024 12:11:36 -0700 Subject: [PATCH] add py r2r --- py/core/main/execution.py | 2 +- poetry.lock => py/poetry.lock | 27 ++++---- py/pyproject.toml | 127 ++++++++++++++++++++++++++++++++++ py/r2r/__init__.py | 16 +++++ py/sdk/retrieval.py | 4 -- 5 files changed, 157 insertions(+), 19 deletions(-) rename poetry.lock => py/poetry.lock (99%) create mode 100644 py/pyproject.toml create mode 100644 py/r2r/__init__.py diff --git a/py/core/main/execution.py b/py/core/main/execution.py index 78915de75..5f9c3f333 100644 --- a/py/core/main/execution.py +++ b/py/core/main/execution.py @@ -16,7 +16,6 @@ VectorSearchSettings, ) from fastapi import UploadFile -from sdk.client import R2RClient from .assembly.builder import R2RBuilder from .assembly.config import R2RConfig @@ -43,6 +42,7 @@ def __init__( self.base_url = base_url if self.client_mode: + from sdk.client import R2RClient self.client = R2RClient(base_url) self.app = None else: diff --git a/poetry.lock b/py/poetry.lock similarity index 99% rename from poetry.lock rename to py/poetry.lock index 54e7c651a..d6b949067 100644 --- a/poetry.lock +++ b/py/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -827,7 +827,7 @@ files = [ name = "distro" version = "1.9.0" description = "Distro - an OS platform information API" -optional = true +optional = false python-versions = ">=3.6" files = [ {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, @@ -1540,13 +1540,13 @@ setuptools = "*" [[package]] name = "importlib-metadata" -version = "8.3.0" +version = "8.4.0" description = "Read metadata from Python packages" optional = true python-versions = ">=3.8" files = [ - {file = "importlib_metadata-8.3.0-py3-none-any.whl", hash = "sha256:42817a4a0be5845d22c6e212db66a94ad261e2318d80b3e0d363894a79df2b67"}, - {file = "importlib_metadata-8.3.0.tar.gz", hash = "sha256:9c8fa6e8ea0f9516ad5c8db9246a731c948193c7754d3babb0114a05b27dd364"}, + {file = "importlib_metadata-8.4.0-py3-none-any.whl", hash = "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1"}, + {file = "importlib_metadata-8.4.0.tar.gz", hash = "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5"}, ] [package.dependencies] @@ -1624,7 +1624,7 @@ i18n = ["Babel (>=2.7)"] name = "jiter" version = "0.5.0" description = "Fast iterable JSON parser." -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "jiter-0.5.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b599f4e89b3def9a94091e6ee52e1d7ad7bc33e238ebb9c4c63f211d74822c3f"}, @@ -2571,7 +2571,7 @@ httpx = ">=0.27.0,<0.28.0" name = "openai" version = "1.41.1" description = "The official Python library for the openai API" -optional = true +optional = false python-versions = ">=3.7.1" files = [ {file = "openai-1.41.1-py3-none-any.whl", hash = "sha256:56fb04105263f79559aff3ceea2e1dd16f8c5385e8238cb66cf0e6888fa8bfcf"}, @@ -2610,11 +2610,11 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.21.0", markers = "python_version == \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""}, {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""}, {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, - {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] [[package]] @@ -2697,8 +2697,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -4416,7 +4416,7 @@ files = [ name = "tqdm" version = "4.66.5" description = "Fast, Extensible Progress Meter" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"}, @@ -4779,11 +4779,10 @@ doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linke test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [extras] -all = [] core = ["aiosqlite", "asyncpg", "bcrypt", "beautifulsoup4", "deepdiff", "fire", "fsspec", "graspologic", "gunicorn", "litellm", "markdown", "neo4j", "ollama", "openai", "openpyxl", "passlib", "poppler-utils", "posthog", "psutil", "pydantic", "pyjwt", "pypdf", "python-docx", "python-multipart", "python-pptx", "pyyaml", "redis", "requests", "sqlalchemy", "toml", "types-requests", "uvicorn", "vecs"] -ingest-movies = ["moviepy", "opencv-python"] +core-ingest-movies = ["moviepy", "opencv-python"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "63c0d66d4c195c646341dcf5e1e863a898d30fc96abf58170eaa1e26faca8de0" +content-hash = "34b93bcd771abd4c1eb4f3ffb79f36a716835e721e46cd0767007201abf6d806" diff --git a/py/pyproject.toml b/py/pyproject.toml new file mode 100644 index 000000000..f96330fbf --- /dev/null +++ b/py/pyproject.toml @@ -0,0 +1,127 @@ +[build-system] +requires = ["poetry-core", "setuptools", "wheel"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "r2r" +readme = "../README.md" +version = "0.3.0" +description = "SciPhi R2R" +authors = ["Owen Colegrove "] +license = "MIT" +include = ["r2r.toml", "compose.yaml", "compose.neo4j.yaml", "compose.ollama.yaml", "compose.postgres.yaml", "pyproject.toml"] +packages = [ + { include = "r2r", from = "." }, + { include = "core", from = "." }, + { include = "sdk", from = "." }, +] + +[tool.poetry.dependencies] +# Python Versions +python = ">=3.9,<3.13" + +httpx = "^0.27.0" +nest-asyncio = "^1.6.0" +fastapi = "^0.109.2" + +# Core dependencies (optional) +openai = { version = "^1.11.1" } + +pydantic = { extras = ["email"], version = "^2.8.2", optional = true } +python-multipart = { version = "^0.0.9", optional = true } +fire = { version = "^0.5.0", optional = true } +gunicorn = { version = "^21.2.0", optional = true } +requests = { version = "^2.31.0", optional = true } +types-requests = { version = "^2.31.0", optional = true } +uvicorn = { version = "^0.27.0.post1", optional = true } +aiosqlite = { version = "^0.20.0", optional = true } +asyncpg = { version = "^0.29.0", optional = true } +redis = { version = "^5.0.4", optional = true } +beautifulsoup4 = { version = "^4.12.3", optional = true } +openpyxl = { version = "^3.1.2", optional = true } +markdown = { version = "^3.6", optional = true } +pypdf = { version = "^4.2.0", optional = true } +python-pptx = { version = "^0.6.23", optional = true } +python-docx = { version = "^1.1.0", optional = true } +opencv-python = { version = "^4.10.0.82", optional = true } +moviepy = { version = "^1.0.3", optional = true } +vecs = { version = "^0.4.0", optional = true } +litellm = { version = "^1.42.3", optional = true } +fsspec = { version = "^2024.6.0", optional = true } +posthog = { version = "^3.5.0", optional = true } +sqlalchemy = { version = "^2.0.30", optional = true } +ollama = { version = "^0.3.1", optional = true } +neo4j = { version = "^5.21.0", optional = true } +passlib = { version = "^1.7.4", optional = true } +bcrypt = { version = "^4.1.3", optional = true } +pyjwt = { version = "^2.8.0", optional = true } +toml = { version = "^0.10.2", optional = true } +pyyaml = { version = "^6.0.1", optional = true } +poppler-utils = { version = "^0.1.0", optional = true } +psutil = { version = "^6.0.0", optional = true } +deepdiff = { version = "^7.0.1", optional = true } +graspologic = { version = "^3.4.1", optional = true } + +[tool.poetry.extras] +core = [ + "pydantic", "python-multipart", "fire", "gunicorn", "requests", "types-requests", + "uvicorn", "aiosqlite", "asyncpg", "redis", "beautifulsoup4", "openpyxl", + "markdown", "pypdf", "python-pptx", "python-docx", "vecs", "litellm", "openai", + "fsspec", "posthog", "sqlalchemy", "ollama", "neo4j", "passlib", "bcrypt", + "pyjwt", "toml", "pyyaml", "poppler-utils", "psutil", "deepdiff", "graspologic" +] +core-ingest-movies = ["opencv-python", "moviepy"] + +[tool.poetry.group.dev.dependencies] +black = "^24.3.0" +codecov = "^2.1.13" +flake8 = "6.1.0" +isort = "5.12.0" +mypy = "^1.5.1" +pre-commit = "^2.9" +pytest = "^8.2.0" +pytest-asyncio = "^0.23.6" +pytest-dependency = "^0.6.0" +pytest-mock = "^3.14.0" +pytest-cov = "^5.0.0" +colorama = "^0.4.6" + +[tool.poetry.scripts] +r2r = "core.cli.cli:main" + +[tool.black] +line-length = 79 +target-version = ['py39'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.mypy] +ignore_missing_imports = true +exclude = 'playground/.*|deprecated/.*|dump/.*|docs/source|vecs/*' + +[[tool.mypy.overrides]] +module = "yaml" +ignore_missing_imports = true + +[tool.pytest.ini_options] +asyncio_mode = "auto" +addopts = "--cov=r2r --cov-report=term-missing --cov-report=xml" +testpaths = [ + "tests", +] +filterwarnings = [ + "ignore::DeprecationWarning", + "ignore::pytest.PytestUnraisableExceptionWarning", +] diff --git a/py/r2r/__init__.py b/py/r2r/__init__.py new file mode 100644 index 000000000..70e023ef8 --- /dev/null +++ b/py/r2r/__init__.py @@ -0,0 +1,16 @@ +from sdk import * + +__all__ = [ + # R2R SDK + "R2RAsyncClient", + "R2RClient", +] + +try: + from core import * + import core + __all__ += core.__all__ +except ImportError: + # Core dependencies not installed + pass + diff --git a/py/sdk/retrieval.py b/py/sdk/retrieval.py index 650d79185..ddd251a01 100644 --- a/py/sdk/retrieval.py +++ b/py/sdk/retrieval.py @@ -1,8 +1,5 @@ from typing import AsyncGenerator, Optional, Union -from core.base import VectorSearchSettings -from core.base.api.models import RAGResponse, SearchResponse - from .models import ( GenerationConfig, KGSearchSettings, @@ -12,7 +9,6 @@ VectorSearchSettings, ) - class RetrievalMethods: @staticmethod async def search(