diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 8b05742b..00000000 --- a/.flake8 +++ /dev/null @@ -1,11 +0,0 @@ -# Taken directly from https://github.com/ambv/black/blob/master/.flake8 -[flake8] -ignore = E203, E266, E501, W503, C901, D104, D100 -max-line-length = 88 -max-complexity = 18 -select = B,C,E,F,W,T4,B9,D -enable-extensions = flake8-docstrings -per-file-ignores = - tests/**:D101,D102,D103,E402 - examples/**:E402 -docstring-convention = numpy \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee006027..971e728b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,12 +6,12 @@ jobs: name: Unit tests - ${{ matrix.PYTHON_VERSION }} runs-on: ubuntu-latest permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" strategy: fail-fast: false matrix: - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11"] + PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12"] services: postgres: image: postgres:11 @@ -45,12 +45,12 @@ jobs: continue-on-error: true - id: google_auth if: steps.check-id-token.outcome == 'success' - name: 'Authenticate to Google Cloud' + name: "Authenticate to Google Cloud" uses: google-github-actions/auth@v1 with: - workload_identity_provider: 'projects/498651197656/locations/global/workloadIdentityPools/qc-minimalkv-gh-actions-pool/providers/github-actions-provider' - service_account: 'sa-github-actions@qc-minimalkv.iam.gserviceaccount.com' - token_format: 'access_token' + workload_identity_provider: "projects/498651197656/locations/global/workloadIdentityPools/qc-minimalkv-gh-actions-pool/providers/github-actions-provider" + service_account: "sa-github-actions@qc-minimalkv.iam.gserviceaccount.com" + token_format: "access_token" - name: Run the unittests shell: bash -x -l {0} run: | diff --git a/.github/workflows/pre-commit-autoupdate.yml b/.github/workflows/pre-commit-autoupdate.yml new file mode 100644 index 00000000..fc4e33fd --- /dev/null +++ b/.github/workflows/pre-commit-autoupdate.yml @@ -0,0 +1,45 @@ +name: pre-commit autoupdate +on: + workflow_dispatch: + schedule: + - cron: "0 6 4 * *" + +defaults: + run: + shell: bash -el {0} + +jobs: + check_update: + name: Check if newer version exists + runs-on: ubuntu-latest + steps: + - name: Checkout branch + uses: actions/checkout@v4 + # We need to checkout with SSH here to have actions run on the PR. + with: + ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} + - name: Set up Conda env + uses: mamba-org/setup-micromamba@db1df3ba9e07ea86f759e98b575c002747e9e757 + with: + environment-name: pre-commit + create-args: >- + -c + conda-forge + pre-commit + mamba + - name: Update pre-commit hooks and run + id: versions + env: + PRE_COMMIT_USE_MAMBA: 1 + run: | + pre-commit autoupdate + pre-commit run -a || true + - uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 + with: + commit-message: "Auto-update pre-commit hooks" + title: "Auto-update pre-commit hooks" + body: | + New versions of the used pre-commit hooks were detected. + This PR updates them to the latest and already ran `pre-commit run -a` for you to fix any changes in formatting. + branch: pre-commit-autoupdate + delete-branch: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31e3c37e..be34607f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,10 @@ repos: - - repo: https://github.com/Quantco/pre-commit-mirrors-black - rev: 23.7.0 + # Run ruff first because autofix behaviour is enabled + - repo: https://github.com/Quantco/pre-commit-mirrors-ruff + rev: "0.1.3" hooks: - - id: black-conda - args: - - --safe - - --target-version=py38 - - repo: https://github.com/Quantco/pre-commit-mirrors-flake8 - rev: 6.1.0 - hooks: - - id: flake8-conda - additional_dependencies: [-c, conda-forge, flake8-docstrings=1.5.0, flake8-rst-docstrings=0.0.14] - - repo: https://github.com/Quantco/pre-commit-mirrors-isort - rev: 5.12.0 - hooks: - - id: isort-conda - additional_dependencies: [-c, conda-forge, toml=0.10.2] + - id: ruff-conda + - id: ruff-format-conda - repo: https://github.com/Quantco/pre-commit-mirrors-mypy rev: "1.5.1" hooks: diff --git a/.readthedocs.yml b/.readthedocs.yml index 82f1f35d..4a018c33 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,5 +10,5 @@ conda: python: install: - - method: setuptools + - method: pip path: . diff --git a/docs/changes.rst b/docs/changes.rst index f86f4a75..85854b22 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,14 @@ Changelog ********* +1.8.2 +===== +* Include Python 3.12 in CI +* Migrate setup.cfg and setup.py into pyproject.toml +* Port to ruff +* Include pre-commit autoupdate workflow +* Determine version in ``docs/conf.py`` automatically + 1.8.1 ===== * Drop `pkg_resources` and use `importlib.metadata` to access package version string. diff --git a/docs/conf.py b/docs/conf.py index 10b297b1..b19f5b4b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,15 +4,21 @@ from sphinx.ext import apidoc +from minimalkv import __version__ as version + +release = version + sys.path.append("../") package = "minimalkv" html_theme = "alabaster" __location__ = os.path.join( - os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())) # type: ignore + os.getcwd(), + os.path.dirname(inspect.getfile(inspect.currentframe())), # type: ignore ) + # Generate module references output_dir = os.path.abspath(os.path.join(__location__, "../docs/_rst")) module_dir = os.path.abspath(os.path.join(__location__, "..", package)) @@ -37,14 +43,6 @@ project = "minimalkv" copyright = "2011-2021, The minimalkv contributors" -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = "0.14.1" -# The full version, including alpha/beta/rc tags. -release = "0.14.1" exclude_trees = ["_build"] diff --git a/minimalkv/_boto.py b/minimalkv/_boto.py index cf6dacc3..5abbc583 100644 --- a/minimalkv/_boto.py +++ b/minimalkv/_boto.py @@ -7,8 +7,11 @@ def _get_s3bucket( create_if_missing=True, ): # TODO: Write docstring. - from boto.s3.connection import S3ResponseError # type: ignore - from boto.s3.connection import OrdinaryCallingFormat, S3Connection + from boto.s3.connection import ( # type: ignore + OrdinaryCallingFormat, + S3Connection, + S3ResponseError, + ) s3_connection_params = { "aws_access_key_id": access_key, @@ -35,5 +38,5 @@ def _get_s3bucket( if create_if_missing: return s3con.create_bucket(bucket) else: - raise OSError(f"Bucket {bucket} does not exist") + raise OSError(f"Bucket {bucket} does not exist") from ex raise diff --git a/minimalkv/_get_store.py b/minimalkv/_get_store.py index e9fb4b23..8002c60d 100644 --- a/minimalkv/_get_store.py +++ b/minimalkv/_get_store.py @@ -10,8 +10,7 @@ def get_store_from_url( url: str, store_cls: Optional[Type[KeyValueStore]] = None ) -> KeyValueStore: - """ - Take a URL and return a minimalkv store according to the parameters in the URL. + """Take a URL and return a minimalkv store according to the parameters in the URL. Parameters ---------- @@ -97,8 +96,7 @@ def get_store_from_url( def _extract_wrappers(parsed_url: SplitResult) -> List[str]: - """ - Extract wrappers from a parsed URL. + """Extract wrappers from a parsed URL. Wrappers allow you to add additional functionality to a store, e.g. encryption. They can be specified in two ways: diff --git a/minimalkv/_key_value_store.py b/minimalkv/_key_value_store.py index 6f645be4..de671abb 100644 --- a/minimalkv/_key_value_store.py +++ b/minimalkv/_key_value_store.py @@ -12,8 +12,7 @@ class KeyValueStore: - """ - Class to access a key-value store. + """Class to access a key-value store. Supported keys are ascii-strings containing alphanumeric characters or symbols out of ``minimalkv._constants.VALID_NON_NUM`` of length not greater than 250. Values @@ -141,8 +140,7 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: raise NotImplementedError def iter_prefixes(self, delimiter: str, prefix: str = "") -> Iterator[str]: - """ - Iterate over unique prefixes in the store up to delimiter, starting with prefix. + """Iterate over unique prefixes in the store up to delimiter, starting with prefix. If ``prefix`` contains ``delimiter``, return the prefix up to the first occurence of delimiter after the prefix. @@ -468,8 +466,7 @@ def __exit__( def _from_parsed_url( cls, parsed_url: SplitResult, query: Dict[str, str] ) -> "KeyValueStore": - """ - Build a ``KeyValueStore`` from a parsed URL. + """Build a ``KeyValueStore`` from a parsed URL. To build a ``KeyValueStore`` from a URL, use :func:`get_store_from_url`. diff --git a/minimalkv/_store_creation.py b/minimalkv/_store_creation.py index 92039818..e11fb5e0 100644 --- a/minimalkv/_store_creation.py +++ b/minimalkv/_store_creation.py @@ -50,7 +50,7 @@ def _create_store_gcs(store_type, params): from minimalkv._hstores import HGoogleCloudStore from minimalkv.net.gcstore import GoogleCloudStore - if type(params["credentials"]) is bytes: + if isinstance(params["credentials"], bytes): account_info = json.loads(params["credentials"].decode()) params["credentials"] = Credentials.from_service_account_info( account_info, diff --git a/minimalkv/_urls.py b/minimalkv/_urls.py index 3df619d3..c31307ff 100644 --- a/minimalkv/_urls.py +++ b/minimalkv/_urls.py @@ -44,14 +44,14 @@ def url2dict(url: str, raise_on_extra_params: bool = False) -> Dict[str, Any]: ) u = urisplit(url) - parsed = dict( - scheme=u.getscheme(), - host=u.gethost(), - port=u.getport(), - path=u.getpath(), - query=u.getquerydict(), - userinfo=u.getuserinfo(), - ) + parsed = { + "scheme": u.getscheme(), + "host": u.gethost(), + "port": u.getport(), + "path": u.getpath(), + "query": u.getquerydict(), + "userinfo": u.getuserinfo(), + } fragment = u.getfragment() params = {"type": parsed["scheme"]} diff --git a/minimalkv/crypt.py b/minimalkv/crypt.py index 7df3d3e8..7d564d8e 100644 --- a/minimalkv/crypt.py +++ b/minimalkv/crypt.py @@ -128,7 +128,7 @@ def get_file(self, key, file): # noqa D try: f = open(file, "wb") except OSError as e: - raise OSError(f"Error opening {file} for writing: {e!r}") + raise OSError(f"Error opening {file} for writing: {e!r}") from e # file is open, now we call ourself again with a proper file try: diff --git a/minimalkv/db/mongo.py b/minimalkv/db/mongo.py index 6acfbaaf..1143b306 100644 --- a/minimalkv/db/mongo.py +++ b/minimalkv/db/mongo.py @@ -34,8 +34,8 @@ def _get(self, key: str) -> bytes: try: item = next(self.db[self.collection].find({"_id": key})) return pickle.loads(item["v"]) - except StopIteration: - raise KeyError(key) + except StopIteration as e: + raise KeyError(key) from e def _open(self, key: str) -> IO: return BytesIO(self._get(key)) diff --git a/minimalkv/db/sql.py b/minimalkv/db/sql.py index f10d76cf..e7aaccb7 100644 --- a/minimalkv/db/sql.py +++ b/minimalkv/db/sql.py @@ -87,4 +87,4 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: # noqa D query = select(self.table.c.key) if prefix != "": query = query.where(self.table.c.key.like(prefix + "%")) - return map(lambda v: str(v[0]), session.execute(query)) + return (str(v[0]) for v in session.execute(query)) diff --git a/minimalkv/decorator.py b/minimalkv/decorator.py index b7de4c0b..553585f7 100644 --- a/minimalkv/decorator.py +++ b/minimalkv/decorator.py @@ -139,8 +139,7 @@ def copy(self, source: str, dest: str): # noqa D class PrefixDecorator(KeyTransformingDecorator): - """ - Prefixes any key with a string before passing it on the decorated store. + """Prefixes any key with a string before passing it on the decorated store. Automatically strips the prefix upon key retrieval. diff --git a/minimalkv/fs.py b/minimalkv/fs.py index a0e4b98a..86c53a82 100644 --- a/minimalkv/fs.py +++ b/minimalkv/fs.py @@ -82,7 +82,7 @@ def _open(self, key: str) -> IO: return f except OSError as e: if 2 == e.errno: - raise KeyError(key) + raise KeyError(key) from e else: raise @@ -97,7 +97,7 @@ def _copy(self, source: str, dest: str) -> str: return dest except OSError as e: if 2 == e.errno: - raise KeyError(source) + raise KeyError(source) from e else: raise @@ -155,7 +155,7 @@ def keys(self, prefix: str = "") -> List[str]: """ root = os.path.abspath(self.root) result = [] - for dp, dn, fn in os.walk(root): + for dp, _, fn in os.walk(root): for f in fn: key = os.path.join(dp, f)[len(root) + 1 :] if key.startswith(prefix): @@ -174,8 +174,7 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: return iter(self.keys(prefix)) def iter_prefixes(self, delimiter: str, prefix: str = "") -> Iterator[str]: - """ - Iterate over unique prefixes in the store up to delimiter, starting with prefix. + """Iterate over unique prefixes in the store up to delimiter, starting with prefix. If ``prefix`` contains ``delimiter``, return the prefix up to the first occurence of delimiter after the prefix. @@ -224,8 +223,7 @@ def _iter_prefixes_efficient( class WebFilesystemStore(FilesystemStore): - """ - FilesystemStore supporting generating URLS for web applications. + """FilesystemStore supporting generating URLS for web applications. The most common use is to make the ``root`` directory of the filesystem store available through a webserver. diff --git a/minimalkv/fsspecstore.py b/minimalkv/fsspecstore.py index 8f28915b..531f6649 100644 --- a/minimalkv/fsspecstore.py +++ b/minimalkv/fsspecstore.py @@ -20,8 +20,7 @@ class FSSpecStoreEntry(io.BufferedIOBase): """A file-like object for reading from an entry in an FSSpecStore.""" def __init__(self, file: "AbstractBufferedFile"): - """ - Initialize an FSSpecStoreEntry. + """Initialize an FSSpecStoreEntry. Parameters ---------- @@ -32,8 +31,7 @@ def __init__(self, file: "AbstractBufferedFile"): self._file = file def seek(self, loc: int, whence: int = 0) -> int: - """ - Set current file location. + """Set current file location. Parameters ---------- @@ -46,9 +44,9 @@ def seek(self, loc: int, whence: int = 0) -> int: raise ValueError("I/O operation on closed file.") try: return self._file.seek(loc, whence) - except ValueError: + except ValueError as e: # Map ValueError to IOError - raise OSError + raise OSError from e def tell(self) -> int: """Return the current offset as int. Always >= 0.""" @@ -90,8 +88,7 @@ def __init__( write_kwargs: Optional[dict] = None, custom_fs: Optional["AbstractFileSystem"] = None, ): - """ - Initialize an FSSpecStore. + """Initialize an FSSpecStore. The underlying fsspec FileSystem is created when the store is used for the first time. @@ -126,8 +123,7 @@ def _prefix_exists(self) -> Union[None, bool]: @property def mkdir_prefix(self): - """ - Whether to create the prefix if it does not exist. + """Whether to create the prefix if it does not exist. .. note:: Deprecated in 2.0.0. """ @@ -135,14 +131,14 @@ def mkdir_prefix(self): "The mkdir_prefix attribute is deprecated!" "It will be renamed to _mkdir_prefix in the next release.", DeprecationWarning, + stacklevel=2, ) return self._mkdir_prefix @property def prefix(self): - """ - Get the prefix used on the ``fsspec`` ``FileSystem`` when storing keys. + """Get the prefix used on the ``fsspec`` ``FileSystem`` when storing keys. .. note:: Deprecated in 2.0.0. """ @@ -150,6 +146,7 @@ def prefix(self): "The prefix attribute is deprecated!" "It will be renamed to _prefix in the next release.", DeprecationWarning, + stacklevel=2, ) return self._prefix @@ -184,10 +181,7 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: all_files_and_dirs = self._fs.find(dir_prefix, prefix=file_prefix) - return map( - lambda k: k.replace(f"{self._prefix}", ""), - all_files_and_dirs, - ) + return (k.replace(f"{self._prefix}", "") for k in all_files_and_dirs) def _delete(self, key: str) -> None: try: @@ -198,16 +192,16 @@ def _delete(self, key: str) -> None: def _open(self, key: str) -> IO: try: return self._fs.open(f"{self._prefix}{key}") - except FileNotFoundError: - raise KeyError(key) + except FileNotFoundError as e: + raise KeyError(key) from e # Required to prevent error when credentials are not sufficient for listing objects def _get_file(self, key: str, file: IO) -> str: try: file.write(self._fs.cat_file(f"{self._prefix}{key}")) return key - except FileNotFoundError: - raise KeyError(key) + except FileNotFoundError as e: + raise KeyError(key) from e def _put_file(self, key: str, file: IO) -> str: self._fs.pipe_file(f"{self._prefix}{key}", file.read(), **self._write_kwargs) diff --git a/minimalkv/git.py b/minimalkv/git.py index 749f3925..87830b7a 100644 --- a/minimalkv/git.py +++ b/minimalkv/git.py @@ -172,8 +172,8 @@ def _get(self, key: str) -> bytes: fn = self.subdir + "/" + key _, blob_id = tree.lookup_path(self.repo.__getitem__, fn.encode("ascii")) blob = self.repo[blob_id] - except KeyError: - raise KeyError(key) + except KeyError as e: + raise KeyError(key) from e return blob.data diff --git a/minimalkv/idgen.py b/minimalkv/idgen.py index 1a5abbeb..10f18ede 100644 --- a/minimalkv/idgen.py +++ b/minimalkv/idgen.py @@ -1,5 +1,4 @@ -""" -In cases where you want to generate IDs automatically, decorators are available. +"""In cases where you want to generate IDs automatically, decorators are available. These should be the outermost decorators, as they change the signature of some of the put methods slightly. diff --git a/minimalkv/memory/redisstore.py b/minimalkv/memory/redisstore.py index 87c1de22..1ace0561 100644 --- a/minimalkv/memory/redisstore.py +++ b/minimalkv/memory/redisstore.py @@ -40,9 +40,7 @@ def keys(self, prefix: str = "") -> List[str]: IOError If there was an error accessing the store. """ - return list( - map(lambda b: b.decode(), self.redis.keys(pattern=re.escape(prefix) + "*")) - ) + return [b.decode() for b in self.redis.keys(pattern=re.escape(prefix) + "*")] def iter_keys(self, prefix="") -> Iterator[str]: """Iterate over all keys in the store starting with prefix. diff --git a/minimalkv/net/_azurestore_new.py b/minimalkv/net/_azurestore_new.py index a63b14bf..a4e4c50c 100644 --- a/minimalkv/net/_azurestore_new.py +++ b/minimalkv/net/_azurestore_new.py @@ -20,8 +20,8 @@ def map_azure_exceptions(key=None, error_codes_pass=()): if error_code is not None and error_code in error_codes_pass: return if error_code == "BlobNotFound": - raise KeyError(key) - raise OSError(str(ex)) + raise KeyError(key) from ex + raise OSError(str(ex)) from ex class AzureBlockBlobStore(KeyValueStore): # noqa D diff --git a/minimalkv/net/_azurestore_old.py b/minimalkv/net/_azurestore_old.py index 610e3c66..6a9b0743 100644 --- a/minimalkv/net/_azurestore_old.py +++ b/minimalkv/net/_azurestore_old.py @@ -22,14 +22,14 @@ def map_azure_exceptions(key=None, exc_pass=()): if ex.__class__.__name__ not in exc_pass: s = str(ex) if s.startswith("The specified container does not exist."): - raise OSError(s) - raise KeyError(key) + raise OSError(s) from ex + raise KeyError(key) from ex except AzureHttpError as ex: if ex.__class__.__name__ not in exc_pass: - raise OSError(str(ex)) + raise OSError(str(ex)) from ex except AzureException as ex: if ex.__class__.__name__ not in exc_pass: - raise OSError(str(ex)) + raise OSError(str(ex)) from ex class AzureBlockBlobStore(KeyValueStore): # noqa D diff --git a/minimalkv/net/boto3store.py b/minimalkv/net/boto3store.py index e1afdfdd..397bf2f9 100644 --- a/minimalkv/net/boto3store.py +++ b/minimalkv/net/boto3store.py @@ -31,8 +31,8 @@ def map_boto3_exceptions(key=None, exc_pass=()): except ClientError as ex: code = ex.response["Error"]["Code"] if code == "404" or code == "NoSuchKey": - raise KeyError(key) - raise OSError(str(ex)) + raise KeyError(key) from ex + raise OSError(str(ex)) from ex class Boto3SimpleKeyFile(io.RawIOBase): # noqa D @@ -120,6 +120,7 @@ def __init__( "The prefix attribute is deprecated and will be removed in the next major release." "Use object_prefix instead.", DeprecationWarning, + stacklevel=2, ) object_prefix = object_prefix or prefix self._object_prefix = object_prefix.strip().lstrip("/") @@ -131,8 +132,7 @@ def __init__( @property def prefix(self) -> str: - """ - Get the prefix used for all keys in this store. + """Get the prefix used for all keys in this store. .. note:: Deprecated in 2.0.0, use :attr:`object_prefix` instead. """ @@ -142,6 +142,7 @@ def prefix(self) -> str: "The `prefix` attribute is deprecated and will be removed in the next major release." "Use `object_prefix` instead.", DeprecationWarning, + stacklevel=2, ) return self._object_prefix @@ -152,9 +153,9 @@ def __new_object(self, name): def iter_keys(self, prefix=""): # noqa D with map_boto3_exceptions(): prefix_len = len(self._object_prefix) - return map( - lambda k: k.key[prefix_len:], - self.bucket.objects.filter(Prefix=self._object_prefix + prefix), + return ( + k.key[prefix_len:] + for k in self.bucket.objects.filter(Prefix=self._object_prefix + prefix) ) def _delete(self, key): @@ -247,8 +248,7 @@ def _url_for(self, key): ) def __eq__(self, other): - """ - Assert that two ``Boto3Store``s are equal. + """Assert that two ``Boto3Store``s are equal. The bucket name and other configuration parameters are compared. See :func:`from_url` for details on the connection parameters. diff --git a/minimalkv/net/botostore.py b/minimalkv/net/botostore.py index af646cc5..b05a7c67 100644 --- a/minimalkv/net/botostore.py +++ b/minimalkv/net/botostore.py @@ -13,11 +13,11 @@ def map_boto_exceptions(key=None, exc_pass=()): yield except StorageResponseError as e: if e.code == "NoSuchKey": - raise KeyError(key) - raise OSError(str(e)) + raise KeyError(key) from e + raise OSError(str(e)) from e except (BotoClientError, BotoServerError) as e: if e.__class__.__name__ not in exc_pass: - raise OSError(str(e)) + raise OSError(str(e)) from e class BotoStore(KeyValueStore, UrlMixin, CopyMixin): # noqa D @@ -46,8 +46,7 @@ def __new_key(self, name): return k def __upload_args(self) -> Dict[str, str]: - """ - Generate a dictionary of arguments to pass to ``set_content_from`` functions. + """Generate a dictionary of arguments to pass to ``set_content_from`` functions. This allows us to save API calls by passing the necessary parameters on with the upload. @@ -76,9 +75,7 @@ def iter_keys(self, prefix="") -> Iterator[str]: """ with map_boto_exceptions(): prefix_len = len(self.prefix) - return map( - lambda k: k.name[prefix_len:], self.bucket.list(self.prefix + prefix) - ) + return (k.name[prefix_len:] for k in self.bucket.list(self.prefix + prefix)) def _has_key(self, key: str) -> bool: with map_boto_exceptions(key=key): @@ -91,7 +88,7 @@ def _delete(self, key: str) -> None: self.bucket.delete_key(self.prefix + key) except StorageResponseError as e: if e.code != "NoSuchKey": - raise OSError(str(e)) + raise OSError(str(e)) from e def _get(self, key: str) -> bytes: k = self.__new_key(key) diff --git a/minimalkv/net/gcstore.py b/minimalkv/net/gcstore.py index 9fa44db2..a8203030 100644 --- a/minimalkv/net/gcstore.py +++ b/minimalkv/net/gcstore.py @@ -42,7 +42,8 @@ def __init__( This was caused by the following error: {error} - """ + """, + stacklevel=2, ) self._credentials = credentials diff --git a/minimalkv/net/s3fsstore.py b/minimalkv/net/s3fsstore.py index d0913148..28ff2c36 100644 --- a/minimalkv/net/s3fsstore.py +++ b/minimalkv/net/s3fsstore.py @@ -88,8 +88,8 @@ def _url_for(self, key) -> str: def _from_parsed_url( cls, parsed_url: SplitResult, query: Dict[str, str] ) -> "S3FSStore": # noqa D - """ - Build an ``S3FSStore`` from a parsed URL. + """Build an ``S3FSStore`` from a parsed URL. + To build an ``S3FSStore`` from a URL, use :func:`get_store_from_url`. URl format: diff --git a/pyproject.toml b/pyproject.toml index 1826ffc8..eea2c983 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,24 +1,64 @@ [build-system] -requires = ['setuptools', 'setuptools-scm', 'wheel'] - -[tool.black] -exclude = ''' -/( - \.eggs - | \.git - | \.venv - | build - | dist -)/ -''' - -[tool.isort] -multi_line_output = 3 -include_trailing_comma = true -line_length = 88 -known_first_party = "minimalkv" -skip_glob = '\.eggs/*,\.git/*,\.venv/*,build/*,dist/*' -default_section = 'THIRDPARTY' +requires = ["setuptools", "setuptools_scm", "wheel"] + +[tool.setuptools_scm] +version_scheme = "post-release" + +[project] +name = "minimalkv" +dynamic = ["version"] +license = { file = "LICENSE" } +description = "A key-value storage for binary data, support many backends." +readme = "README.rst" +authors = [{name = "Data Engineering Collective", email = "minimalkv@uwekorn.com"}] +classifiers = [ + "License :: OSI Approved :: BSD License", + "License :: OSI Approved :: BSD-3 Clause", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Development Status :: 5 - Production/Stable", + "Operating System :: OS Independent", +] +dependencies = ["uritools"] +requires-python = ">=3.8" + +[project.urls] +repository = "https://github.com/data-engineering-collective/minimalkv" + +[tool.setuptools.packages.find] +include = ["minimalkv"] + +[tool.setuptools.package-data] +minimalkv = ["py.typed"] + +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +ignore = ["E203", "E266", "E501", "C901", "D104", "D100"] +select = ["B", "C", "E", "F", "W", "B9", "D", "I"] + +[tool.ruff.lint.isort] +force-wrap-aliases = true +combine-as-imports = true +known-first-party = ["minimalkv"] + +[tool.ruff.per-file-ignores] +"tests/*" = ["D101", "D102", "D103", "E402"] +"tests/test_azure_store.py" = ["B018"] +"tests/storefact/test_urls.py" = ["C408"] + +[tool.ruff.pydocstyle] +convention = "numpy" + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" [tool.mypy] python_version = 3.8 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 526aeb28..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal = 0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 136559c2..00000000 --- a/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python - -from os import path - -from setuptools import find_packages, setup - -here = path.abspath(path.dirname(__file__)) - -with open("README.rst") as f: - long_description = f.read() - -setup( - name="minimalkv", - use_scm_version=True, - setup_requires=["setuptools_scm"], - description=("A key-value storage for binary data, support many " "backends."), - long_description=long_description, - author="Data Engineering Collective", - author_email="minimalkv@uwekorn.com", - url="https://github.com/data-engineering-collective/minimalkv", - license="BSD-3-clause", - packages=find_packages(exclude=["test"]), - install_requires=["uritools"], - python_requires=">=3.8", - package_data={"minimalkv": ["py.typed"]}, - classifiers=[ - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Development Status :: 5 - Production/Stable", - "Operating System :: OS Independent", - ], -) diff --git a/tests/basic_store.py b/tests/basic_store.py index 20798c27..cdb85634 100644 --- a/tests/basic_store.py +++ b/tests/basic_store.py @@ -431,12 +431,12 @@ def test_uuid_decorator(self, ustore, value): def test_advertises_ttl_features(self, store): assert store.ttl_support is True assert hasattr(store, "ttl_support") - assert getattr(store, "ttl_support") is True + assert store.ttl_support is True def test_advertises_ttl_features_through_decorator(self, dstore): assert dstore.ttl_support is True assert hasattr(dstore, "ttl_support") - assert getattr(dstore, "ttl_support") is True + assert dstore.ttl_support is True def test_can_pass_ttl_through_decorator(self, dstore, key, value): dstore.put(key, value, ttl_secs=10) diff --git a/tests/bucket_manager.py b/tests/bucket_manager.py index c2f8f78a..2a150dc1 100644 --- a/tests/bucket_manager.py +++ b/tests/bucket_manager.py @@ -55,8 +55,7 @@ def boto3_bucket( is_secure=None, **kwargs, ): - """ - Create a boto3 bucket. + """Create a boto3 bucket. The bucket is deleted after the consuming function returns. """ @@ -85,8 +84,7 @@ def boto3_bucket_reference( port=None, is_secure=None, ): - """ - Create a boto3 bucket reference. + """Create a boto3 bucket reference. The bucket is not created. """ diff --git a/tests/store_creation/test_creation_boto3store.py b/tests/store_creation/test_creation_boto3store.py index 3043679b..525a08f8 100644 --- a/tests/store_creation/test_creation_boto3store.py +++ b/tests/store_creation/test_creation_boto3store.py @@ -42,8 +42,7 @@ def test_equal_access(): def s3fsstores_equal(store1, store2): - """ - Return whether two ``S3FSStore``s are equal. + """Return whether two ``S3FSStore``s are equal. The bucket name and other configuration parameters are compared. See :func:`from_url` for details on the connection parameters. diff --git a/tests/test_filesystem_store.py b/tests/test_filesystem_store.py index d8350add..1901b0ed 100644 --- a/tests/test_filesystem_store.py +++ b/tests/test_filesystem_store.py @@ -3,9 +3,11 @@ import tempfile from io import BytesIO from unittest.mock import Mock -from urllib.parse import quote as url_quote -from urllib.parse import unquote as url_unquote -from urllib.parse import urlparse +from urllib.parse import ( + quote as url_quote, + unquote as url_unquote, + urlparse, +) import pytest from basic_store import BasicStore diff --git a/tests/test_gcloud_store.py b/tests/test_gcloud_store.py index 1db23e3f..f171c5ba 100644 --- a/tests/test_gcloud_store.py +++ b/tests/test_gcloud_store.py @@ -87,7 +87,7 @@ def get_bucket_from_store(gcstore: GoogleCloudStore) -> Bucket: def get_client_from_store(gcstore: GoogleCloudStore) -> Client: - if type(gcstore._credentials) is str: + if isinstance(gcstore._credentials, str): client = Client.from_service_account_json(gcstore._credentials) else: client = Client(credentials=gcstore._credentials, project=gcstore.project_name) diff --git a/tests/test_get_store_from_url.py b/tests/test_get_store_from_url.py index e1a072e0..25a3cc8d 100644 --- a/tests/test_get_store_from_url.py +++ b/tests/test_get_store_from_url.py @@ -2,8 +2,10 @@ import pytest -from minimalkv._get_store import get_store -from minimalkv._get_store import get_store_from_url as get_store_from_url_new +from minimalkv._get_store import ( + get_store, + get_store_from_url as get_store_from_url_new, +) from minimalkv._hstores import HDictStore from minimalkv._key_value_store import KeyValueStore from minimalkv._urls import url2dict diff --git a/tests/test_prefix_decorator.py b/tests/test_prefix_decorator.py index eae5fa70..0a1b3dcc 100644 --- a/tests/test_prefix_decorator.py +++ b/tests/test_prefix_decorator.py @@ -48,7 +48,7 @@ def test_put_returns_correct_key(self, store, prefix, key, value): def test_put_sets_prefix(self, store, prefix, key, value): full_key = prefix + key - key == store.put(key, value) + assert key == store.put(key, value) assert store._dstore.get(full_key) == value @@ -57,7 +57,7 @@ def test_put_file_returns_correct_key(self, store, prefix, key, value): def test_put_file_sets_prefix(self, store, prefix, key, value): full_key = prefix + key - key == store.put_file(key, BytesIO(value)) + assert key == store.put_file(key, BytesIO(value)) assert store._dstore.get(full_key) == value