From 7f105efce6b0425acba9c0777bbf9dc8ecab4766 Mon Sep 17 00:00:00 2001 From: Johan Olsson Date: Wed, 22 Nov 2023 21:27:35 +0100 Subject: [PATCH 1/2] Changing generic `IO` type to `BinaryIO`. Since virtually every implementation expects the underlying data type of the file-like object to be bytes, it is more appropriate to use `BinaryIO` in favor of the broader `IO` type. --- docs/changes.rst | 1 + minimalkv/_key_value_store.py | 26 +++++++++++++------------- minimalkv/_mixins.py | 13 ++++++++----- minimalkv/cache.py | 14 +++++++------- minimalkv/db/mongo.py | 6 +++--- minimalkv/db/sql.py | 6 +++--- minimalkv/fs.py | 6 +++--- minimalkv/fsspecstore.py | 8 ++++---- minimalkv/git.py | 6 +++--- minimalkv/idgen.py | 10 +++++----- minimalkv/memory/redisstore.py | 11 +++++++---- minimalkv/net/botostore.py | 10 +++++----- minimalkv/net/gcstore.py | 8 ++++---- 13 files changed, 66 insertions(+), 59 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 71712e2c..6e5599eb 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -4,6 +4,7 @@ Changelog 1.8.4 ===== * Removing invalid BSD-3 Clause license classifier. +* Changed generic `IO` type to `BinaryIO`. 1.8.3 ===== diff --git a/minimalkv/_key_value_store.py b/minimalkv/_key_value_store.py index d43647eb..95596fab 100644 --- a/minimalkv/_key_value_store.py +++ b/minimalkv/_key_value_store.py @@ -1,6 +1,6 @@ from io import BytesIO from types import TracebackType -from typing import IO, Dict, Iterator, List, Optional, Type, Union +from typing import BinaryIO, Dict, Iterator, List, Optional, Type, Union from uritools import SplitResult @@ -92,7 +92,7 @@ def get(self, key: str) -> bytes: self._check_valid_key(key) return self._get(key) - def get_file(self, key: str, file: Union[str, IO]) -> str: + def get_file(self, key: str, file: Union[str, BinaryIO]) -> str: """Write data at key to file. Like :meth:`~mininmalkv.KeyValueStore.put_file`, this method allows backends to @@ -106,7 +106,7 @@ def get_file(self, key: str, file: Union[str, IO]) -> str: ---------- key : str The key to be read. - file : file-like or str + file : BinaryIO or str Output filename or file-like object with a ``write`` method. Raises @@ -188,7 +188,7 @@ def keys(self, prefix: str = "") -> List[str]: """ return list(self.iter_keys(prefix)) - def open(self, key: str) -> IO: + def open(self, key: str) -> BinaryIO: """Open record at key. Parameters @@ -198,7 +198,7 @@ def open(self, key: str) -> IO: Returns ------- - file: file-like + file: BinaryIO Read-only file-like object for reading data at key. Raises @@ -240,7 +240,7 @@ def put(self, key: str, data: bytes) -> str: raise OSError("Provided data is not of type bytes") return self._put(key, data) - def put_file(self, key: str, file: Union[str, IO]) -> str: + def put_file(self, key: str, file: Union[str, BinaryIO]) -> str: """Store contents of file at key. Store data from a file into key. ``file`` can be a string, which will be @@ -253,7 +253,7 @@ def put_file(self, key: str, file: Union[str, IO]) -> str: ---------- key : str Key where to store data in file. - file : file-like or str + file : BinaryIO or str A filename or a file-like object with a read method. Returns @@ -313,14 +313,14 @@ def _get(self, key: str) -> bytes: return buf.getvalue() - def _get_file(self, key: str, file: IO) -> str: + def _get_file(self, key: str, file: BinaryIO) -> str: """Write data at key to file-like object file. Parameters ---------- key : str Key of data to be written to file. - file : file-like + file : BinaryIO File-like object with a *write* method to be written. """ bufsize = 1024 * 1024 @@ -365,7 +365,7 @@ def _has_key(self, key: str) -> bool: """ return key in self.keys() - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: """Open record at key. Parameters @@ -375,7 +375,7 @@ def _open(self, key: str) -> IO: Returns ------- - file: file-like + file: BinaryIO Opened file. """ raise NotImplementedError @@ -398,14 +398,14 @@ def _put(self, key: str, data: bytes) -> str: """ return self._put_file(key, BytesIO(data)) - def _put_file(self, key: str, file: IO) -> str: + def _put_file(self, key: str, file: BinaryIO) -> str: """Store data from file-like object at key. Parameters ---------- key : str Key at which to store contents of file. - file : file-like + file : BinaryIO File-like object to store data from. Returns diff --git a/minimalkv/_mixins.py b/minimalkv/_mixins.py index 3260e988..71bee5f3 100644 --- a/minimalkv/_mixins.py +++ b/minimalkv/_mixins.py @@ -1,5 +1,5 @@ from io import BytesIO -from typing import IO, Callable, Optional, Union +from typing import BinaryIO, Callable, Optional, Union from minimalkv._constants import FOREVER, NOT_SET, VALID_KEY_RE_EXTENDED @@ -161,7 +161,7 @@ def put( def put_file( self, key: str, - file: Union[str, IO], + file: Union[str, BinaryIO], ttl_secs: Optional[Union[float, int, str]] = None, ) -> str: """Store contents of file at key. @@ -181,7 +181,7 @@ def put_file( ---------- key : str Key where to store data in file. - file : file-like or str + file : BinaryIO or str A filename or an object with a read method. ttl_secs : str or numeric or None, optional, default = None Number of seconds until the key expires. @@ -232,7 +232,10 @@ def _put( return self._put_file(key, BytesIO(data), ttl_secs) def _put_file( - self, key: str, file: IO, ttl_secs: Optional[Union[str, float, int]] = None + self, + key: str, + file: BinaryIO, + ttl_secs: Optional[Union[str, float, int]] = None, ): """Store contents of file at key. @@ -240,7 +243,7 @@ def _put_file( ---------- key : str Key under which data should be stored. - file : file-like + file : BinaryIO File-like object with a ``read`` method. ttl_secs : str or numeric or None, optional, default = None Number of seconds until the key expires. diff --git a/minimalkv/cache.py b/minimalkv/cache.py index bdca7918..84cdc884 100644 --- a/minimalkv/cache.py +++ b/minimalkv/cache.py @@ -1,4 +1,4 @@ -from typing import IO, Union +from typing import BinaryIO, Union from minimalkv._key_value_store import KeyValueStore from minimalkv.decorator import StoreDecorator @@ -81,7 +81,7 @@ def get(self, key: str) -> bytes: # cache error, ignore completely and return from backend return self._dstore.get(key) - def get_file(self, key: str, file: Union[str, IO]) -> str: + def get_file(self, key: str, file: Union[str, BinaryIO]) -> str: """Write data at key to file. If a cache miss occurs, the value is retrieved, stored in the cache and @@ -98,7 +98,7 @@ def get_file(self, key: str, file: Union[str, IO]) -> str: ---------- key : str The key to be read. - file : file-like or str + file : BinaryIO or str Output filename or file-like object with a ``write`` method. """ @@ -114,7 +114,7 @@ def get_file(self, key: str, file: Union[str, IO]) -> str: # if an IOError occured, file pointer may be dirty - cannot proceed # safely - def open(self, key: str) -> IO: + def open(self, key: str) -> BinaryIO: """Open record at key. If a cache miss occurs, the value is retrieved, stored in the cache, @@ -133,7 +133,7 @@ def open(self, key: str) -> IO: Returns ------- - file: file-like + file: BinaryIO Read-only file-like object for reading data at key. """ @@ -205,7 +205,7 @@ def put(self, key: str, data: bytes) -> str: finally: self.cache.delete(key) - def put_file(self, key: str, file: Union[str, IO]) -> str: + def put_file(self, key: str, file: Union[str, BinaryIO]) -> str: """Store contents of file at key. Will store the value in the backing store. Afterwards delete the (original) @@ -215,7 +215,7 @@ def put_file(self, key: str, file: Union[str, IO]) -> str: ---------- key : str Key where to store data in file. - file : file-like or str + file : BinaryIO or str A filename or a file-like object with a read method. Returns diff --git a/minimalkv/db/mongo.py b/minimalkv/db/mongo.py index 1143b306..af438cb3 100644 --- a/minimalkv/db/mongo.py +++ b/minimalkv/db/mongo.py @@ -1,7 +1,7 @@ import pickle import re from io import BytesIO -from typing import IO, Iterator +from typing import BinaryIO, Iterator from bson.binary import Binary @@ -37,7 +37,7 @@ def _get(self, key: str) -> bytes: except StopIteration as e: raise KeyError(key) from e - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: return BytesIO(self._get(key)) def _put(self, key: str, value: bytes) -> str: @@ -46,7 +46,7 @@ def _put(self, key: str, value: bytes) -> str: ) return key - def _put_file(self, key: str, file: IO) -> str: + def _put_file(self, key: str, file: BinaryIO) -> str: return self._put(key, file.read()) def iter_keys(self, prefix: str = "") -> Iterator[str]: diff --git a/minimalkv/db/sql.py b/minimalkv/db/sql.py index e7aaccb7..138bc25e 100644 --- a/minimalkv/db/sql.py +++ b/minimalkv/db/sql.py @@ -1,5 +1,5 @@ from io import BytesIO -from typing import IO, Iterator +from typing import BinaryIO, Iterator from sqlalchemy import Column, LargeBinary, String, Table, exists, select from sqlalchemy.orm import Session @@ -39,7 +39,7 @@ def _get(self, key: str) -> bytes: return rv - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: return BytesIO(self._get(key)) def _copy(self, source: str, dest: str): @@ -79,7 +79,7 @@ def _put(self, key: str, data: bytes) -> str: session.commit() return key - def _put_file(self, key: str, file: IO) -> str: + def _put_file(self, key: str, file: BinaryIO) -> str: return self._put(key, file.read()) def iter_keys(self, prefix: str = "") -> Iterator[str]: # noqa D diff --git a/minimalkv/fs.py b/minimalkv/fs.py index 86c53a82..72bca9ca 100644 --- a/minimalkv/fs.py +++ b/minimalkv/fs.py @@ -2,7 +2,7 @@ import os.path import shutil import urllib.parse -from typing import IO, Any, Callable, Iterator, List, Optional, Union, cast +from typing import Any, BinaryIO, Callable, Iterator, List, Optional, Union, cast from minimalkv._key_value_store import KeyValueStore from minimalkv._mixins import CopyMixin, UrlMixin @@ -76,7 +76,7 @@ def _fix_permissions(self, filename: str) -> None: def _has_key(self, key: str) -> bool: return os.path.exists(self._build_filename(key)) - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: try: f = open(self._build_filename(key), "rb") return f @@ -109,7 +109,7 @@ def _ensure_dir_exists(self, path: str) -> None: if not os.path.isdir(path): raise e - def _put_file(self, key: str, file: IO, *args, **kwargs) -> str: + def _put_file(self, key: str, file: BinaryIO, *args, **kwargs) -> str: bufsize = self.bufsize target = self._build_filename(key) diff --git a/minimalkv/fsspecstore.py b/minimalkv/fsspecstore.py index 531f6649..dcd178f6 100644 --- a/minimalkv/fsspecstore.py +++ b/minimalkv/fsspecstore.py @@ -1,6 +1,6 @@ import io import warnings -from typing import IO, TYPE_CHECKING, Iterator, Optional, Union +from typing import TYPE_CHECKING, BinaryIO, Iterator, Optional, Union from minimalkv.net._net_common import LAZY_PROPERTY_ATTR_PREFIX, lazy_property @@ -189,21 +189,21 @@ def _delete(self, key: str) -> None: except FileNotFoundError: pass - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: try: return self._fs.open(f"{self._prefix}{key}") except FileNotFoundError as e: raise KeyError(key) from e # Required to prevent error when credentials are not sufficient for listing objects - def _get_file(self, key: str, file: IO) -> str: + def _get_file(self, key: str, file: BinaryIO) -> str: try: file.write(self._fs.cat_file(f"{self._prefix}{key}")) return key except FileNotFoundError as e: raise KeyError(key) from e - def _put_file(self, key: str, file: IO) -> str: + def _put_file(self, key: str, file: BinaryIO) -> str: self._fs.pipe_file(f"{self._prefix}{key}", file.read(), **self._write_kwargs) return key diff --git a/minimalkv/git.py b/minimalkv/git.py index 87830b7a..faad0400 100644 --- a/minimalkv/git.py +++ b/minimalkv/git.py @@ -1,7 +1,7 @@ import re import time from io import BytesIO -from typing import IO, Iterator, List, Optional, Union +from typing import BinaryIO, Iterator, List, Optional, Union from dulwich.objects import Blob, Commit, Tree from dulwich.repo import Repo @@ -197,10 +197,10 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: # noqa D if o.path.decode("ascii").startswith(prefix): yield o.path.decode("ascii") - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: return BytesIO(self._get(key)) - def _put_file(self, key: str, file: IO) -> str: + def _put_file(self, key: str, file: BinaryIO) -> str: # FIXME: it may be worth to try to move large files directly into the # store here return self._put(key, file.read()) diff --git a/minimalkv/idgen.py b/minimalkv/idgen.py index 10f18ede..02396b8c 100644 --- a/minimalkv/idgen.py +++ b/minimalkv/idgen.py @@ -17,7 +17,7 @@ import os import tempfile import uuid -from typing import IO, Optional, Union +from typing import BinaryIO, Optional, Union from minimalkv.decorator import StoreDecorator @@ -71,7 +71,7 @@ def put(self, key: Optional[str], data: bytes, *args, **kwargs): return self._dstore.put(key, data, *args, **kwargs) # type: ignore - def put_file(self, key: Optional[str], file: Union[str, IO], *args, **kwargs): + def put_file(self, key: Optional[str], file: Union[str, BinaryIO], *args, **kwargs): """Store contents of file at key. Store data from a file into key. ``file`` can be a string, which will be @@ -85,7 +85,7 @@ def put_file(self, key: Optional[str], file: Union[str, IO], *args, **kwargs): key : str or None Key where to store data in file. If None, the hash of data is used. - file : file-like or str + file : BinaryIO or str A filename or a file-like object with a read method. Returns @@ -198,7 +198,7 @@ def put(self, key: Optional[str], data: bytes, *args, **kwargs) -> str: return self._dstore.put(self._template.format(key), data, *args, **kwargs) # type: ignore - def put_file(self, key: Optional[str], file: Union[str, IO], *args, **kwargs): + def put_file(self, key: Optional[str], file: Union[str, BinaryIO], *args, **kwargs): """Store contents of file at key. Store data from a file into key. ``file`` can be a string, which will be @@ -211,7 +211,7 @@ def put_file(self, key: Optional[str], file: Union[str, IO], *args, **kwargs): ---------- key : str or None The key under which the data is to be stored. If None, a uuid is generated. - file : file-like or str + file : BinaryIO or str A filename or a file-like object with a read method. Returns diff --git a/minimalkv/memory/redisstore.py b/minimalkv/memory/redisstore.py index 1ace0561..d51a22e7 100644 --- a/minimalkv/memory/redisstore.py +++ b/minimalkv/memory/redisstore.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import re from io import BytesIO -from typing import IO, TYPE_CHECKING, Iterator, List, Optional, Union +from typing import TYPE_CHECKING, BinaryIO, Iterator, List, Optional, Union if TYPE_CHECKING: from redis import StrictRedis @@ -63,11 +63,11 @@ def _get(self, key: str) -> bytes: raise KeyError(key) return val - def _get_file(self, key: str, file: IO) -> str: + def _get_file(self, key: str, file: BinaryIO) -> str: file.write(self._get(key)) return key - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: return BytesIO(self._get(key)) def _put( @@ -95,7 +95,10 @@ def _put( return key def _put_file( - self, key: str, file: IO, ttl_secs: Optional[Union[str, int, float]] = None + self, + key: str, + file: BinaryIO, + ttl_secs: Optional[Union[str, int, float]] = None, ) -> str: self._put(key, file.read(), ttl_secs) return key diff --git a/minimalkv/net/botostore.py b/minimalkv/net/botostore.py index b05a7c67..e76fbff4 100644 --- a/minimalkv/net/botostore.py +++ b/minimalkv/net/botostore.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from typing import IO, Dict, Iterator, cast +from typing import BinaryIO, Dict, Iterator, cast from minimalkv import CopyMixin, KeyValueStore, UrlMixin @@ -95,7 +95,7 @@ def _get(self, key: str) -> bytes: with map_boto_exceptions(key=key): return k.get_contents_as_string() - def _get_file(self, key: str, file: IO) -> str: + def _get_file(self, key: str, file: BinaryIO) -> str: k = self.__new_key(key) with map_boto_exceptions(key=key): return k.get_contents_to_file(file) @@ -105,7 +105,7 @@ def _get_filename(self, key: str, filename: str) -> str: with map_boto_exceptions(key=key): return k.get_contents_to_filename(filename) - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: from boto.s3.keyfile import KeyFile class SimpleKeyFile(KeyFile): # noqa D @@ -124,7 +124,7 @@ def readable(self): # noqa D k = self.__new_key(key) with map_boto_exceptions(key=key): - return cast(IO, SimpleKeyFile(k)) + return cast(BinaryIO, SimpleKeyFile(k)) def _copy(self, source: str, dest: str) -> None: if not self._has_key(source): @@ -140,7 +140,7 @@ def _put(self, key: str, data: bytes) -> str: k.set_contents_from_string(data, **self.__upload_args()) return key - def _put_file(self, key: str, file: IO) -> str: + def _put_file(self, key: str, file: BinaryIO) -> str: k = self.__new_key(key) with map_boto_exceptions(key=key): k.set_contents_from_file(file, **self.__upload_args()) diff --git a/minimalkv/net/gcstore.py b/minimalkv/net/gcstore.py index a8203030..93d52d01 100644 --- a/minimalkv/net/gcstore.py +++ b/minimalkv/net/gcstore.py @@ -1,6 +1,6 @@ import json import warnings -from typing import IO, cast +from typing import BinaryIO, cast from minimalkv.fsspecstore import FSSpecStore, FSSpecStoreEntry @@ -65,14 +65,14 @@ def _create_filesystem(self) -> "GCSFileSystem": default_location=self.bucket_creation_location, ) - def _open(self, key: str) -> IO: + def _open(self, key: str) -> BinaryIO: from google.cloud.exceptions import NotFound if self._prefix_exists is False: raise NotFound(f"Could not find bucket: {self.bucket_name}") - return cast(IO, FSSpecStoreEntry(super()._open(key))) + return cast(BinaryIO, FSSpecStoreEntry(super()._open(key))) - def _get_file(self, key: str, file: IO) -> str: + def _get_file(self, key: str, file: BinaryIO) -> str: from google.cloud.exceptions import NotFound if self._prefix_exists is False: From 6f67d27cb2ebe964c11119addb3d88b5e4a127d7 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Tue, 28 Nov 2023 21:19:52 +0100 Subject: [PATCH 2/2] Update changes.rst --- docs/changes.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index 6e5599eb..6318567c 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,10 +1,13 @@ Changelog ********* +1.8.5 +===== +* Changed generic `IO` type to `BinaryIO`. + 1.8.4 ===== * Removing invalid BSD-3 Clause license classifier. -* Changed generic `IO` type to `BinaryIO`. 1.8.3 =====