From 2258c9233f86f3010f93eb60414206fa8367ab2c Mon Sep 17 00:00:00 2001 From: Calin Crisan Date: Wed, 15 Nov 2023 18:43:38 +0200 Subject: [PATCH] utils/json: Generalize JSON extra types support --- qtoggleserver/drivers/persist/json.py | 8 ++- qtoggleserver/drivers/persist/redis.py | 4 +- .../frontend/js/devices/add-device-form.js | 5 +- qtoggleserver/persist/__init__.py | 19 +++--- qtoggleserver/utils/json.py | 60 ++++++++++++++++--- 5 files changed, 72 insertions(+), 24 deletions(-) diff --git a/qtoggleserver/drivers/persist/json.py b/qtoggleserver/drivers/persist/json.py index aa3c99f1..077ce855 100644 --- a/qtoggleserver/drivers/persist/json.py +++ b/qtoggleserver/drivers/persist/json.py @@ -253,7 +253,7 @@ def _load(self) -> UnindexedData: try: with open(self._file_path, 'rb') as f: data = f.read() - return json_utils.loads(data, allow_extended_types=True) + return json_utils.loads(data, extra_types=json_utils.EXTRA_TYPES_EXTENDED) except Exception as e: if not self._use_backup: raise @@ -266,7 +266,7 @@ def _load(self) -> UnindexedData: logger.warning('loading from backup %s', backup_file_path) with open(backup_file_path, 'rb') as f: - return json_utils.loads(f.read(), allow_extended_types=True) + return json_utils.loads(f.read(), extra_types=json_utils.EXTRA_TYPES_EXTENDED) return {} @@ -284,7 +284,9 @@ def _save(self, data: UnindexedData) -> None: logger.debug('saving to %s', self._file_path) with open(self._file_path, 'wb') as f: - data = json_utils.dumps(data, allow_extended_types=True, indent=4 if self._pretty_format else None) + data = json_utils.dumps( + data, extra_types=json_utils.EXTRA_TYPES_EXTENDED, indent=4 if self._pretty_format else None + ) f.write(data.encode()) @staticmethod diff --git a/qtoggleserver/drivers/persist/redis.py b/qtoggleserver/drivers/persist/redis.py index b1fe92f9..909574ca 100644 --- a/qtoggleserver/drivers/persist/redis.py +++ b/qtoggleserver/drivers/persist/redis.py @@ -287,11 +287,11 @@ def _record_to_db(cls, record: Record) -> GenericJSONDict: @staticmethod def _value_to_db(value: Any) -> str: - return json_utils.dumps(value, allow_extended_types=True) + return json_utils.dumps(value, extra_types=json_utils.EXTRA_TYPES_EXTENDED) @staticmethod def _value_from_db(value: str) -> Any: - return json_utils.loads(value, allow_extended_types=True) + return json_utils.loads(value, extra_types=json_utils.EXTRA_TYPES_EXTENDED) @staticmethod def _make_record_key(collection: str, id_: Id) -> str: diff --git a/qtoggleserver/frontend/js/devices/add-device-form.js b/qtoggleserver/frontend/js/devices/add-device-form.js index 6bb4df0d..e2ff3c53 100644 --- a/qtoggleserver/frontend/js/devices/add-device-form.js +++ b/qtoggleserver/frontend/js/devices/add-device-form.js @@ -90,10 +90,7 @@ class AddDeviceForm extends PageForm { }).catch(function (error) { /* Retry with /api path, which should be a default location for qToggleServer implementations */ - if (error instanceof BaseAPI.APIError && - RETRY_API_ERROR_CODES.includes(error.code) && - url.path === '/') { - + if (error instanceof BaseAPI.APIError && error.status === 404 && url.path === '/') { logger.debug('retrying with /api suffix') url.path = '/api' data.url = url.toString() diff --git a/qtoggleserver/persist/__init__.py b/qtoggleserver/persist/__init__.py index 76c1eded..5743d887 100644 --- a/qtoggleserver/persist/__init__.py +++ b/qtoggleserver/persist/__init__.py @@ -59,7 +59,7 @@ async def query( 'querying %s (%s) where %s (sort=%s, limit=%s)', collection, json_utils.dumps(fields) if fields else 'all fields', - json_utils.dumps(filt, allow_extended_types=True), + json_utils.dumps(filt, extra_types=json_utils.EXTRA_TYPES_EXTENDED), json_utils.dumps(sort), json_utils.dumps(limit), ) @@ -123,7 +123,7 @@ async def set_value(name: str, value: Any) -> None: considering the fist (and only) record.""" if logger.getEffectiveLevel() <= logging.DEBUG: - logger.debug('setting %s to %s', name, json_utils.dumps(value, allow_extended_types=True)) + logger.debug('setting %s to %s', name, json_utils.dumps(value, extra_types=json_utils.EXTRA_TYPES_EXTENDED)) driver = await _get_driver() record = {'value': value} @@ -152,7 +152,9 @@ async def insert(collection: str, record: Record) -> Id: Return the associated record ID.""" if logger.getEffectiveLevel() <= logging.DEBUG: - logger.debug('inserting %s into %s', json_utils.dumps(record, allow_extended_types=True), collection) + logger.debug( + 'inserting %s into %s', json_utils.dumps(record, extra_types=json_utils.EXTRA_TYPES_EXTENDED), collection + ) driver = await _get_driver() return await driver.insert(collection, record) @@ -170,8 +172,8 @@ async def update(collection: str, record_part: Record, filt: Optional[dict[str, logger.debug( 'updating %s where %s with %s', collection, - json_utils.dumps(filt or {}, allow_extended_types=True), - json_utils.dumps(record_part, allow_extended_types=True) + json_utils.dumps(filt or {}, extra_types=json_utils.EXTRA_TYPES_EXTENDED), + json_utils.dumps(record_part, extra_types=json_utils.EXTRA_TYPES_EXTENDED) ) driver = await _get_driver() @@ -191,7 +193,7 @@ async def replace(collection: str, id_: Id, record: Record) -> bool: logger.debug( 'replacing record with id %s with %s in %s', id_, - json_utils.dumps(record, allow_extended_types=True), + json_utils.dumps(record, extra_types=json_utils.EXTRA_TYPES_EXTENDED), collection ) @@ -219,7 +221,10 @@ async def remove(collection: str, filt: Optional[dict[str, Any]] = None) -> int: Return the total number of records that were removed.""" if logger.getEffectiveLevel() <= logging.DEBUG: - logger.debug('removing from %s where %s', collection, json_utils.dumps(filt or {}, allow_extended_types=True)) + logger.debug( + 'removing from %s where %s', + collection, json_utils.dumps(filt or {}, extra_types=json_utils.EXTRA_TYPES_EXTENDED), + ) driver = await _get_driver() count = await driver.remove(collection, filt or {}) diff --git a/qtoggleserver/utils/json.py b/qtoggleserver/utils/json.py index 7db1f86f..a782b1e7 100644 --- a/qtoggleserver/utils/json.py +++ b/qtoggleserver/utils/json.py @@ -16,6 +16,15 @@ DATE_TYPE = '__d' DATETIME_TYPE = '__dt' +DATETIME_FORMAT_ISO = '%Y-%m-%dT%H:%M:%SZ' +DATE_FORMAT_ISO = '%Y-%m-%d' +DATETIME_FORMAT_ISO_LEN = len(DATETIME_FORMAT_ISO) +DATE_FORMAT_ISO_LEN = len(DATE_FORMAT_ISO) + +EXTRA_TYPES_NONE = '' +EXTRA_TYPES_ISO = 'iso' +EXTRA_TYPES_EXTENDED = 'extended' + def _replace_nan_inf_rec(obj: Any, replace_value: Any) -> Any: if isinstance(obj, dict): @@ -54,7 +63,18 @@ def _resolve_refs_rec(obj: Any, root_obj: Any) -> Any: return obj -def encode_default_json(obj: Any) -> Any: +def encode_default_json_iso(obj: Any) -> Any: + if isinstance(obj, datetime.datetime): + return obj.strftime(DATETIME_FORMAT_ISO) + elif isinstance(obj, datetime.date): + return obj.strftime(DATE_FORMAT) + elif isinstance(obj, (set, tuple)): + return list(obj) + else: + raise TypeError() + + +def encode_default_json_extended(obj: Any) -> Any: if isinstance(obj, datetime.datetime): return { TYPE_FIELD: DATETIME_TYPE, @@ -71,7 +91,24 @@ def encode_default_json(obj: Any) -> Any: raise TypeError() -def decode_json_hook(obj: dict) -> Any: +def decode_json_hook_iso(obj: dict) -> Any: + for k, v in obj.items(): + if isinstance(v, str): + if len(v) == DATETIME_FORMAT_ISO_LEN: + try: + obj[k] = datetime.datetime.strptime(v, DATETIME_FORMAT_ISO) + except ValueError: + pass + elif len(v) == DATE_FORMAT_ISO_LEN: + try: + obj[k] = datetime.datetime.strptime(v, DATE_FORMAT_ISO).date() + except ValueError: + pass + + return obj + + +def decode_json_hook_extended(obj: dict) -> Any: __t = obj.get(TYPE_FIELD) if __t is not None: __v = obj.get(VALUE_FIELD) @@ -89,7 +126,7 @@ def decode_json_hook(obj: dict) -> Any: return obj -def dumps(obj: Any, allow_extended_types: bool = False, **kwargs) -> str: +def dumps(obj: Any, extra_types: str = EXTRA_TYPES_NONE, **kwargs) -> str: # Treat primitive types separately to gain just a bit of performance if isinstance(obj, str): return '"' + obj + '"' @@ -103,8 +140,10 @@ def dumps(obj: Any, allow_extended_types: bool = False, **kwargs) -> str: elif obj is None: return 'null' else: - if allow_extended_types: - return json.dumps(obj, default=encode_default_json, allow_nan=True, **kwargs) + if extra_types == EXTRA_TYPES_EXTENDED: + return json.dumps(obj, default=encode_default_json_extended, allow_nan=True, **kwargs) + elif extra_types == EXTRA_TYPES_ISO: + return json.dumps(obj, default=encode_default_json_iso, allow_nan=False, **kwargs) else: try: return json.dumps(obj, allow_nan=False, **kwargs) @@ -114,10 +153,15 @@ def dumps(obj: Any, allow_extended_types: bool = False, **kwargs) -> str: return json.dumps(obj, allow_nan=False, **kwargs) -def loads(s: Union[str, bytes], resolve_refs: bool = False, allow_extended_types: bool = False, **kwargs) -> Any: - object_hook = decode_json_hook if allow_extended_types else None - obj = json.loads(s, object_hook=object_hook, **kwargs) +def loads(s: Union[str, bytes], resolve_refs: bool = False, extra_types: str = EXTRA_TYPES_NONE, **kwargs) -> Any: + if extra_types == EXTRA_TYPES_EXTENDED: + object_hook = decode_json_hook_extended + elif extra_types == EXTRA_TYPES_ISO: + object_hook = decode_json_hook_iso + else: + object_hook = None + obj = json.loads(s, object_hook=object_hook, **kwargs) if resolve_refs: obj = _resolve_refs_rec(obj, root_obj=obj)