From da6ee79d37f4a35e1faa4a7e5332048ff1dcdad6 Mon Sep 17 00:00:00 2001 From: Seamile Date: Tue, 20 Jun 2023 10:24:01 +0800 Subject: [PATCH 01/12] add interrupt handler --- jsonfmt.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jsonfmt.py b/jsonfmt.py index 063c232..6d8ec34 100755 --- a/jsonfmt.py +++ b/jsonfmt.py @@ -10,6 +10,7 @@ from io import TextIOBase from pydoc import pager from shutil import get_terminal_size +from signal import signal, SIGINT from sys import stdin, stdout, stderr, exit as sys_exit from typing import Any, List, IO, Optional, Sequence, Tuple, Union from unittest.mock import patch @@ -278,6 +279,14 @@ def parse_cmdline_args(args: Optional[Sequence[str]] = None): return parser.parse_args(args) +def handle_interrupt(signum, _): + print_err('user canceled!') + sys_exit(0) + + +signal(SIGINT, handle_interrupt) + + def main(): args = parse_cmdline_args() From 38f08e92dcbde08c74caa248f5b69b520886014a Mon Sep 17 00:00:00 2001 From: Seamile Date: Sat, 12 Aug 2023 18:43:30 +0800 Subject: [PATCH 02/12] add xml_to_dict --- jsonfmt/__init__.py | 0 jsonfmt.py => jsonfmt/jsonfmt.py | 30 +++++++---------------- jsonfmt/utils.py | 16 +++++++++++++ jsonfmt/xml2.py | 41 ++++++++++++++++++++++++++++++++ pyproject.toml | 12 +++++----- test/test.py | 27 +++++++++++---------- 6 files changed, 87 insertions(+), 39 deletions(-) create mode 100644 jsonfmt/__init__.py rename jsonfmt.py => jsonfmt/jsonfmt.py (95%) create mode 100644 jsonfmt/utils.py create mode 100644 jsonfmt/xml2.py diff --git a/jsonfmt/__init__.py b/jsonfmt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jsonfmt.py b/jsonfmt/jsonfmt.py similarity index 95% rename from jsonfmt.py rename to jsonfmt/jsonfmt.py index 6d8ec34..af9cfe7 100755 --- a/jsonfmt.py +++ b/jsonfmt/jsonfmt.py @@ -2,20 +2,20 @@ '''JSON Formatter''' import json -import re -import toml -import yaml from argparse import ArgumentParser from functools import partial from io import TextIOBase from pydoc import pager from shutil import get_terminal_size -from signal import signal, SIGINT -from sys import stdin, stdout, stderr, exit as sys_exit -from typing import Any, List, IO, Optional, Sequence, Tuple, Union +from signal import SIGINT, signal +from sys import exit as sys_exit +from sys import stderr, stdin, stdout +from typing import IO, Any, List, Optional, Sequence, Tuple, Union from unittest.mock import patch import pyperclip +import toml +import yaml from jmespath import compile as jcompile from jmespath.exceptions import JMESPathError from jmespath.parser import ParsedResult as JMESPath @@ -26,10 +26,10 @@ from pygments.formatters import TerminalFormatter from pygments.lexers import JsonLexer, TOMLLexer, YamlLexer -__version__ = '0.2.6' +from .utils import load_value + +__version__ = '0.2.7' -NUMERIC = re.compile(r'-?\d+$|-?\d+\.\d+$|^-?\d+\.?\d+e-?\d+$') -DICT_OR_LIST = re.compile(r'^\{.*\}$|^\[.*\]$') QueryPath = Union[JMESPath, JSONPath] @@ -109,18 +109,6 @@ def key_or_idx(obj: Any, key: str): return py_obj, key_or_idx(py_obj, _keys[-1]) -def load_value(value: str): - if NUMERIC.match(value): - return eval(value) - elif DICT_OR_LIST.match(value): - try: - return eval(value) - except Exception: - return value - else: - return value - - def modify_pyobj(py_obj: Any, sets: List[str], pops: List[str]): '''add, modify or pop items for PyObj''' for kv in sets: diff --git a/jsonfmt/utils.py b/jsonfmt/utils.py new file mode 100644 index 0000000..bfc20c0 --- /dev/null +++ b/jsonfmt/utils.py @@ -0,0 +1,16 @@ +import re + +NUMERIC = re.compile(r'-?\d+$|-?\d+\.\d+$|^-?\d+\.?\d+e-?\d+$') +DICT_OR_LIST = re.compile(r'^\{.*\}$|^\[.*\]$') + + +def load_value(value: str): + if NUMERIC.match(value): + return eval(value) + elif DICT_OR_LIST.match(value): + try: + return eval(value) + except Exception: + return value + else: + return value diff --git a/jsonfmt/xml2.py b/jsonfmt/xml2.py new file mode 100644 index 0000000..f811aa5 --- /dev/null +++ b/jsonfmt/xml2.py @@ -0,0 +1,41 @@ +import xml.etree.ElementTree as ET +from typing import Any + +from utils import load_value + + +def element_to_dict(element: ET.Element) -> Any: + result: dict = {f'@{k}': load_value(v) for k, v in element.attrib.items()} + + if len(element) == 0: + value = load_value(element.text.strip()) if element.text else '' + if result and value: + result.update({'@text': value}) + + return result if result else value + + for child in element: + child_dict = element_to_dict(child) + if child.tag in result: + previous = result[child.tag] + if isinstance(previous, list): + previous.append(child_dict) + else: + result[child.tag] = [previous, child_dict] + else: + result[child.tag] = child_dict + + return result + + +def xml_to_dict(xml_text: str) -> dict[str, Any] | None: + try: + root = ET.fromstring(xml_text.strip()) + except ET.ParseError: + return None + + return {root.tag: element_to_dict(root)} + + +def dict_to_xml(dict) -> str: + return '' diff --git a/pyproject.toml b/pyproject.toml index 919d503..2e4a544 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,11 @@ build-backend = "setuptools.build_meta" name = "jsonfmt" dynamic = ["dependencies", "version"] requires-python = ">=3.6" -license = {text = "MIT License"} -description = "A simple tool for formatting JSON object." +license = { text = "MIT License" } +description = "A powerful tool for pretty-printing, querying and conversion JSON documents." readme = "README.md" keywords = ["json", "formatter", "pretty-print", "highlight", "jmespath"] -authors = [{name = "Seamile", email = "lanhuermao@gmail.com"}] +authors = [{ name = "Seamile", email = "lanhuermao@gmail.com" }] classifiers = [ "Development Status :: 4 - Beta", "Environment :: Console", @@ -34,8 +34,8 @@ repository = "https://github.com/seamile/jsonfmt" documentation = "https://seamile.github.io/jsonfmt/" [tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} -version = {attr = "jsonfmt.__version__" } +dependencies = { file = ["requirements.txt"] } +version = { attr = "jsonfmt.__version__" } [project.scripts] -jsonfmt = "jsonfmt:main" +jsonfmt = "jsonfmt.jsonfmt:main" diff --git a/test/test.py b/test/test.py index 775d1a9..4886f12 100644 --- a/test/test.py +++ b/test/test.py @@ -3,21 +3,22 @@ import sys import tempfile import unittest -import pyperclip from argparse import Namespace from copy import deepcopy from functools import partial from io import StringIO +from unittest.mock import patch + +import pyperclip from jmespath import compile as jcompile from jsonpath_ng import parse as jparse from pygments import highlight from pygments.formatters import TerminalFormatter -from pygments.lexers import JsonLexer, YamlLexer, TOMLLexer -from unittest.mock import patch +from pygments.lexers import JsonLexer, TOMLLexer, YamlLexer BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, BASE_DIR) -import jsonfmt +from jsonfmt import jsonfmt JSON_FILE = f'{BASE_DIR}/test/example.json' with open(JSON_FILE) as json_fp: @@ -402,7 +403,9 @@ def test_main_convert(self): self.assertEqual(jsonfmt.stdout.read(), colored_output) @patch.multiple(sys, argv=['jsonfmt', '-oc']) - @patch.multiple(jsonfmt, stdin=FakeStdIn('{"a": "asfd", "b": [1, 2, 3]}'), stdout=FakeStdOut(tty=False)) + @patch.multiple(jsonfmt, + stdin=FakeStdIn('{"a": "asfd", "b": [1, 2, 3]}'), + stdout=FakeStdOut(tty=False)) def test_main_overview(self): jsonfmt.main() self.assertEqual(jsonfmt.stdout.read().strip(), '{"a":"...","b":[]}') @@ -421,20 +424,17 @@ def test_main_overwrite_to_original_file(self): @patch.multiple(jsonfmt, stdout=FakeStdOut(), stderr=FakeStdErr()) def test_main_copy_to_clipboard(self): if jsonfmt.is_clipboard_available(): - with patch("sys.argv", - ['jsonfmt', '-Ccs', JSON_FILE]): + with patch("sys.argv", ['jsonfmt', '-Ccs', JSON_FILE]): jsonfmt.main() copied_text = pyperclip.paste().strip() self.assertEqual(copied_text, JSON_TEXT.strip()) - with patch("sys.argv", - ['jsonfmt', '-Cs', TOML_FILE]): + with patch("sys.argv", ['jsonfmt', '-Cs', TOML_FILE]): jsonfmt.main() copied_text = pyperclip.paste().strip() self.assertEqual(copied_text, TOML_TEXT.strip()) - with patch("sys.argv", - ['jsonfmt', '-Cs', YAML_FILE]): + with patch("sys.argv", ['jsonfmt', '-Cs', YAML_FILE]): jsonfmt.main() copied_text = pyperclip.paste().strip() self.assertEqual(copied_text, YAML_TEXT.strip()) @@ -448,7 +448,10 @@ def test_main_clipboard_unavailable(self): self.assertEqual(jsonfmt.stderr.read(), errmsg) self.assertEqual(jsonfmt.stdout.read(), color(JSON_TEXT, 'json')) - @patch.multiple(sys, argv=['jsonfmt', '--set', 'age=32; box=[1,2,3]', '--pop', 'money; actions.1']) + @patch.multiple(sys, + argv=['jsonfmt', + '--set', 'age=32; box=[1,2,3]', + '--pop', 'money; actions.1']) @patch.multiple(jsonfmt, stdin=FakeStdIn(JSON_TEXT), stdout=FakeStdOut(tty=False)) def test_main_modify_and_pop(self): try: From 87d3a32b9b357ed1046a67b30a2ed972880ce1f6 Mon Sep 17 00:00:00 2001 From: Seamile Date: Wed, 16 Aug 2023 14:43:33 +0800 Subject: [PATCH 03/12] add dict_to_xml --- jsonfmt/jsonfmt.py | 21 ++++++--------------- jsonfmt/utils.py | 12 +++++++++++- jsonfmt/{xml2.py => x2d.py} | 24 ++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 18 deletions(-) rename jsonfmt/{xml2.py => x2d.py} (55%) diff --git a/jsonfmt/jsonfmt.py b/jsonfmt/jsonfmt.py index af9cfe7..b9c5866 100755 --- a/jsonfmt/jsonfmt.py +++ b/jsonfmt/jsonfmt.py @@ -2,14 +2,14 @@ '''JSON Formatter''' import json -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace from functools import partial from io import TextIOBase from pydoc import pager from shutil import get_terminal_size from signal import SIGINT, signal from sys import exit as sys_exit -from sys import stderr, stdin, stdout +from sys import stdin, stdout from typing import IO, Any, List, Optional, Sequence, Tuple, Union from unittest.mock import patch @@ -26,22 +26,13 @@ from pygments.formatters import TerminalFormatter from pygments.lexers import JsonLexer, TOMLLexer, YamlLexer -from .utils import load_value +from .utils import load_value, print_err, print_inf __version__ = '0.2.7' - QueryPath = Union[JMESPath, JSONPath] -def print_inf(msg: Any): - print(f'\033[1;94mjsonfmt:\033[0m \033[0;94m{msg}\033[0m', file=stderr) - - -def print_err(msg: Any): - print(f'\033[1;91mjsonfmt:\033[0m \033[0;91m{msg}\033[0m', file=stderr) - - class FormatError(Exception): pass @@ -132,8 +123,8 @@ def modify_pyobj(py_obj: Any, sets: List[str], pops: List[str]): continue -def get_overview(py_obj: Any): - def clip(value: Any): +def get_overview(py_obj: Any) -> Any: + def clip(value: Any) -> Any: if isinstance(value, str): return '...' elif isinstance(value, (list, tuple)): @@ -232,7 +223,7 @@ def process(input_fp: IO, qpath: Optional[QueryPath], to_fmt: Optional[str], output(output_fp, formated_text, to_fmt, cp2clip) -def parse_cmdline_args(args: Optional[Sequence[str]] = None): +def parse_cmdline_args(args: Optional[Sequence[str]] = None) -> Namespace: parser = ArgumentParser('jsonfmt') parser.add_argument('-c', dest='compact', action='store_true', help='suppress all whitespace separation') diff --git a/jsonfmt/utils.py b/jsonfmt/utils.py index bfc20c0..266a4ad 100644 --- a/jsonfmt/utils.py +++ b/jsonfmt/utils.py @@ -1,10 +1,12 @@ import re +from sys import stderr +from typing import Any NUMERIC = re.compile(r'-?\d+$|-?\d+\.\d+$|^-?\d+\.?\d+e-?\d+$') DICT_OR_LIST = re.compile(r'^\{.*\}$|^\[.*\]$') -def load_value(value: str): +def load_value(value: str) -> Any: if NUMERIC.match(value): return eval(value) elif DICT_OR_LIST.match(value): @@ -14,3 +16,11 @@ def load_value(value: str): return value else: return value + + +def print_inf(msg: Any): + print(f'\033[1;94mjsonfmt:\033[0m \033[0;94m{msg}\033[0m', file=stderr) + + +def print_err(msg: Any): + print(f'\033[1;91mjsonfmt:\033[0m \033[0;91m{msg}\033[0m', file=stderr) diff --git a/jsonfmt/xml2.py b/jsonfmt/x2d.py similarity index 55% rename from jsonfmt/xml2.py rename to jsonfmt/x2d.py index f811aa5..ebe8628 100644 --- a/jsonfmt/xml2.py +++ b/jsonfmt/x2d.py @@ -37,5 +37,25 @@ def xml_to_dict(xml_text: str) -> dict[str, Any] | None: return {root.tag: element_to_dict(root)} -def dict_to_xml(dict) -> str: - return '' +def _dict_to_xml(dictionary, parent): + for key, value in dictionary.items(): + if isinstance(value, dict): + child = ET.SubElement(parent, key) + _dict_to_xml(value, child) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + child = ET.SubElement(parent, key) + _dict_to_xml(item, child) + else: + child = ET.SubElement(parent, key) + child.text = str(item) + else: + child = ET.SubElement(parent, key) + child.text = str(value) + + +def dict_to_xml(dictionary, root_tag='root') -> Any: + root = ET.Element(root_tag) + _dict_to_xml(dictionary, root) + return ET.tostring(root, encoding='utf-8').decode('utf-8') From 9c118be367c10ab934703b75abfe414ab92aedae Mon Sep 17 00:00:00 2001 From: Seamile Date: Tue, 12 Mar 2024 17:10:37 +0800 Subject: [PATCH 04/12] fix bug --- jsonfmt/x2d.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jsonfmt/x2d.py b/jsonfmt/x2d.py index ebe8628..dca32d3 100644 --- a/jsonfmt/x2d.py +++ b/jsonfmt/x2d.py @@ -1,7 +1,7 @@ import xml.etree.ElementTree as ET from typing import Any -from utils import load_value +from .utils import load_value, print_err def element_to_dict(element: ET.Element) -> Any: @@ -31,8 +31,8 @@ def element_to_dict(element: ET.Element) -> Any: def xml_to_dict(xml_text: str) -> dict[str, Any] | None: try: root = ET.fromstring(xml_text.strip()) - except ET.ParseError: - return None + except ET.ParseError as err: + print_err(err) return {root.tag: element_to_dict(root)} From 3350cc25e13d9a423e0a673a08fb0192ec5f2fc5 Mon Sep 17 00:00:00 2001 From: Seamile Date: Fri, 5 Apr 2024 14:23:33 +0800 Subject: [PATCH 05/12] 1. add sort_dict 2. complete xml conversion feature --- jsonfmt/utils.py | 12 ++++ jsonfmt/x2d.py | 61 ------------------ jsonfmt/xml2py.py | 155 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 61 deletions(-) delete mode 100644 jsonfmt/x2d.py create mode 100644 jsonfmt/xml2py.py diff --git a/jsonfmt/utils.py b/jsonfmt/utils.py index 38c5c92..22f02f4 100644 --- a/jsonfmt/utils.py +++ b/jsonfmt/utils.py @@ -1,6 +1,7 @@ import re import sys from typing import Any +from collections import OrderedDict NUMERIC = re.compile(r'-?\d+$|-?\d+\.\d+$|^-?\d+\.?\d+e-?\d+$') DICT_OR_LIST = re.compile(r'^\{.*\}$|^\[.*\]$') @@ -18,6 +19,17 @@ def load_value(value: str) -> Any: return value +def sort_dict(py_obj: Any) -> Any: + '''sort the dicts in py_obj by keys''' + if isinstance(py_obj, dict): + sorted_items = sorted((key, sort_dict(value)) for key, value in py_obj.items()) + return OrderedDict(sorted_items) + elif isinstance(py_obj, list): + return [sort_dict(item) for item in py_obj] + else: + return py_obj + + def print_inf(msg: Any): print(f'\033[0;94m{msg}\033[0m', file=sys.stderr) diff --git a/jsonfmt/x2d.py b/jsonfmt/x2d.py deleted file mode 100644 index dca32d3..0000000 --- a/jsonfmt/x2d.py +++ /dev/null @@ -1,61 +0,0 @@ -import xml.etree.ElementTree as ET -from typing import Any - -from .utils import load_value, print_err - - -def element_to_dict(element: ET.Element) -> Any: - result: dict = {f'@{k}': load_value(v) for k, v in element.attrib.items()} - - if len(element) == 0: - value = load_value(element.text.strip()) if element.text else '' - if result and value: - result.update({'@text': value}) - - return result if result else value - - for child in element: - child_dict = element_to_dict(child) - if child.tag in result: - previous = result[child.tag] - if isinstance(previous, list): - previous.append(child_dict) - else: - result[child.tag] = [previous, child_dict] - else: - result[child.tag] = child_dict - - return result - - -def xml_to_dict(xml_text: str) -> dict[str, Any] | None: - try: - root = ET.fromstring(xml_text.strip()) - except ET.ParseError as err: - print_err(err) - - return {root.tag: element_to_dict(root)} - - -def _dict_to_xml(dictionary, parent): - for key, value in dictionary.items(): - if isinstance(value, dict): - child = ET.SubElement(parent, key) - _dict_to_xml(value, child) - elif isinstance(value, list): - for item in value: - if isinstance(item, dict): - child = ET.SubElement(parent, key) - _dict_to_xml(item, child) - else: - child = ET.SubElement(parent, key) - child.text = str(item) - else: - child = ET.SubElement(parent, key) - child.text = str(value) - - -def dict_to_xml(dictionary, root_tag='root') -> Any: - root = ET.Element(root_tag) - _dict_to_xml(dictionary, root) - return ET.tostring(root, encoding='utf-8').decode('utf-8') diff --git a/jsonfmt/xml2py.py b/jsonfmt/xml2py.py new file mode 100644 index 0000000..c563a7a --- /dev/null +++ b/jsonfmt/xml2py.py @@ -0,0 +1,155 @@ +import re +import xml.etree.ElementTree as ET +from collections.abc import Mapping +from copy import deepcopy +from typing import Any, Optional, Self +from xml.dom.minidom import parseString + +from .utils import load_value, sort_dict + +RESERVED_CHARS = re.compile(r'[<>&"\']') + + +class Element(ET.Element): + def __init__(self, + tag: str, + attrib={}, + text: Optional[str] = None, + tail: Optional[str] = None, + **extra) -> None: + super().__init__(tag, attrib, **extra) + self.text = text + self.tail = tail + self.parent: Optional[Self] = None + + def __str__(self): + def _(ele, n=0): + indent = ' ' * n + line = f'{indent}{ele.tag} : ATTRIB={ele.attrib} TEXT={ele.text}\n' + for e in ele: + line += _(e, n + 1) # type: ignore + return line + return _(self) + + @classmethod + def makeelement(cls, tag, attrib, text=None, tail=None) -> Self: + """Create a new element with the same type.""" + return cls(tag, attrib, text, tail) + + @classmethod + def from_xml(cls, xml: str) -> Self: + def clone(src: ET.Element, dst: Self): + for child in src: + _child = dst.spawn(child.tag, child.attrib, child.text, child.tail) + clone(child, _child) + + root = ET.fromstring(xml.strip()) + ele = cls(root.tag, root.attrib, root.text, root.tail) + clone(root, ele) + return ele + + def to_xml(self, + minimal: Optional[bool] = None, + indent: Optional[int | str] = 2 + ) -> str: + ele = deepcopy(self) + for e in ele.iter(): + if len(e) > 0: + e.text = e.text.strip() if isinstance(e.text, str) else None + e.tail = e.tail.strip() if isinstance(e.tail, str) else None + + xml = ET.tostring(ele, 'unicode') + if minimal or indent is None: + return xml + else: + doc = parseString(xml) + if isinstance(indent, int) or indent.isdecimal(): + indent = ' ' * int(indent) + else: + indent = '\t' + return doc.toprettyxml(indent=indent) + + def spawn(self, tag: str, attrib={}, text=None, tail=None, **extra) -> Self: + """Create and append a new child element to the current element.""" + attrib = {**attrib, **extra} + child = self.makeelement(tag, attrib, text, tail) + child.parent = self + self.append(child) + return child + + def _get_attrs(self): + attrs = {f'@{k}': load_value(v) for k, v in self.attrib.items()} + + if len(self) == 0: + if not self.text: + return attrs or None + else: + value = load_value(self.text.strip()) + if attrs and value: + attrs['@text'] = value + return attrs or value + else: + for child in self: + child_attrs = child._get_attrs() # type: ignore + if child.tag in attrs: + # Make a list for duplicate tags + previous = attrs[child.tag] + if isinstance(previous, list): + previous.append(child_attrs) + else: + attrs[child.tag] = [previous, child_attrs] + else: + attrs[child.tag] = child_attrs + return attrs + + def to_py(self) -> Any: + '''Convert into Python object''' + attrs = self._get_attrs() + return attrs if self.tag == 'root' else {self.tag: attrs} + + def _set_attrs(self, py_obj: Any): + if isinstance(py_obj, Mapping): + for key, value in py_obj.items(): + if key == '@text': + self.text = str(value) + elif key[0] == '@': + self.set(key[1:], str(value)) + else: + self.spawn(key)._set_attrs(value) + elif isinstance(py_obj, (list, tuple, set)): + for i, item in enumerate(py_obj): + ele = self if i == 0 else self.parent.spawn(self.tag) # type: ignore + if isinstance(item, Mapping): + ele._set_attrs(item) + elif isinstance(item, (list, tuple, set)): + ele._set_attrs(str(item)) + else: + ele.text = str(item) + else: + self.text = str(py_obj) + + @classmethod + def from_py(cls, py_obj: Any): + if not isinstance(py_obj, dict) or len(py_obj) != 1: + element = cls('root') + else: + tag, value = list(py_obj.items())[0] + if isinstance(value, Mapping): + element = cls(tag) + py_obj = value + else: + element = cls('root') + element._set_attrs(py_obj) + return element + + +def loads(xml: str) -> Any: + '''Load and convert an XML string into a Python object''' + return Element.from_xml(xml).to_py() + + +def dumps(py_obj: Any, indent: str = 't', minimal: bool = False, + sort_keys: bool = False) -> str: + if sort_keys: + py_obj = sort_dict(py_obj) + return Element.from_py(py_obj).to_xml(indent=indent, minimal=minimal) From f715ab459341482dbb7c49ff0d4d4a6bdbd5df22 Mon Sep 17 00:00:00 2001 From: Seamile Date: Fri, 5 Apr 2024 15:04:44 +0800 Subject: [PATCH 06/12] 1. improve the load_value() to safe_eval() 2. fix bug --- jsonfmt/jsonfmt.py | 63 +++++++++++++++++----------------------------- jsonfmt/utils.py | 21 ++++++---------- jsonfmt/xml2py.py | 16 ++++++------ 3 files changed, 38 insertions(+), 62 deletions(-) diff --git a/jsonfmt/jsonfmt.py b/jsonfmt/jsonfmt.py index 14cec27..2bcd7c0 100755 --- a/jsonfmt/jsonfmt.py +++ b/jsonfmt/jsonfmt.py @@ -6,7 +6,6 @@ import os import sys from argparse import ArgumentParser -from collections import OrderedDict from functools import partial from pydoc import pager from shutil import get_terminal_size @@ -27,9 +26,8 @@ from pygments.formatters import TerminalFormatter from pygments.lexers import JsonLexer, TOMLLexer, YamlLexer -from . import __version__ +from . import __version__, utils from .diff import compare -from .utils import exit_with_error, load_value, print_err, print_inf QueryPath = Union[JMESPath, JSONPath] TEMP_CLIPBOARD = io.StringIO() @@ -55,7 +53,7 @@ def parse_querypath(querypath: Optional[str], querylang: Optional[str]): elif querylang in ['jmespath', 'jsonpath']: parsers = [{'jmespath': parse_jmespath, 'jsonpath': parse_jsonpath}[querylang]] else: - exit_with_error(f'invalid querylang: "{querylang}"') + utils.exit_with_error(f'invalid querylang: "{querylang}"') for parse in parsers: try: @@ -63,7 +61,7 @@ def parse_querypath(querypath: Optional[str], querylang: Optional[str]): except (JMESPathError, JSONPathError, AttributeError): pass - exit_with_error(f'invalid querypath expression: "{querypath}"') + utils.exit_with_error(f'invalid querypath expression: "{querypath}"') def extract_elements(qpath: QueryPath, py_obj: Any) -> Any: @@ -122,17 +120,6 @@ def key_or_idx(obj: Any, key: str): return py_obj, key_or_idx(py_obj, _keys[-1]) -def sort_dict(py_obj: Any) -> Any: - '''sort the dicts in py_obj by keys''' - if isinstance(py_obj, dict): - sorted_items = sorted((key, sort_dict(value)) for key, value in py_obj.items()) - return OrderedDict(sorted_items) - elif isinstance(py_obj, list): - return [sort_dict(item) for item in py_obj] - else: - return py_obj - - def modify_pyobj(py_obj: Any, sets: List[str], pops: List[str]): '''add, modify or pop items for PyObj''' for kv in sets: @@ -140,11 +127,11 @@ def modify_pyobj(py_obj: Any, sets: List[str], pops: List[str]): keys, value = kv.split('=') bottom, last_k = traverse_to_bottom(py_obj, keys) if isinstance(bottom, list) and len(bottom) <= last_k: # type: ignore - bottom.append(load_value(value)) + bottom.append(utils.safe_eval(value)) else: - bottom[last_k] = load_value(value) # type: ignore + bottom[last_k] = utils.safe_eval(value) # type: ignore except (IndexError, KeyError, ValueError, TypeError): - print_err(f'invalid key path: {kv}') + utils.print_err(f'invalid key path: {kv}') continue for keys in pops: @@ -152,7 +139,7 @@ def modify_pyobj(py_obj: Any, sets: List[str], pops: List[str]): bottom, last_k = traverse_to_bottom(py_obj, keys) bottom.pop(last_k) except (IndexError, KeyError): - print_err(f'invalid key path: {keys}') + utils.print_err(f'invalid key path: {keys}') continue @@ -189,7 +176,9 @@ def format_to_text(py_obj: Any, fmt: str, *, if not isinstance(py_obj, dict): msg = 'the pyobj must be a Mapping when format to toml' raise FormatError(msg) - result = toml.dumps(sort_dict(py_obj) if sort_keys else py_obj) + result = toml.dumps(utils.sort_dict(py_obj) if sort_keys else py_obj) + elif fmt == 'xml': + result = '' elif fmt == 'yaml': _indent = None if indent == 't' else int(indent) result = yaml.safe_dump(py_obj, allow_unicode=not escape, indent=_indent, @@ -238,7 +227,7 @@ def output(output_fp: IO, text: str, fmt: str): output_fp.truncate() output_fp.write(text) output_fp.close() - print_inf(f'result written to {os.path.basename(output_fp.name)}') + utils.print_inf(f'result written to {os.path.basename(output_fp.name)}') elif isinstance(output_fp, io.StringIO) and output_fp.tell() != 0: output_fp.write('\n\n') output_fp.write(text) @@ -316,18 +305,18 @@ def main(_args: Optional[Sequence[str]] = None): # check if the clipboard is available if args.cp2clip and not is_clipboard_available(): - exit_with_error('clipboard is not available') + utils.exit_with_error('clipboard is not available') # check the input files files = args.files or [sys.stdin] n_files = len(files) if n_files < 1: - exit_with_error('no data file specified') + utils.exit_with_error('no data file specified') # check the diff mode diff_mode: bool = args.diff or args.difftool if diff_mode and len(files) != 2: - exit_with_error('less than two files') + utils.exit_with_error('less than two files') sort_keys = True if diff_mode else args.sort_keys del_tmpfile = False if args.difftool == 'code' else True @@ -353,19 +342,14 @@ def main(_args: Optional[Sequence[str]] = None): # output the result output_fp = get_output_fp(input_fp, args.cp2clip, diff_mode, args.overview, args.overwrite, del_tmpfile) - output(output_fp, formated, fmt) - if args.diff or args.difftool: - diff_files.append(output_fp) - except (FormatError, JMESPathError, JSONPathError) as err: - exit_with_error(err) - except FileNotFoundError: - exit_with_error(f'no such file: {file}') - except PermissionError: - exit_with_error(f'permission denied: {file}') - except KeyboardInterrupt: - exit_with_error('user canceled') + if diff_mode: + diff_files.append(output_fp.name) + except (FormatError, JMESPathError, JSONPathError, OSError) as err: + utils.exit_with_error(err) + except KeyboardInterrupt: + utils.exit_with_error('user canceled') finally: input_fp = locals().get('input_fp') if isinstance(input_fp, io.TextIOBase): @@ -374,13 +358,12 @@ def main(_args: Optional[Sequence[str]] = None): if args.cp2clip: TEMP_CLIPBOARD.seek(0) pyperclip.copy(TEMP_CLIPBOARD.read()) - print_inf('result copied to clipboard') + utils.print_inf('result copied to clipboard') elif diff_mode: try: - path1, path2 = [f.name for f in diff_files] - compare(path1, path2, args.difftool) + compare(diff_files[0], diff_files[1], args.difftool) except (OSError, ValueError) as err: - exit_with_error(err) + utils.exit_with_error(err) if __name__ == "__main__": diff --git a/jsonfmt/utils.py b/jsonfmt/utils.py index 22f02f4..ff699d1 100644 --- a/jsonfmt/utils.py +++ b/jsonfmt/utils.py @@ -1,21 +1,14 @@ -import re import sys -from typing import Any +from ast import literal_eval from collections import OrderedDict - -NUMERIC = re.compile(r'-?\d+$|-?\d+\.\d+$|^-?\d+\.?\d+e-?\d+$') -DICT_OR_LIST = re.compile(r'^\{.*\}$|^\[.*\]$') +from typing import Any -def load_value(value: str) -> Any: - if NUMERIC.match(value): - return eval(value) - elif DICT_OR_LIST.match(value): - try: - return eval(value) - except Exception: - return value - else: +def safe_eval(value: str) -> Any: + '''Safely evaluates the provided string expression as a Python literal''' + try: + return literal_eval(value) + except (ValueError, SyntaxError): return value diff --git a/jsonfmt/xml2py.py b/jsonfmt/xml2py.py index c563a7a..d57d8d1 100644 --- a/jsonfmt/xml2py.py +++ b/jsonfmt/xml2py.py @@ -5,7 +5,7 @@ from typing import Any, Optional, Self from xml.dom.minidom import parseString -from .utils import load_value, sort_dict +from .utils import safe_eval, sort_dict RESERVED_CHARS = re.compile(r'[<>&"\']') @@ -77,27 +77,27 @@ def spawn(self, tag: str, attrib={}, text=None, tail=None, **extra) -> Self: self.append(child) return child - def _get_attrs(self): - attrs = {f'@{k}': load_value(v) for k, v in self.attrib.items()} + def _get_attrs(self) -> Optional[dict[str, Any]]: + attrs = {f'@{k}': safe_eval(v) for k, v in self.attrib.items()} if len(self) == 0: if not self.text: return attrs or None else: - value = load_value(self.text.strip()) + value = safe_eval(self.text.strip()) if attrs and value: attrs['@text'] = value return attrs or value else: - for child in self: + for n, child in enumerate(self, start=1): child_attrs = child._get_attrs() # type: ignore if child.tag in attrs: # Make a list for duplicate tags previous = attrs[child.tag] - if isinstance(previous, list): - previous.append(child_attrs) - else: + if n == 2: attrs[child.tag] = [previous, child_attrs] + else: + previous.append(child_attrs) else: attrs[child.tag] = child_attrs return attrs From ad76e999c62ced134753498e83754e9b9f80669c Mon Sep 17 00:00:00 2001 From: Seamile Date: Fri, 5 Apr 2024 16:00:50 +0800 Subject: [PATCH 07/12] add XML format support --- jsonfmt/jsonfmt.py | 14 ++++++++------ test/example.xml | 13 +++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 test/example.xml diff --git a/jsonfmt/jsonfmt.py b/jsonfmt/jsonfmt.py index 2bcd7c0..049af6b 100755 --- a/jsonfmt/jsonfmt.py +++ b/jsonfmt/jsonfmt.py @@ -24,9 +24,9 @@ from jsonpath_ng.exceptions import JSONPathError from pygments import highlight from pygments.formatters import TerminalFormatter -from pygments.lexers import JsonLexer, TOMLLexer, YamlLexer +from pygments.lexers import JsonLexer, TOMLLexer, XmlLexer, YamlLexer -from . import __version__, utils +from . import __version__, utils, xml2py from .diff import compare QueryPath = Union[JMESPath, JSONPath] @@ -85,6 +85,7 @@ def parse_to_pyobj(text: str, qpath: Optional[QueryPath]) -> Tuple[Any, str]: loads_methods: dict[str, Callable] = { 'json': json.loads, 'toml': toml.loads, + 'xml': xml2py.loads, 'yaml': partial(yaml.load, Loader=yaml.Loader), } @@ -178,7 +179,7 @@ def format_to_text(py_obj: Any, fmt: str, *, raise FormatError(msg) result = toml.dumps(utils.sort_dict(py_obj) if sort_keys else py_obj) elif fmt == 'xml': - result = '' + result = xml2py.dumps(py_obj, indent, compact, sort_keys) elif fmt == 'yaml': _indent = None if indent == 't' else int(indent) result = yaml.safe_dump(py_obj, allow_unicode=not escape, indent=_indent, @@ -209,7 +210,8 @@ def output(output_fp: IO, text: str, fmt: str): if hasattr(output_fp, 'name') and output_fp.name == '': if output_fp.isatty(): # highlight the text when output to TTY divice - Lexer = {'json': JsonLexer, 'toml': TOMLLexer, 'yaml': YamlLexer}[fmt] + Lexer = {'json': JsonLexer, 'toml': TOMLLexer, + 'xml': XmlLexer, 'yaml': YamlLexer}[fmt] colored_text = highlight(text, Lexer(), TerminalFormatter()) win_w, win_h = get_terminal_size() # use pager when line-hight > screen hight or @@ -274,7 +276,7 @@ def parse_cmdline_args() -> ArgumentParser: help='Suppress all whitespace separation (most compact), only valid for JSON') parser.add_argument('-e', dest='escape', action='store_true', help='escape non-ASCII characters') - parser.add_argument('-f', dest='format', choices=['json', 'toml', 'yaml'], + parser.add_argument('-f', dest='format', choices=['json', 'toml', 'xml', 'yaml'], help='the format to output (default: same as input)') parser.add_argument('-i', dest='indent', metavar='{0-8 or t}', choices='012345678t', default='2', @@ -347,7 +349,7 @@ def main(_args: Optional[Sequence[str]] = None): if diff_mode: diff_files.append(output_fp.name) except (FormatError, JMESPathError, JSONPathError, OSError) as err: - utils.exit_with_error(err) + utils.print_err(err) except KeyboardInterrupt: utils.exit_with_error('user canceled') finally: diff --git a/test/example.xml b/test/example.xml new file mode 100644 index 0000000..d5fdb45 --- /dev/null +++ b/test/example.xml @@ -0,0 +1,13 @@ + + + + + HR + 50000 + + + + Tech + 60000 + + From b2610571a80cbd31246c521babe04d5ca8047042 Mon Sep 17 00:00:00 2001 From: Seamile Date: Mon, 8 Apr 2024 11:05:42 +0800 Subject: [PATCH 08/12] 1. fix bugs in xmltopy 2. temp files are not deleted in DiffMode --- jsonfmt/jsonfmt.py | 7 +++-- jsonfmt/xml2py.py | 67 ++++++++++++++++++++++++++-------------------- test/example.json | 2 +- test/example.toml | 8 ++++-- test/example.xml | 33 ++++++++++++++--------- test/example.yaml | 7 +++-- 6 files changed, 74 insertions(+), 50 deletions(-) diff --git a/jsonfmt/jsonfmt.py b/jsonfmt/jsonfmt.py index 049af6b..b5fdb5a 100755 --- a/jsonfmt/jsonfmt.py +++ b/jsonfmt/jsonfmt.py @@ -191,13 +191,13 @@ def format_to_text(py_obj: Any, fmt: str, *, def get_output_fp(input_file: IO, cp2clip: bool, diff: bool, - overview: bool, overwrite: bool, del_tmpfile=True) -> IO: + overview: bool, overwrite: bool) -> IO: if cp2clip: return TEMP_CLIPBOARD elif diff: name = f"_{os.path.basename(input_file.name)}" return NamedTemporaryFile(mode='w+', prefix='jf-', suffix=name, - delete=del_tmpfile, delete_on_close=False) + delete=False, delete_on_close=False) elif input_file is sys.stdin or overview: return sys.stdout elif overwrite: @@ -320,7 +320,6 @@ def main(_args: Optional[Sequence[str]] = None): if diff_mode and len(files) != 2: utils.exit_with_error('less than two files') sort_keys = True if diff_mode else args.sort_keys - del_tmpfile = False if args.difftool == 'code' else True # get sets and pops sets = [k.strip() for k in args.set.split(';')] if args.set else [] @@ -343,7 +342,7 @@ def main(_args: Optional[Sequence[str]] = None): sort_keys=sort_keys, sets=sets, pops=pops) # output the result output_fp = get_output_fp(input_fp, args.cp2clip, diff_mode, - args.overview, args.overwrite, del_tmpfile) + args.overview, args.overwrite) output(output_fp, formated, fmt) if diff_mode: diff --git a/jsonfmt/xml2py.py b/jsonfmt/xml2py.py index d57d8d1..e324979 100644 --- a/jsonfmt/xml2py.py +++ b/jsonfmt/xml2py.py @@ -1,4 +1,3 @@ -import re import xml.etree.ElementTree as ET from collections.abc import Mapping from copy import deepcopy @@ -7,10 +6,12 @@ from .utils import safe_eval, sort_dict -RESERVED_CHARS = re.compile(r'[<>&"\']') +class _list(list): + pass -class Element(ET.Element): + +class XmlElement(ET.Element): def __init__(self, tag: str, attrib={}, @@ -37,16 +38,20 @@ def makeelement(cls, tag, attrib, text=None, tail=None) -> Self: return cls(tag, attrib, text, tail) @classmethod - def from_xml(cls, xml: str) -> Self: - def clone(src: ET.Element, dst: Self): - for child in src: - _child = dst.spawn(child.tag, child.attrib, child.text, child.tail) - clone(child, _child) + def clone(cls, src: Self | ET.Element, dst: Optional[Self] = None) -> Self: + if dst is None: + dst = cls(src.tag, src.attrib, src.text, src.tail) + + for child in src: + _child = dst.spawn(child.tag, child.attrib, child.text, child.tail) + cls.clone(child, _child) + return dst + + @classmethod + def from_xml(cls, xml: str) -> Self: root = ET.fromstring(xml.strip()) - ele = cls(root.tag, root.attrib, root.text, root.tail) - clone(root, ele) - return ele + return cls.clone(root) def to_xml(self, minimal: Optional[bool] = None, @@ -89,13 +94,13 @@ def _get_attrs(self) -> Optional[dict[str, Any]]: attrs['@text'] = value return attrs or value else: - for n, child in enumerate(self, start=1): + for child in self: child_attrs = child._get_attrs() # type: ignore if child.tag in attrs: # Make a list for duplicate tags previous = attrs[child.tag] - if n == 2: - attrs[child.tag] = [previous, child_attrs] + if not isinstance(previous, _list): + attrs[child.tag] = _list([previous, child_attrs]) else: previous.append(child_attrs) else: @@ -114,42 +119,46 @@ def _set_attrs(self, py_obj: Any): self.text = str(value) elif key[0] == '@': self.set(key[1:], str(value)) + elif isinstance(value, list): + for v in value: + self.spawn(key)._set_attrs(v) else: self.spawn(key)._set_attrs(value) elif isinstance(py_obj, (list, tuple, set)): - for i, item in enumerate(py_obj): - ele = self if i == 0 else self.parent.spawn(self.tag) # type: ignore - if isinstance(item, Mapping): - ele._set_attrs(item) - elif isinstance(item, (list, tuple, set)): - ele._set_attrs(str(item)) - else: - ele.text = str(item) + if self.parent is None: + return self.spawn(self.tag)._set_attrs(py_obj) + else: + for i, item in enumerate(py_obj): + ele = self.parent[i] if len(self.parent) > i else self.parent.spawn(self.tag) + if isinstance(item, (list, tuple, set)): + ele.text = str(item) + else: + ele._set_attrs(item) # type: ignore else: self.text = str(py_obj) @classmethod def from_py(cls, py_obj: Any): if not isinstance(py_obj, dict) or len(py_obj) != 1: - element = cls('root') + root = cls('root') else: tag, value = list(py_obj.items())[0] if isinstance(value, Mapping): - element = cls(tag) + root = cls(tag) py_obj = value else: - element = cls('root') - element._set_attrs(py_obj) - return element + root = cls('root') + root._set_attrs(py_obj) + return root def loads(xml: str) -> Any: '''Load and convert an XML string into a Python object''' - return Element.from_xml(xml).to_py() + return XmlElement.from_xml(xml).to_py() def dumps(py_obj: Any, indent: str = 't', minimal: bool = False, sort_keys: bool = False) -> str: if sort_keys: py_obj = sort_dict(py_obj) - return Element.from_py(py_obj).to_xml(indent=indent, minimal=minimal) + return XmlElement.from_py(py_obj).to_xml(indent=indent, minimal=minimal) diff --git a/test/example.json b/test/example.json index d906864..0c440ac 100644 --- a/test/example.json +++ b/test/example.json @@ -1 +1 @@ -{"name":"Bob","age":23,"gender":"纯爷们","money":3.1415926,"actions":[{"name":"eat","calorie":294.9,"date":"2021-03-02"},{"name":"sport","calorie":-375,"date":"2023-04-27"}]} +{"name":"Bob","age":23,"gender":"纯爷们","money":3.1415926,"actions":[{"name":"eat","calorie":1294.9,"date":"2021-03-02"},{"name":"sport","calorie":-2375,"date":"2023-04-27"},{"name":"sleep","calorie":-420.5,"date":"2023-05-15"}]} diff --git a/test/example.toml b/test/example.toml index 2818b07..0ba2268 100644 --- a/test/example.toml +++ b/test/example.toml @@ -4,11 +4,15 @@ gender = "纯爷们" money = 3.1415926 [[actions]] name = "eat" -calorie = 294.9 +calorie = 1294.9 date = "2021-03-02" [[actions]] name = "sport" -calorie = -375 +calorie = -2375 date = "2023-04-27" +[[actions]] +name = "sleep" +calorie = -420.5 +date = "2023-05-15" diff --git a/test/example.xml b/test/example.xml index d5fdb45..86d4ec0 100644 --- a/test/example.xml +++ b/test/example.xml @@ -1,13 +1,22 @@ - - - - HR - 50000 - - - - Tech - 60000 - - + + Bob + 23 + 纯爷们 + 3.1415926 + + eat + 1294.9 + 2021-03-02 + + + sport + -2375 + 2023-04-27 + + + sleep + -420.5 + 2023-05-15 + + diff --git a/test/example.yaml b/test/example.yaml index d7159e6..dc289d1 100644 --- a/test/example.yaml +++ b/test/example.yaml @@ -4,8 +4,11 @@ gender: 纯爷们 money: 3.1415926 actions: - name: eat - calorie: 294.9 + calorie: 1294.9 date: '2021-03-02' - name: sport - calorie: -375 + calorie: -2375 date: '2023-04-27' +- name: sleep + calorie: -420.5 + date: '2023-05-15' From 91ac55f2251d8df3cd1853926824d27e0833f885 Mon Sep 17 00:00:00 2001 From: Seamile Date: Mon, 8 Apr 2024 17:35:16 +0800 Subject: [PATCH 09/12] 1. modify docs 2. fix bugs --- README.md | 201 ++++++++++++++++++++++++------------------- README_CN.md | 208 +++++++++++++++++++++++++-------------------- jsonfmt/jsonfmt.py | 14 +-- jsonfmt/xml2py.py | 7 ++ 4 files changed, 245 insertions(+), 185 deletions(-) diff --git a/README.md b/README.md index f0d20a4..05e6d38 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ -### +### **_jsonfmt_** (JSON Formatter) is a simple yet powerful JSON processing tool. @@ -22,7 +22,7 @@ As we all know, Python has a built-in tool for formatting JSON data: `python -m 🎨 It can not only print JSON data in a pretty way, -🔄 But also convert JSON, TOML, and YAML data formats to each other, +🔄 But also convert JSON, TOML, XML and YAML data formats to each other, 🔎 And even extract content from JSON data using JMESPATH or JSONPATH. @@ -68,7 +68,7 @@ $ pip install jsonfmt **Positional Arguments** -`files`: The data files to process, supporting JSON / TOML / YAML formats. +`files`: The data files to process, supporting JSON / TOML / XML / YAML formats. **Options** @@ -80,7 +80,7 @@ $ pip install jsonfmt - `-O`: OverwriteMode, which will overwrite the original file with the formated text. - `-c`: Suppress all whitespace separation (most compact), only valid for JSON. - `-e`: Escape all characters to ASCII codes. -- `-f`: The format to output (default: same as input data format, options: `json` / `toml` / `yaml`). +- `-f`: The format to output (default: same as input data format, options: `json` / `toml` / `xml` / `yaml`). - `-i`: Number of spaces for indentation (default: 2, range: 0~8, set to 't' to use Tab as indentation). - `-l`: Query language for extracting data (default: auto-detect, options: jmespath / jsonpath). - `-p QUERYPATH`: JMESPath or JSONPath query path. @@ -103,19 +103,24 @@ In order to demonstrate the features of jsonfmt, we need to first create a test "actions": [ { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" }, { "name": "sport", - "calorie": -375, + "calorie": -2375, "date": "2023-04-27" + }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" } ] } ``` -Then, convert this data to TOML and YAML formats, and save them as example.toml and example.yaml respectively. +Then, convert this data to TOML, XML and YAML formats, and save them as example.toml, example.xml and example.yaml respectively. These data files can be found in the *test* folder of the source code: @@ -123,6 +128,7 @@ These data files can be found in the *test* folder of the source code: test/ |- example.json |- example.toml +|- example.xml |- example.yaml ``` @@ -148,14 +154,19 @@ Output: { "actions": [ { - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02", "name": "eat" }, { - "calorie": -375, + "calorie": -2375, "date": "2023-04-27", "name": "sport" + }, + { + "calorie": -420.5, + "date": "2023-05-15", + "name": "sleep" } ], "age": 23, @@ -227,16 +238,16 @@ JMESPath can elegantly use simple syntax to extract part of the content from JSO ```json { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" } ``` -- Filter all items with `calorie > 0` from `actions`. +- Filter all items with `calorie < 0` from `actions`. ```shell # Here, `0` means 0 is a number - $ jf -p 'actions[?calorie>`0`]' test/example.json + $ jf -p 'actions[?calorie<`0`]' test/example.json ``` Output: @@ -244,9 +255,14 @@ JMESPath can elegantly use simple syntax to extract part of the content from JSO ```json [ { - "name": "eat", - "calorie": 294.9, - "date": "2021-03-02" + "name": "sport", + "calorie": -2375, + "date": "2023-04-27" + }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" } ] ``` @@ -268,7 +284,7 @@ JMESPath can elegantly use simple syntax to extract part of the content from JSO "money", "actions" ], - "actions_len": 2 + "actions_len": 3 } ``` @@ -284,11 +300,15 @@ JMESPath can elegantly use simple syntax to extract part of the content from JSO [ { "foo": "sport", - "bar": -375 + "bar": -2375 + }, + { + "foo": "sleep", + "bar": -420.5 }, { "foo": "eat", - "bar": 294.9 + "bar": 1294.9 } ] ``` @@ -316,13 +336,14 @@ Some queries that are difficult to handle with JMESPath can be easily achieved w [ "Bob", "eat", - "sport" + "sport", + "sleep" ] ``` -#### Querying TOML and YAML +#### Querying TOML, XML and YAML -One of the powerful features of jsonfmt is that you can process TOML and YAML in exactly the same way as JSON, and freely convert the result format. You can even process these three formats simultaneously in one command. +One of the powerful features of jsonfmt is that you can process TOML, XML and YAML in exactly the same way as JSON, and freely convert the result format. You can even process these four formats simultaneously in one command. - Read data from a toml file and output in YAML format @@ -339,7 +360,7 @@ One of the powerful features of jsonfmt is that you can process TOML and YAML in - gender - money - actions - actions_len: 2 + actions_len: 3 ``` - Process three formats at once @@ -354,57 +375,44 @@ One of the powerful features of jsonfmt is that you can process TOML and YAML in 1. test/example.json { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" } 2. test/example.toml name = "eat" - calorie = 294.9 + calorie = 1294.9 date = "2021-03-02" - 3. test/example.yaml + 3. test/example.xml + + + eat + 1294.9 + 2021-03-02 + + + 4. test/example.yaml name: eat - calorie: 294.9 + calorie: 1294.9 date: '2021-03-02' ``` ### 4. Format Conversion -*jsonfmt* supports processing JSON, TOML, and YAML formats. Each format can be converted to other formats by specifying the "-f" option. +*jsonfmt* supports processing JSON, TOML, XML and YAML formats. Each format can be converted to other formats by specifying the "-f" option.
Note:
-In TOML, `null` values are invalid. Therefore, when converting from other formats to TOML, all null values will be removed. - -#### JSON to TOML -```shell -$ jf test/example.json -f toml -``` +1. `null` is not supported in TOML. Therefore, all `null` values will be deleted when converting from other formats to TOML. -Output: +2. XML does not support multi-dimensional arrays. Therefore, if the original data contains multi-dimensional arrays, a wrong data will be generated during the conversion to XML format. -```toml -name = "Bob" -age = 23 -gender = "纯爷们" -money = 3.1415926 -[[actions]] -name = "eat" -calorie = 294.9 -date = "2021-03-02" - -[[actions]] -name = "sport" -calorie = -375 -date = "2023-04-27" -``` - -#### TOML to YAML +#### Example 1. JSON to YAML ```shell -$ jf test/example.toml -f yaml +$ jf test/example.json -f yaml ``` Output: @@ -416,40 +424,47 @@ gender: 纯爷们 money: 3.1415926 actions: - name: eat - calorie: 294.9 + calorie: 1294.9 date: '2021-03-02' - name: sport - calorie: -375 + calorie: -2375 date: '2023-04-27' +- name: sleep + calorie: -420.5 + date: '2023-05-15' ``` -#### YAML to JSON +#### Example 2. TOML to XML ```shell -$ jf test/example.yaml -f json +$ jf test/example.toml -f xml ``` Output: -```json -{ - "name": "Bob", - "age": 23, - "gender": "纯爷们", - "money": 3.1415926, - "actions": [ - { - "name": "eat", - "calorie": 294.9, - "date": "2021-03-02" - }, - { - "name": "sport", - "calorie": -375, - "date": "2023-04-27" - } - ] -} +```xml + + + Bob + 23 + 纯爷们 + 3.1415926 + + eat + 1294.9 + 2021-03-02 + + + sport + -2375 + 2023-04-27 + + + sleep + -420.5 + 2023-05-15 + + ``` @@ -463,8 +478,6 @@ By default, jsonfmt will first check if git is installed on the computer. If git In DiffMode, jsonfmt will first format the data to be compared (at this time, the `-s` option will be automatically enabled), and save the result to a temporary file, and then call the specified tool for diff comparison. -Once the comparison is complete, the temporary file will be automatically deleted. However, if VS Code is selected as the diff-tool, the temporary file will not be immediately deleted. Instead, it will be removed by the operating system during the cleanup process. - #### Example 1: Compare two JSON files ```shell @@ -596,11 +609,11 @@ Output: ```json { - "actions": [], + "name": "...", "age": 23, "gender": "...", "money": 3.1415926, - "name": "..." + "actions": [] } ``` @@ -630,7 +643,7 @@ For items in a list, use `key[i]` or `key.i` to specify. If the index is greater ```shell # Add country = China, and append an item to actions -$ jf --set 'country=China; actions[2]={"name": "drink"}' test/example.json +$ jf --set 'country=China; actions[3]={"name": "drink"}' test/example.json ``` Output: @@ -644,14 +657,19 @@ Output: "actions": [ { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" }, { "name": "sport", - "calorie": -375, + "calorie": -2375, "date": "2023-04-27" }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" + }, { "name": "drink" } @@ -678,13 +696,18 @@ Output: "actions": [ { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" }, { "name": "swim", - "calorie": -375, + "calorie": -2375, "date": "2023-04-27" + }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" } ] } @@ -707,8 +730,13 @@ Output: "actions": [ { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" + }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" } ] } @@ -740,7 +768,6 @@ $ jf -s -i 4 --set 'name=Alex' -O test/example.json ## TODO -- [ ] Add XML format support -- [ ] Add INI format support - [ ] Add URL support to directly compare data from two APIs +- [ ] Add INI format support - [ ] Add merge mode to combine multiple JSON or other formatted data into one diff --git a/README_CN.md b/README_CN.md index d3db392..6eb6fa1 100644 --- a/README_CN.md +++ b/README_CN.md @@ -14,7 +14,7 @@ -### +### **_jsonfmt_**（JSON Formatter）是一款简单而强大的 JSON 处理工具。 @@ -22,7 +22,7 @@ 🎨 它不仅可以用来漂亮的打印 JSON 数据， -🔄 也可以将 JSON、TOML、YAML 数据进行互相转化， +🔄 也可以将 JSON、TOML、XML、YAML 数据进行互相转化， 🔎 还可以通过 JMESPATH 或 JSONPATH 来提取 JSON 中的内容。 @@ -68,7 +68,7 @@ $ pip install jsonfmt **位置参数** -`files`: 要处理的数据文件，支持 JSON / TOML / YAML 格式。 +`files`: 要处理的数据文件，支持 JSON / TOML / XML / YAML 格式。 **可选参数** @@ -80,7 +80,7 @@ $ pip install jsonfmt - `-O`: 覆盖模式，会将处理后的内容覆盖到原文件。 - `-c`: 删除 JSON 中的所有空白字符（对其他数据格式无效） - `-e`: 将所有字符转义成 ASCII 码 -- `-f`: 输出格式（默认值：与传入的数据格式相同，可选项：`json` / `toml` / `yaml`） +- `-f`: 输出格式（默认值：与传入的数据格式相同，可选项：`json` / `toml` / `xml` / `yaml`） - `-i`: 缩进的空格数（默认值：2，范围：0~8，设置 t 时会以 Tab 作为缩进符） - `-l`: 提取数据时的查询语言（默认：自动识别，可选项：jmespath / jsonpath） - `-p QUERYPATH`: JMESPath 或 JSONPath 查询路径 @@ -103,19 +103,24 @@ $ pip install jsonfmt "actions": [ { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" }, { "name": "sport", - "calorie": -375, + "calorie": -2375, "date": "2023-04-27" + }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" } ] } ``` -然后，再将这份数据转换为 TOML 和 YAML 格式，分别保存到文件 example.toml 和 example.yaml 中。 +然后，再将这份数据转换为 TOML、XML 和 YAML 格式，分别保存到文件 example.toml、example.xml 和 example.yaml 中。 这些数据文件可以在源码的 *test* 文件夹中找到: @@ -123,6 +128,7 @@ $ pip install jsonfmt test/ |- example.json |- example.toml +|- example.xml |- example.yaml ``` @@ -148,14 +154,19 @@ $ jf -s -i 4 test/example.json { "actions": [ { - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02", "name": "eat" }, { - "calorie": -375, + "calorie": -2375, "date": "2023-04-27", "name": "sport" + }, + { + "calorie": -420.5, + "date": "2023-05-15", + "name": "sleep" } ], "age": 23, @@ -227,16 +238,16 @@ JMESPath 可以优雅地使用简单的语法从 JSON 数据中提取一部分 ```json { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" } ``` -- 过滤 `actions` 中所有 `calorie > 0` 的项。 +- 过滤 `actions` 中所有 `calorie < 0` 的项。 ```shell # 此处的 `0` 表示 0 是一个数字 - $ jf -p 'actions[?calorie>`0`]' test/example.json + $ jf -p 'actions[?calorie<`0`]' test/example.json ``` 输出： @@ -244,9 +255,14 @@ JMESPath 可以优雅地使用简单的语法从 JSON 数据中提取一部分 ```json [ { - "name": "eat", - "calorie": 294.9, - "date": "2021-03-02" + "name": "sport", + "calorie": -2375, + "date": "2023-04-27" + }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" } ] ``` @@ -268,7 +284,7 @@ JMESPath 可以优雅地使用简单的语法从 JSON 数据中提取一部分 "money", "actions" ], - "actions_len": 2 + "actions_len": 3 } ``` @@ -284,11 +300,15 @@ JMESPath 可以优雅地使用简单的语法从 JSON 数据中提取一部分 [ { "foo": "sport", - "bar": -375 + "bar": -2375 + }, + { + "foo": "sleep", + "bar": -420.5 }, { "foo": "eat", - "bar": 294.9 + "bar": 1294.9 } ] ``` @@ -316,15 +336,16 @@ JSONPath 的设计灵感来源于 XPath。因此它可以像 XPath 那样通过 [ "Bob", "eat", - "sport" + "sport", + "sleep" ] ``` 在执行查询时，您可以不指定 `-l` 选项。jsonfmt 会先尝试使用 JMESPath 语法去解析 `-p QUERYPATH` -#### 查询 TOML 和 YAML +#### 查询 TOML、XML 和 YAML -jsonfmt 的众多强大功能之一就是，您可以使用与 JSON 完全同样的方式来处理 TOML 和 YAML，并任意转换结果的格式。甚至可以在单个命令中同时处理这三种格式。 +jsonfmt 的众多强大功能之一就是，您可以使用与 JSON 完全同样的方式来处理 TOML、XML 和 YAML，并任意转换结果的格式。甚至可以在单个命令中同时处理这四种格式。 - 从 toml 文件读取数据，并以 YAML 格式输出 @@ -341,13 +362,13 @@ jsonfmt 的众多强大功能之一就是，您可以使用与 JSON 完全同样 - gender - money - actions - actions_len: 2 + actions_len: 3 ``` -- 同时处理三种格式 +- 同时处理四种格式 ```shell - $ jf -p 'actions[0]' test/example.json test/example.toml test/example.yaml + $ jf -p 'actions[0]' test/example.json test/example.toml test/example.xml test/example.yaml ``` 输出： @@ -356,57 +377,43 @@ jsonfmt 的众多强大功能之一就是，您可以使用与 JSON 完全同样 1. test/example.json { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" } 2. test/example.toml name = "eat" - calorie = 294.9 + calorie = 1294.9 date = "2021-03-02" - 3. test/example.yaml + 3. test/example.xml + + + eat + 1294.9 + 2021-03-02 + + + 4. test/example.yaml name: eat - calorie: 294.9 + calorie: 1294.9 date: '2021-03-02' ``` ### 4. 格式转换 -*jsonfmt* 支持 JSON、TOML 和 YAML 格式的处理。每种格式都可以通过指定 "-f" 选项转换为其他格式。 +*jsonfmt* 支持 JSON、TOML、XML 和 YAML 格式的处理。每种格式都可以通过指定 "-f" 选项转换为其他格式。
注意:
-在 TOML 中,`null` 值是无效的。因此，从其他格式转换为 TOML 时，所有的 null 值都将被删除。 - -#### JSON 转换为 TOML - -```shell -$ jf test/example.json -f toml -``` - -输出： - -```toml -name = "Bob" -age = 23 -gender = "纯爷们" -money = 3.1415926 -[[actions]] -name = "eat" -calorie = 294.9 -date = "2021-03-02" -[[actions]] -name = "sport" -calorie = -375 -date = "2023-04-27" -``` +1. TOML 中不存在 `null` 值。因此从其他格式转换为 TOML 时，所有的 null 值都将被删除。 +2. XML 不支持多维数组。所以在向 XML 格式转换时，如果原数据中存在多维数组，则会产生错误的数据。 -#### TOML 转换为 YAML +#### 例1. JSON 转换为 YAML ```shell -$ jf test/example.toml -f yaml +$ jf test/example.json -f yaml ``` 输出： @@ -418,40 +425,47 @@ gender: 纯爷们 money: 3.1415926 actions: - name: eat - calorie: 294.9 + calorie: 1294.9 date: '2021-03-02' - name: sport - calorie: -375 + calorie: -2375 date: '2023-04-27' +- name: sleep + calorie: -420.5 + date: '2023-05-15' ``` -#### YAML 转换为 JSON +#### 例2. TOML 转换为 XML ```shell -$ jf test/example.yaml -f json +$ jf test/example.toml -f xml ``` 输出： -```json -{ - "name": "Bob", - "age": 23, - "gender": "纯爷们", - "money": 3.1415926, - "actions": [ - { - "name": "eat", - "calorie": 294.9, - "date": "2021-03-02" - }, - { - "name": "sport", - "calorie": -375, - "date": "2023-04-27" - } - ] -} +```xml + + + Bob + 23 + 纯爷们 + 3.1415926 + + eat + 1294.9 + 2021-03-02 + + + sport + -2375 + 2023-04-27 + + + sleep + -420.5 + 2023-05-15 + + ``` @@ -465,8 +479,6 @@ jsonfmt 默认支持多种差异对比工具，如：`diff`、`vimdiff`、`git` 在差异对比模式下，jsonfmt 会先将需要对比的数据进行格式化处理（此时 `-s` 选项会被自动激活），并将结果保存到临时文件中，然后再调用指定的工具进行差异对比。 -对比结束后，这个临时文件会被自动删除。如果选择 VS Code 作为差异对比工具，那么这个的临时文件不会被立即删除，它会由操作系统在执行清理操作时删除。 - #### 例1. 对比两个 JSON 文件 ```shell @@ -601,11 +613,11 @@ $ jf -o test/test.json ```json { - "actions": [], + "name": "...", "age": 23, "gender": "...", "money": 3.1415926, - "name": "..." + "actions": [] } ``` @@ -636,7 +648,7 @@ $ jf -C test/example.json ```shell # 添加 country = China，并为 actions 追加一项 -$ jf --set 'country=China; actions[2]={"name": "drink"}' test/example.json +$ jf --set 'country=China; actions[3]={"name": "drink"}' test/example.json ``` 输出： @@ -650,14 +662,19 @@ $ jf --set 'country=China; actions[2]={"name": "drink"}' test/example.json "actions": [ { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" }, { "name": "sport", - "calorie": -375, + "calorie": -2375, "date": "2023-04-27" }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" + }, { "name": "drink" } @@ -684,13 +701,18 @@ $ jf --set 'money=1000; actions[1].name=swim' test/example.json "actions": [ { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" }, { "name": "swim", - "calorie": -375, + "calorie": -2375, "date": "2023-04-27" + }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" } ] } @@ -713,8 +735,13 @@ $ jf --pop 'gender; actions[1]' test/example.json "actions": [ { "name": "eat", - "calorie": 294.9, + "calorie": 1294.9, "date": "2021-03-02" + }, + { + "name": "sleep", + "calorie": -420.5, + "date": "2023-05-15" } ] } @@ -748,7 +775,6 @@ $ jf -s -i 4 --set 'name=Alex' -O test/example.json ## TODO -- [ ] 增加 XML 格式支持 -- [ ] 增加 INI 格式支持 - [ ] 增加 URL 支持，可以直接对比来自两个 API 的数据 -- [ ] 增加 merge 模式，将多个 JSON 或其他格式的数据合并成一个 +- [ ] 增加 INI 格式支持 +- [ ] 增加 merge 模式，将多个 JSON 或其他格式的数据按 key 进行合并 diff --git a/jsonfmt/jsonfmt.py b/jsonfmt/jsonfmt.py index b5fdb5a..b120ea3 100755 --- a/jsonfmt/jsonfmt.py +++ b/jsonfmt/jsonfmt.py @@ -82,15 +82,15 @@ def extract_elements(qpath: QueryPath, py_obj: Any) -> Any: def parse_to_pyobj(text: str, qpath: Optional[QueryPath]) -> Tuple[Any, str]: '''read json, toml or yaml from IO and then match sub-element by jmespath''' # parse json, toml or yaml to python object - loads_methods: dict[str, Callable] = { - 'json': json.loads, - 'toml': toml.loads, - 'xml': xml2py.loads, - 'yaml': partial(yaml.load, Loader=yaml.Loader), - } + loads_methods: list[tuple[str, Callable]] = [ + ('json', json.loads), + ('toml', toml.loads), + ('xml', xml2py.loads), + ('yaml', partial(yaml.load, Loader=yaml.Loader)), + ] # try to load the text to be parsed - for fmt, fn_loads in loads_methods.items(): + for fmt, fn_loads in loads_methods: try: py_obj = fn_loads(text) break diff --git a/jsonfmt/xml2py.py b/jsonfmt/xml2py.py index e324979..1ef7cc7 100644 --- a/jsonfmt/xml2py.py +++ b/jsonfmt/xml2py.py @@ -94,6 +94,7 @@ def _get_attrs(self) -> Optional[dict[str, Any]]: attrs['@text'] = value return attrs or value else: + _tags = [] # tags of type "_list" for child in self: child_attrs = child._get_attrs() # type: ignore if child.tag in attrs: @@ -101,10 +102,16 @@ def _get_attrs(self) -> Optional[dict[str, Any]]: previous = attrs[child.tag] if not isinstance(previous, _list): attrs[child.tag] = _list([previous, child_attrs]) + _tags.append(child.tag) else: previous.append(child_attrs) else: attrs[child.tag] = child_attrs + + # recover "_list" to "list" + for k in _tags: + attrs[k] = list(attrs[k]) + return attrs def to_py(self) -> Any: From 6e175765c81229cdc6343c5417950ced1b35c39e Mon Sep 17 00:00:00 2001 From: Seamile Date: Tue, 9 Apr 2024 12:08:20 +0800 Subject: [PATCH 10/12] update test data --- test/another.json | 18 ++++++++++++++++++ test/example.json | 2 +- test/example.toml | 6 +++--- test/example.xml | 6 +++--- test/example.yaml | 6 +++--- test/todo1.json | 6 ------ test/todo2.json | 1 - test/todo3.toml | 4 ---- 8 files changed, 28 insertions(+), 21 deletions(-) create mode 100644 test/another.json delete mode 100644 test/todo1.json delete mode 100644 test/todo2.json delete mode 100644 test/todo3.toml diff --git a/test/another.json b/test/another.json new file mode 100644 index 0000000..97b19fe --- /dev/null +++ b/test/another.json @@ -0,0 +1,18 @@ +{ + "name": "Tom", + "age": 23, + "gender": "male", + "money": 3.1415926, + "actions": [ + { + "name": "thinking", + "calorie": 1294.9, + "date": "2021-03-02" + }, + { + "name": "sleeping", + "calorie": -420.5, + "date": "2023-05-15" + } + ] +} diff --git a/test/example.json b/test/example.json index 0c440ac..3f211b1 100644 --- a/test/example.json +++ b/test/example.json @@ -1 +1 @@ -{"name":"Bob","age":23,"gender":"纯爷们","money":3.1415926,"actions":[{"name":"eat","calorie":1294.9,"date":"2021-03-02"},{"name":"sport","calorie":-2375,"date":"2023-04-27"},{"name":"sleep","calorie":-420.5,"date":"2023-05-15"}]} +{"name":"Bob","age":23,"gender":"纯爷们","money":3.1415926,"actions":[{"name":"eating","calorie":1294.9,"date":"2021-03-02"},{"name":"sporting","calorie":-2375,"date":"2023-04-27"},{"name":"sleeping","calorie":-420.5,"date":"2023-05-15"}]} diff --git a/test/example.toml b/test/example.toml index 0ba2268..3ea083b 100644 --- a/test/example.toml +++ b/test/example.toml @@ -3,16 +3,16 @@ age = 23 gender = "纯爷们" money = 3.1415926 [[actions]] -name = "eat" +name = "eating" calorie = 1294.9 date = "2021-03-02" [[actions]] -name = "sport" +name = "sporting" calorie = -2375 date = "2023-04-27" [[actions]] -name = "sleep" +name = "sleeping" calorie = -420.5 date = "2023-05-15" diff --git a/test/example.xml b/test/example.xml index 86d4ec0..8b09094 100644 --- a/test/example.xml +++ b/test/example.xml @@ -5,17 +5,17 @@ 纯爷们 3.1415926 - eat + eating 1294.9 2021-03-02 - sport + sporting -2375 2023-04-27 - sleep + sleeping -420.5 2023-05-15 diff --git a/test/example.yaml b/test/example.yaml index dc289d1..9ce9c78 100644 --- a/test/example.yaml +++ b/test/example.yaml @@ -3,12 +3,12 @@ age: 23 gender: 纯爷们 money: 3.1415926 actions: -- name: eat +- name: eating calorie: 1294.9 date: '2021-03-02' -- name: sport +- name: sporting calorie: -2375 date: '2023-04-27' -- name: sleep +- name: sleeping calorie: -420.5 date: '2023-05-15' diff --git a/test/todo1.json b/test/todo1.json deleted file mode 100644 index b0c7830..0000000 --- a/test/todo1.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "id": 1, - "userId": 1072, - "title": "delectus aut autem", - "completed": false -} diff --git a/test/todo2.json b/test/todo2.json deleted file mode 100644 index 7357df8..0000000 --- a/test/todo2.json +++ /dev/null @@ -1 +0,0 @@ -{"userId":1092,"id":2,"title":"molestiae perspiciatis ipsa","completed":false} diff --git a/test/todo3.toml b/test/todo3.toml deleted file mode 100644 index 5baa4b6..0000000 --- a/test/todo3.toml +++ /dev/null @@ -1,4 +0,0 @@ -id = 3 -userId = 1072 -completed = false -title = "fugiat veniam minus" From a2d10903de360b9d904b0d92f49dd2c422410d84 Mon Sep 17 00:00:00 2001 From: Seamile Date: Wed, 10 Apr 2024 14:13:19 +0800 Subject: [PATCH 11/12] 1. modify the test files 2. modify test data 3. modify the docs --- README.md | 201 +++++++++++------- README_CN.md | 201 +++++++++++------- jsonfmt/diff.py | 4 +- jsonfmt/jsonfmt.py | 22 +- jsonfmt/xml2py.py | 20 +- test/test_diff.py | 106 ++++++++++ test/{test.py => test_jsonfmt.py} | 334 +++++++++++++++++------------- test/test_utils.py | 42 ++++ test/test_xml2py.py | 157 ++++++++++++++ 9 files changed, 781 insertions(+), 306 deletions(-) create mode 100644 test/test_diff.py rename test/{test.py => test_jsonfmt.py} (54%) create mode 100644 test/test_utils.py create mode 100644 test/test_xml2py.py diff --git a/README.md b/README.md index 05e6d38..ade5710 100644 --- a/README.md +++ b/README.md @@ -102,17 +102,17 @@ In order to demonstrate the features of jsonfmt, we need to first create a test "money": 3.1415926, "actions": [ { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" }, { - "name": "sport", + "name": "sporting", "calorie": -2375, "date": "2023-04-27" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" } @@ -156,17 +156,17 @@ Output: { "calorie": 1294.9, "date": "2021-03-02", - "name": "eat" + "name": "eating" }, { "calorie": -2375, "date": "2023-04-27", - "name": "sport" + "name": "sporting" }, { "calorie": -420.5, "date": "2023-05-15", - "name": "sleep" + "name": "sleeping" } ], "age": 23, @@ -237,7 +237,7 @@ JMESPath can elegantly use simple syntax to extract part of the content from JSO ```json { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" } @@ -255,12 +255,12 @@ JMESPath can elegantly use simple syntax to extract part of the content from JSO ```json [ { - "name": "sport", + "name": "sporting", "calorie": -2375, "date": "2023-04-27" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" } @@ -299,15 +299,15 @@ JMESPath can elegantly use simple syntax to extract part of the content from JSO ```json [ { - "foo": "sport", + "foo": "sporting", "bar": -2375 }, { - "foo": "sleep", + "foo": "sleeping", "bar": -420.5 }, { - "foo": "eat", + "foo": "eating", "bar": 1294.9 } ] @@ -335,9 +335,9 @@ Some queries that are difficult to handle with JMESPath can be easily achieved w ```json [ "Bob", - "eat", - "sport", - "sleep" + "eating", + "sporting", + "sleeping" ] ``` @@ -374,26 +374,26 @@ One of the powerful features of jsonfmt is that you can process TOML, XML and YA ```yaml 1. test/example.json { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" } 2. test/example.toml - name = "eat" + name = "eating" calorie = 1294.9 date = "2021-03-02" 3. test/example.xml - eat + eating 1294.9 2021-03-02 4. test/example.yaml - name: eat + name: eating calorie: 1294.9 date: '2021-03-02' ``` @@ -423,13 +423,13 @@ age: 23 gender: 纯爷们 money: 3.1415926 actions: -- name: eat +- name: eating calorie: 1294.9 date: '2021-03-02' -- name: sport +- name: sporting calorie: -2375 date: '2023-04-27' -- name: sleep +- name: sleeping calorie: -420.5 date: '2023-05-15' ``` @@ -450,17 +450,17 @@ Output: 纯爷们 3.1415926 - eat + eating 1294.9 2021-03-02 - sport + sporting -2375 2023-04-27 - sleep + sleeping -420.5 2023-05-15 @@ -481,23 +481,38 @@ In DiffMode, jsonfmt will first format the data to be compared (at this time, th #### Example 1: Compare two JSON files ```shell -$ jf -d test/todo1.json test/todo2.json +$ jf -d test/example.json test/another.json ``` Output: ```diff ---- /tmp/.../jf-jjn86s7r_todo1.json 2024-03-23 18:22:00 -+++ /tmp/.../jf-vik3bqsu_todo2.json 2024-03-23 18:22:00 -@@ -1,6 +1,6 @@ - { -- "userId": 1072, -- "id": 1, -- "title": "delectus aut autem", -+ "userId": 1092, -+ "id": 2, -+ "title": "molestiae perspiciatis ipsa", - "completed": false +--- /tmp/.../jf-jjn86s7r_example.json 2024-03-23 18:22:00 ++++ /tmp/.../jf-vik3bqsu_another.json 2024-03-23 18:22:00 +@@ -3,21 +3,16 @@ + { + "calorie": 1294.9, + "date": "2021-03-02", +- "name": "eating" ++ "name": "thinking" + }, + { +- "calorie": -2375, +- "date": "2023-04-27", +- "name": "sporting" +- }, +- { + "calorie": -420.5, + "date": "2023-05-15", + "name": "sleeping" + } + ], + "age": 23, +- "gender": "纯爷们", ++ "gender": "male", + "money": 3.1415926, +- "name": "Bob" ++ "name": "Tom" } ``` @@ -506,18 +521,35 @@ Output: The `-D DIFFTOOL` option can specify a diff comparison tool. As long as its command format matches `command [options] file1 file2`, it doesn't matter whether it's in jsonfmt's default supported tool list or not. ```shell -$ jf -D sdiff test/todo1.json test/todo2.json +$ jf -D sdiff test/example.json test/another.json ``` Output: ``` -{ { - "userId": 1072, | "userId": 1092, - "id": 1, | "id": 2, - "title": "delectus aut autem", | "title": "molestiae perspiciatis ipsa", - "completed": false "completed": false -} } +{ { + "actions": [ "actions": [ + { { + "calorie": 1294.9, "calorie": 1294.9, + "date": "2021-03-02", "date": "2021-03-02", + "name": "eating" | "name": "thinking" + }, }, + { { + "calorie": -2375, < + "date": "2023-04-27", < + "name": "sporting" < + }, < + { < + "calorie": -420.5, "calorie": -420.5, + "date": "2023-05-15", "date": "2023-05-15", + "name": "sleeping" "name": "sleeping" + } } + ], ], + "age": 23, "age": 23, + "gender": "纯爷们", | "gender": "male", + "money": 3.1415926, "money": 3.1415926, + "name": "Bob" | "name": "Tom" +} } ``` #### Example 3: Specify options for the selected tool @@ -525,20 +557,30 @@ Output: If you need to pass parameters to the diff-tool, you can use `-D 'DIFFTOOL OPTIONS'`. ```shell -$ jf -D 'diff --ignore-case --color=always' test/todo1.json test/todo2.json +$ jf -D 'diff --ignore-case --color=always' test/example.json test/another.json ``` Output: ```diff -3,5c3,5 -< "id": 1, -< "title": "delectus aut autem", -< "userId": 1072 +6c6 +< "name": "eating" --- -> "id": 2, -> "title": "molestiae perspiciatis ipsa", -> "userId": 1092 +> "name": "thinking" +9,13d8 +< "calorie": -2375, +< "date": "2023-04-27", +< "name": "sporting" +< }, +< { +20c15 +< "gender": "纯爷们", +--- +> "gender": "male", +22c17 +< "name": "Bob" +--- +> "name": "Tom" ``` #### Example 4: Compare data in different formats @@ -546,21 +588,36 @@ Output: For data from different sources, their formats, indentation, and key order may be different. In this case, you can use `-i` and `-f` together for diff comparison. ```shell -$ jf -d -i 4 -f toml test/todo1.json test/todo3.toml +$ jf -d -i 4 -f toml test/example.toml test/another.json ``` Output: ```diff ---- /var/.../jf-qw9vm33n_todo1.json 2024-03-23 18:29:17 -+++ /var/.../jf-dqb_fl4x_todo3.toml 2024-03-23 18:29:17 -@@ -1,4 +1,4 @@ - completed = false --id = 1 --title = "delectus aut autem" -+id = 3 -+title = "fugiat veniam minus" - userId = 1072 +--- /var/.../jf-qw9vm33n_example.toml 2024-03-23 18:29:17 ++++ /var/.../jf-dqb_fl4x_another.json 2024-03-23 18:29:17 +@@ -1,18 +1,13 @@ + age = 23 +-gender = "纯爷们" ++gender = "male" + money = 3.1415926 +-name = "Bob" ++name = "Tom" + [[actions]] + calorie = 1294.9 + date = "2021-03-02" +-name = "eating" ++name = "thinking" + + [[actions]] +-calorie = -2375 +-date = "2023-04-27" +-name = "sporting" +- +-[[actions]] + calorie = -420.5 + date = "2023-05-15" + name = "sleeping" ``` ### 6. Handle Large JSON Data Conveniently @@ -602,7 +659,7 @@ Sometimes we only want to see an overview of the JSON data without caring about If the root node of the JSON data is a list, only its first child element will be preserved in the overview. ```shell -$ jf -o test/test.json +$ jf -o test/example.json ``` Output: @@ -643,7 +700,7 @@ For items in a list, use `key[i]` or `key.i` to specify. If the index is greater ```shell # Add country = China, and append an item to actions -$ jf --set 'country=China; actions[3]={"name": "drink"}' test/example.json +$ jf --set 'country=China; actions[3]={"name": "drinking"}' test/example.json ``` Output: @@ -656,22 +713,22 @@ Output: "money": 3.1415926, "actions": [ { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" }, { - "name": "sport", + "name": "sporting", "calorie": -2375, "date": "2023-04-27" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" }, { - "name": "drink" + "name": "drinking" } ], "country": "China" @@ -695,7 +752,7 @@ Output: "money": 1000, "actions": [ { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" }, @@ -705,7 +762,7 @@ Output: "date": "2023-04-27" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" } @@ -729,12 +786,12 @@ Output: "money": 3.1415926, "actions": [ { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" } diff --git a/README_CN.md b/README_CN.md index 6eb6fa1..25cd8b9 100644 --- a/README_CN.md +++ b/README_CN.md @@ -102,17 +102,17 @@ $ pip install jsonfmt "money": 3.1415926, "actions": [ { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" }, { - "name": "sport", + "name": "sporting", "calorie": -2375, "date": "2023-04-27" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" } @@ -156,17 +156,17 @@ $ jf -s -i 4 test/example.json { "calorie": 1294.9, "date": "2021-03-02", - "name": "eat" + "name": "eating" }, { "calorie": -2375, "date": "2023-04-27", - "name": "sport" + "name": "sporting" }, { "calorie": -420.5, "date": "2023-05-15", - "name": "sleep" + "name": "sleeping" } ], "age": 23, @@ -237,7 +237,7 @@ JMESPath 可以优雅地使用简单的语法从 JSON 数据中提取一部分 ```json { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" } @@ -255,12 +255,12 @@ JMESPath 可以优雅地使用简单的语法从 JSON 数据中提取一部分 ```json [ { - "name": "sport", + "name": "sporting", "calorie": -2375, "date": "2023-04-27" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" } @@ -299,15 +299,15 @@ JMESPath 可以优雅地使用简单的语法从 JSON 数据中提取一部分 ```json [ { - "foo": "sport", + "foo": "sporting", "bar": -2375 }, { - "foo": "sleep", + "foo": "sleeping", "bar": -420.5 }, { - "foo": "eat", + "foo": "eating", "bar": 1294.9 } ] @@ -335,9 +335,9 @@ JSONPath 的设计灵感来源于 XPath。因此它可以像 XPath 那样通过 ```json [ "Bob", - "eat", - "sport", - "sleep" + "eating", + "sporting", + "sleeping" ] ``` @@ -376,26 +376,26 @@ jsonfmt 的众多强大功能之一就是，您可以使用与 JSON 完全同样 ```yaml 1. test/example.json { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" } 2. test/example.toml - name = "eat" + name = "eating" calorie = 1294.9 date = "2021-03-02" 3. test/example.xml - eat + eating 1294.9 2021-03-02 4. test/example.yaml - name: eat + name: eating calorie: 1294.9 date: '2021-03-02' ``` @@ -424,13 +424,13 @@ age: 23 gender: 纯爷们 money: 3.1415926 actions: -- name: eat +- name: eating calorie: 1294.9 date: '2021-03-02' -- name: sport +- name: sporting calorie: -2375 date: '2023-04-27' -- name: sleep +- name: sleeping calorie: -420.5 date: '2023-05-15' ``` @@ -451,17 +451,17 @@ $ jf test/example.toml -f xml 纯爷们 3.1415926 - eat + eating 1294.9 2021-03-02 - sport + sporting -2375 2023-04-27 - sleep + sleeping -420.5 2023-05-15 @@ -482,23 +482,38 @@ jsonfmt 默认支持多种差异对比工具，如：`diff`、`vimdiff`、`git` #### 例1. 对比两个 JSON 文件 ```shell -$ jf -d test/todo1.json test/todo2.json +$ jf -d test/example.json test/another.json ``` 输出： ```diff ---- /tmp/.../jf-jjn86s7r_todo1.json 2024-03-23 18:22:00 -+++ /tmp/.../jf-vik3bqsu_todo2.json 2024-03-23 18:22:00 -@@ -1,6 +1,6 @@ - { -- "userId": 1072, -- "id": 1, -- "title": "delectus aut autem", -+ "userId": 1092, -+ "id": 2, -+ "title": "molestiae perspiciatis ipsa", - "completed": false +--- /tmp/.../jf-jjn86s7r_example.json 2024-03-23 18:22:00 ++++ /tmp/.../jf-vik3bqsu_another.json 2024-03-23 18:22:00 +@@ -3,21 +3,16 @@ + { + "calorie": 1294.9, + "date": "2021-03-02", +- "name": "eating" ++ "name": "thinking" + }, + { +- "calorie": -2375, +- "date": "2023-04-27", +- "name": "sporting" +- }, +- { + "calorie": -420.5, + "date": "2023-05-15", + "name": "sleeping" + } + ], + "age": 23, +- "gender": "纯爷们", ++ "gender": "male", + "money": 3.1415926, +- "name": "Bob" ++ "name": "Tom" } ``` @@ -507,18 +522,35 @@ $ jf -d test/todo1.json test/todo2.json `-D DIFFTOOL` 选项可以指定一款差异对比工具。只要其命令格式满足 `command [options] file1 file2` 即可，无论它是否在 jsonfmt 默认支持的工具列表中。 ```shell -$ jf -D sdiff test/todo1.json test/todo2.json +$ jf -D sdiff test/example.json test/another.json ``` 输出： ``` -{ { - "userId": 1072, | "userId": 1092, - "id": 1, | "id": 2, - "title": "delectus aut autem", | "title": "molestiae perspiciatis ipsa", - "completed": false "completed": false -} } +{ { + "actions": [ "actions": [ + { { + "calorie": 1294.9, "calorie": 1294.9, + "date": "2021-03-02", "date": "2021-03-02", + "name": "eating" | "name": "thinking" + }, }, + { { + "calorie": -2375, < + "date": "2023-04-27", < + "name": "sporting" < + }, < + { < + "calorie": -420.5, "calorie": -420.5, + "date": "2023-05-15", "date": "2023-05-15", + "name": "sleeping" "name": "sleeping" + } } + ], ], + "age": 23, "age": 23, + "gender": "纯爷们", | "gender": "male", + "money": 3.1415926, "money": 3.1415926, + "name": "Bob" | "name": "Tom" +} } ``` #### 例3. 为选定的工具指定参数 @@ -526,20 +558,30 @@ $ jf -D sdiff test/todo1.json test/todo2.json 如果需要向差异对比工具传递参数，可以使用 `-D 'DIFFTOOL OPTIONS'` 来操作。 ```shell -$ jf -D 'diff --ignore-case --color=always' test/todo1.json test/todo2.json +$ jf -D 'diff --ignore-case --color=always' test/example.json test/another.json ``` 输出： ```diff -3,5c3,5 -< "id": 1, -< "title": "delectus aut autem", -< "userId": 1072 +6c6 +< "name": "eating" --- -> "id": 2, -> "title": "molestiae perspiciatis ipsa", -> "userId": 1092 +> "name": "thinking" +9,13d8 +< "calorie": -2375, +< "date": "2023-04-27", +< "name": "sporting" +< }, +< { +20c15 +< "gender": "纯爷们", +--- +> "gender": "male", +22c17 +< "name": "Bob" +--- +> "name": "Tom" ``` #### 例4. 对比不同格式的数据 @@ -547,21 +589,36 @@ $ jf -D 'diff --ignore-case --color=always' test/todo1.json test/todo2.json 对于不同来源的数据，其格式、缩进，以及键的顺序可能都不一样，这时可以使用 `-i`、`-f` 配合来进行差异对比。 ```shell -$ jf -d -i 4 -f toml test/todo1.json test/todo3.toml +$ jf -d -i 4 -f toml test/example.toml test/another.json ``` 输出： ```diff ---- /var/.../jf-qw9vm33n_todo1.json 2024-03-23 18:29:17 -+++ /var/.../jf-dqb_fl4x_todo3.toml 2024-03-23 18:29:17 -@@ -1,4 +1,4 @@ - completed = false --id = 1 --title = "delectus aut autem" -+id = 3 -+title = "fugiat veniam minus" - userId = 1072 +--- /var/.../jf-qw9vm33n_example.toml 2024-03-23 18:29:17 ++++ /var/.../jf-dqb_fl4x_another.json 2024-03-23 18:29:17 +@@ -1,18 +1,13 @@ + age = 23 +-gender = "纯爷们" ++gender = "male" + money = 3.1415926 +-name = "Bob" ++name = "Tom" + [[actions]] + calorie = 1294.9 + date = "2021-03-02" +-name = "eating" ++name = "thinking" + + [[actions]] +-calorie = -2375 +-date = "2023-04-27" +-name = "sporting" +- +-[[actions]] + calorie = -420.5 + date = "2023-05-15" + name = "sleeping" ``` @@ -606,7 +663,7 @@ $ curl -s https://jsonplaceholder.typicode.com/users | jf 如果 JSON 数据的根节点是一个列表，概览中仅保留它的第一个子元素。 ```shell -$ jf -o test/test.json +$ jf -o test/example.json ``` 输出： @@ -648,7 +705,7 @@ $ jf -C test/example.json ```shell # 添加 country = China，并为 actions 追加一项 -$ jf --set 'country=China; actions[3]={"name": "drink"}' test/example.json +$ jf --set 'country=China; actions[3]={"name": "drinking"}' test/example.json ``` 输出： @@ -661,22 +718,22 @@ $ jf --set 'country=China; actions[3]={"name": "drink"}' test/example.json "money": 3.1415926, "actions": [ { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" }, { - "name": "sport", + "name": "sporting", "calorie": -2375, "date": "2023-04-27" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" }, { - "name": "drink" + "name": "drinking" } ], "country": "China" @@ -700,7 +757,7 @@ $ jf --set 'money=1000; actions[1].name=swim' test/example.json "money": 1000, "actions": [ { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" }, @@ -710,7 +767,7 @@ $ jf --set 'money=1000; actions[1].name=swim' test/example.json "date": "2023-04-27" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" } @@ -734,12 +791,12 @@ $ jf --pop 'gender; actions[1]' test/example.json "money": 3.1415926, "actions": [ { - "name": "eat", + "name": "eating", "calorie": 1294.9, "date": "2021-03-02" }, { - "name": "sleep", + "name": "sleeping", "calorie": -420.5, "date": "2023-05-15" } diff --git a/jsonfmt/diff.py b/jsonfmt/diff.py index c184e6b..525df57 100644 --- a/jsonfmt/diff.py +++ b/jsonfmt/diff.py @@ -6,14 +6,14 @@ from pygments.formatters import TerminalFormatter from pygments.lexers import DiffLexer -from .utils import print_err +from .utils import print_inf def cmp_by_diff(path1: str, path2: str): '''use diff to compare the difference between two files''' stat, result = getstatusoutput(f'diff -u {path1} {path2}') if stat == 0: - print_err(result) + print_inf('no difference') else: output = highlight(result, DiffLexer(), TerminalFormatter()) print(output) diff --git a/jsonfmt/jsonfmt.py b/jsonfmt/jsonfmt.py index b120ea3..01e6933 100755 --- a/jsonfmt/jsonfmt.py +++ b/jsonfmt/jsonfmt.py @@ -10,7 +10,7 @@ from pydoc import pager from shutil import get_terminal_size from tempfile import NamedTemporaryFile, _TemporaryFileWrapper -from typing import IO, Any, Callable, List, Optional, Sequence, Tuple, Union +from typing import IO, Any, Callable, List, Optional, Tuple, Union from unittest.mock import patch import pyperclip @@ -97,7 +97,7 @@ def parse_to_pyobj(text: str, qpath: Optional[QueryPath]) -> Tuple[Any, str]: except Exception: continue else: - raise FormatError("no json, toml or yaml found in the text") + raise FormatError("no supported format found") if qpath is None: return py_obj, fmt @@ -228,13 +228,12 @@ def output(output_fp: IO, text: str, fmt: str): output_fp.seek(0) output_fp.truncate() output_fp.write(text) - output_fp.close() - utils.print_inf(f'result written to {os.path.basename(output_fp.name)}') - elif isinstance(output_fp, io.StringIO) and output_fp.tell() != 0: + elif output_fp is TEMP_CLIPBOARD and TEMP_CLIPBOARD.tell() != 0: output_fp.write('\n\n') output_fp.write(text) else: output_fp.write(text) + output_fp.flush() def process(input_fp: IO, qpath: Optional[QueryPath], to_fmt: Optional[str], *, @@ -298,22 +297,20 @@ def parse_cmdline_args() -> ArgumentParser: return parser -def main(_args: Optional[Sequence[str]] = None): +def main(): parser = parse_cmdline_args() - args = parser.parse_args(_args) + args = parser.parse_args() # check and parse the querypath querypath = parse_querypath(args.querypath, args.querylang) # check if the clipboard is available if args.cp2clip and not is_clipboard_available(): - utils.exit_with_error('clipboard is not available') + utils.exit_with_error('clipboard unavailable') # check the input files files = args.files or [sys.stdin] n_files = len(files) - if n_files < 1: - utils.exit_with_error('no data file specified') # check the diff mode diff_mode: bool = args.diff or args.difftool @@ -347,6 +344,9 @@ def main(_args: Optional[Sequence[str]] = None): if diff_mode: diff_files.append(output_fp.name) + elif args.overwrite: + utils.print_inf(f'result written to {os.path.basename(output_fp.name)}') + except (FormatError, JMESPathError, JSONPathError, OSError) as err: utils.print_err(err) except KeyboardInterrupt: @@ -361,6 +361,8 @@ def main(_args: Optional[Sequence[str]] = None): pyperclip.copy(TEMP_CLIPBOARD.read()) utils.print_inf('result copied to clipboard') elif diff_mode: + if len(diff_files) < 2: + utils.exit_with_error('not enougth files to compare') try: compare(diff_files[0], diff_files[1], args.difftool) except (OSError, ValueError) as err: diff --git a/jsonfmt/xml2py.py b/jsonfmt/xml2py.py index 1ef7cc7..77b2e1d 100644 --- a/jsonfmt/xml2py.py +++ b/jsonfmt/xml2py.py @@ -23,15 +23,6 @@ def __init__(self, self.tail = tail self.parent: Optional[Self] = None - def __str__(self): - def _(ele, n=0): - indent = ' ' * n - line = f'{indent}{ele.tag} : ATTRIB={ele.attrib} TEXT={ele.text}\n' - for e in ele: - line += _(e, n + 1) # type: ignore - return line - return _(self) - @classmethod def makeelement(cls, tag, attrib, text=None, tail=None) -> Self: """Create a new element with the same type.""" @@ -94,6 +85,11 @@ def _get_attrs(self) -> Optional[dict[str, Any]]: attrs['@text'] = value return attrs or value else: + if self.text: + value = safe_eval(self.text.strip()) + if value: + attrs['@text'] = value + _tags = [] # tags of type "_list" for child in self: child_attrs = child._get_attrs() # type: ignore @@ -136,7 +132,11 @@ def _set_attrs(self, py_obj: Any): return self.spawn(self.tag)._set_attrs(py_obj) else: for i, item in enumerate(py_obj): - ele = self.parent[i] if len(self.parent) > i else self.parent.spawn(self.tag) + if len(self.parent) > i: + ele = self.parent[i] + else: + ele = self.parent.spawn(self.tag) + if isinstance(item, (list, tuple, set)): ele.text = str(item) else: diff --git a/test/test_diff.py b/test/test_diff.py new file mode 100644 index 0000000..0ba9edf --- /dev/null +++ b/test/test_diff.py @@ -0,0 +1,106 @@ +import os +import sys +import unittest +from unittest.mock import Mock, patch + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, BASE_DIR) +from jsonfmt import diff as df + + +class TestDiff(unittest.TestCase): + + def setUp(self) -> None: + self.mock_call = Mock() + self.mock_getstatusoutput = Mock() + self.mock_system = Mock() + + def test_cmp_by_diff(self): + + self.mock_getstatusoutput.return_value = (0, '') + with patch.multiple(df, getstatusoutput=self.mock_getstatusoutput): + df.cmp_by_diff('file1.txt', 'file2.txt') + self.assertTrue(self.mock_getstatusoutput.called) + self.assertEqual(self.mock_getstatusoutput.call_args[0][0], + 'diff -u file1.txt file2.txt') + + self.mock_getstatusoutput.return_value = (1, '') + with patch.multiple(df, getstatusoutput=self.mock_getstatusoutput): + df.cmp_by_diff('file1.txt', 'file2.txt') + self.assertTrue(self.mock_getstatusoutput.called) + + def test_cmp_by_fc(self): + with patch.multiple(df, call=self.mock_call): + os.environ["WINDIR"] = 'c' + df.cmp_by_fc('file1.txt', 'file2.txt') + self.assertTrue(self.mock_call.called) + self.assertEqual(self.mock_call.call_args[0][0][1:], ['/n', 'file1.txt', 'file2.txt']) + + def test_cmp_by_code(self): + with patch.multiple(df, call=self.mock_call): + df.cmp_by_code('file1.txt', 'file2.txt') + self.assertTrue(self.mock_call.called) + self.assertEqual(self.mock_call.call_args[0][0], ['code', '--diff', 'file1.txt', 'file2.txt']) + + def test_cmp_by_git(self): + + self.mock_getstatusoutput.return_value = (0, 'vimdiff') + with patch.multiple(df, call=self.mock_call, getstatusoutput=self.mock_getstatusoutput): + df.cmp_by_git('file1.txt', 'file2.txt') + self.assertTrue(self.mock_call.called) + self.assertEqual(self.mock_call.call_args[0][0], ['vimdiff', 'file1.txt', 'file2.txt']) + + self.mock_getstatusoutput.return_value = (1, '') + with patch.multiple(df, call=self.mock_call, getstatusoutput=self.mock_getstatusoutput): + df.cmp_by_git('file1.txt', 'file2.txt') + self.assertTrue(self.mock_call.called) + self.assertEqual(self.mock_call.call_args[0][0], + ['git', 'diff', '--color=always', '--no-index', 'file1.txt', 'file2.txt']) + + def test_cmp_by_others(self): + with patch.multiple(df, call=self.mock_call): + df.cmp_by_others('foo --bar', 'file1.txt', 'file2.txt') + self.assertTrue(self.mock_call.called) + self.assertEqual(self.mock_call.call_args[0][0], + ['foo', '--bar', 'file1.txt', 'file2.txt']) + + @patch('os.name', 'posix') + def test_has_command_posix(self): + self.mock_system.return_value = 0 + with patch('os.system', self.mock_system): + for cmd in ['ls', 'pwd', 'cat']: + self.assertTrue(df.has_command(cmd)) + self.assertEqual(self.mock_system.call_args[0][0], f'hash {cmd} > /dev/null 2>&1') + + @patch('os.name', 'nt') + def test_has_command_nt(self): + self.mock_system.return_value = 0 + with patch('os.system', self.mock_system): + for cmd in ['dir', 'cd', 'type']: + self.assertTrue(df.has_command(cmd)) + self.assertEqual(self.mock_system.call_args[0][0], f'where {cmd}') + + @patch('os.name', 'unsupported_os') + def test_has_command_unsupported_os(self): + with self.assertRaises(OSError): + df.has_command('ls') + + @patch('os.name', 'posix') + def test_command_not_found(self): + self.mock_system.return_value = 1 + with patch('os.system', self.mock_system): + for command in ['nonexistent_command', 'invalid_command']: + self.assertFalse(df.has_command(command)) + + @patch('os.name', 'posix') + def test_compare(self): + tools = [None, 'git', 'code', 'diff', 'fc', 'unknow-diff-tool'] + self.mock_system.return_value = 0 + for tool in tools: + with patch.multiple(df, call=self.mock_call), patch('os.system', self.mock_system): + df.compare('file1.txt', 'file2.txt', tool) + self.assertTrue(self.mock_call.called) + + self.mock_system.return_value = 1 + with patch('os.system', self.mock_system), self.assertRaises(ValueError): + df.compare('file1.txt', 'file2.txt', None) diff --git a/test/test.py b/test/test_jsonfmt.py similarity index 54% rename from test/test.py rename to test/test_jsonfmt.py index 4886f12..eed4c79 100644 --- a/test/test.py +++ b/test/test_jsonfmt.py @@ -4,6 +4,7 @@ import tempfile import unittest from argparse import Namespace +from contextlib import contextmanager from copy import deepcopy from functools import partial from io import StringIO @@ -14,7 +15,7 @@ from jsonpath_ng import parse as jparse from pygments import highlight from pygments.formatters import TerminalFormatter -from pygments.lexers import JsonLexer, TOMLLexer, YamlLexer +from pygments.lexers import JsonLexer, TOMLLexer, XmlLexer, YamlLexer BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, BASE_DIR) @@ -28,27 +29,27 @@ with open(TOML_FILE) as toml_fp: TOML_TEXT = toml_fp.read() +XML_FILE = f'{BASE_DIR}/test/example.xml' +with open(XML_FILE) as xml_fp: + XML_TEXT = xml_fp.read() + YAML_FILE = f'{BASE_DIR}/test/example.yaml' with open(YAML_FILE) as yaml_fp: YAML_TEXT = yaml_fp.read() def color(text, fmt): - fn = { - 'json': partial(highlight, - lexer=JsonLexer(), - formatter=TerminalFormatter()), - 'toml': partial(highlight, - lexer=TOMLLexer(), - formatter=TerminalFormatter()), - 'yaml': partial(highlight, - lexer=YamlLexer(), - formatter=TerminalFormatter()), - }[fmt] + functions = { + 'json': partial(highlight, lexer=JsonLexer(), formatter=TerminalFormatter()), + 'toml': partial(highlight, lexer=TOMLLexer(), formatter=TerminalFormatter()), + 'xml': partial(highlight, lexer=XmlLexer(), formatter=TerminalFormatter()), + 'yaml': partial(highlight, lexer=YamlLexer(), formatter=TerminalFormatter()), + } + fn = functions[fmt] return fn(text) -class FakeStdStream(StringIO): +class FakeStream(StringIO): def __init__(self, initial_value='', newline='\n', tty=True): super().__init__(initial_value, newline) @@ -66,21 +67,21 @@ def read(self): return content -class FakeStdIn(FakeStdStream): +class StdIn(FakeStream): name = '' def fileno(self) -> int: return 0 -class FakeStdOut(FakeStdStream): +class StdOut(FakeStream): name = '' def fileno(self) -> int: return 1 -class FakeStdErr(FakeStdStream): +class StdErr(FakeStream): name = '' def fileno(self) -> int: @@ -92,18 +93,45 @@ def setUp(self): self.maxDiff = None self.py_obj = json.loads(JSON_TEXT) + @contextmanager + def assertNotRaises(self, exc_type): + try: + yield None + except exc_type: + raise self.failureException('{} raised'.format(exc_type.__name__)) + def test_is_clipboard_available(self): available = jsonfmt.is_clipboard_available() self.assertIsInstance(available, bool) + def test_parse_querypath(self): + jmespath, jsonpath = 'actions.name', '$..name' + # test parse jmespath + res1 = jsonfmt.parse_querypath(jmespath, None) + res2 = jsonfmt.parse_querypath(jmespath, 'jmespath') + + self.assertEqual(res1, res2) + # test parse jsonpath + res3 = jsonfmt.parse_querypath(jsonpath, None) + res4 = jsonfmt.parse_querypath(jsonpath, 'jsonpath') + self.assertEqual(res3, res4) + + # test wrong args + self.assertEqual(jsonfmt.parse_querypath(None, None), None) + with self.assertRaises(SystemExit): + jsonfmt.parse_querypath('as*df', None) + + with self.assertRaises(SystemExit): + jsonfmt.parse_querypath(jsonpath, 'wrong') + def test_parse_to_pyobj_with_jmespath(self): # normal parameters test matched_obj = jsonfmt.parse_to_pyobj(JSON_TEXT, jcompile("actions[:].calorie")) - self.assertEqual(matched_obj, ([294.9, -375], 'json')) + self.assertEqual(matched_obj, ([1294.9, -2375, -420.5], 'json')) matched_obj = jsonfmt.parse_to_pyobj(TOML_TEXT, jcompile("actions[*].name")) - self.assertEqual(matched_obj, (['eat', 'sport'], 'toml')) + self.assertEqual(matched_obj, (['eating', 'sporting', 'sleeping'], 'toml')) matched_obj = jsonfmt.parse_to_pyobj(YAML_TEXT, jcompile("actions[*].date")) - self.assertEqual(matched_obj, (['2021-03-02', '2023-04-27'], 'yaml')) + self.assertEqual(matched_obj, (['2021-03-02', '2023-04-27', '2023-05-15'], 'yaml')) # test not exists key matched_obj = jsonfmt.parse_to_pyobj(TOML_TEXT, jcompile("not_exist_key")) self.assertEqual(matched_obj, (None, 'toml')) @@ -116,9 +144,9 @@ def test_parse_to_pyobj_with_jsonpath(self): matched_obj = jsonfmt.parse_to_pyobj(JSON_TEXT, jparse("age")) self.assertEqual(matched_obj, (23, 'json')) matched_obj = jsonfmt.parse_to_pyobj(TOML_TEXT, jparse("$..name")) - self.assertEqual(matched_obj, (['Bob', 'eat', 'sport'], 'toml')) + self.assertEqual(matched_obj, (['Bob', 'eating', 'sporting', 'sleeping'], 'toml')) matched_obj = jsonfmt.parse_to_pyobj(YAML_TEXT, jparse("actions[*].date")) - self.assertEqual(matched_obj, (['2021-03-02', '2023-04-27'], 'yaml')) + self.assertEqual(matched_obj, (['2021-03-02', '2023-04-27', '2023-05-15'], 'yaml')) # test not exists key matched_obj = jsonfmt.parse_to_pyobj(TOML_TEXT, jparse("not_exist_key")) self.assertEqual(matched_obj, (None, 'toml')) @@ -142,12 +170,12 @@ def test_modify_pyobj_for_adding(self): jsonfmt.modify_pyobj(obj, ['new=value'], []) self.assertEqual(obj['new'], 'value') # add single value to list - jsonfmt.modify_pyobj(obj, ['actions[20]={"K":"V"}'], []) - self.assertEqual(obj['actions'][2], {"K": "V"}) + jsonfmt.modify_pyobj(obj, ['actions[10]={"A":"B"}'], []) + self.assertEqual(obj['actions'][-1], {"A": "B"}) # add multiple values at once - jsonfmt.modify_pyobj(obj, ['new=[1,2,3]', 'actions[50]={"K":"V"}'], []) + jsonfmt.modify_pyobj(obj, ['new=[1,2,3]', 'actions[11]={"X":"Y"}'], []) self.assertEqual(obj['new'], [1, 2, 3]) - self.assertEqual(obj['actions'][2], {"K": "V"}) + self.assertEqual(obj['actions'][-1], {"X": "Y"}) def test_modify_pyobj_for_modifying(self): # test modify values @@ -163,6 +191,7 @@ def test_modify_pyobj_for_modifying(self): def test_modify_pyobj_for_popping(self): # test pop values obj = deepcopy(self.py_obj) + n_actions = len(obj['actions']) # pop single value jsonfmt.modify_pyobj(obj, [], ['age']) self.assertNotIn('age', obj) @@ -170,7 +199,7 @@ def test_modify_pyobj_for_popping(self): jsonfmt.modify_pyobj(obj, [], ['money', 'actions[1]', 'actions.0.date']) self.assertNotIn('money', obj) self.assertNotIn('date', obj['actions'][0]) - self.assertEqual(1, len(obj['actions'])) + self.assertEqual(n_actions - 1, len(obj['actions'])) def test_modify_pyobj_for_all(self): # test adding, popping and modifying simultaneously @@ -185,13 +214,13 @@ def test_modify_pyobj_for_all(self): # test exceptions obj = deepcopy(self.py_obj) - with patch('jsonfmt.stderr', FakeStdErr()): + with patch('sys.stderr', StdErr()): # modifying jsonfmt.modify_pyobj(obj, ['aa.bb=empty'], []) - self.assertIn('invalid key path', jsonfmt.stderr.read()) + self.assertIn('invalid key path', sys.stderr.read()) # popping jsonfmt.modify_pyobj(obj, [], ['actions[3]']) - self.assertIn('invalid key path', jsonfmt.stderr.read()) + self.assertIn('invalid key path', sys.stderr.read()) def test_get_overview(self): # test dict obj @@ -215,9 +244,9 @@ def test_get_overview(self): obj = deepcopy(self.py_obj['actions']) expected_2 = [ { - "calorie": 294.9, - "date": "...", - "name": "..." + "name": "...", + "calorie": 1294.9, + "date": "..." } ] overview = jsonfmt.get_overview(obj) @@ -228,105 +257,105 @@ def test_format_to_text(self): # format to json (compacted) j_compacted = jsonfmt.format_to_text(py_obj, 'json', compact=True, escape=True, - indent=4, sort_keys=False) + indent='4', sort_keys=False) self.assertEqual(j_compacted.strip(), '{"name":"\\u7ea6\\u7ff0","age":30}') # format to json (indentation) j_indented = jsonfmt.format_to_text(py_obj, 'json', compact=False, escape=False, - indent=4, sort_keys=True) - self.assertEqual(j_indented.strip(), '{\n "age": 30,\n "name": "约翰"\n}') + indent='t', sort_keys=True) + self.assertEqual(j_indented.strip(), '{\n\t"age": 30,\n\t"name": "约翰"\n}') # format to toml toml_text = jsonfmt.format_to_text(self.py_obj, 'toml', compact=False, escape=False, - indent=4, sort_keys=False) + indent='4', sort_keys=False) self.assertEqual(toml_text.strip(), TOML_TEXT.strip()) + # format to xml + xml_text = jsonfmt.format_to_text(py_obj, 'xml', + compact=True, escape=False, + indent='t', sort_keys=True) + result = '30约翰' + self.assertEqual(xml_text.strip(), result) + # format to yaml yaml_text = jsonfmt.format_to_text(self.py_obj, 'yaml', compact=False, escape=False, - indent=2, sort_keys=True) + indent='2', sort_keys=False) self.assertEqual(yaml_text.strip(), YAML_TEXT.strip()) # test exceptions with self.assertRaises(jsonfmt.FormatError): jsonfmt.format_to_text([1, 2, 3], 'toml', compact=False, escape=False, - indent=4, sort_keys=False) + indent='4', sort_keys=False) with self.assertRaises(jsonfmt.FormatError): - jsonfmt.format_to_text(py_obj, 'xml', + jsonfmt.format_to_text(py_obj, 'unknow', compact=False, escape=False, - indent=4, sort_keys=False) + indent='4', sort_keys=False) def test_output(self): # output JSON to clipboard - if jsonfmt.is_clipboard_available(): - with patch('jsonfmt.stdout', FakeStdOut()): - jsonfmt.output(jsonfmt.stdout, JSON_TEXT, 'json', True) - self.assertEqual(pyperclip.paste(), JSON_TEXT) + jsonfmt.TEMP_CLIPBOARD.seek(0) + jsonfmt.TEMP_CLIPBOARD.truncate() + jsonfmt.output(jsonfmt.TEMP_CLIPBOARD, JSON_TEXT, 'json') + jsonfmt.output(jsonfmt.TEMP_CLIPBOARD, XML_TEXT, 'xml') + jsonfmt.TEMP_CLIPBOARD.seek(0) + self.assertEqual(jsonfmt.TEMP_CLIPBOARD.read(), + JSON_TEXT + '\n\n' + XML_TEXT) # output TOML to file (temp file) with tempfile.NamedTemporaryFile(mode='r+') as tmpfile: - jsonfmt.output(tmpfile, TOML_TEXT, 'toml', False) + jsonfmt.output(tmpfile, TOML_TEXT, 'toml') tmpfile.seek(0) self.assertEqual(tmpfile.read(), TOML_TEXT) # output YAML to stdout (mock) - with patch('jsonfmt.stdout', FakeStdOut()): - jsonfmt.output(jsonfmt.stdout, YAML_TEXT, 'yaml', False) - self.assertEqual(jsonfmt.stdout.read(), color(YAML_TEXT, 'yaml')) + with patch('sys.stdout', StdOut()): + jsonfmt.output(sys.stdout, YAML_TEXT, 'yaml') + self.assertEqual(sys.stdout.read(), color(YAML_TEXT, 'yaml')) # output unknow format - with self.assertRaises(KeyError), patch('jsonfmt.stdout', FakeStdOut()): - jsonfmt.output(jsonfmt.stdout, YAML_TEXT, 'xml', False) + with self.assertRaises(KeyError), patch('sys.stdout', StdOut()): + jsonfmt.output(sys.stdout, YAML_TEXT, 'null') def test_parse_cmdline_args(self): # test default parameters default_args = Namespace( - compact=False, cp2clip=False, + diff=False, + difftool=None, + overview=False, + overwrite=False, + compact=False, escape=False, format=None, indent='2', - overview=False, - overwrite=False, - querylang='jmespath', + querylang=None, querypath=None, sort_keys=False, set=None, pop=None, files=[] ) - actual_args = jsonfmt.parse_cmdline_args(args=[]) - self.assertEqual(actual_args, default_args) + + with patch('sys.argv', ['jf']): + actual_args = jsonfmt.parse_cmdline_args().parse_args() + self.assertEqual(actual_args, default_args) # test specified parameters - args = [ - '-c', - '-C', - '-e', - '-f', 'toml', - '-i', '4', - '-o', - '-O', - '-l', 'jsonpath', - '-p', 'path.to.json', - '--set', 'a; b', - '--pop', 'c; d', - '-s', - 'file1.json', - 'file2.json' - ] expected_args = Namespace( + cp2clip=False, + diff=True, + difftool=None, + overview=False, + overwrite=False, compact=True, - cp2clip=True, escape=True, format='toml', indent='4', - overview=True, - overwrite=True, querylang='jsonpath', querypath='path.to.json', sort_keys=True, @@ -334,83 +363,95 @@ def test_parse_cmdline_args(self): pop='c; d', files=['file1.json', 'file2.json'] ) - - actual_args = jsonfmt.parse_cmdline_args(args=args) - self.assertEqual(actual_args, expected_args) + with patch('sys.argv', ['jf', '-d', '-c', '-e', '-f', 'toml', '-i', '4', + '-l', 'jsonpath', '-p', 'path.to.json', '-s', + '--set', 'a; b', '--pop', 'c; d', + 'file1.json', 'file2.json']): + actual_args = jsonfmt.parse_cmdline_args().parse_args() + self.assertEqual(actual_args, expected_args) ############################################################################ # main test # ############################################################################ - @patch.multiple(sys, argv=['jsonfmt', '-i', 't', '-p', 'actions[*].name', + @patch.multiple(sys, argv=['jf', '-i', 't', '-p', 'actions[*].name', JSON_FILE, YAML_FILE]) - @patch.multiple(jsonfmt, stdout=FakeStdOut()) + @patch.multiple(sys, stdout=StdOut()) def test_main_with_file(self): - expected_output = color('[\n\t"eat",\n\t"sport"\n]', 'json') - expected_output += '----------------\n' - expected_output += color('- eat\n- sport', 'yaml') + json_output = color('[\n\t"eating",\n\t"sporting",\n\t"sleeping"\n]', 'json') + yaml_output = color('- eating\n- sporting\n- sleeping', 'yaml') jsonfmt.main() - self.assertEqual(jsonfmt.stdout.read(), expected_output) + output = sys.stdout.read() + self.assertIn(json_output, output) + self.assertIn(yaml_output, output) - @patch.multiple(sys, argv=['jsonfmt', '-f', 'yaml']) - @patch.multiple(jsonfmt, stdin=FakeStdIn('["a", "b"]'), stdout=FakeStdOut()) def test_main_with_stdin(self): - expected_output = color('- a\n- b', 'yaml') - jsonfmt.main() - self.assertEqual(jsonfmt.stdout.read(), expected_output) + with patch.multiple(sys, argv=['jf', '-f', 'yaml'], + stdin=StdIn('["a", "b"]'), stdout=StdOut()): + expected_output = color('- a\n- b', 'yaml') + jsonfmt.main() + self.assertEqual(sys.stdout.read(), expected_output) - @patch.multiple(jsonfmt, stderr=FakeStdErr()) + @patch.multiple(sys, stderr=StdErr()) def test_main_invalid_input(self): # test not exist file and wrong format - with patch.multiple(sys, argv=['jsonfmt', 'not_exist_file.json', __file__]): + with patch.multiple(sys, argv=['jf', 'not_exist_file.json', __file__]): jsonfmt.main() - errmsg = jsonfmt.stderr.read() - self.assertIn('no such file: not_exist_file.json', errmsg) - self.assertIn('no json, toml or yaml found', errmsg) + errmsg = sys.stderr.read() + self.assertIn("No such file or directory: 'not_exist_file.json'", errmsg) + self.assertIn('no supported format found', errmsg) - @patch.multiple(jsonfmt, stderr=FakeStdErr()) + with patch.multiple(sys, argv=['jf']), \ + patch('sys.stdin.read', side_effect=KeyboardInterrupt), \ + self.assertRaises(SystemExit): + jsonfmt.main() + self.assertIn('user canceled', sys.stderr.read()) + + @patch.multiple(sys, stderr=StdErr()) def test_main_querying(self): # test empty jmespath - with patch('sys.argv', ['jsonfmt', JSON_FILE, '-p', '$.-[=]']),\ + with patch('sys.argv', ['jf', JSON_FILE, '-p', '$.-[=]']), \ self.assertRaises(SystemExit): jsonfmt.main() - self.assertIn('invalid querypath expression', jsonfmt.stderr.read()) + self.assertIn('invalid querypath expression', sys.stderr.read()) # test empty jmespath - with patch('sys.argv', ['jsonfmt', JSON_FILE, '-l', 'jsonpath', '-p', ' ']),\ + with patch('sys.argv', ['jf', JSON_FILE, '-l', 'jsonpath', '-p', ' ']), \ self.assertRaises(SystemExit): jsonfmt.main() - self.assertIn('invalid querypath expression', jsonfmt.stderr.read()) + self.assertIn('invalid querypath expression', sys.stderr.read()) - @patch('jsonfmt.stdout', FakeStdOut()) + @patch('sys.stdout', StdOut()) def test_main_convert(self): # test json to toml - with patch.multiple(sys, argv=['jsonfmt', '-f', 'toml', JSON_FILE]): - colored_output = color(TOML_TEXT, 'toml') + with patch.multiple(sys, argv=['jf', '-f', 'toml', JSON_FILE]): jsonfmt.main() - self.assertEqual(jsonfmt.stdout.read(), colored_output) + self.assertEqual(sys.stdout.read(), color(TOML_TEXT, 'toml')) - # test toml to yaml - with patch.multiple(sys, argv=['jsonfmt', '-s', '-f', 'yaml', TOML_FILE]): - colored_output = color(YAML_TEXT, 'yaml') + # test toml to xml + with patch.multiple(sys, argv=['jf', '-f', 'xml', TOML_FILE]): jsonfmt.main() - self.assertEqual(jsonfmt.stdout.read(), colored_output) + self.assertEqual(sys.stdout.read(), color(XML_TEXT, 'xml')) + + # test xml to yaml + with patch.multiple(sys, argv=['jf', '-f', 'yaml', XML_FILE]): + jsonfmt.main() + self.assertEqual(sys.stdout.read(), color(YAML_TEXT, 'yaml')) # test yaml to json - with patch.multiple(sys, argv=['jsonfmt', '-c', '-f', 'json', YAML_FILE]): - colored_output = color(JSON_TEXT, 'json') + with patch.multiple(sys, argv=['jf', '-c', '-f', 'json', YAML_FILE]): jsonfmt.main() - self.assertEqual(jsonfmt.stdout.read(), colored_output) + self.assertEqual(sys.stdout.read(), color(JSON_TEXT, 'json')) - @patch.multiple(sys, argv=['jsonfmt', '-oc']) - @patch.multiple(jsonfmt, - stdin=FakeStdIn('{"a": "asfd", "b": [1, 2, 3]}'), - stdout=FakeStdOut(tty=False)) + @patch.multiple(sys, argv=['jf', '-oc']) + @patch.multiple(sys, + stdin=StdIn('{"a": "asfd", "b": [1, 2, 3]}'), + stdout=StdOut(tty=False)) def test_main_overview(self): jsonfmt.main() - self.assertEqual(jsonfmt.stdout.read().strip(), '{"a":"...","b":[]}') + self.assertEqual(sys.stdout.read().strip(), '{"a":"...","b":[]}') - @patch('sys.argv', ['jsonfmt', '-Ocsf', 'json', TOML_FILE]) + @patch('sys.argv', ['jf', '-Ocf', 'json', TOML_FILE]) def test_main_overwrite_to_original_file(self): try: jsonfmt.main() @@ -421,50 +462,63 @@ def test_main_overwrite_to_original_file(self): with open(TOML_FILE, 'w') as toml_fp: toml_fp.write(TOML_TEXT) - @patch.multiple(jsonfmt, stdout=FakeStdOut(), stderr=FakeStdErr()) + @patch.multiple(sys, argv=['jf', '-Cc', JSON_FILE, TOML_FILE]) def test_main_copy_to_clipboard(self): if jsonfmt.is_clipboard_available(): - with patch("sys.argv", ['jsonfmt', '-Ccs', JSON_FILE]): - jsonfmt.main() - copied_text = pyperclip.paste().strip() - self.assertEqual(copied_text, JSON_TEXT.strip()) - - with patch("sys.argv", ['jsonfmt', '-Cs', TOML_FILE]): - jsonfmt.main() - copied_text = pyperclip.paste().strip() - self.assertEqual(copied_text, TOML_TEXT.strip()) - - with patch("sys.argv", ['jsonfmt', '-Cs', YAML_FILE]): - jsonfmt.main() - copied_text = pyperclip.paste().strip() - self.assertEqual(copied_text, YAML_TEXT.strip()) + pyperclip.copy('') + jsonfmt.main() + copied_text = pyperclip.paste().strip() + self.assertEqual(copied_text, JSON_TEXT.strip() + '\n\n\n' + TOML_TEXT.strip()) @patch.multiple(jsonfmt, is_clipboard_available=lambda: False) - @patch.multiple(jsonfmt, stdout=FakeStdOut(), stderr=FakeStdErr()) - @patch.multiple(sys, argv=['jsonfmt', JSON_FILE, '-cC']) + @patch.multiple(sys, stderr=StdErr()) + @patch.multiple(sys, argv=['jf', JSON_FILE, '-cC']) def test_main_clipboard_unavailable(self): errmsg = '\033[1;91mjsonfmt:\033[0m \033[0;91mclipboard unavailable\033[0m\n' - jsonfmt.main() - self.assertEqual(jsonfmt.stderr.read(), errmsg) - self.assertEqual(jsonfmt.stdout.read(), color(JSON_TEXT, 'json')) + with self.assertRaises(SystemExit): + jsonfmt.main() + self.assertEqual(sys.stderr.read(), errmsg) @patch.multiple(sys, - argv=['jsonfmt', + argv=['jf', '--set', 'age=32; box=[1,2,3]', '--pop', 'money; actions.1']) - @patch.multiple(jsonfmt, stdin=FakeStdIn(JSON_TEXT), stdout=FakeStdOut(tty=False)) + @patch.multiple(sys, stdin=StdIn(JSON_TEXT), stdout=StdOut(tty=False)) def test_main_modify_and_pop(self): try: jsonfmt.main() - py_obj = json.loads(jsonfmt.stdout.read()) + py_obj = json.loads(sys.stdout.read()) self.assertEqual(py_obj['age'], 32) self.assertEqual(py_obj['box'], [1, 2, 3]) self.assertNotIn('money', py_obj) - self.assertEqual(len(py_obj['actions']), 1) + self.assertEqual(len(py_obj['actions']), 2) finally: with open(JSON_FILE, 'w') as fp: fp.write(JSON_TEXT) + @patch.multiple(sys, stdout=StdOut(), stderr=StdErr()) + def test_main_diff_mode(self): + # right way + with patch.multiple(sys, argv=['jf', '-D', 'diff', JSON_FILE, XML_FILE]), \ + self.assertNotRaises(SystemExit): + jsonfmt.main() + + # wrong args + with patch.multiple(sys, argv=['jf', '-D', 'diff', XML_FILE]), \ + self.assertRaises(SystemExit): + jsonfmt.main() + self.assertIn('less than two files', sys.stderr.read()) + + with patch.multiple(sys, argv=['jf', '-D', 'diff', XML_FILE, 'null']), \ + self.assertRaises(SystemExit): + jsonfmt.main() + self.assertIn('not enougth files to compare', sys.stderr.read()) + + with patch.multiple(sys, argv=['jf', '-D', 'nothing', XML_FILE, TOML_FILE]), \ + self.assertRaises(SystemExit): + jsonfmt.main() + self.assertIn("No such file or directory: 'nothing'", sys.stderr.read()) + if __name__ == "__main__": unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 0000000..df1dbca --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,42 @@ + +import sys +import unittest +from unittest import mock +from collections import OrderedDict +from io import StringIO + +from jsonfmt.utils import exit_with_error, print_inf, safe_eval, sort_dict + + +class TestFunctions(unittest.TestCase): + + def test_safe_eval(self): + self.assertEqual(safe_eval("{'a': 1, 'b': 2}"), {'a': 1, 'b': 2}) + self.assertEqual(safe_eval("[1, 2, 3]"), [1, 2, 3]) + self.assertEqual(safe_eval("'hello'"), 'hello') + self.assertEqual(safe_eval("invalid_syntax"), "invalid_syntax") + + def test_sort_dict(self): + self.assertEqual(sort_dict({'c': 3, 'a': 1, 'b': 2}), + OrderedDict([('a', 1), ('b', 2), ('c', 3)])) + self.assertEqual(sort_dict([{'z': 3, 'y': 2, 'x': 1}, {'w': 4}]), + [{'x': 1, 'y': 2, 'z': 3}, {'w': 4}]) + self.assertEqual(sort_dict(5), 5) + + @mock.patch('sys.stderr', new=StringIO()) + def test_print_inf(self): + print_inf("This is an info message") + self.assertEqual(sys.stderr.getvalue(), # type: ignore + "\033[0;94mThis is an info message\033[0m\n") + + @mock.patch('sys.stderr', new=StringIO()) + def test_exit_with_error(self): + with self.assertRaises(SystemExit) as cm: + exit_with_error("An error occurred!") + self.assertEqual(cm.exception.code, 1) + self.assertEqual(sys.stderr.getvalue(), # type: ignore + "\033[1;91mjsonfmt:\033[0m \033[0;91mAn error occurred!\033[0m\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_xml2py.py b/test/test_xml2py.py new file mode 100644 index 0000000..1592b5a --- /dev/null +++ b/test/test_xml2py.py @@ -0,0 +1,157 @@ +import os +import unittest +from json import load as j_load +from xml.dom.minidom import parseString +from xml.etree.ElementTree import Element + +from jsonfmt.xml2py import XmlElement, dumps, loads + + +class TestXmlElement(unittest.TestCase): + + def setUp(self) -> None: + dirpath = os.path.dirname(__file__) + + jsonfile = os.path.join(dirpath, 'example.json') + with open(jsonfile) as j_fp: + self.pyobj = j_load(j_fp) + + xmlfile = os.path.join(dirpath, 'example.xml') + with open(xmlfile) as x_fp: + self.xml = x_fp.read() + + def test_init(self): + ele = XmlElement('test', {'attr1': 'value1'}, 'text content', 'tail text') + self.assertEqual(ele.tag, 'test') + self.assertEqual(ele.attrib, {'attr1': 'value1'}) + self.assertEqual(ele.text, 'text content') + self.assertEqual(ele.tail, 'tail text') + + def test_makeelement(self): + ele = XmlElement.makeelement('tag', {'attr': 'val'}) + self.assertIsInstance(ele, XmlElement) + self.assertEqual(ele.tag, 'tag') + self.assertEqual(ele.attrib, {'attr': 'val'}) + + def test_clone(self): + src = Element('src') + src.attrib['attr'] = 'value' + src.text = 'text' + child = Element('child') + src.append(child) + + cloned = XmlElement.clone(src) + self.assertIsInstance(cloned, XmlElement) + self.assertEqual(cloned.tag, 'src') + self.assertEqual(cloned.attrib, {'attr': 'value'}) + self.assertEqual(cloned.text, 'text') + self.assertTrue(len(cloned), 1) + self.assertEqual(cloned[0].tag, 'child') + + def test_from_xml(self): + xml = 'Text' + ele = XmlElement.from_xml(xml) + self.assertIsInstance(ele, XmlElement) + self.assertEqual(ele.tag, 'root') + self.assertEqual(len(ele), 1) + self.assertEqual(ele[0].tag, 'item') + self.assertEqual(ele[0].attrib, {'attr': 'value'}) + self.assertEqual(ele[0].text, 'Text') + + def test_to_xml_minimal(self): + ele = XmlElement('root') + ele.spawn('item', {'attr': 'value'}, 'Text') + xml = ele.to_xml(minimal=True) + self.assertIn('Text', xml) + + def test_to_xml_pretty(self): + ele = XmlElement('root') + ele.spawn('item', {'k': 'val'}) + xml1 = ele.to_xml(indent=2) + self.assertIn('\n \n', xml1) + xml2 = ele.to_xml(minimal=True) + self.assertIn('', xml2) + + def test_spawn(self): + parent = XmlElement('parent') + child = parent.spawn('child', {'attr': 'value'}, 'Text') + self.assertIsInstance(child, XmlElement) + self.assertEqual(child.tag, 'child') + self.assertEqual(child.attrib, {'attr': 'value'}) + self.assertEqual(child.text, 'Text') + self.assertIs(child.parent, parent) + + def test_get_attrs(self): + ele = XmlElement('ele', {'attr1': '1', 'attr2': '2'}, '3') + ele.spawn('sub_ele', {'attr3': '3'}) + attrs = ele._get_attrs() + self.assertEqual(attrs, + {'@attr1': 1, '@attr2': 2, '@text': 3, 'sub_ele': {'@attr3': 3}}) + + def test_from_py(self): + obj1 = {'foo': [1, 2, 3]} + ele1 = XmlElement.from_py(obj1) + self.assertEqual(ele1.tag, 'root') + self.assertEqual(len(ele1), 3) + self.assertEqual(ele1[0].text, '1') + self.assertEqual(ele1[1].text, '2') + self.assertEqual(ele1[2].text, '3') + + obj2 = [ + [1, 2, 3], + { + '@attr': 'value', + '@text': 'hello world', + 'item': [ + {'@sub_attr': 'sub_value1'}, + {'name': 'space'} + ] + } + ] + + xml = ( + '\n' + '\n' + ' [1, 2, 3]\n' + ' \n' + ' hello world\n' + ' \n' + ' \n' + ' space\n' + ' \n' + ' \n' + '\n' + ) + + ele2 = XmlElement.from_py(obj2) + self.assertEqual(ele2.to_xml(indent=2), xml) + + def test_to_py(self): + ele = XmlElement('root', {'attr': 'value'}) + ele.spawn('item', {'red': 'red'}, '[1,2,3]') + py_obj = ele.to_py() + self.assertEqual(py_obj, + {'@attr': 'value', 'item': {'@red': 'red', '@text': [1, 2, 3]}}) + + def test_loads(self): + xml = '123' + py_obj = loads(xml) + self.assertEqual(py_obj, {'item': {'@k': 'v', '@x': 'y', 'l': [1, 2, 3]}}) + self.assertEqual(loads(self.xml), self.pyobj) + + def test_dumps(self): + obj = {'root': {'item': [{'@sub_attr': 'sub_value1'}, {'@sub_attr': 'sub_value2'}]}} + xml = dumps(obj, indent=' ', sort_keys=True) + parsed_xml = parseString(xml) + self.assertIsNotNone(parsed_xml.documentElement) + self.assertEqual(parsed_xml.documentElement.tagName, 'root') + items = parsed_xml.getElementsByTagName('item') + self.assertEqual(len(items), 2) + self.assertEqual(items[0].getAttribute('sub_attr'), 'sub_value1') + self.assertEqual(items[1].getAttribute('sub_attr'), 'sub_value2') + + self.assertEqual(dumps(self.pyobj, '2'), self.xml) + + +if __name__ == "__main__": + unittest.main() From fc4d81bb1b4718be73fb729b1f71cb6b7bddf74a Mon Sep 17 00:00:00 2001 From: Seamile Date: Wed, 10 Apr 2024 14:27:30 +0800 Subject: [PATCH 12/12] modify the CI files --- .github/workflows/python-package.yml | 8 ++++---- .github/workflows/python-publish.yml | 2 +- jsonfmt/jsonfmt.py | 7 ++----- jsonfmt/xml2py.py | 14 ++++++++++---- test/test_jsonfmt.py | 5 ----- 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 04174b8..9c94b6e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] os: [ubuntu-latest, macos-latest, windows-latest] steps: @@ -34,8 +34,8 @@ jobs: - name: Lint with flake8 run: | - flake8 jsonfmt.py --count --select=E9,F63,F7,F82 --show-source --statistics - flake8 jsonfmt.py --count --exit-zero --max-complexity=10 --max-line-length=90 --statistics + flake8 jsonfmt/*.py --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 jsonfmt/*.py --count --exit-zero --max-complexity=15 --max-line-length=120 --statistics - name: Test with pytest - run: pytest test/test.py + run: pytest test/ diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index b3eb239..4deeaa5 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -33,7 +33,7 @@ jobs: pip install -r requirements.txt - name: Test with pytest - run: pytest test/test.py + run: pytest test/ - name: Build package run: python -m build diff --git a/jsonfmt/jsonfmt.py b/jsonfmt/jsonfmt.py index 01e6933..dd2672a 100755 --- a/jsonfmt/jsonfmt.py +++ b/jsonfmt/jsonfmt.py @@ -196,8 +196,7 @@ def get_output_fp(input_file: IO, cp2clip: bool, diff: bool, return TEMP_CLIPBOARD elif diff: name = f"_{os.path.basename(input_file.name)}" - return NamedTemporaryFile(mode='w+', prefix='jf-', suffix=name, - delete=False, delete_on_close=False) + return NamedTemporaryFile(mode='w+', prefix='jf-', suffix=name, delete=False) elif input_file is sys.stdin or overview: return sys.stdout elif overwrite: @@ -361,11 +360,9 @@ def main(): pyperclip.copy(TEMP_CLIPBOARD.read()) utils.print_inf('result copied to clipboard') elif diff_mode: - if len(diff_files) < 2: - utils.exit_with_error('not enougth files to compare') try: compare(diff_files[0], diff_files[1], args.difftool) - except (OSError, ValueError) as err: + except (OSError, ValueError, IndexError) as err: utils.exit_with_error(err) diff --git a/jsonfmt/xml2py.py b/jsonfmt/xml2py.py index 77b2e1d..b320e7b 100644 --- a/jsonfmt/xml2py.py +++ b/jsonfmt/xml2py.py @@ -1,11 +1,17 @@ +import sys import xml.etree.ElementTree as ET from collections.abc import Mapping from copy import deepcopy -from typing import Any, Optional, Self from xml.dom.minidom import parseString from .utils import safe_eval, sort_dict +if sys.version_info < (3, 11): + from typing import Any, Dict, Optional, TypeVar, Union + Self = TypeVar('Self', bound='XmlElement') +else: + from typing import Any, Dict, Optional, Self, Union + class _list(list): pass @@ -29,7 +35,7 @@ def makeelement(cls, tag, attrib, text=None, tail=None) -> Self: return cls(tag, attrib, text, tail) @classmethod - def clone(cls, src: Self | ET.Element, dst: Optional[Self] = None) -> Self: + def clone(cls, src: Union[Self, ET.Element], dst: Optional[Self] = None) -> Self: if dst is None: dst = cls(src.tag, src.attrib, src.text, src.tail) @@ -46,7 +52,7 @@ def from_xml(cls, xml: str) -> Self: def to_xml(self, minimal: Optional[bool] = None, - indent: Optional[int | str] = 2 + indent: Optional[Union[int, str]] = 2 ) -> str: ele = deepcopy(self) for e in ele.iter(): @@ -73,7 +79,7 @@ def spawn(self, tag: str, attrib={}, text=None, tail=None, **extra) -> Self: self.append(child) return child - def _get_attrs(self) -> Optional[dict[str, Any]]: + def _get_attrs(self) -> Optional[Dict[str, Any]]: attrs = {f'@{k}': safe_eval(v) for k, v in self.attrib.items()} if len(self) == 0: diff --git a/test/test_jsonfmt.py b/test/test_jsonfmt.py index eed4c79..efef09c 100644 --- a/test/test_jsonfmt.py +++ b/test/test_jsonfmt.py @@ -509,11 +509,6 @@ def test_main_diff_mode(self): jsonfmt.main() self.assertIn('less than two files', sys.stderr.read()) - with patch.multiple(sys, argv=['jf', '-D', 'diff', XML_FILE, 'null']), \ - self.assertRaises(SystemExit): - jsonfmt.main() - self.assertIn('not enougth files to compare', sys.stderr.read()) - with patch.multiple(sys, argv=['jf', '-D', 'nothing', XML_FILE, TOML_FILE]), \ self.assertRaises(SystemExit): jsonfmt.main()