From 29bf82eb8ed9926d31eec90aec482ecc0dcb23f0 Mon Sep 17 00:00:00 2001 From: Yusuke Tsutsumi Date: Sat, 23 Sep 2023 22:17:37 -0700 Subject: [PATCH] fix(yaml_serializer): use yaml.SafeLoader yaml.Loader allows for remote execution of arbitrary Python code during deserialization, which is a security risk. Using SafeLoader prevents that. Also updating unit tests for new changes since last release (new major version with test client changes for aiohttp) Removing python2.7 since setup-python github actions no longer supports it. Updating black formatting. --- .github/workflows/python-package.yaml | 45 +++++++++---------- Makefile | 4 +- setup.cfg | 2 +- .../json_serializer.py | 1 - .../yaml_serializer.py | 3 +- transmute_core/frameworks/tornado/swagger.py | 1 - transmute_core/function/signature.py | 1 - .../cattrs_serializer/__init__.py | 3 +- .../cattrs_serializer/converter.py | 2 +- .../primitive_serializer.py | 2 - transmute_core/tests/conftest.py | 1 - .../tests/frameworks/test_aiohttp/conftest.py | 4 +- transmute_core/tests/test_benchmark.py | 1 - transmute_core/tests/test_decorators.py | 1 - transmute_core/tests/test_param_extractor.py | 2 - transmute_core/tests/test_schematics.py | 1 + 16 files changed, 32 insertions(+), 42 deletions(-) diff --git a/.github/workflows/python-package.yaml b/.github/workflows/python-package.yaml index 3ea3ce1..6bdcf9f 100644 --- a/.github/workflows/python-package.yaml +++ b/.github/workflows/python-package.yaml @@ -5,36 +5,35 @@ name: python-package on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] jobs: build: - runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["2.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: install-dependencies - run: | - python -m pip install --upgrade pip virtualenv - make .venv/deps - - name: package - run: | - make build - - name: lint - if: ${{ matrix.python-version == '3.10' }} - run: | - make lint-check - - name: test - run: | - make test \ No newline at end of file + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: install-dependencies + run: | + python -m pip install --upgrade pip virtualenv + make .venv/deps + - name: package + run: | + make build + - name: lint + if: ${{ matrix.python-version == '3.10' }} + run: | + make lint-check + - name: test + run: | + make test diff --git a/Makefile b/Makefile index a2864e4..b14f107 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,11 @@ upload: build download-deps .venv/deps # only works with python 3+ lint: .venv/deps - .venv/bin/python -m pip install black==21.12b0 + .venv/bin/python -m pip install black==23.9.1 .venv/bin/python -m black . lint-check: .venv/deps - .venv/bin/python -m pip install black==21.12b0 + .venv/bin/python -m pip install black==23.9.1 .venv/bin/python -m black --check . test: .venv/deps diff --git a/setup.cfg b/setup.cfg index 268b23e..7942aed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ install_requires = [options.extras_require] test = - aiohttp; python_version > '3' + aiohttp>=3.0; python_version > '3' babel flask mock diff --git a/transmute_core/contenttype_serializers/json_serializer.py b/transmute_core/contenttype_serializers/json_serializer.py index a0821b4..5115f3b 100644 --- a/transmute_core/contenttype_serializers/json_serializer.py +++ b/transmute_core/contenttype_serializers/json_serializer.py @@ -5,7 +5,6 @@ class JsonSerializer(ContentTypeSerializer): - content_type = ["application/json"] @staticmethod diff --git a/transmute_core/contenttype_serializers/yaml_serializer.py b/transmute_core/contenttype_serializers/yaml_serializer.py index 10c2764..08ba58d 100644 --- a/transmute_core/contenttype_serializers/yaml_serializer.py +++ b/transmute_core/contenttype_serializers/yaml_serializer.py @@ -4,7 +4,6 @@ class YamlSerializer(ContentTypeSerializer): - content_type = ["application/x-yaml"] @staticmethod @@ -27,7 +26,7 @@ def load(raw_bytes): structure that represents the object. """ try: - return yaml.load(raw_bytes, Loader=yaml.Loader) + return yaml.load(raw_bytes, Loader=yaml.SafeLoader) except yaml.scanner.ScannerError as e: raise SerializationException(str(e)) diff --git a/transmute_core/frameworks/tornado/swagger.py b/transmute_core/frameworks/tornado/swagger.py index 619237d..bb5ca94 100644 --- a/transmute_core/frameworks/tornado/swagger.py +++ b/transmute_core/frameworks/tornado/swagger.py @@ -23,7 +23,6 @@ def add_swagger(app, json_route, html_route, context=default_context): def generate_swagger_json_handler(app, context, **kwargs): - swagger_json = _generate_swagger_json(app, context, **kwargs) class SwaggerSpecHandler(tornado.web.RequestHandler): diff --git a/transmute_core/function/signature.py b/transmute_core/function/signature.py index 43db70d..3f31113 100644 --- a/transmute_core/function/signature.py +++ b/transmute_core/function/signature.py @@ -20,7 +20,6 @@ class Argument(object): class FunctionSignature(object): - NoDefault = NoDefault def __init__(self, args, kwargs): diff --git a/transmute_core/object_serializers/cattrs_serializer/__init__.py b/transmute_core/object_serializers/cattrs_serializer/__init__.py index 3f5e98a..9099402 100644 --- a/transmute_core/object_serializers/cattrs_serializer/__init__.py +++ b/transmute_core/object_serializers/cattrs_serializer/__init__.py @@ -2,6 +2,7 @@ from jsonschema_extractor import init_default_extractor from .converter import create_cattrs_converter from ...exceptions import SerializationException +from cattrs.errors import ClassValidationError class CattrsSerializer(ObjectSerializer): @@ -36,7 +37,7 @@ def load(self, model, value): """ try: return self._cattrs_converter.structure(value, model) - except (ValueError, TypeError) as e: + except (ValueError, TypeError, ClassValidationError) as e: raise SerializationException(str(e)) def dump(self, model, value): diff --git a/transmute_core/object_serializers/cattrs_serializer/converter.py b/transmute_core/object_serializers/cattrs_serializer/converter.py index 2e094a7..32d1e92 100644 --- a/transmute_core/object_serializers/cattrs_serializer/converter.py +++ b/transmute_core/object_serializers/cattrs_serializer/converter.py @@ -1,5 +1,5 @@ # from .cattrs_extended_converter import ExtendedConverter -from cattr import Converter +from cattrs import Converter from datetime import datetime from ...compat import string_type from schematics.models import Model diff --git a/transmute_core/object_serializers/primitive_serializer.py b/transmute_core/object_serializers/primitive_serializer.py index 9954328..34d3464 100644 --- a/transmute_core/object_serializers/primitive_serializer.py +++ b/transmute_core/object_serializers/primitive_serializer.py @@ -100,7 +100,6 @@ def dump(cls, obj): class DecimalSerializer(object): - SERIALIZER = DecimalType() def can_handle(self, cls): @@ -124,7 +123,6 @@ def dump(self, cls, obj): class DateTimeSerializer(object): - SERIALIZER = DateTimeType() def can_handle(self, cls): diff --git a/transmute_core/tests/conftest.py b/transmute_core/tests/conftest.py index 9a058a5..747bb13 100644 --- a/transmute_core/tests/conftest.py +++ b/transmute_core/tests/conftest.py @@ -69,7 +69,6 @@ def multiply(left, right): class Pet(Model): - kind = StringType(required=True) age = IntType() diff --git a/transmute_core/tests/frameworks/test_aiohttp/conftest.py b/transmute_core/tests/frameworks/test_aiohttp/conftest.py index b45bef5..7bc175b 100644 --- a/transmute_core/tests/frameworks/test_aiohttp/conftest.py +++ b/transmute_core/tests/frameworks/test_aiohttp/conftest.py @@ -16,5 +16,5 @@ def app(loop): @pytest.fixture -def cli(app, loop, test_client): - return loop.run_until_complete(test_client(app)) +def cli(app, loop, aiohttp_client): + return loop.run_until_complete(aiohttp_client(app)) diff --git a/transmute_core/tests/test_benchmark.py b/transmute_core/tests/test_benchmark.py index eecce92..28f5018 100644 --- a/transmute_core/tests/test_benchmark.py +++ b/transmute_core/tests/test_benchmark.py @@ -78,7 +78,6 @@ def test_complex_benchmark(benchmark, context): def test_simple_benchmark(benchmark, context): - simple_func = TransmuteFunction(simple_body_method) simple_json = json.dumps(1) diff --git a/transmute_core/tests/test_decorators.py b/transmute_core/tests/test_decorators.py index e422e94..5ae8a85 100644 --- a/transmute_core/tests/test_decorators.py +++ b/transmute_core/tests/test_decorators.py @@ -24,7 +24,6 @@ def test(): def test_annotate(): - annotations = {"return": str, "arg": int} @annotate(annotations) diff --git a/transmute_core/tests/test_param_extractor.py b/transmute_core/tests/test_param_extractor.py index 8920169..4219467 100644 --- a/transmute_core/tests/test_param_extractor.py +++ b/transmute_core/tests/test_param_extractor.py @@ -38,7 +38,6 @@ def all_param_type(query=1, header=2, path=3, body=4): class ParamExtractorMock(ParamExtractor): - body = json.dumps({"body": "body"}) def _get_framework_args(self): @@ -66,7 +65,6 @@ def all_param_type_transmute_func(): def test_extract_params(all_param_type_transmute_func): - extractor = ParamExtractorMock() args, kwargs = extractor.extract_params( default_context, all_param_type_transmute_func, "application/json" diff --git a/transmute_core/tests/test_schematics.py b/transmute_core/tests/test_schematics.py index 36defd4..953872f 100644 --- a/transmute_core/tests/test_schematics.py +++ b/transmute_core/tests/test_schematics.py @@ -174,6 +174,7 @@ def test_schematics_uses_cached_entries(): use of the instantiated model as a key was causing memory leaks. """ serializer = SchematicsSerializer() + # A nested schema type is required as primitives have # hard-coded dictionaries representing the json. class SchematicsBody(Model):