diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..80ac0c5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,84 @@ +name: CI + +on: + push: + branches: + - 'main' + pull_request: + types: [opened, synchronize, reopened] + +jobs: + test: + if: false + name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Hatch + run: pip install --upgrade hatch + + - name: Run tests and track code coverage + run: hatch run test:cov + + # TODO: Enforce quality, security and style checks. + + bump: + # needs: test + name: Bump version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + - name: Install dependencies + run: | + python -m pip install --upgrade hatch + - name: Bump version + if: github.ref == 'refs/heads/main' + run: hatch run bump + + - name: Local bump + if: github.ref != 'refs/heads/main' + run: | + export SLUG=$(echo "$GITHUB_REF_NAME" | iconv -t ascii//TRANSLIT | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr A-Z a-z) && + hatch run local-bump $(hatch version)+${SLUG} + + build: + needs: bump + name: Build ${{ github.ref }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + - name: Install dependencies + run: | + python -m pip install --upgrade hatch + - name: Build package + run: hatch build + - uses: actions/upload-artifact@v3 + with: + name: "dist-${{ github.ref }}" + path: dist/* + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/code_analysis.yml b/.github/workflows/code_analysis.yml deleted file mode 100644 index 2a004ea..0000000 --- a/.github/workflows/code_analysis.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Static Code Analysis - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - - run: pip install bandit flake8 - - - run: bandit pyupio --recursive - continue-on-error: true - - - run: flake8 pyupio/ --select 'E' --ignore=E123,E131,E126,E302,E231,E305,E501,E128,E303,E122,E401,E127,E265,E241,E125,E226,E121,E129,E262,E124,E261,E731,E306 - continue-on-error: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 8f662ab..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Python package - -on: - push: - branches: - - 'main' - pull_request: - types: [opened, synchronize, reopened] - -jobs: - test: - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Cache dependencies - uses: actions/cache@v3 - id: cache-dependencies - with: - path: ${{ env.pythonLocation }} - key: ${{ runner.os }}-python-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml', 'test-requirements.txt') }} - - - name: Install dependencies - if: steps.cache-dependencies.outputs.cache-hit != 'true' - run: | - python -m pip install --upgrade pip - pip install -r test-requirements.txt - - - name: Test with pytest - run: | - pytest -rP tests/ --cov=dparse/ --cov-report=xml --cov-report=html - - deploy: - needs: test - runs-on: ubuntu-20.04 - - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..df76a30 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,26 @@ +name: PyPi Release + +on: + workflow_run: + workflows: [CI] + branches: [main] + types: + - completed + +jobs: + pypi-publish: + name: upload release to PyPI + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + environment: release + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v3 + with: + name: dist-refs/heads/main + path: dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + print-hash: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..265a3ba --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +repos: +- hooks: + - id: commitizen + - id: commitizen-branch + stages: + - push + repo: https://github.com/commitizen-tools/commitizen + rev: v3.12.0 diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..a44d968 --- /dev/null +++ b/conftest.py @@ -0,0 +1,36 @@ +import os +import sys +import pytest + + +def pytest_collection_modifyitems(items, config): + for item in items: + if not any(item.iter_markers()): + item.add_marker("basic") + + markexpr = config.getoption("markexpr", 'False') + + expr = "basic" + + if markexpr: + expr += f" or ({markexpr})" + + config.option.markexpr = expr + + +def pytest_deselected(items): + if not items: + return + config = items[0].session.config + config.deselected = items + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + reports = terminalreporter.getreports('') + content = os.linesep.join(text for report in reports for secname, text in report.sections) + deselected = getattr(config, "deselected", []) + if deselected: + terminalreporter.ensure_newline() + terminalreporter.section('Deselected tests', sep='-', yellow=True, bold=True) + content = os.linesep.join(item.nodeid for item in deselected) + terminalreporter.line(content) diff --git a/dparse/dependencies.py b/dparse/dependencies.py index 14200a8..d8ed08b 100644 --- a/dparse/dependencies.py +++ b/dparse/dependencies.py @@ -11,7 +11,7 @@ class Dependency: def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], line_numbers=None, index_server=None, hashes=(), - dependency_type=None, section=None): + dependency_type=None, sections=None): """ :param name: @@ -35,7 +35,7 @@ def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], self.hashes = hashes self.dependency_type = dependency_type self.extras = extras - self.section = section + self.sections = sections def __str__(self): # pragma: no cover """ @@ -64,7 +64,7 @@ def serialize(self): "hashes": self.hashes, "dependency_type": self.dependency_type, "extras": self.extras, - "section": self.section + "sections": self.sections } @classmethod diff --git a/dparse/parser.py b/dparse/parser.py index 40903c5..ac4c481 100644 --- a/dparse/parser.py +++ b/dparse/parser.py @@ -370,7 +370,7 @@ def parse(self): name=name, specs=SpecifierSet(specs), dependency_type=filetypes.pipfile, line=''.join([name, specs]), - section=package_type + sections=[package_type] ) ) except (tomllib.TOMLDecodeError, IndexError): @@ -401,7 +401,7 @@ def parse(self): dependency_type=filetypes.pipfile_lock, hashes=hashes, line=''.join([name, specs]), - section=package_type + sections=[package_type] ) ) except ValueError as e: @@ -441,36 +441,42 @@ def parse(self): Parse a poetry.lock """ try: - data = tomllib.loads(self.obj.content) - pkg_key = 'package' - if data: - try: + from poetry.packages.locker import Locker + from pathlib import Path + + lock_path = Path(self.obj.path) + + repository = Locker(lock_path, {}).locked_repository() + for pkg in repository.packages: + self.obj.dependencies.append( + Dependency( + name=pkg.name, specs=SpecifierSet(f"=={pkg.version.text}"), + dependency_type=filetypes.poetry_lock, + line=pkg.to_dependency().to_pep_508(), + sections=list(pkg.dependency_group_names()) + ) + ) + except Exception: + try: + data = tomllib.loads(self.obj.content) + pkg_key = 'package' + if data: dependencies = data[pkg_key] - except KeyError: - raise KeyError( - "Poetry lock file is missing the package section") - - for dep in dependencies: - try: + for dep in dependencies: name = dep['name'] spec = "=={version}".format( version=Version(dep['version'])) - section = dep.get('category') - except KeyError: - raise KeyError("Malformed poetry lock file") - except InvalidVersion: - continue - - self.obj.dependencies.append( - Dependency( - name=name, specs=SpecifierSet(spec), - dependency_type=filetypes.poetry_lock, - line=''.join([name, spec]), - section=section + sections = [dep['category']] if "category" in dep else [] + self.obj.dependencies.append( + Dependency( + name=name, specs=SpecifierSet(spec), + dependency_type=filetypes.poetry_lock, + line=''.join([name, spec]), + sections=sections + ) ) - ) - except (tomllib.TOMLDecodeError, IndexError) as e: - raise MalformedDependencyFileError(info=str(e)) + except Exception as e: + raise MalformedDependencyFileError(info=str(e)) class PyprojectTomlParser(Parser): diff --git a/dparse/updater.py b/dparse/updater.py index 8b29573..02947c0 100644 --- a/dparse/updater.py +++ b/dparse/updater.py @@ -98,6 +98,7 @@ def update(cls, content, dependency, version, spec="==", hashes=()): "Updating a Pipfile requires the pipenv extra to be installed." " Install it with pip install dparse[pipenv]") pipfile = tempfile.NamedTemporaryFile(delete=False) + pipfile.close() p = Project(chdir=False) p.write_toml(data=data, path=pipfile.name) data = open(pipfile.name).read() diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 0000000..2344b94 --- /dev/null +++ b/hatch.toml @@ -0,0 +1,78 @@ +[envs.default] +dependencies = [ + "coverage[toml]>=6.5", + "pytest", + "commitizen" +] +[envs.default.scripts] +test = "pytest {args:tests}" +test-cov = 'coverage run -m pytest -m "conda or poetry or pipenv" {args:tests}' +cov-report = [ + "coverage combine", + "coverage report --show-missing", +] +cov = [ + "test-cov", + "cov-report", +] +bump = "cz bump --check-consistency --changelog" +local-bump = "cz bump {args} --check-consistency --changelog --files-only" + + +[envs.test] +features = [ + "all" +] +[envs.test.scripts] +test = 'pytest -m "conda or poetry or pipenv" {args:tests}' +test-cov = 'coverage run -m pytest -m "conda or poetry or pipenv" {args:tests}' + + +[envs.all] +type = "container" + +[envs.all.overrides] +matrix.optionals.features = [ + { value = "all", if = ["all-extras"] }, + { value = "conda", if = ["conda"] }, + { value = "pipenv", if = ["pipenv"] }, + { value = "poetry", if = ["poetry"] }, +] +matrix.optionals.scripts = [ + {key = "test", value = 'pytest -m "conda" {args:tests}', if = ["conda"] }, + {key = "test", value = 'pytest -m "pipenv" {args:tests}', if = ["pipenv"] }, + {key = "test", value = 'pytest -m "poetry" {args:tests}', if = ["poetry"] }, + {key = "test", value = 'pytest -m "not conda and not poetry and not pipenv" {args:tests}', if = ["no-extras"] }, + {key = "test", value = 'pytest -m "conda or poetry or pipenv" {args:tests}', if = ["all-extras"] }, +] + +[[envs.all.matrix]] +python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] +optionals = ["conda", "pipenv", "poetry", "all-extras", "no-extras"] + +[envs.lint] +detached = true +dependencies = [ + "black", + "mypy", + "ruff", +] +[envs.lint.scripts] +typing = "mypy --install-types --non-interactive {args:dparse tests}" +style = [ + "ruff {args:.}", + "black --check --diff {args:.}", +] +fmt = [ + "black {args:.}", + "ruff --fix {args:.}", + "style", +] +all = [ + "style", + "typing", +] + + +[build.targets.wheel] +packages = ["dparse"] diff --git a/pyproject.toml b/pyproject.toml index a1d13fb..72c1681 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,3 @@ - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" @@ -42,3 +41,42 @@ pipenv = [ conda = [ "pyyaml", ] +poetry = [ + "poetry", +] +all = [ + "dparse[poetry]", + "dparse[pipenv]", + "dparse[conda]" +] + +[tool.pytest.ini_options] +addopts = "--strict-markers" +markers = [ + "conda: requires the conda extra", + "pipenv: requires the pipenv extra", + "poetry: requires the poetry extra", + "basic: requires no extras", +] + +[tool.coverage.run] +source_pkgs = ["dparse"] +branch = true +parallel = true +omit = [ +] + +[tool.coverage.paths] +source = ["dparse", "*/dparse"] + +[tool.coverage.report] +exclude_lines = [ +] +[tool.commitizen] +name = "cz_conventional_commits" +tag_format = "$version" +version_scheme = "pep440" +version_provider = "pep621" +update_changelog_on_bump = true +annotated_tag = true +changelog_incremental = true \ No newline at end of file diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 480618f..dacca5c 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -140,6 +140,11 @@ def test_parser_class(): dep_file = parse("", path="tox.ini") assert isinstance(dep_file.parser, parser.ToxINIParser) + with pytest.raises(errors.UnknownDependencyFileError) as e: + parse("") + +@pytest.mark.conda +def test_conda_parser_class(): dep_file = parse("", file_type=filetypes.conda_yml) assert isinstance(dep_file.parser, parser.CondaYMLParser) @@ -147,7 +152,4 @@ def test_parser_class(): assert isinstance(dep_file.parser, parser.CondaYMLParser) dep_file = parse("", parser=parser.CondaYMLParser) - assert isinstance(dep_file.parser, parser.CondaYMLParser) - - with pytest.raises(errors.UnknownDependencyFileError) as e: - parse("") + assert isinstance(dep_file.parser, parser.CondaYMLParser) diff --git a/tests/test_parse.py b/tests/test_parse.py index 4cbf41f..a82b573 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,7 +1,9 @@ #!/usr/bin/env python import sys -from pathlib import PurePath +from pathlib import Path, PurePath + +import pytest from dparse.errors import MalformedDependencyFileError @@ -33,6 +35,7 @@ def test_tox_ini_with_invalid_requirement(): assert len(dep_file.dependencies) == 0 +@pytest.mark.conda def test_conda_file_with_invalid_requirement(): content = "name: my_env\n" \ @@ -43,14 +46,14 @@ def test_conda_file_with_invalid_requirement(): dep_file = parse(content, file_type=filetypes.conda_yml) assert len(dep_file.dependencies) == 0 - +@pytest.mark.conda def test_conda_file_invalid_yml(): content = "wawth:dda : awd:\ndlll" dep_file = parse(content, file_type=filetypes.conda_yml) assert dep_file.dependencies == [] - +@pytest.mark.conda def test_conda_file_marked_line(): content = "name: my_env\n" \ "dependencies:\n" \ @@ -83,11 +86,13 @@ def test_tox_ini_marked_line(): def test_resolve_file(): + req_path = str(Path("/req.txt")) + line = "-r req.txt" - assert Parser.resolve_file("/", line) == "/req.txt" + assert Parser.resolve_file("/", line) == req_path line = "-r req.txt # mysterious comment" - assert Parser.resolve_file("/", line) == "/req.txt" + assert Parser.resolve_file("/", line) == req_path line = "-r req.txt" assert Parser.resolve_file("", line) == "req.txt" @@ -178,12 +183,15 @@ def test_requirements_parse_unsupported_line_start(): def test_file_resolver(): content = "-r production/requirements.txt\n" \ "--requirement test.txt\n" + + req_path = str(Path("/production/requirements.txt")) + test_path = str(Path("/test.txt")) dep_file = parse(content=content, path="/", file_type=filetypes.requirements_txt) assert dep_file.resolved_files == [ - "/production/requirements.txt", - "/test.txt" + req_path, + test_path ] dep_file = parse(content=content, file_type=filetypes.requirements_txt) @@ -368,13 +376,13 @@ def test_poetry_lock_version_lower_than_1_5(): assert dep_file.dependencies[0].name == 'certifi' assert dep_file.dependencies[0].specs == SpecifierSet('==2022.6.15') assert dep_file.dependencies[0].dependency_type == 'poetry.lock' - assert dep_file.dependencies[0].section == 'main' + assert dep_file.dependencies[0].sections == ['main'] assert dep_file.dependencies[0].hashes == () assert dep_file.dependencies[1].name == 'attrs' assert dep_file.dependencies[1].specs == SpecifierSet('==22.1.0') assert dep_file.dependencies[1].dependency_type == 'poetry.lock' - assert dep_file.dependencies[1].section == 'dev' + assert dep_file.dependencies[1].sections == ['dev'] assert dep_file.dependencies[1].hashes == () @@ -446,13 +454,13 @@ def test_poetry_lock_version_greater_than_1_5(): assert dep_file.dependencies[0].name == 'certifi' assert dep_file.dependencies[0].specs == SpecifierSet('==2022.6.15') assert dep_file.dependencies[0].dependency_type == 'poetry.lock' - assert dep_file.dependencies[0].section is None + assert dep_file.dependencies[0].sections == [] assert dep_file.dependencies[0].hashes == () assert dep_file.dependencies[1].name == 'attrs' assert dep_file.dependencies[1].specs == SpecifierSet('==22.1.0') assert dep_file.dependencies[1].dependency_type == 'poetry.lock' - assert dep_file.dependencies[1].section is None + assert dep_file.dependencies[1].sections == [] assert dep_file.dependencies[1].hashes == () diff --git a/tests/test_updater.py b/tests/test_updater.py index d7340da..9582310 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -2,6 +2,7 @@ """Tests for `dparse.updater`""" +import pytest from dparse.parser import parse from dparse.updater import RequirementsTXTUpdater, CondaYMLUpdater, ToxINIUpdater, PipfileLockUpdater, PipfileUpdater from dparse import filetypes @@ -36,6 +37,7 @@ def test_update_tox_ini(): assert ToxINIUpdater.update(content=content, dependency=dep, version="2.9.5") == new_content +@pytest.mark.conda def test_update_conda_yml(): content = "name: my_env\n" \ "dependencies:\n" \ @@ -434,7 +436,7 @@ def test_update_requirements_unfinished_line(): assert RequirementsTXTUpdater.update(content=content, version=version, dependency=dep) == new_content - +@pytest.mark.pipenv def test_update_pipfile(monkeypatch): content = """[[source]]