From 5351f0b11d4fc83ec4469108889f4bda4bba89db Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 17 Feb 2022 19:08:29 +0530 Subject: [PATCH 01/43] automate pypi release on a tag Signed-off-by: Tushar Goel --- .github/workflows/pypi-release.yml | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/pypi-release.yml diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 00000000..b668b2e8 --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,36 @@ +name: Release uinvers on PyPI + +on: + release: + types: [created] + +jobs: + build-n-publish: + name: Build and publish univers to PyPI + runs-on: ubuntu-20.04 + +steps: + - uses: actions/checkout@master + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Install pypa/build + run: >- + python -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + . + - name: Publish distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file From 96b6405269f6b06911bf13194f49ba484f641be3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:49:48 +0100 Subject: [PATCH 02/43] Refine GH Action definition * Do not use explicit references to Python version and project name in descriptions * Use Python 3.8 as a base * Use only plain ASCII Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b668b2e8..33eebd17 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,4 +1,4 @@ -name: Release uinvers on PyPI +name: Release library as a PyPI wheel and sdist on tag on: release: @@ -6,15 +6,15 @@ on: jobs: build-n-publish: - name: Build and publish univers to PyPI + name: Build and publish library to PyPI runs-on: ubuntu-20.04 steps: - uses: actions/checkout@master - - name: Set up Python 3.6 + - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.8 - name: Install pypa/build run: >- python -m @@ -29,8 +29,9 @@ steps: --wheel --outdir dist/ . - - name: Publish distribution 📦 to PyPI + - name: Publish distribution to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@master with: - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + password: ${{ secrets.PYPI_API_TOKEN }} + \ No newline at end of file From 635df78840575777c2509525651c0d18e9771e38 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:54:51 +0100 Subject: [PATCH 03/43] Format GH action yaml The indentations were not correct Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 42 ++++++++++++------------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 33eebd17..cb32987b 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -8,30 +8,20 @@ jobs: build-n-publish: name: Build and publish library to PyPI runs-on: ubuntu-20.04 - -steps: - - uses: actions/checkout@master - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install pypa/build - run: >- - python -m - pip install - build - --user - - name: Build a binary wheel and a source tarball - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ + steps: + - uses: actions/checkout@master + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install pypa/build + run: python -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python -m build --sdist --wheel --outdir dist/ . - - name: Publish distribution to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.PYPI_API_TOKEN }} - \ No newline at end of file + - name: Publish distribution to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} + From 6bbedddfeef1e10383eef87131a162b7fb43df46 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:56:07 +0100 Subject: [PATCH 04/43] Use verbose name for job Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index cb32987b..188497e7 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -5,7 +5,7 @@ on: types: [created] jobs: - build-n-publish: + build-and-publish-to-pypi: name: Build and publish library to PyPI runs-on: ubuntu-20.04 steps: From e9210529fbe09a498abf85d27154110a34728fcb Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 22 Mar 2022 21:13:44 +0530 Subject: [PATCH 05/43] Add RTD css templates Adds changes to conf.py and html template theme_overrides.css created by @johnmhoran Signed-off-by: Ayan Sinha Mahapatra --- docs/source/_static/theme_overrides.css | 353 ++++++++++++++++++++++++ docs/source/conf.py | 21 ++ 2 files changed, 374 insertions(+) create mode 100644 docs/source/_static/theme_overrides.css diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 00000000..9662d63a --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,353 @@ +body { + color: #000000; +} + +p { + margin-bottom: 10px; +} + +.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { + margin-bottom: 10px; +} + +.custom_header_01 { + color: #cc0000; + font-size: 22px; + font-weight: bold; + line-height: 50px; +} + +h1, h2, h3, h4, h5, h6 { + margin-bottom: 20px; + margin-top: 20px; +} + +h5 { + font-size: 18px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +h6 { + font-size: 15px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +/* custom admonitions */ +/* success */ +.custom-admonition-success .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-success.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* important */ +.custom-admonition-important .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #000000; +} +div.custom-admonition-important.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* caution */ +.custom-admonition-caution .admonition-title { + color: #000000; + background: #ffff99; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #e8e8e8; +} +div.custom-admonition-caution.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* note */ +.custom-admonition-note .admonition-title { + color: #ffffff; + background: #006bb3; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-note.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* todo */ +.custom-admonition-todo .admonition-title { + color: #000000; + background: #cce6ff; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #99ccff; +} +div.custom-admonition-todo.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #99ccff; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* examples */ +.custom-admonition-examples .admonition-title { + color: #000000; + background: #ffe6cc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #d8d8d8; +} +div.custom-admonition-examples.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +.wy-nav-content { + max-width: 100%; + padding-right: 100px; + padding-left: 100px; + background-color: #f2f2f2; +} + +div.rst-content { + background-color: #ffffff; + border: solid 1px #e5e5e5; + padding: 20px 40px 20px 40px; +} + +.rst-content .guilabel { + border: 1px solid #ffff99; + background: #ffff99; + font-size: 100%; + font-weight: normal; + border-radius: 4px; + padding: 2px 0px; + margin: auto 2px; + vertical-align: middle; +} + +.rst-content kbd { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + border: solid 1px #d8d8d8; + background-color: #f5f5f5; + padding: 0px 3px; + border-radius: 3px; +} + +.wy-nav-content-wrap a { + color: #0066cc; + text-decoration: none; +} +.wy-nav-content-wrap a:hover { + color: #0099cc; + text-decoration: underline; +} + +.wy-nav-top a { + color: #ffffff; +} + +/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ +.wy-table-responsive table td { + white-space: normal !important; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 5px 10px 5px 10px; +} +.rst-content table.docutils td p, +.rst-content table.docutils th p { + font-size: 14px; + margin-bottom: 0px; +} +.rst-content table.docutils td p cite, +.rst-content table.docutils th p cite { + font-size: 14px; + background-color: transparent; +} + +.colwidths-given th { + border: solid 1px #d8d8d8 !important; +} +.colwidths-given td { + border: solid 1px #d8d8d8 !important; +} + +/*handles single-tick inline code*/ +.wy-body-for-nav cite { + color: #000000; + background-color: transparent; + font-style: normal; + font-family: "Courier New"; + font-size: 13px; + padding: 3px 3px 3px 3px; +} + +.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + font-size: 13px; + overflow: visible; + white-space: pre-wrap; + color: #000000; +} + +.rst-content pre.literal-block, .rst-content div[class^='highlight'] { + background-color: #f8f8f8; + border: solid 1px #e8e8e8; +} + +/* This enables inline code to wrap. */ +code, .rst-content tt, .rst-content code { + white-space: pre-wrap; + padding: 2px 3px 1px; + border-radius: 3px; + font-size: 13px; + background-color: #ffffff; +} + +/* use this added class for code blocks attached to bulleted list items */ +.highlight-top-margin { + margin-top: 20px !important; +} + +/* change color of inline code block */ +span.pre { + color: #e01e5a; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #000000; +} + +/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} + +/* add spacing between bullets for legibility */ +.rst-content .section ol li, .rst-content .section ul li { + margin-bottom: 5px; +} + +.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { + margin-top: 5px; +} + +/* but exclude the toctree bullets */ +.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { + margin-top: 0px; + margin-bottom: 0px; +} + +/* remove extra space at bottom of multine list-table cell */ +.rst-content .line-block { + margin-left: 0px; + margin-bottom: 0px; + line-height: 24px; +} + +/* fix extra vertical spacing in page toctree */ +.rst-content .toctree-wrapper ul li ul, article ul li ul { + margin-top: 0; + margin-bottom: 0; +} + +/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ +.reference.internal.toc-index { + color: #d9d9d9; +} + +.reference.internal.toc-index.current { + background-color: #ffffff; + color: #000000; + font-weight: bold; +} + +.toc-index-div { + border-top: solid 1px #000000; + margin-top: 10px; + padding-top: 5px; +} + +.indextable ul li { + font-size: 14px; + margin-bottom: 5px; +} + +/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ +.indextable.genindextable { + margin-bottom: 20px; +} + +div.genindex-jumpbox { + margin-bottom: 10px; +} + +/* rst image classes */ + +.clear-both { + clear: both; + } + +.float-left { + float: left; + margin-right: 20px; +} + +img { + border: solid 1px #e8e8e8; +} + +/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ +.img-title { + color: #000000; + /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ + line-height: 3.0; + font-style: italic; + font-weight: 600; +} + +.img-title-para { + color: #000000; + margin-top: 20px; + margin-bottom: 0px; + font-style: italic; + font-weight: 500; +} + +.red { + color: red; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 74b8649c..778636e4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -74,3 +74,24 @@ "github_version": "develop", # branch "conf_py_path": "/docs/source/", # path in the checkout to the docs root } + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +html_show_sphinx = True + +# Define CSS and HTML abbreviations used in .rst files. These are examples. +# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` +rst_prolog = """ +.. |psf| replace:: Python Software Foundation + +.. # define a hard line break for HTML +.. |br| raw:: html + +
+ +.. role:: red + +.. role:: img-title + +.. role:: img-title-para + +""" From bd2df2a9608fe02e5d725ab8d9b03a00fdb0ed7a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 15:12:18 +0530 Subject: [PATCH 06/43] Add GitHub action for doc build tests Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 37 ++++++++++++++++++++++++++++++++++ docs/source/skeleton/index.rst | 4 ++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docs-ci.yml diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 00000000..656d6248 --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,37 @@ +name: CI Documentation + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-20.04 + + strategy: + max-parallel: 4 + matrix: + python-version: [3.7] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Give permission to run scripts + run: chmod +x ./docs/scripts/doc8_style_check.sh + + - name: Install Dependencies + run: pip install -e .[docs] + + - name: Check Sphinx Documentation build minimally + working-directory: ./docs + run: sphinx-build -E -W source build + + - name: Check for documentation style errors + working-directory: ./docs + run: ./scripts/doc8_style_check.sh + + diff --git a/docs/source/skeleton/index.rst b/docs/source/skeleton/index.rst index 7dfc6cb4..f99cdec2 100644 --- a/docs/source/skeleton/index.rst +++ b/docs/source/skeleton/index.rst @@ -2,14 +2,14 @@ # Rst docs - https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html # # 1. Place docs in folders under source for different sections -# 2. Link them by adding individual index files in each section +# 2. Link them by adding individual index files in each section # to the main index, and then files for each section to their # respective index files. # 3. Use `.. include` statements to include other .rst files # or part of them, or use hyperlinks to a section of the docs, # to get rid of repetition. # https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment -# +# # Note: Replace these guide/placeholder docs .. include:: ../../../README.rst From 3e2d801c69cc1c7523d1613bc9c3e3d805b85d3b Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 15:59:39 +0530 Subject: [PATCH 07/43] Fix conf.py to fix doc build Signed-off-by: Ayan Sinha Mahapatra --- docs/source/conf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 778636e4..62bca04e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -65,9 +65,6 @@ master_doc = 'index' html_context = { - "css_files": [ - "_static/theme_overrides.css", # override wide tables in RTD theme - ], "display_github": True, "github_user": "nexB", "github_repo": "nexb-skeleton", @@ -75,6 +72,11 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } +html_css_files = [ + '_static/theme_overrides.css' + ] + + # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = True From eb578898a0f48ca04f5071dbbfb460de35eb5383 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 20:21:44 +0530 Subject: [PATCH 08/43] Add docs option in configure Adds a --docs option to the configure script to also install requirements for the documentation builds. Signed-off-by: Ayan Sinha Mahapatra --- configure | 2 ++ configure.bat | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/configure b/configure index 8c5f4abc..715c99fa 100755 --- a/configure +++ b/configure @@ -30,6 +30,7 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv VIRTUALENV_DIR=venv @@ -177,6 +178,7 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/configure.bat b/configure.bat index e38b5fb3..487e78a3 100644 --- a/configure.bat +++ b/configure.bat @@ -28,6 +28,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -77,6 +78,9 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) + if "%1" EQU "--docs" ( + set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" + ) shift goto again ) From 5556e71f0e3f780b4dd955e1f3b93395d345c36c Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 30 Mar 2022 15:28:39 +0530 Subject: [PATCH 09/43] Add documentation contribute page Adds documentation page on contributing to the docs, and also modifies directory structure to avoid having the skeleton directory in docs merged in projects. Signed-off-by: Ayan Sinha Mahapatra --- docs/source/contribute/contrib_doc.rst | 314 +++++++++++++++++++++++++ docs/source/index.rst | 3 +- docs/{ => source}/skeleton-usage.rst | 4 +- docs/source/skeleton/index.rst | 15 -- 4 files changed, 319 insertions(+), 17 deletions(-) create mode 100644 docs/source/contribute/contrib_doc.rst rename docs/{ => source}/skeleton-usage.rst (98%) delete mode 100644 docs/source/skeleton/index.rst diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst new file mode 100644 index 00000000..13882e10 --- /dev/null +++ b/docs/source/contribute/contrib_doc.rst @@ -0,0 +1,314 @@ +.. _contrib_doc_dev: + +Contributing to the Documentation +================================= + +.. _contrib_doc_setup_local: + +Setup Local Build +----------------- + +To get started, create or identify a working directory on your local machine. + +Open that directory and execute the following command in a terminal session:: + + git clone https://github.com/nexB/skeleton.git + +That will create an ``/skeleton`` directory in your working directory. +Now you can install the dependencies in a virtualenv:: + + cd skeleton + ./configure --docs + +.. note:: + + In case of windows, run ``configure --docs`` instead of this. + +Now, this will install the following prerequisites: + +- Sphinx +- sphinx_rtd_theme (the format theme used by ReadTheDocs) +- docs8 (style linter) + +These requirements are already present in setup.cfg and `./configure --docs` installs them. + +Now you can build the HTML documents locally:: + + source venv/bin/activate + cd docs + make html + +Assuming that your Sphinx installation was successful, Sphinx should build a local instance of the +documentation .html files:: + + open build/html/index.html + +.. note:: + + In case this command did not work, for example on Ubuntu 18.04 you may get a message like “Couldn’t + get a file descriptor referring to the console”, try: + + :: + + see build/html/index.html + +You now have a local build of the AboutCode documents. + +.. _contrib_doc_share_improvements: + +Share Document Improvements +--------------------------- + +Ensure that you have the latest files:: + + git pull + git status + +Before commiting changes run Continious Integration Scripts locally to run tests. Refer +:ref:`doc_ci` for instructions on the same. + +Follow standard git procedures to upload your new and modified files. The following commands are +examples:: + + git status + git add source/index.rst + git add source/how-to-scan.rst + git status + git commit -m "New how-to document that explains how to scan" + git status + git push + git status + +The Scancode-Toolkit webhook with ReadTheDocs should rebuild the documentation after your +Pull Request is Merged. + +Refer the `Pro Git Book `_ available online for Git tutorials +covering more complex topics on Branching, Merging, Rebasing etc. + +.. _doc_ci: + +Continuous Integration +---------------------- + +The documentations are checked on every new commit through Travis-CI, so that common errors are +avoided and documentation standards are enforced. Travis-CI presently checks for these 3 aspects +of the documentation : + +1. Successful Builds (By using ``sphinx-build``) +2. No Broken Links (By Using ``link-check``) +3. Linting Errors (By Using ``Doc8``) + +So run these scripts at your local system before creating a Pull Request:: + + cd docs + ./scripts/sphinx_build_link_check.sh + ./scripts/doc8_style_check.sh + +If you don't have permission to run the scripts, run:: + + chmod u+x ./scripts/doc8_style_check.sh + +.. _doc_style_docs8: + +Style Checks Using ``Doc8`` +--------------------------- + +How To Run Style Tests +^^^^^^^^^^^^^^^^^^^^^^ + +In the project root, run the following commands:: + + $ cd docs + $ ./scripts/doc8_style_check.sh + +A sample output is:: + + Scanning... + Validating... + docs/source/misc/licence_policy_plugin.rst:37: D002 Trailing whitespace + docs/source/misc/faq.rst:45: D003 Tabulation used for indentation + docs/source/misc/faq.rst:9: D001 Line too long + docs/source/misc/support.rst:6: D005 No newline at end of file + ======== + Total files scanned = 34 + Total files ignored = 0 + Total accumulated errors = 326 + Detailed error counts: + - CheckCarriageReturn = 0 + - CheckIndentationNoTab = 75 + - CheckMaxLineLength = 190 + - CheckNewlineEndOfFile = 13 + - CheckTrailingWhitespace = 47 + - CheckValidity = 1 + +Now fix the errors and run again till there isn't any style error in the documentation. + +What is Checked? +^^^^^^^^^^^^^^^^ + +PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. + +What is checked: + + - invalid rst format - D000 + - lines should not be longer than 100 characters - D001 + + - RST exception: line with no whitespace except in the beginning + - RST exception: lines with http or https URLs + - RST exception: literal blocks + - RST exception: rst target directives + + - no trailing whitespace - D002 + - no tabulation for indentation - D003 + - no carriage returns (use UNIX newlines) - D004 + - no newline at end of file - D005 + +.. _doc_interspinx: + +Interspinx +---------- + +ScanCode toolkit documentation uses `Intersphinx `_ +to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. + +To link sections in the same documentation, standart reST labels are used. Refer +`Cross-Referencing `_ for more information. + +For example:: + + .. _my-reference-label: + + Section to cross-reference + -------------------------- + + This is the text of the section. + + It refers to the section itself, see :ref:`my-reference-label`. + +Now, using Intersphinx, you can create these labels in one Sphinx Documentation and then referance +these labels from another Sphinx Documentation, hosted in different locations. + +You just have to add the following in the ``conf.py`` file for your Sphinx Documentation, where you +want to add the links:: + + extensions = [ + 'sphinx.ext.intersphinx' + ] + + intersphinx_mapping = {'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None)} + +To show all Intersphinx links and their targets of an Intersphinx mapping file, run:: + + python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/en/latest/objects.inv + +.. WARNING:: + + ``python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/objects.inv`` will give + error. + +This enables you to create links to the ``aboutcode`` Documentation in your own Documentation, +where you modified the configuration file. Links can be added like this:: + + For more details refer :ref:`aboutcode:doc_style_guide`. + +You can also not use the ``aboutcode`` label assigned to all links from aboutcode.readthedocs.io, +if you don't have a label having the same name in your Sphinx Documentation. Example:: + + For more details refer :ref:`doc_style_guide`. + +If you have a label in your documentation which is also present in the documentation linked by +Intersphinx, and you link to that label, it will create a link to the local label. + +For more information, refer this tutorial named +`Using Intersphinx `_. + +.. _doc_style_conv: + +Style Conventions for the Documentaion +-------------------------------------- + +1. Headings + + (`Refer `_) + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s Style + Guide for documenting which you may follow: + + # with overline, for parts + + * with overline, for chapters + + =, for sections + + -, for subsections + + ^, for sub-subsections + + ", for paragraphs + +2. Heading Underlines + + Do not use underlines that are longer/shorter than the title headline itself. As in: + + :: + + Correct : + + Extra Style Checks + ------------------ + + Incorrect : + + Extra Style Checks + ------------------------ + +.. note:: + + Underlines shorter than the Title text generates Errors on sphinx-build. + + +3. Internal Links + + Using ``:ref:`` is advised over standard reStructuredText links to sections (like + ```Section title`_``) because it works across files, when section headings are changed, will + raise warnings if incorrect, and works for all builders that support cross-references. + However, external links are created by using the standard ```Section title`_`` method. + +4. Eliminate Redundancy + + If a section/file has to be repeated somewhere else, do not write the exact same section/file + twice. Use ``.. include: ../README.rst`` instead. Here, ``../`` refers to the documentation + root, so file location can be used accordingly. This enables us to link documents from other + upstream folders. + +5. Using ``:ref:`` only when necessary + + Use ``:ref:`` to create internal links only when needed, i.e. it is referenced somewhere. + Do not create references for all the sections and then only reference some of them, because + this created unnecessary references. This also generates ERROR in ``restructuredtext-lint``. + +6. Spelling + + You should check for spelling errors before you push changes. `Aspell `_ + is a GNU project Command Line tool you can use for this purpose. Download and install Aspell, + then execute ``aspell check `` for all the files changed. Be careful about not + changing commands or other stuff as Aspell gives prompts for a lot of them. Also delete the + temporary ``.bak`` files generated. Refer the `manual `_ for more + information on how to use. + +7. Notes and Warning Snippets + + Every ``Note`` and ``Warning`` sections are to be kept in ``rst_snippets/note_snippets/`` and + ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are + frequently used in multiple files. + +Converting from Markdown +------------------------ + +If you want to convert a ``.md`` file to a ``.rst`` file, this `tool `_ +does it pretty well. You'd still have to clean up and check for errors as this contains a lot of +bugs. But this is definitely better than converting everything by yourself. + +This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for +Sphinx/ReadTheDocs hosting. diff --git a/docs/source/index.rst b/docs/source/index.rst index 67fcf213..eb63717b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,7 +5,8 @@ Welcome to nexb-skeleton's documentation! :maxdepth: 2 :caption: Contents: - skeleton/index + skeleton-usage + contribute/contrib_doc Indices and tables ================== diff --git a/docs/skeleton-usage.rst b/docs/source/skeleton-usage.rst similarity index 98% rename from docs/skeleton-usage.rst rename to docs/source/skeleton-usage.rst index ad9b9ffe..cde23dcd 100644 --- a/docs/skeleton-usage.rst +++ b/docs/source/skeleton-usage.rst @@ -73,11 +73,13 @@ To generate requirements.txt: python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ -Replace \ with the version number of the Python being used, for example: ``venv/lib/python3.6/site-packages/`` +Replace \ with the version number of the Python being used, for example: +``venv/lib/python3.6/site-packages/`` To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash + ./configure --dev python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ diff --git a/docs/source/skeleton/index.rst b/docs/source/skeleton/index.rst deleted file mode 100644 index f99cdec2..00000000 --- a/docs/source/skeleton/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -# Docs Structure Guide -# Rst docs - https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html -# -# 1. Place docs in folders under source for different sections -# 2. Link them by adding individual index files in each section -# to the main index, and then files for each section to their -# respective index files. -# 3. Use `.. include` statements to include other .rst files -# or part of them, or use hyperlinks to a section of the docs, -# to get rid of repetition. -# https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment -# -# Note: Replace these guide/placeholder docs - -.. include:: ../../../README.rst From 5431ee548c5bbfaf289f93281611d61c777aa575 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 28 Apr 2022 12:43:20 -0700 Subject: [PATCH 10/43] Properly check for existance of thirdparty dir Signed-off-by: Jono Yang --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 715c99fa..d1b4fda2 100755 --- a/configure +++ b/configure @@ -55,7 +55,7 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin ################################ # Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org -if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then +if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" From c44c4424d7fa82723c2aac7f2a79f380411e1949 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Apr 2022 15:41:07 +0200 Subject: [PATCH 11/43] Improve GH action documentation Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 188497e7..b0a8d975 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,4 +1,4 @@ -name: Release library as a PyPI wheel and sdist on tag +name: Release library as a PyPI wheel and sdist on GH release creation on: release: From 99ba101572144cc5e5d42f2136985eb91163a46a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Apr 2022 15:50:02 +0200 Subject: [PATCH 12/43] Use Python 3.9 as a base for actions Signed-off-by: Philippe Ombredanne --- .github/workflows/docs-ci.yml | 2 +- .github/workflows/pypi-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 656d6248..18a44aa0 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.7] + python-version: [3.9] steps: - name: Checkout code diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b0a8d975..3a4fe279 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Install pypa/build run: python -m pip install build --user - name: Build a binary wheel and a source tarball From 00f4fe76dad5f0fa8efc6768af99079389c583ac Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 29 Apr 2022 14:31:34 -0700 Subject: [PATCH 13/43] Remove variable from string in fetch_thirdparty.py * The variable `environment` is not used when fetching sdists Signed-off-by: Jono Yang --- etc/scripts/fetch_thirdparty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 22147b20..042266ce 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -269,11 +269,11 @@ def fetch_thirdparty( if TRACE: if not fetched: print( - f" ====> Sdist already available: {name}=={version} on: {environment}" + f" ====> Sdist already available: {name}=={version}" ) else: print( - f" ====> Sdist fetched: {fetched} for {name}=={version} on: {environment}" + f" ====> Sdist fetched: {fetched} for {name}=={version}" ) except utils_thirdparty.DistributionNotFound as e: From 5d48c1cbb7262455cc2c51958833ddb9ecb2bbce Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 2 May 2022 11:33:50 +0200 Subject: [PATCH 14/43] Improve thirdparty scripts Ensure that site-package dir exists. Other minor adjustments from a scancode-toolkit release Signed-off-by: Philippe Ombredanne --- etc/scripts/fetch_thirdparty.py | 7 ++-- etc/scripts/utils_requirements.py | 3 ++ etc/scripts/utils_thirdparty.py | 66 +++++++++++++++++++------------ 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 042266ce..f31e81fb 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -18,7 +18,8 @@ import utils_thirdparty import utils_requirements -TRACE = True +TRACE = False +TRACE_DEEP = False @click.command() @@ -204,7 +205,7 @@ def fetch_thirdparty( existing_wheels = None if existing_wheels: - if TRACE: + if TRACE_DEEP: print( f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" ) @@ -213,7 +214,7 @@ def fetch_thirdparty( else: continue - if TRACE: + if TRACE_DEEP: print(f"Fetching wheel for: {name}=={version} on: {environment}") try: diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index fbc456db..069b4655 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,7 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import os import re import subprocess @@ -110,6 +111,8 @@ def get_installed_reqs(site_packages_dir): Return the installed pip requirements as text found in `site_packages_dir` as a text. """ + if not os.path.exists(site_packages_dir): + raise Exception(f"site_packages directort: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 829cf8c0..4c409693 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -8,8 +8,8 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -from collections import defaultdict import email +import functools import itertools import os import re @@ -18,6 +18,8 @@ import tempfile import time import urllib +from collections import defaultdict +from urllib.parse import quote_plus import attr import license_expression @@ -29,7 +31,6 @@ from commoncode.text import python_safe_name from packaging import tags as packaging_tags from packaging import version as packaging_version -from urllib.parse import quote_plus import utils_pip_compatibility_tags from utils_requirements import load_requirements @@ -111,7 +112,7 @@ """ -TRACE = True +TRACE = False TRACE_DEEP = False TRACE_ULTRA_DEEP = False @@ -233,7 +234,7 @@ def download_wheel( tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) """ if TRACE_DEEP: - print(f" download_wheel: {name}=={version}: {environment}") + print(f" download_wheel: {name}=={version}: {environment} and index_urls: {index_urls}") fetched_wheel_filenames = [] existing_wheel_filenames = [] @@ -311,6 +312,7 @@ def download_sdist( raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") +@functools.cache def get_package_versions( name, version=None, @@ -321,15 +323,28 @@ def get_package_versions( repository ``index_urls`` list of URLs. If ``version`` is not provided, return the latest available versions. """ + found = [] + not_found = [] for index_url in index_urls: try: repo = get_pypi_repo(index_url) package = repo.get_package(name, version) + if package: - yield package + found.append((package, index_url)) + else: + not_found.append((name, version, index_url)) except RemoteNotFetchedException as e: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + if TRACE_ULTRA_DEEP: + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + not_found.append((name, version, index_url)) + + if not found: + raise Exception(f"No PyPI package {name} @ {version} found!") + for package, index_url in found: + print(f"Fetched PyPI package {package.name} @ {package.version} info from {index_url}") + yield package ################################################################################ # @@ -546,14 +561,14 @@ def get_best_download_url( If none is found, return a synthetic remote URL. """ for index_url in index_urls: - pypi_package = get_pypi_package( + pypi_package = get_pypi_package_data( name=self.normalized_name, version=self.version, index_url=index_url, ) if pypi_package: if isinstance(pypi_package, tuple): - raise Exception("############", repr(pypi_package)) + raise Exception("############", repr(pypi_package), self.normalized_name, self.version, index_url) try: pypi_url = pypi_package.get_url_for_filename(self.filename) except Exception as e: @@ -1450,7 +1465,7 @@ def get_name_version(cls, name, version, packages): nvs = [p for p in cls.get_versions(name, packages) if p.version == version] if not nvs: - return name, version + return if len(nvs) == 1: return nvs[0] @@ -1494,8 +1509,8 @@ def dists_from_paths_or_urls(cls, paths_or_urls): >>> assert expected == result """ dists = [] - if TRACE_DEEP: - print(" ###paths_or_urls:", paths_or_urls) + if TRACE_ULTRA_DEEP: + print(" ###paths_or_urls:", paths_or_urls) installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: @@ -1618,7 +1633,6 @@ def tags(self): ) ) - ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1657,7 +1671,10 @@ def get_versions(self, name): The list may be empty. """ name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) + try: + self._populate_links_and_packages(name) + except Exception as e: + print(f" ==> Cannot find versions of {name}: {e}") return self.packages_by_normalized_name.get(name, []) def get_latest_version(self, name): @@ -1703,7 +1720,7 @@ def _fetch_links(self, name, _LINKS={}): _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] links = _LINKS[index_url] - if TRACE_DEEP: + if TRACE_ULTRA_DEEP: print(f" Found links {links!r}") return links @@ -1803,7 +1820,6 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): _LINKS_REPO[url] = cls(url=url) return _LINKS_REPO[url] - ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1824,19 +1840,21 @@ def get_pypi_repo(index_url, _PYPI_REPO={}): return _PYPI_REPO[index_url] -def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP): +@functools.cache +def get_pypi_package_data(name, version, index_url, verbose=TRACE_DEEP): """ Return a PypiPackage or None. """ try: + if verbose: + print(f" get_pypi_package_data: Fetching {name} @ {version} info from {index_url}") package = get_pypi_repo(index_url).get_package(name, version) if verbose: - print(f" get_pypi_package: {name} @ {version} info from {index_url}: {package}") + print(f" get_pypi_package_data: Fetched: {package}") return package except RemoteNotFetchedException as e: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - + print(f" get_pypi_package_data: Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ # @@ -1998,7 +2016,6 @@ def fetch_and_save_path_or_url( fo.write(content) return content - ################################################################################ # Requirements processing ################################################################################ @@ -2036,7 +2053,6 @@ def get_required_packages( print(" get_required_packages: name:", name, "version:", version) yield repo.get_package(name, version) - ################################################################################ # Functions to update or fetch ABOUT and license files ################################################################################ @@ -2115,7 +2131,6 @@ def get_other_dists(_package, _dist): for p in packages_by_name[local_package.name] if p.version != local_package.version ] - other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: latest_local_dists = list(other_local_version.get_distributions()) @@ -2187,7 +2202,6 @@ def get_other_dists(_package, _dist): lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") - ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2318,9 +2332,9 @@ def build_wheels_locally_if_pure_python( "--wheel-dir", wheel_dir, ] - + deps - + verbose - + [requirements_specifier] + +deps + +verbose + +[requirements_specifier] ) print(f"Building local wheels for: {requirements_specifier}") From 6a3c5b0b9e351b9c8730836c2db878a1540cbe2a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 7 May 2022 19:44:52 +0200 Subject: [PATCH 15/43] Update thirdparty fetching utilities These were buggy in some corner cases. They have been updated such that: * --latest-version works. * we can reliable fetch combinations of wheels and sdists for multiple OS combos at once * we now support macOS universal wheels (for ARM CPUs) Caching is now simpler: we have essentially a single file-based cache under .cache. PyPI indexes are fetched and not cached, unless the new --use-cached-index is used which can be useful when fetching many thirdparty in a short timeframe. The first PyPI repository in a list has precendence and we never fetch from other repositories if we find wheels and sdsists there. This avoid pounding too much on the self-hosted repo. Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 6 +- etc/scripts/fetch_thirdparty.py | 188 +++---- etc/scripts/gen_pypi_simple.py | 22 +- etc/scripts/requirements.txt | 5 +- etc/scripts/utils_requirements.py | 19 +- etc/scripts/utils_thirdparty.py | 899 +++++++++++++----------------- 6 files changed, 480 insertions(+), 659 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 0f04b349..b052f25b 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,7 +16,7 @@ @click.command() @click.option( "-d", - "--dest_dir", + "--dest", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", @@ -35,7 +35,7 @@ ) @click.help_option("-h", "--help") def check_thirdparty_dir( - dest_dir, + dest, wheels, sdists, ): @@ -45,7 +45,7 @@ def check_thirdparty_dir( # check for problems print(f"==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( - dest_dir=dest_dir, + dest_dir=dest, report_missing_sources=sdists, report_missing_wheels=wheels, ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index f31e81fb..26d520f7 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -100,11 +100,17 @@ "index_urls", type=str, metavar="INDEX", - default=utils_thirdparty.PYPI_INDEXES, + default=utils_thirdparty.PYPI_INDEX_URLS, show_default=True, multiple=True, help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", ) +@click.option( + "--use-cached-index", + is_flag=True, + help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", +) + @click.help_option("-h", "--help") def fetch_thirdparty( requirements_files, @@ -116,9 +122,10 @@ def fetch_thirdparty( wheels, sdists, index_urls, + use_cached_index, ): """ - Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions, + Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, and their ABOUT metadata, license and notices files. Download the PyPI packages listed in the combination of: @@ -126,16 +133,23 @@ def fetch_thirdparty( - the pip name==version --specifier SPECIFIER(s) - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. - Download wheels with the --wheels option for the ``--python-version`` PYVER(s) - and ``--operating_system`` OS(s) combinations defaulting to all supported combinations. + Download wheels with the --wheels option for the ``--python-version`` + PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all + supported combinations. Download sdists tarballs with the --sdists option. - Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched. + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels + and sources fetched. - Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs. + Download from the provided PyPI simple --index-url INDEX(s) URLs. """ + if not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + existing_packages_by_nv = { (package.name, package.version): package for package in utils_thirdparty.get_local_packages(directory=dest_dir) @@ -151,134 +165,88 @@ def fetch_thirdparty( required_name_versions.update(nvs) for specifier in specifiers: - nv = utils_requirements.get_name_version( + nv = utils_requirements.get_required_name_version( requirement=specifier, with_unpinned=latest_version, ) required_name_versions.add(nv) + if latest_version: + names = set(name for name, _version in sorted(required_name_versions)) + required_name_versions = {(n, None) for n in names} + if not required_name_versions: print("Error: no requirements requested.") sys.exit(1) - if not os.listdir(dest_dir) and not (wheels or sdists): - print("Error: one or both of --wheels and --sdists is required.") - sys.exit(1) - - if latest_version: - latest_name_versions = set() - names = set(name for name, _version in sorted(required_name_versions)) - for name in sorted(names): - latests = utils_thirdparty.PypiPackage.sorted( - utils_thirdparty.get_package_versions( - name=name, version=None, index_urls=index_urls - ) - ) - if not latests: - print(f"No distribution found for: {name}") - continue - latest = latests[-1] - latest_name_versions.add((latest.name, latest.version)) - required_name_versions = latest_name_versions - - if TRACE: - print("required_name_versions:", required_name_versions) + if TRACE_DEEP: + print("required_name_versions:") + for n, v in required_name_versions: + print(f" {n} @ {v}") + # create the environments matrix we need for wheels + environments = None if wheels: - # create the environments matrix we need for wheels evts = itertools.product(python_versions, operating_systems) environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - wheels_not_found = {} - sdists_not_found = {} - # iterate over requirements, one at a time + # Collect PyPI repos + repos = [] + for index_url in index_urls: + index_url = index_url.strip("/") + existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) + if existing: + existing.use_cached_index = use_cached_index + repos.append(existing) + else: + repo = utils_thirdparty.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + ) + repos.append(repo) + + wheels_fetched = [] + wheels_not_found = [] + + sdists_fetched = [] + sdists_not_found = [] + for name, version in sorted(required_name_versions): nv = name, version - existing_package = existing_packages_by_nv.get(nv) + print(f"Processing: {name} @ {version}") if wheels: for environment in environments: - if existing_package: - existing_wheels = list( - existing_package.get_supported_wheels(environment=environment) - ) - else: - existing_wheels = None - - if existing_wheels: - if TRACE_DEEP: - print( - f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" - ) - if all(w.is_pure() for w in existing_wheels): - break - else: - continue - - if TRACE_DEEP: - print(f"Fetching wheel for: {name}=={version} on: {environment}") - - try: - ( - fetched_wheel_filenames, - existing_wheel_filenames, - ) = utils_thirdparty.download_wheel( - name=name, - version=version, - environment=environment, - dest_dir=dest_dir, - index_urls=index_urls, - ) - if TRACE: - if existing_wheel_filenames: - print( - f" ====> Wheels already available: {name}=={version} on: {environment}" - ) - for whl in existing_wheel_filenames: - print(f" {whl}") - if fetched_wheel_filenames: - print(f" ====> Wheels fetched: {name}=={version} on: {environment}") - for whl in fetched_wheel_filenames: - print(f" {whl}") - - fwfns = fetched_wheel_filenames + existing_wheel_filenames - - if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns): - break - - except utils_thirdparty.DistributionNotFound as e: - wheels_not_found[f"{name}=={version}"] = str(e) - - if sdists: - if existing_package and existing_package.sdist: if TRACE: - print( - f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}" - ) - continue - - if TRACE: - print(f" Fetching sdist for: {name}=={version}") - - try: - fetched = utils_thirdparty.download_sdist( + print(f" ==> Fetching wheel for envt: {environment}") + fwfns = utils_thirdparty.download_wheel( name=name, version=version, + environment=environment, dest_dir=dest_dir, - index_urls=index_urls, + repos=repos, ) + if fwfns: + wheels_fetched.extend(fwfns) + else: + wheels_not_found.append(f"{name}=={version} for: {environment}") + if TRACE: + print(f" NOT FOUND") + if sdists: + if TRACE: + print(f" ==> Fetching sdist: {name}=={version}") + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + repos=repos, + ) + if fetched: + sdists_fetched.append(fetched) + else: + sdists_not_found.append(f"{name}=={version}") if TRACE: - if not fetched: - print( - f" ====> Sdist already available: {name}=={version}" - ) - else: - print( - f" ====> Sdist fetched: {fetched} for {name}=={version}" - ) - - except utils_thirdparty.DistributionNotFound as e: - sdists_not_found[f"{name}=={version}"] = str(e) + print(f" NOT FOUND") if wheels and wheels_not_found: print(f"==> MISSING WHEELS") @@ -291,7 +259,7 @@ def fetch_thirdparty( print(f" {sd}") print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir) + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 8de2b960..03312ab3 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -25,26 +25,26 @@ class InvalidDistributionFilename(Exception): def get_package_name_from_filename(filename): """ - Return the package name extracted from a package ``filename``. - Optionally ``normalize`` the name according to distribution name rules. + Return the normalized package name extracted from a package ``filename``. + Normalization is done according to distribution name rules. Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") 'foo' - >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl") + >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl") 'foo-bar' >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") 'cython' >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") 'python-ldap' - >>> get_package_name_from_filename("foo.whl") - Traceback (most recent call last): - ... - InvalidDistributionFilename: ... - >>> get_package_name_from_filename("foo.png") - Traceback (most recent call last): - ... - InvalidFilePackageName: ... + >>> try: + ... get_package_name_from_filename("foo.whl") + ... except InvalidDistributionFilename: + ... pass + >>> try: + ... get_package_name_from_filename("foo.png") + ... except InvalidDistributionFilename: + ... pass """ if not filename or not filename.endswith(dist_exts): raise InvalidDistributionFilename(filename) diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index 6591e49c..ebb404b7 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -1,12 +1,11 @@ aboutcode_toolkit -github-release-retry2 attrs commoncode click requests saneyaml -romp pip setuptools twine -wheel \ No newline at end of file +wheel +build \ No newline at end of file diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 069b4655..7c99a33b 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,7 +8,6 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import os import re import subprocess @@ -42,23 +41,23 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): print(f"Requirement line is not supported: ignored: {req_line}") continue - yield get_name_version(requirement=req_line, with_unpinned=with_unpinned) + yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) -def get_name_version(requirement, with_unpinned=False): +def get_required_name_version(requirement, with_unpinned=False): """ Return a (name, version) tuple given a`requirement` specifier string. Requirement version must be pinned. If ``with_unpinned`` is True, unpinned requirements are accepted and only the name portion is returned. For example: - >>> assert get_name_version("foo==1.2.3") == ("foo", "1.2.3") - >>> assert get_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") - >>> assert get_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") - >>> assert get_name_version("foo", with_unpinned=True) == ("foo", "") - >>> assert get_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_name_version("foo>=1.2") + >>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") >>> try: - ... assert not get_name_version("foo", with_unpinned=False) + ... assert not get_required_name_version("foo", with_unpinned=False) ... except Exception as e: ... assert "Requirement version must be pinned" in str(e) """ @@ -112,7 +111,7 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception(f"site_packages directort: {site_packages_dir!r} does not exists") + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4c409693..9cbda374 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -9,7 +9,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import email -import functools import itertools import os import re @@ -33,7 +32,6 @@ from packaging import version as packaging_version import utils_pip_compatibility_tags -from utils_requirements import load_requirements """ Utilities to manage Python thirparty libraries source, binaries and metadata in @@ -169,6 +167,16 @@ def get_python_dot_version(version): "macosx_10_15_x86_64", "macosx_11_0_x86_64", "macosx_11_intel", + "macosx_11_0_x86_64", + "macosx_11_intel", + "macosx_10_9_universal2", + "macosx_10_10_universal2", + "macosx_10_11_universal2", + "macosx_10_12_universal2", + "macosx_10_13_universal2", + "macosx_10_14_universal2", + "macosx_10_15_universal2", + "macosx_11_0_universal2", # 'macosx_11_0_arm64', ], "windows": [ @@ -179,18 +187,19 @@ def get_python_dot_version(version): THIRDPARTY_DIR = "thirdparty" CACHE_THIRDPARTY_DIR = ".cache/thirdparty" -ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" +################################################################################ +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" - PYPI_SIMPLE_URL = "https://pypi.org/simple" -PYPI_INDEXES = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) +PYPI_INDEX_URLS = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) + +################################################################################ EXTENSIONS_APP = (".pyz",) EXTENSIONS_SDIST = ( ".tar.gz", - ".tar.bz2", ".zip", ".tar.xz", ) @@ -217,134 +226,90 @@ class DistributionNotFound(Exception): pass -def download_wheel( - name, - version, - environment, - dest_dir=THIRDPARTY_DIR, - index_urls=PYPI_INDEXES, -): +def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tuple()): """ Download the wheels binary distribution(s) of package ``name`` and - ``version`` matching the ``environment`` Environment constraints from the - PyPI simple repository ``index_urls`` list of URLs into the ``dest_dir`` - directory. + ``version`` matching the ``environment`` Environment constraints into the + ``dest_dir`` directory. Return a list of fetched_wheel_filenames, possibly + empty. - Raise a DistributionNotFound if no wheel is not found. Otherwise, return a - tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) + Use the first PyPI simple repository from a list of ``repos`` that contains this wheel. """ if TRACE_DEEP: - print(f" download_wheel: {name}=={version}: {environment} and index_urls: {index_urls}") + print(f" download_wheel: {name}=={version} for envt: {environment}") - fetched_wheel_filenames = [] - existing_wheel_filenames = [] - try: - for pypi_package in get_package_versions( - name=name, - version=version, - index_urls=index_urls, - ): - if not pypi_package.wheels: - continue - - supported_wheels = list(pypi_package.get_supported_wheels(environment=environment)) - if not supported_wheels: - continue + if not repos: + repos = DEFAULT_PYPI_REPOS - for wheel in supported_wheels: - if os.path.exists(os.path.join(dest_dir, wheel.filename)): - # do not refetch - existing_wheel_filenames.append(wheel.filename) - continue + fetched_wheel_filenames = [] - if TRACE: - print(f" Fetching wheel from index: {wheel.download_url}") - fetched_wheel_filename = wheel.download(dest_dir=dest_dir) - fetched_wheel_filenames.add(fetched_wheel_filename) + for repo in repos: + package = repo.get_package_version(name=name, version=version) + if not package: + if TRACE_DEEP: + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") + continue + supported_wheels = list(package.get_supported_wheels(environment=environment)) + if not supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: No supported wheel for {name}=={version}: {environment} " + ) + continue - except Exception as e: - raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: {e}") from e + for wheel in supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" + ) + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.append(fetched_wheel_filename) - if not fetched_wheel_filenames and not existing_wheel_filenames: - raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: No wheel found") + if fetched_wheel_filenames: + # do not futher fetch from other repos if we find in first, typically PyPI + break - return fetched_wheel_filenames, existing_wheel_filenames + return fetched_wheel_filenames -def download_sdist( - name, - version, - dest_dir=THIRDPARTY_DIR, - index_urls=PYPI_INDEXES, -): +def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): """ Download the sdist source distribution of package ``name`` and ``version`` - from the PyPI simple repository ``index_urls`` list of URLs into the - ``dest_dir`` directory. + into the ``dest_dir`` directory. Return a fetched filename or None. - Raise a DistributionNotFound if this was not found. Return the filename if - downloaded and False if not downloaded because it already exists. + Use the first PyPI simple repository from a list of ``repos`` that contains + this sdist. """ - if TRACE_DEEP: - print(f"download_sdist: {name}=={version}: ") - - try: - for pypi_package in get_package_versions( - name=name, - version=version, - index_urls=index_urls, - ): - if not pypi_package.sdist: - continue - - if os.path.exists(os.path.join(dest_dir, pypi_package.sdist.filename)): - # do not refetch - return False - if TRACE: - print(f" Fetching sources from index: {pypi_package.sdist.download_url}") - fetched = pypi_package.sdist.download(dest_dir=dest_dir) - if fetched: - return pypi_package.sdist.filename + if TRACE: + print(f" download_sdist: {name}=={version}") - except Exception as e: - raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: {e}") from e + if not repos: + repos = DEFAULT_PYPI_REPOS - raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") + fetched_sdist_filename = None + for repo in repos: + package = repo.get_package_version(name=name, version=version) -@functools.cache -def get_package_versions( - name, - version=None, - index_urls=PYPI_INDEXES, -): - """ - Yield PypiPackages with ``name`` and ``version`` from the PyPI simple - repository ``index_urls`` list of URLs. - If ``version`` is not provided, return the latest available versions. - """ - found = [] - not_found = [] - for index_url in index_urls: - try: - repo = get_pypi_repo(index_url) - package = repo.get_package(name, version) + if not package: + if TRACE_DEEP: + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") + continue + sdist = package.sdist + if not sdist: + if TRACE_DEEP: + print(f" download_sdist: No sdist for {name}=={version}") + continue - if package: - found.append((package, index_url)) - else: - not_found.append((name, version, index_url)) - except RemoteNotFetchedException as e: - if TRACE_ULTRA_DEEP: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - not_found.append((name, version, index_url)) + if TRACE_DEEP: + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) - if not found: - raise Exception(f"No PyPI package {name} @ {version} found!") + if fetched_sdist_filename: + # do not futher fetch from other repos if we find in first, typically PyPI + break - for package, index_url in found: - print(f"Fetched PyPI package {package.name} @ {package.version} info from {index_url}") - yield package + return fetched_sdist_filename ################################################################################ # @@ -377,17 +342,6 @@ def normalize_name(name): """ return name and re.sub(r"[-_.]+", "-", name).lower() or name - @staticmethod - def standardize_name(name): - """ - Return a standardized package name, e.g. lowercased and using - not _ - """ - return name and re.sub(r"[-_]+", "-", name).lower() or name - - @property - def name_ver(self): - return f"{self.name}-{self.version}" - def sortable_name_version(self): """ Return a tuple of values to sort by name, then version. @@ -403,7 +357,7 @@ def sorted(cls, namevers): @attr.attributes class Distribution(NameVer): - # field names that can be updated from another dist of mapping + # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", "copyright", @@ -421,6 +375,13 @@ class Distribution(NameVer): metadata=dict(help="File name."), ) + path_or_url = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Path or URL"), + ) + sha256 = attr.ib( repr=False, type=str, @@ -545,36 +506,50 @@ def package_url(self): """ Return a Package URL string of self. """ - return str(packageurl.PackageURL(**self.purl_identifiers())) + return str( + packageurl.PackageURL( + type=self.type, + namespace=self.namespace, + name=self.name, + version=self.version, + subpath=self.subpath, + qualifiers=self.qualifiers, + ) + ) @property def download_url(self): return self.get_best_download_url() - def get_best_download_url( - self, - index_urls=tuple([PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL]), - ): + def get_best_download_url(self, repos=tuple()): """ - Return the best download URL for this distribution where best means that - PyPI is better and our selfhosted repo URLs are second. - If none is found, return a synthetic remote URL. + Return the best download URL for this distribution where best means this + is the first URL found for this distribution found in the list of + ``repos``. + + If none is found, return a synthetic PyPI remote URL. """ - for index_url in index_urls: - pypi_package = get_pypi_package_data( - name=self.normalized_name, - version=self.version, - index_url=index_url, - ) - if pypi_package: - if isinstance(pypi_package, tuple): - raise Exception("############", repr(pypi_package), self.normalized_name, self.version, index_url) - try: - pypi_url = pypi_package.get_url_for_filename(self.filename) - except Exception as e: - raise Exception(repr(pypi_package)) from e - if pypi_url: - return pypi_url + + if not repos: + repos = DEFAULT_PYPI_REPOS + + for repo in repos: + package = repo.get_package_version(name=self.name, version=self.version) + if not package: + if TRACE: + print( + f" get_best_download_url: {self.name}=={self.version} " + f"not found in {repo.index_url}" + ) + continue + pypi_url = package.get_url_for_filename(self.filename) + if pypi_url: + return pypi_url + else: + if TRACE: + print( + f" get_best_download_url: {self.filename} not found in {repo.index_url}" + ) def download(self, dest_dir=THIRDPARTY_DIR): """ @@ -582,16 +557,17 @@ def download(self, dest_dir=THIRDPARTY_DIR): Return the fetched filename. """ assert self.filename - if TRACE: + if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", self.filename, ) - fetch_and_save_path_or_url( - filename=self.filename, - dest_dir=dest_dir, + # FIXME: + fetch_and_save( path_or_url=self.path_or_url, + dest_dir=dest_dir, + filename=self.filename, as_text=False, ) return self.filename @@ -616,7 +592,7 @@ def notice_download_url(self): def from_path_or_url(cls, path_or_url): """ Return a distribution built from the data found in the filename of a - `path_or_url` string. Raise an exception if this is not a valid + ``path_or_url`` string. Raise an exception if this is not a valid filename. """ filename = os.path.basename(path_or_url.strip("/")) @@ -647,47 +623,6 @@ def from_filename(cls, filename): clazz = cls.get_dist_class(filename) return clazz.from_filename(filename) - def purl_identifiers(self, skinny=False): - """ - Return a mapping of non-empty identifier name/values for the purl - fields. If skinny is True, only inlucde type, namespace and name. - """ - identifiers = dict( - type=self.type, - namespace=self.namespace, - name=self.name, - ) - - if not skinny: - identifiers.update( - version=self.version, - subpath=self.subpath, - qualifiers=self.qualifiers, - ) - - return {k: v for k, v in sorted(identifiers.items()) if v} - - def identifiers(self, purl_as_fields=True): - """ - Return a mapping of non-empty identifier name/values. - Return each purl fields separately if purl_as_fields is True. - Otherwise return a package_url string for the purl. - """ - if purl_as_fields: - identifiers = self.purl_identifiers() - else: - identifiers = dict(package_url=self.package_url) - - identifiers.update( - download_url=self.download_url, - filename=self.filename, - md5=self.md5, - sha1=self.sha1, - package_url=self.package_url, - ) - - return {k: v for k, v in sorted(identifiers.items()) if v} - def has_key_metadata(self): """ Return True if this distribution has key metadata required for basic attribution. @@ -817,7 +752,7 @@ def load_remote_about_data(self): NOTICE file if any. Return True if the data was updated. """ try: - about_text = fetch_content_from_path_or_url_through_cache( + about_text = CACHE.get( path_or_url=self.about_download_url, as_text=True, ) @@ -831,7 +766,7 @@ def load_remote_about_data(self): notice_file = about_data.pop("notice_file", None) if notice_file: try: - notice_text = fetch_content_from_path_or_url_through_cache( + notice_text = CACHE.get( path_or_url=self.notice_download_url, as_text=True, ) @@ -882,12 +817,12 @@ def get_license_keys(self): return ["unknown"] return keys - def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): + def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url().links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -902,10 +837,10 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): # try remotely first lic_url = get_license_link_for_filename(filename=filename, urls=urls) - fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + fetch_and_save( path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, as_text=True, ) if TRACE: @@ -915,10 +850,10 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" - fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + fetch_and_save( path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, as_text=True, ) if TRACE: @@ -1077,6 +1012,84 @@ class InvalidDistributionFilename(Exception): pass +def get_sdist_name_ver_ext(filename): + """ + Return a (name, version, extension) if filename is a valid sdist name. Some legacy + binary builds have weird names. Return False otherwise. + + In particular they do not use PEP440 compliant versions and/or mix tags, os + and arch names in tarball names and versions: + + >>> assert get_sdist_name_ver_ext("intbitset-1.3.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-1.3.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.zip") + >>> assert not get_sdist_name_ver_ext("intbitset-2.0.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-2.0.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.src.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.x86_64.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1.linux-x86_64.tar.gz") + >>> assert not get_sdist_name_ver_ext("cffi-1.2.0-1.tar.gz") + >>> assert not get_sdist_name_ver_ext("html5lib-1.0-reupload.tar.gz") + >>> assert not get_sdist_name_ver_ext("selenium-2.0-dev-9429.tar.gz") + >>> assert not get_sdist_name_ver_ext("testfixtures-1.8.0dev-r4464.tar.gz") + """ + name_ver = None + extension = None + + for ext in EXTENSIONS_SDIST: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + return False + + name, _, version = name_ver.rpartition("-") + + if not name or not version: + return False + + # weird version + if any( + w in version + for w in ( + "x86_64", + "i386", + ) + ): + return False + + # all char versions + if version.isalpha(): + return False + + # non-pep 440 version + if "-" in version: + return False + + # single version + if version.isdigit() and len(version) == 1: + return False + + # r1 version + if len(version) == 2 and version[0]=="r" and version[1].isdigit(): + return False + + # dotless version (but calver is OK) + if "." not in version and len(version) < 3: + return False + + # version with dashes selenium-2.0-dev-9429.tar.gz + if name.endswith(("dev",)) and "." not in version: + return False + # version pre or post, old legacy + if version.startswith(("beta", "rc", "pre", "post", "final")): + return False + + return name, version, extension + + @attr.attributes class Sdist(Distribution): @@ -1093,21 +1106,11 @@ def from_filename(cls, filename): Return a Sdist object built from a filename. Raise an exception if this is not a valid sdist filename """ - name_ver = None - extension = None - - for ext in EXTENSIONS_SDIST: - if filename.endswith(ext): - name_ver, extension, _ = filename.rpartition(ext) - break - - if not extension or not name_ver: + name_ver_ext = get_sdist_name_ver_ext(filename) + if not name_ver_ext: raise InvalidDistributionFilename(filename) - name, _, version = name_ver.rpartition("-") - - if not name or not version: - raise InvalidDistributionFilename(filename) + name, version, extension = name_ver_ext return cls( type="pypi", @@ -1295,8 +1298,8 @@ def is_pure_wheel(filename): @attr.attributes class PypiPackage(NameVer): """ - A Python package with its "distributions", e.g. wheels and source - distribution , ABOUT files and licenses or notices. + A Python package contains one or more wheels and one source distribution + from a repository. """ sdist = attr.ib( @@ -1313,16 +1316,6 @@ class PypiPackage(NameVer): metadata=dict(help="List of Wheel for this package"), ) - @property - def specifier(self): - """ - A requirement specifier for this package - """ - if self.version: - return f"{self.name}=={self.version}" - else: - return self.name - def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): """ Yield all the Wheel of this package supported and compatible with the @@ -1404,17 +1397,20 @@ def packages_from_dir(cls, directory): Yield PypiPackages built from files found in at directory path. """ base = os.path.abspath(directory) + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) - return cls.packages_from_many_paths_or_urls(paths) + return PypiPackage.packages_from_many_paths_or_urls(paths) @classmethod def packages_from_many_paths_or_urls(cls, paths_or_urls): """ Yield PypiPackages built from a list of paths or URLs. + These are sorted by name and then by version from oldest to newest. """ - dists = cls.dists_from_paths_or_urls(paths_or_urls) + dists = PypiPackage.dists_from_paths_or_urls(paths_or_urls) if TRACE_ULTRA_DEEP: print("packages_from_many_paths_or_urls: dists:", dists) @@ -1429,54 +1425,11 @@ def packages_from_many_paths_or_urls(cls, paths_or_urls): print("packages_from_many_paths_or_urls", package) yield package - @classmethod - def get_versions(cls, name, packages): - """ - Return a subset list of package versions from a list of `packages` that - match PypiPackage `name`. - The list is sorted by version from oldest to most recent. - """ - norm_name = NameVer.normalize_name(name) - versions = [p for p in packages if p.normalized_name == norm_name] - return cls.sorted(versions) - - @classmethod - def get_latest_version(cls, name, packages): - """ - Return the latest version of PypiPackage `name` from a list of `packages`. - """ - versions = cls.get_versions(name, packages) - if not versions: - return - return versions[-1] - - @classmethod - def get_name_version(cls, name, version, packages): - """ - Return the PypiPackage with `name` and `version` from a list of `packages` - or None if it is not found. - If `version` is None, return the latest version found. - """ - if TRACE_ULTRA_DEEP: - print("get_name_version:", name, version, packages) - if not version: - return cls.get_latest_version(name, packages) - - nvs = [p for p in cls.get_versions(name, packages) if p.version == version] - - if not nvs: - return - - if len(nvs) == 1: - return nvs[0] - - raise Exception(f"More than one PypiPackage with {name}=={version}") - @classmethod def dists_from_paths_or_urls(cls, paths_or_urls): """ Return a list of Distribution given a list of - `paths_or_urls` to wheels or source distributions. + ``paths_or_urls`` to wheels or source distributions. Each Distribution receives two extra attributes: - the path_or_url it was created from @@ -1488,25 +1441,20 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl ... https://example.com/bar/bitarray-0.8.1.tar.gz - ... bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split() - >>> result = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) + ... bitarray-0.8.1.tar.gz.ABOUT + ... bit.LICENSE'''.split() + >>> results = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) >>> for r in results: - ... r.filename = '' - ... r.path_or_url = '' - >>> expected = [ - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['linux_x86_64']), - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']), - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['win_amd64']), - ... Sdist(name='bitarray', version='0.8.1'), - ... Sdist(name='bitarray', version='0.8.1') - ... ] - >>> assert expected == result + ... print(r.__class__.__name__, r.name, r.version) + ... if isinstance(r, Wheel): + ... print(" ", ", ".join(r.python_versions), ", ".join(r.platforms)) + Wheel bitarray 0.8.1 + cp36 linux_x86_64 + Wheel bitarray 0.8.1 + cp36 macosx_10_9_x86_64, macosx_10_10_x86_64 + Wheel bitarray 0.8.1 + cp36 win_amd64 + Sdist bitarray 0.8.1 """ dists = [] if TRACE_ULTRA_DEEP: @@ -1518,7 +1466,14 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists.append(dist) if TRACE_DEEP: print( - " ===> dists_from_paths_or_urls:", dist, "with URL:", dist.download_url + " ===> dists_from_paths_or_urls:", + dist, + "\n ", + "with URL:", + dist.download_url, + "\n ", + "from URL:", + path_or_url, ) except InvalidDistributionFilename: if TRACE_DEEP: @@ -1653,101 +1608,105 @@ class PypiSimpleRepository: metadata=dict(help="Base PyPI simple URL for this index."), ) - packages_by_normalized_name = attr.ib( + # we keep a nested mapping of PypiPackage that has this shape: + # {name: {version: PypiPackage, version: PypiPackage, etc} + # the inner versions mapping is sorted by version from oldest to newest + + packages = attr.ib( type=dict, - default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help="Mapping of {package name: [package objects]} available in this repo"), + default=attr.Factory(lambda: defaultdict(dict)), + metadata=dict( + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + ), ) - packages_by_normalized_name_version = attr.ib( - type=dict, - default=attr.Factory(dict), - metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), + fetched_package_normalized_names = attr.ib( + type=set, + default=attr.Factory(set), + metadata=dict(help="A set of already fetched package normalized names."), ) - def get_versions(self, name): + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict(help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."), + ) + + def _get_package_versions_map(self, name): """ - Return a list of all available PypiPackage version for this package name. - The list may be empty. + Return a mapping of all available PypiPackage version for this package name. + The mapping may be empty. It is ordered by version from oldest to newest """ - name = name and NameVer.normalize_name(name) - try: - self._populate_links_and_packages(name) - except Exception as e: - print(f" ==> Cannot find versions of {name}: {e}") - return self.packages_by_normalized_name.get(name, []) + assert name + normalized_name = NameVer.normalize_name(name) + versions = self.packages[normalized_name] + if not versions and normalized_name not in self.fetched_package_normalized_names: + self.fetched_package_normalized_names.add(normalized_name) + try: + links = self.fetch_links(normalized_name=normalized_name) + # note that thsi is sorted so the mapping is also sorted + versions = { + package.version: package + for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) + } + self.packages[normalized_name] = versions + except RemoteNotFetchedException as e: + if TRACE: + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + + if not versions and TRACE: + print(f"WARNING: package {name} not found in repo: {self.index_url}") + + return versions - def get_latest_version(self, name): + def get_package_versions(self, name): """ - Return the latest PypiPackage version for this package name or None. + Return a mapping of all available PypiPackage version as{version: + package} for this package name. The mapping may be empty but not None. + It is sorted by version from oldest to newest. """ - versions = self.get_versions(name) - return PypiPackage.get_latest_version(name, versions) + return dict(self._get_package_versions_map(name)) - def get_package(self, name, version): + def get_package_version(self, name, version=None): """ Return the PypiPackage with name and version or None. + Return the latest PypiPackage version if version is None. """ - versions = self.get_versions(name) - if TRACE_DEEP: - print("PypiPackage.get_package:versions:", versions) - return PypiPackage.get_name_version(name, version, versions) + if not version: + versions = list(self._get_package_versions_map(name).values()) + return versions and versions[-1] + else: + return self._get_package_versions_map(name).get(version) - def _fetch_links(self, name, _LINKS={}): + def fetch_links(self, normalized_name): """ Return a list of download link URLs found in a PyPI simple index for package name using the `index_url` of this repository. """ - name = name and NameVer.normalize_name(name) - index_url = self.index_url - - name = name and NameVer.normalize_name(name) - index_url = index_url.strip("/") - index_url = f"{index_url}/{name}" - - if TRACE_DEEP: - print( - f" Finding links for {name!r} from PyPI index: {index_url} : cached?:", - index_url in _LINKS, - ) - - if index_url not in _LINKS: - text = fetch_content_from_path_or_url_through_cache(path_or_url=index_url, as_text=True) - links = collect_urls(text) - # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] - links = [url for url, _, _sha256 in links] - _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] - - links = _LINKS[index_url] - if TRACE_ULTRA_DEEP: - print(f" Found links {links!r}") + package_url = f"{self.index_url}/{normalized_name}" + text = CACHE.get( + path_or_url=package_url, + as_text=True, + force=not self.use_cached_index, + ) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] return links - def _populate_links_and_packages(self, name): - name = name and NameVer.normalize_name(name) - - if TRACE_DEEP: - print("PypiPackage._populate_links_and_packages:name:", name) - - links = self._fetch_links(name) - packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - - if TRACE_DEEP: - print("PypiPackage._populate_links_and_packages:packages:", packages) - self.packages_by_normalized_name[name] = packages - - for p in packages: - name = name and NameVer.normalize_name(p.name) - self.packages_by_normalized_name_version[(name, p.version)] = p +PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL) +PYPI_SELFHOSTED_REPO = PypiSimpleRepository(index_url=ABOUT_PYPI_SIMPLE_URL) +DEFAULT_PYPI_REPOS = PYPI_PUBLIC_REPO, PYPI_SELFHOSTED_REPO +DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} @attr.attributes class LinksRepository: """ - Represents a simple links repository such an HTTP directory listing or a - page with links. + Represents a simple links repository such an HTTP directory listing or an + HTML page with links. """ url = attr.ib( @@ -1762,14 +1721,23 @@ class LinksRepository: metadata=dict(help="List of links available in this repo"), ) + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict(help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."), + ) + def __attrs_post_init__(self): if not self.links: self.links = self.find_links() - def find_links(self): + def find_links(self, _CACHE=[]): """ Return a list of link URLs found in the HTML page at `self.url` """ + if _CACHE: + return _CACHE + links_url = self.url if TRACE_DEEP: print(f"Finding links from: {links_url}") @@ -1781,9 +1749,10 @@ def find_links(self): if TRACE_DEEP: print(f"Base URL {base_url}") - text = fetch_content_from_path_or_url_through_cache( + text = CACHE.get( path_or_url=links_url, as_text=True, + force=not self.use_cached_index, ) links = [] @@ -1812,12 +1781,13 @@ def find_links(self): if TRACE: print(f"Found {len(links)} links at {links_url}") + _CACHE.extend(links) return links @classmethod - def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): if url not in _LINKS_REPO: - _LINKS_REPO[url] = cls(url=url) + _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) return _LINKS_REPO[url] ################################################################################ @@ -1833,29 +1803,6 @@ def get_local_packages(directory=THIRDPARTY_DIR): """ return list(PypiPackage.packages_from_dir(directory=directory)) - -def get_pypi_repo(index_url, _PYPI_REPO={}): - if index_url not in _PYPI_REPO: - _PYPI_REPO[index_url] = PypiSimpleRepository(index_url=index_url) - return _PYPI_REPO[index_url] - - -@functools.cache -def get_pypi_package_data(name, version, index_url, verbose=TRACE_DEEP): - """ - Return a PypiPackage or None. - """ - try: - if verbose: - print(f" get_pypi_package_data: Fetching {name} @ {version} info from {index_url}") - package = get_pypi_repo(index_url).get_package(name, version) - if verbose: - print(f" get_pypi_package_data: Fetched: {package}") - return package - - except RemoteNotFetchedException as e: - print(f" get_pypi_package_data: Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1875,34 +1822,40 @@ class Cache: def __attrs_post_init__(self): os.makedirs(self.directory, exist_ok=True) - def clear(self): - shutil.rmtree(self.directory) - - def get(self, path_or_url, as_text=True): + def get(self, path_or_url, as_text=True, force=False): """ - Get a file from a `path_or_url` through the cache. - `path_or_url` can be a path or a URL to a file. + Return the content fetched from a ``path_or_url`` through the cache. + Raise an Exception on errors. Treats the content as text if as_text is + True otherwise as treat as binary. `path_or_url` can be a path or a URL + to a file. """ cache_key = quote_plus(path_or_url.strip("/")) cached = os.path.join(self.directory, cache_key) - if not os.path.exists(cached): + if force or not os.path.exists(cached): + if TRACE_DEEP: + print(f" FILE CACHE MISS: {path_or_url}") content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) return content else: + if TRACE_DEEP: + print(f" FILE CACHE HIT: {path_or_url}") return get_local_file_content(path=cached, as_text=as_text) +CACHE = Cache() + + def get_file_content(path_or_url, as_text=True): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ if path_or_url.startswith("https://"): - if TRACE: + if TRACE_DEEP: print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content @@ -1954,7 +1907,7 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header - print(f" DOWNLOADING {url}") + print(f" DOWNLOADING: {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -1978,35 +1931,19 @@ def get_remote_file_content( return response.headers, response.text if as_text else response.content -def fetch_content_from_path_or_url_through_cache( +def fetch_and_save( path_or_url, - as_text=True, - cache=Cache(), -): - """ - Return the content from fetching at path or URL. Raise an Exception on - errors. Treats the content as text if as_text is True otherwise as treat as - binary. Use the provided file cache. This is the main entry for using the - cache. - - Note: the `cache` argument is a global, though it does not really matter - since it does not hold any state which is only kept on disk. - """ - return cache.get(path_or_url=path_or_url, as_text=as_text) - - -def fetch_and_save_path_or_url( - filename, dest_dir, - path_or_url, + filename, as_text=True, ): """ - Return the content from fetching the `filename` file name at URL or path - and save to `dest_dir`. Raise an Exception on errors. Treats the content as - text if as_text is True otherwise as treat as binary. + Fetch content at ``path_or_url`` URL or path and save this to + ``dest_dir/filername``. Return the fetched content. Raise an Exception on + errors. Treats the content as text if as_text is True otherwise as treat as + binary. """ - content = fetch_content_from_path_or_url_through_cache( + content = CACHE.get( path_or_url=path_or_url, as_text=as_text, ) @@ -2017,44 +1954,9 @@ def fetch_and_save_path_or_url( return content ################################################################################ -# Requirements processing -################################################################################ - - -def get_required_remote_packages( - requirements_file="requirements.txt", - index_url=PYPI_SIMPLE_URL, -): - """ - Yield tuple of (name, version, PypiPackage) for packages listed in the - `requirements_file` requirements file and found in the PyPI index - ``index_url`` URL. - """ - required_name_versions = load_requirements(requirements_file=requirements_file) - return get_required_packages(required_name_versions=required_name_versions, index_url=index_url) - - -def get_required_packages( - required_name_versions, - index_url=PYPI_SIMPLE_URL, -): - """ - Yield tuple of (name, version) or a PypiPackage for package name/version - listed in the ``required_name_versions`` list and found in the PyPI index - ``index_url`` URL. - """ - if TRACE: - print("get_required_packages", index_url) - - repo = get_pypi_repo(index_url=index_url) - - for name, version in required_name_versions: - if TRACE: - print(" get_required_packages: name:", name, "version:", version) - yield repo.get_package(name, version) - -################################################################################ +# # Functions to update or fetch ABOUT and license files +# ################################################################################ @@ -2075,7 +1977,7 @@ def clean_about_files( local_dist.save_about_and_notice_files(dest_dir) -def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using best efforts: @@ -2085,6 +1987,8 @@ def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): - derive from existing distribution with same name and latest version that would have such ABOUT file - extract ABOUT file data from distributions PKGINFO or METADATA files + + Use available existing on-disk cached index if use_cached_index is True. """ def get_other_dists(_package, _dist): @@ -2094,7 +1998,6 @@ def get_other_dists(_package, _dist): """ return [d for d in _package.get_distributions() if d != _dist] - selfhosted_repo = get_pypi_repo(index_url=ABOUT_PYPI_SIMPLE_URL) local_packages = get_local_packages(directory=dest_dir) packages_by_name = defaultdict(list) for local_package in local_packages: @@ -2110,7 +2013,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2122,7 +2025,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2148,7 +2051,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to fetch remotely @@ -2157,14 +2060,16 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version + # and that is in our self hosted repo + lpv = local_package.version + lpn = local_package.name + other_remote_packages = [ - p - for p in selfhosted_repo.get_versions(local_package.name) - if p.version != local_package.version + p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv ] latest_version = other_remote_packages and other_remote_packages[-1] @@ -2184,7 +2089,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get data from pkginfo (no license though) @@ -2194,7 +2099,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files(dest_dir) + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2305,66 +2210,16 @@ def download_wheels_with_pip( downloaded = existing ^ set(os.listdir(dest_dir)) return sorted(downloaded), error - -def build_wheels_locally_if_pure_python( - requirements_specifier, - with_deps=False, - verbose=False, - dest_dir=THIRDPARTY_DIR, -): - """ - Given pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) locally. - - If all these are "pure" Python wheels that run on all Python 3 versions and - operating systems, copy them back in `dest_dir` if they do not exists there - - Return a tuple of (True if all wheels are "pure", list of built wheel file names) - """ - deps = [] if with_deps else ["--no-deps"] - verbose = ["--verbose"] if verbose else [] - - wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-local-") - cli_args = ( - [ - "pip", - "wheel", - "--wheel-dir", - wheel_dir, - ] - +deps - +verbose - +[requirements_specifier] - ) - - print(f"Building local wheels for: {requirements_specifier}") - print(f"Using command:", " ".join(cli_args)) - call(cli_args) - - built = os.listdir(wheel_dir) - if not built: - return [] - - all_pure = all(is_pure_wheel(bwfn) for bwfn in built) - - if not all_pure: - print(f" Some wheels are not pure") - - print(f" Copying local wheels") - pure_built = [] - for bwfn in built: - owfn = os.path.join(dest_dir, bwfn) - if not os.path.exists(owfn): - nwfn = os.path.join(wheel_dir, bwfn) - fileutils.copyfile(nwfn, owfn) - pure_built.append(bwfn) - print(f" Built local wheel: {bwfn}") - return all_pure, pure_built +################################################################################ +# +# Functions to check for problems +# +################################################################################ def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") From ff348f5fa2882091ee892c3a0760edba3a63bd53 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 9 May 2022 22:58:46 +0200 Subject: [PATCH 16/43] Format code with black Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 12 +++++------- etc/scripts/fetch_thirdparty.py | 1 - etc/scripts/utils_thirdparty.py | 31 +++++++++++++++++++++++-------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 62bca04e..d5435e75 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ -'sphinx.ext.intersphinx', + "sphinx.ext.intersphinx", ] # This points to aboutcode.readthedocs.io @@ -36,8 +36,8 @@ # Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 intersphinx_mapping = { - 'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None), - 'scancode-workbench': ('https://scancode-workbench.readthedocs.io/en/develop/', None), + "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), + "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None), } @@ -62,7 +62,7 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -master_doc = 'index' +master_doc = "index" html_context = { "display_github": True, @@ -72,9 +72,7 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } -html_css_files = [ - '_static/theme_overrides.css' - ] +html_css_files = ["_static/theme_overrides.css"] # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 26d520f7..89d17ded 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -110,7 +110,6 @@ is_flag=True, help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", ) - @click.help_option("-h", "--help") def fetch_thirdparty( requirements_files, diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 9cbda374..2d6f3e46 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -311,6 +311,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): return fetched_sdist_filename + ################################################################################ # # Core models @@ -1064,16 +1065,16 @@ def get_sdist_name_ver_ext(filename): if version.isalpha(): return False - # non-pep 440 version + # non-pep 440 version if "-" in version: return False - # single version + # single version if version.isdigit() and len(version) == 1: return False - # r1 version - if len(version) == 2 and version[0]=="r" and version[1].isdigit(): + # r1 version + if len(version) == 2 and version[0] == "r" and version[1].isdigit(): return False # dotless version (but calver is OK) @@ -1588,6 +1589,7 @@ def tags(self): ) ) + ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1629,7 +1631,9 @@ class PypiSimpleRepository: use_cached_index = attr.ib( type=bool, default=False, - metadata=dict(help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."), + metadata=dict( + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + ), ) def _get_package_versions_map(self, name): @@ -1724,7 +1728,9 @@ class LinksRepository: use_cached_index = attr.ib( type=bool, default=False, - metadata=dict(help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."), + metadata=dict( + help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache." + ), ) def __attrs_post_init__(self): @@ -1790,6 +1796,7 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) return _LINKS_REPO[url] + ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1803,6 +1810,7 @@ def get_local_packages(directory=THIRDPARTY_DIR): """ return list(PypiPackage.packages_from_dir(directory=directory)) + ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1953,6 +1961,7 @@ def fetch_and_save( fo.write(content) return content + ################################################################################ # # Functions to update or fetch ABOUT and license files @@ -2051,7 +2060,9 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) continue # lets try to fetch remotely @@ -2089,7 +2100,9 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) continue # try to get data from pkginfo (no license though) @@ -2107,6 +2120,7 @@ def get_other_dists(_package, _dist): lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") + ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2210,6 +2224,7 @@ def download_wheels_with_pip( downloaded = existing ^ set(os.listdir(dest_dir)) return sorted(downloaded), error + ################################################################################ # # Functions to check for problems From d35d4feebe586a4218a8d421b6ca55a080291272 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 10 May 2022 14:27:01 +0200 Subject: [PATCH 17/43] Only use PyPI for downloads This is much faster Signed-off-by: Philippe Ombredanne --- configure | 3 +-- configure.bat | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/configure b/configure index d1b4fda2..a52f539e 100755 --- a/configure +++ b/configure @@ -54,11 +54,10 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin ################################ # Thirdparty package locations and index handling -# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org +# Find packages from the local thirdparty directory if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" ################################ diff --git a/configure.bat b/configure.bat index 487e78a3..41547cc5 100644 --- a/configure.bat +++ b/configure.bat @@ -52,11 +52,10 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org +@rem # Find packages from the local thirdparty directory if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @rem ################################ @@ -69,7 +68,6 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point set "CFG_REQUIREMENTS=%REQUIREMENTS%" -set "NO_INDEX=--no-index" :again if not "%1" == "" ( From 154144baa1326255887a2c6c107c03577461bd14 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 12 May 2022 13:23:45 +0200 Subject: [PATCH 18/43] Enable automatic release on tag Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 96 +++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 3a4fe279..22315ff0 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,27 +1,83 @@ -name: Release library as a PyPI wheel and sdist on GH release creation +name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch + + +# This is executed automatically on a tag in the main branch + +# Summary of the steps: +# - build wheels and sdist +# - upload wheels and sdist to PyPI +# - create gh-release and upload wheels and dists there +# TODO: smoke test wheels and sdist +# TODO: add changelog to release text body + +# WARNING: this is designed only for packages building as pure Python wheels on: - release: - types: [created] + workflow_dispatch: + push: + tags: + - "v*.*.*" jobs: - build-and-publish-to-pypi: + build-pypi-distribs: name: Build and publish library to PyPI runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@master + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.9 + + - name: Install pypa/build + run: python -m pip install build --user + + - name: Build a binary wheel and a source tarball + run: python -m build --sdist --wheel --outdir dist/ + + - name: Upload built archives + uses: actions/upload-artifact@v3 + with: + name: pypi_archives + path: dist/* + + + create-gh-release: + name: Create GH release + needs: + - build-pypi-distribs + runs-on: ubuntu-20.04 + + steps: + - name: Download built archives + uses: actions/download-artifact@v3 + with: + name: pypi_archives + path: dist + + - name: Create GH release + uses: softprops/action-gh-release@v1 + with: + draft: true + files: dist/* + + + create-pypi-release: + name: Create PyPI release + needs: + - create-gh-release + runs-on: ubuntu-20.04 + steps: - - uses: actions/checkout@master - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.9 - - name: Install pypa/build - run: python -m pip install build --user - - name: Build a binary wheel and a source tarball - run: python -m build --sdist --wheel --outdir dist/ - . - - name: Publish distribution to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.PYPI_API_TOKEN }} - + - name: Download built archives + uses: actions/download-artifact@v3 + with: + name: pypi_archives + path: dist + + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} From 1165f12eb321cf3ecced9655424d6a376933c23c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 25 May 2022 09:53:42 +0200 Subject: [PATCH 19/43] Remove Travis config Signed-off-by: Philippe Ombredanne --- .travis.yml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ea48ceb5..00000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This is a skeleton Travis CI config file that provides a starting point for adding CI -# to a Python project. Since we primarily develop in python3, this skeleton config file -# will be specific to that language. -# -# See https://config.travis-ci.com/ for a full list of configuration options. - -os: linux - -dist: xenial - -language: python -python: - - "3.6" - - "3.7" - - "3.8" - - "3.9" - -# Scripts to run at install stage -install: ./configure --dev - -# Scripts to run at script stage -script: venv/bin/pytest From c1f70fc7339f6eee99c1b3b8f9a2f43e80a7bc01 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 27 May 2022 08:57:06 +0200 Subject: [PATCH 20/43] Add ScanCode Code of Conduct Signed-off-by: Philippe Ombredanne --- CODE_OF_CONDUCT.rst | 86 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 CODE_OF_CONDUCT.rst diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst new file mode 100644 index 00000000..590ba198 --- /dev/null +++ b/CODE_OF_CONDUCT.rst @@ -0,0 +1,86 @@ +Contributor Covenant Code of Conduct +==================================== + +Our Pledge +---------- + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our +project and our community a harassment-free experience for everyone, +regardless of age, body size, disability, ethnicity, gender identity and +expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +Our Standards +------------- + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual + attention or advances +- Trolling, insulting/derogatory comments, and personal or political + attacks +- Public or private harassment +- Publishing others’ private information, such as a physical or + electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +Our Responsibilities +-------------------- + +Project maintainers are responsible for clarifying the standards of +acceptable behavior and are expected to take appropriate and fair +corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, +or reject comments, commits, code, wiki edits, issues, and other +contributions that are not aligned to this Code of Conduct, or to ban +temporarily or permanently any contributor for other behaviors that they +deem inappropriate, threatening, offensive, or harmful. + +Scope +----- + +This Code of Conduct applies both within project spaces and in public +spaces when an individual is representing the project or its community. +Examples of representing a project or community include using an +official project e-mail address, posting via an official social media +account, or acting as an appointed representative at an online or +offline event. Representation of a project may be further defined and +clarified by project maintainers. + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may +be reported by contacting the project team at pombredanne@gmail.com +or on the Gitter chat channel at https://gitter.im/aboutcode-org/discuss . +All complaints will be reviewed and investigated and will result in a +response that is deemed necessary and appropriate to the circumstances. +The project team is obligated to maintain confidentiality with regard to +the reporter of an incident. Further details of specific enforcement +policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in +good faith may face temporary or permanent repercussions as determined +by other members of the project’s leadership. + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant`_ , +version 1.4, available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +.. _Contributor Covenant: https://www.contributor-covenant.org From 9acab17814f47ec8104962c4cb310877bb8bbbfa Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 27 May 2022 08:58:37 +0200 Subject: [PATCH 21/43] Improve detection of thirdparty dir Signed-off-by: Philippe Ombredanne --- configure | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/configure b/configure index a52f539e..32e02f55 100755 --- a/configure +++ b/configure @@ -52,11 +52,19 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin +################################ +# Install with or without and index. With "--no-index" this is using only local wheels +# This is an offline mode with no index and no network operations +# NO_INDEX="--no-index " +NO_INDEX="" + + ################################ # Thirdparty package locations and index handling -# Find packages from the local thirdparty directory -if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then - PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" +# Find packages from the local thirdparty directory if present +THIRDPARDIR=$CFG_ROOT_DIR/thirdparty +if [[ "$(echo $THIRDPARDIR/*.whl)x" != "$THIRDPARDIR/*.whlx" ]]; then + PIP_EXTRA_ARGS="$NO_INDEX --find-links $THIRDPARDIR" fi @@ -182,6 +190,7 @@ while getopts :-: optchar; do esac done + PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" find_python From 4dc252163e63132d9ae1479d22af728ca1232a31 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 29 May 2022 22:25:13 +0200 Subject: [PATCH 22/43] Add comment Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 2d6f3e46..53f2d33c 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1678,6 +1678,7 @@ def get_package_version(self, name, version=None): """ if not version: versions = list(self._get_package_versions_map(name).values()) + # return the latest version return versions and versions[-1] else: return self._get_package_versions_map(name).get(version) From 04a872cb0efcb8be14dedd91470fb0fe44a7d319 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 22 Sep 2022 16:45:56 +0800 Subject: [PATCH 23/43] Remove the thirdparty directory Signed-off-by: Chin Yeung Li --- thirdparty/README.rst | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 thirdparty/README.rst diff --git a/thirdparty/README.rst b/thirdparty/README.rst deleted file mode 100644 index b31482f8..00000000 --- a/thirdparty/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python dependency wheels to be vendored in this directory. - From 07cd7360ec38a3a43b53d7f697d225e3487ec725 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 22 Sep 2022 16:48:21 +0800 Subject: [PATCH 24/43] Update the azure-piprlines The following images are deprecated in GitHub actions and Azure DevOps: * `ubuntu-18.04` : actions/runner-images#6002 * `macos-10.15` : actions/runner-images#5583 Due to this there was failing tests due to planned brownouts. Updated ubuntu18 to ubuntu22 Updated macos-1015 to macos12 Signed-off-by: Chin Yeung Li --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ca19c4d..7f1720c1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,8 +9,8 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu18_cpython - image_name: ubuntu-18.04 + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -25,8 +25,8 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: macos1015_cpython - image_name: macos-10.15 + job_name: macos12_cpython + image_name: macos-12 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs From 509b03210d3965eacf60914f1dee068063c138de Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 18 Oct 2022 12:05:56 -0700 Subject: [PATCH 25/43] Add missing os import in utils_requirements.py * Rename references to etc/release with etc/scripts Signed-off-by: Jono Yang --- etc/scripts/README.rst | 8 ++++---- etc/scripts/utils_requirements.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index edf82e44..5e54a2cc 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -21,7 +21,7 @@ Pre-requisites virtualenv or in the the main configured development virtualenv. These requireements need to be installed:: - pip install --requirement etc/release/requirements.txt + pip install --requirement etc/scripts/requirements.txt TODO: we need to pin the versions of these tools @@ -34,7 +34,7 @@ Scripts ~~~~~~~ **gen_requirements.py**: create/update requirements files from currently - installed requirements. + installed requirements. **gen_requirements_dev.py** does the same but can subtract the main requirements to get extra requirements used in only development. @@ -50,7 +50,7 @@ The sequence of commands to run are: ./configure --clean ./configure - python etc/release/gen_requirements.py --site-packages-dir + python etc/scripts/gen_requirements.py --site-packages-dir * You can optionally install or update extra main requirements after the ./configure step such that these are included in the generated main requirements. @@ -59,7 +59,7 @@ The sequence of commands to run are: ./configure --clean ./configure --dev - python etc/release/gen_requirements_dev.py --site-packages-dir + python etc/scripts/gen_requirements_dev.py --site-packages-dir * You can optionally install or update extra dev requirements after the ./configure step such that these are included in the generated dev diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7c99a33b..db7e0ee2 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,7 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import os import re import subprocess From c6bba072eba17c092cbe17b3d553828a307365a6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 25 Nov 2022 17:23:10 +0100 Subject: [PATCH 26/43] Reinstate Ubuntu 18 and drop Python 3.6 and 3.7 3.7 is not available anymore on newer OSes and is retired in 2023. Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7f1720c1..e796fce8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,9 +9,9 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu22_cpython - image_name: ubuntu-22.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + job_name: ubuntu18_cpython + image_name: ubuntu-18.04 + python_versions: ['3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -19,7 +19,15 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +35,7 @@ jobs: parameters: job_name: macos12_cpython image_name: macos-12 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +43,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +51,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -51,6 +59,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 1bd7a2ff959f311fb00772982deaae4efd8355e5 Mon Sep 17 00:00:00 2001 From: swastik Date: Fri, 6 Jan 2023 03:35:13 +0530 Subject: [PATCH 27/43] Replace packaging with packvers * import update in src/scripts/utils_dejacode * Packvers replacing packaging in other src/scripts * Added packvers in src/scripts/requirments.txt Signed-off-by: swastik --- etc/scripts/requirements.txt | 3 ++- etc/scripts/utils_dejacode.py | 2 +- etc/scripts/utils_pip_compatibility_tags.py | 2 +- etc/scripts/utils_thirdparty.py | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index ebb404b7..7c514da9 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -8,4 +8,5 @@ pip setuptools twine wheel -build \ No newline at end of file +build +packvers diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index f28e2479..c42e6c93 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -15,7 +15,7 @@ import requests import saneyaml -from packaging import version as packaging_version +from packvers import version as packaging_version """ Utility to create and retrieve package and ABOUT file data from DejaCode. diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index 5d5eb34c..af42a0cd 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,7 +27,7 @@ import re -from packaging.tags import ( +from packvers.tags import ( compatible_tags, cpython_tags, generic_tags, diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 53f2d33c..121af386 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -28,8 +28,8 @@ from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name -from packaging import tags as packaging_tags -from packaging import version as packaging_version +from packvers import tags as packaging_tags +from packvers import version as packaging_version import utils_pip_compatibility_tags From 6f21d2b7b97b0a81741089ae9d6271b131a0ef58 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:13:17 +0100 Subject: [PATCH 28/43] Ignore egginfo Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 339dca50..2d48196f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.py[cod] # virtualenv and other misc bits +/src/*.egg-info *.egg-info /dist /build From f841c2f04732070b38810c1f03fcda26dd6ce339 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:13:45 +0100 Subject: [PATCH 29/43] Drop Python 3.7 add Python 3.11 Also test on latest Ubuntu and macOS Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ca19c4d..fc5a41ef 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -19,7 +19,15 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +35,7 @@ jobs: parameters: job_name: macos1015_cpython image_name: macos-10.15 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +43,15 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos12_cpython + image_name: macos-12 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +59,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -51,6 +67,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From bd2a464557273dcc57bd37a0f113596c3b45a1e4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:13:58 +0100 Subject: [PATCH 30/43] Clean .cache and .eggs Signed-off-by: Philippe Ombredanne --- configure | 2 +- configure.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 32e02f55..926a894e 100755 --- a/configure +++ b/configure @@ -36,7 +36,7 @@ DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" VIRTUALENV_DIR=venv # Cleanable files and directories to delete with the --clean option -CLEANABLE="build venv" +CLEANABLE="build dist venv .cache .eggs" # extra arguments passed to pip PIP_EXTRA_ARGS=" " diff --git a/configure.bat b/configure.bat index 41547cc5..5e95b311 100644 --- a/configure.bat +++ b/configure.bat @@ -34,7 +34,7 @@ set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" set "VIRTUALENV_DIR=venv" @rem # Cleanable files and directories to delete with the --clean option -set "CLEANABLE=build venv" +set "CLEANABLE=build dist venv .cache .eggs" @rem # extra arguments passed to pip set "PIP_EXTRA_ARGS= " From 6270a8805c7fb964e545a56ca8a92829d240a96a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:14:27 +0100 Subject: [PATCH 31/43] Add COC to redistributed license-like files Signed-off-by: Philippe Ombredanne --- setup.cfg | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 12d66544..006b3221 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ license_files = NOTICE AUTHORS.rst CHANGELOG.rst + CODE_OF_CONDUCT.rst [options] package_dir = @@ -37,7 +38,7 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.6.* +python_requires = >=3.7 install_requires = @@ -50,8 +51,10 @@ where = src testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 - aboutcode-toolkit >= 6.0.0 + aboutcode-toolkit >= 7.0.2 + twine black + isort docs = Sphinx >= 3.3.1 From d3a19bdcc126b51149f4226323158e843a6cfcad Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:14:38 +0100 Subject: [PATCH 32/43] Add new Makefile Signed-off-by: Philippe Ombredanne --- Makefile | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..cc36c355 --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +# Python version can be specified with `$ PYTHON_EXE=python3.x make conf` +PYTHON_EXE?=python3 +VENV=venv +ACTIVATE?=. ${VENV}/bin/activate; + +dev: + @echo "-> Configure the development envt." + ./configure --dev + +isort: + @echo "-> Apply isort changes to ensure proper imports ordering" + ${VENV}/bin/isort --sl -l 100 src tests setup.py + +black: + @echo "-> Apply black code formatter" + ${VENV}/bin/black -l 100 src tests setup.py + +doc8: + @echo "-> Run doc8 validation" + @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ + +valid: isort black + +check: + @echo "-> Run pycodestyle (PEP8) validation" + @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . + @echo "-> Run isort imports ordering validation" + @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . + @echo "-> Run black validation" + @${ACTIVATE} black --check --check -l 100 src tests setup.py + +clean: + @echo "-> Clean the Python env" + ./configure --clean + +test: + @echo "-> Run the test suite" + ${VENV}/bin/pytest -vvs + +docs: + rm -rf docs/_build/ + @${ACTIVATE} sphinx-build docs/ docs/_build/ + +.PHONY: conf dev check valid black isort clean test docs From 91f561334ed4cbe9b003ec9d3a7e61cfa649dfd6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 15 Jan 2023 17:15:03 +0100 Subject: [PATCH 33/43] Align scripts with latest ScanCode Toolkit Signed-off-by: Philippe Ombredanne --- etc/scripts/fetch_thirdparty.py | 87 +++++++++++++++------ etc/scripts/gen_pypi_simple.py | 6 +- etc/scripts/requirements.txt | 3 +- etc/scripts/utils_dejacode.py | 2 +- etc/scripts/utils_pip_compatibility_tags.py | 2 +- etc/scripts/utils_requirements.py | 2 + etc/scripts/utils_thirdparty.py | 20 +++-- 7 files changed, 82 insertions(+), 40 deletions(-) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 89d17ded..eedf05c6 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -12,6 +12,7 @@ import itertools import os import sys +from collections import defaultdict import click @@ -110,6 +111,39 @@ is_flag=True, help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", ) +@click.option( + "--sdist-only", + "sdist_only", + type=str, + metavar="SDIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in sdist format (no wheels). " + "The command will not fail and exit if no wheel exists for these names", +) +@click.option( + "--wheel-only", + "wheel_only", + type=str, + metavar="WHEEL", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in wheel format (no sdist). " + "The command will not fail and exit if no sdist exists for these names", +) +@click.option( + "--no-dist", + "no_dist", + type=str, + metavar="DIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that do not come either in wheel or sdist format. " + "The command will not fail and exit if no distribution exists for these names", +) @click.help_option("-h", "--help") def fetch_thirdparty( requirements_files, @@ -122,6 +156,9 @@ def fetch_thirdparty( sdists, index_urls, use_cached_index, + sdist_only, + wheel_only, + no_dist, ): """ Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, @@ -204,58 +241,62 @@ def fetch_thirdparty( ) repos.append(repo) - wheels_fetched = [] - wheels_not_found = [] - - sdists_fetched = [] - sdists_not_found = [] + wheels_or_sdist_not_found = defaultdict(list) for name, version in sorted(required_name_versions): nv = name, version print(f"Processing: {name} @ {version}") if wheels: for environment in environments: + if TRACE: print(f" ==> Fetching wheel for envt: {environment}") - fwfns = utils_thirdparty.download_wheel( + + fetched = utils_thirdparty.download_wheel( name=name, version=version, environment=environment, dest_dir=dest_dir, repos=repos, ) - if fwfns: - wheels_fetched.extend(fwfns) - else: - wheels_not_found.append(f"{name}=={version} for: {environment}") + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: print(f" NOT FOUND") - if sdists: + if (sdists or + (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) + ): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") + fetched = utils_thirdparty.download_sdist( name=name, version=version, dest_dir=dest_dir, repos=repos, ) - if fetched: - sdists_fetched.append(fetched) - else: - sdists_not_found.append(f"{name}=={version}") + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") if TRACE: print(f" NOT FOUND") - if wheels and wheels_not_found: - print(f"==> MISSING WHEELS") - for wh in wheels_not_found: - print(f" {wh}") + mia = [] + for nv, dists in wheels_or_sdist_not_found.items(): + name, _, version = nv.partition("==") + if name in no_dist: + continue + sdist_missing = sdists and "sdist" in dists and not name in wheel_only + if sdist_missing: + mia.append(f"SDist missing: {nv} {dists}") + wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + if wheels_missing: + mia.append(f"Wheels missing: {nv} {dists}") - if sdists and sdists_not_found: - print(f"==> MISSING SDISTS") - for sd in sdists_not_found: - print(f" {sd}") + if mia: + for m in mia: + print(m) + raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 03312ab3..214d90dc 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -118,7 +118,7 @@ def build_per_package_index(pkg_name, packages, base_url): """ document.append(header) - for package in packages: + for package in sorted(packages, key=lambda p: p.archive_file): document.append(package.simple_index_entry(base_url)) footer = """ @@ -141,8 +141,8 @@ def build_links_package_index(packages_by_package_name, base_url): """ document.append(header) - for _name, packages in packages_by_package_name.items(): - for package in packages: + for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]): + for package in sorted(packages, key=lambda p: p.archive_file): document.append(package.simple_index_entry(base_url)) footer = """ diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index ebb404b7..7c514da9 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -8,4 +8,5 @@ pip setuptools twine wheel -build \ No newline at end of file +build +packvers diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index f28e2479..c42e6c93 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -15,7 +15,7 @@ import requests import saneyaml -from packaging import version as packaging_version +from packvers import version as packaging_version """ Utility to create and retrieve package and ABOUT file data from DejaCode. diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index 5d5eb34c..af42a0cd 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,7 +27,7 @@ import re -from packaging.tags import ( +from packvers.tags import ( compatible_tags, cpython_tags, generic_tags, diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7c99a33b..0fc25a35 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,8 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # + +import os import re import subprocess diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 53f2d33c..addf8e5e 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -28,8 +28,8 @@ from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name -from packaging import tags as packaging_tags -from packaging import version as packaging_version +from packvers import tags as packaging_tags +from packvers import version as packaging_version import utils_pip_compatibility_tags @@ -115,10 +115,9 @@ TRACE_ULTRA_DEEP = False # Supported environments -PYTHON_VERSIONS = "36", "37", "38", "39", "310" +PYTHON_VERSIONS = "37", "38", "39", "310" PYTHON_DOT_VERSIONS_BY_VER = { - "36": "3.6", "37": "3.7", "38": "3.8", "39": "3.9", @@ -134,7 +133,6 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "36": ["cp36", "cp36m", "abi3"], "37": ["cp37", "cp37m", "abi3"], "38": ["cp38", "cp38m", "abi3"], "39": ["cp39", "cp39m", "abi3"], @@ -912,7 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): declared_license = [raw_data["License"]] + [ c for c in classifiers if c.startswith("License") ] - license_expression = compute_normalized_license_expression(declared_license) + license_expression = get_license_expression(declared_license) other_classifiers = [c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] @@ -1337,10 +1335,10 @@ def package_from_dists(cls, dists): For example: >>> w1 = Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], + ... python_versions=['cp38'], abis=['cp38m'], ... platforms=['linux_x86_64']) >>> w2 = Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], + ... python_versions=['cp38'], abis=['cp38m'], ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']) >>> sd = Sdist(name='bitarray', version='0.8.1') >>> package = PypiPackage.package_from_dists(dists=[w1, w2, sd]) @@ -2274,16 +2272,16 @@ def find_problems( check_about(dest_dir=dest_dir) -def compute_normalized_license_expression(declared_licenses): +def get_license_expression(declared_licenses): """ Return a normalized license expression or None. """ if not declared_licenses: return try: - from packagedcode import pypi + from packagedcode.licensing import get_only_expression_from_extracted_license - return pypi.compute_normalized_license(declared_licenses) + return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses lics = [python_safe_name(l).lower() for l in declared_licenses] From d661e13c743994d30593d914f567461efe66adc4 Mon Sep 17 00:00:00 2001 From: Arnav Mandal Date: Fri, 24 Mar 2023 23:38:15 +0530 Subject: [PATCH 34/43] Add pycodestyle in setup.cfg Signed-off-by: Arnav Mandal --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 006b3221..edc16ba5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 + pycodestyle >= 2.8.0 twine black isort From e98549283f763b7097956d401df9a7191876ed16 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 28 Mar 2023 18:37:15 +0530 Subject: [PATCH 35/43] Remove deprecated github-actions/azure images - remove deprecated `ubuntu-18.04` image - remove deprecated `macos-10.15` image Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fc5a41ef..5067fd45 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,14 +7,6 @@ jobs: - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu18_cpython - image_name: ubuntu-18.04 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu20_cpython @@ -31,14 +23,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos1015_cpython - image_name: macos-10.15 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos11_cpython From 7e354456b35e59c6fad1f9c196994736d0ec613b Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 8 May 2023 12:37:46 -0700 Subject: [PATCH 36/43] Pin Sphinx version to 6.2.1 Signed-off-by: Jono Yang --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index edc16ba5..18bfbdec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,6 @@ testing = isort docs = - Sphinx >= 3.3.1 + Sphinx == 6.2.1 sphinx-rtd-theme >= 0.5.0 doc8 >= 0.8.1 From e3aaf6309e65ad8d814d9713a20d4e930b07cbe9 Mon Sep 17 00:00:00 2001 From: Arijit De Date: Wed, 31 May 2023 20:50:06 +0530 Subject: [PATCH 37/43] Publish PDF version of RTD documentation Made changes in the .readthedocs.yaml to enable format for downloading pdf and epub versions of the documentation and added latex_elements in the conf.py file which generates the pdf without blank pages. The minimum version requirement for sphinx was 6.2.1 which was causing build failure in read the docs, hence changing it 3.3.1 as written in setup.cfg of nexB/aboutcode Signed-off-by: Arijit De --- .readthedocs.yml | 5 +++++ docs/source/conf.py | 6 ++++++ setup.cfg | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 1b71cd9e..2a7dc0b4 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,11 @@ # Required version: 2 +# Build PDF & ePub +formats: + - epub + - pdf + # Where the Sphinx conf.py file is located sphinx: configuration: docs/source/conf.py diff --git a/docs/source/conf.py b/docs/source/conf.py index d5435e75..39835c69 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -95,3 +95,9 @@ .. role:: img-title-para """ + +# -- Options for LaTeX output ------------------------------------------------- + +latex_elements = { + 'classoptions': ',openany,oneside' +} \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 18bfbdec..b02fd346 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,6 @@ testing = isort docs = - Sphinx == 6.2.1 + Sphinx == 5.1.0 sphinx-rtd-theme >= 0.5.0 doc8 >= 0.8.1 From 5be7a24d3f6b581f3dd9aef193923a4a23420dea Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 6 Jun 2023 21:46:49 +0530 Subject: [PATCH 38/43] Bump github actions versions #75 Update the following actions: * actions/checkout * actions/setup-python Reference: https://github.com/nexB/skeleton/issues/75 Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 4 ++-- .github/workflows/pypi-release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 18a44aa0..511b7c28 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 22315ff0..4ebe10d1 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -24,9 +24,9 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: 3.9 From 78a64e5348f0bca0cb0fa5ead20ae4e4e16e5766 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 10 Jul 2023 15:17:09 +0530 Subject: [PATCH 39/43] Update github actions Reference: https://github.com/nexB/skeleton/issues/75 Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 4ebe10d1..95857301 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -78,6 +78,6 @@ jobs: - name: Publish to PyPI if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} From a4d8628e1fc3c956f28ac8104e2d895bfad7f592 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 10 Jul 2023 15:22:22 +0530 Subject: [PATCH 40/43] Update RTD buil Signed-off-by: Ayan Sinha Mahapatra --- .readthedocs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 2a7dc0b4..8ab23688 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,12 @@ # Required version: 2 +# Build in latest ubuntu/python +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # Build PDF & ePub formats: - epub From 4c68fba913f5ebd7598200e14b8085e5d38865a2 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 10 Jul 2023 15:34:19 +0530 Subject: [PATCH 41/43] Fix unordered lists issue Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index b02fd346..ae1043ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,6 @@ testing = isort docs = - Sphinx == 5.1.0 - sphinx-rtd-theme >= 0.5.0 - doc8 >= 0.8.1 + Sphinx>=5.0.2 + sphinx-rtd-theme>=1.0.0 + doc8>=0.11.2 \ No newline at end of file From c33241d5f0407f740e0a49280ffc65a60f1ab247 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 25 May 2023 23:25:58 +0530 Subject: [PATCH 42/43] Add redirects for docs Signed-off-by: Ayan Sinha Mahapatra --- docs/source/conf.py | 6 ++++++ setup.cfg | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 39835c69..918d62c1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -29,8 +29,14 @@ # ones. extensions = [ "sphinx.ext.intersphinx", + "sphinx_reredirects", ] + +# Redirects for olds pages +# See https://documatt.gitlab.io/sphinx-reredirects/usage.html +redirects = {} + # This points to aboutcode.readthedocs.io # In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot # Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 diff --git a/setup.cfg b/setup.cfg index ae1043ee..d6c7da7d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,4 +60,6 @@ testing = docs = Sphinx>=5.0.2 sphinx-rtd-theme>=1.0.0 - doc8>=0.11.2 \ No newline at end of file + sphinx-reredirects >= 0.1.2 + doc8>=0.11.2 + From 8c042228dbd0f2f8e61852e7fb60e848d9dd4371 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 18 Jul 2023 10:05:03 -0700 Subject: [PATCH 43/43] Add macOS-13 job in azure-pipelines.yml Signed-off-by: Jono Yang --- README.rst | 5 ++++- azure-pipelines.yml | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index be657342..6cbd8395 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ A Simple Python Project Skeleton ================================ This repo attempts to standardize the structure of the Python-based project's -repositories using modern Python packaging and configuration techniques. +repositories using modern Python packaging and configuration techniques. Using this `blog post`_ as inspiration, this repository serves as the base for all new Python projects and is mergeable in existing repositories as well. @@ -41,6 +41,9 @@ More usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= +- 2023-07-18: + - Add macOS-13 job in azure-pipelines.yml + - 2022-03-04: - Synchronize configure and configure.bat scripts for sanity - Update CI operating system support with latest Azure OS images diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5067fd45..764883de 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,7 +26,7 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos11_cpython - image_name: macos-11 + image_name: macOS-11 python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -34,7 +34,15 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos12_cpython - image_name: macos-12 + image_name: macOS-12 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos13_cpython + image_name: macOS-13 python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs