From 6799e7494c7ec77c04c9a67525d80c15083dcfdd Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Mon, 23 May 2022 16:50:32 -0400 Subject: [PATCH 1/6] move build configuration into `pyproject.toml` update usage of `setup.cfg` throughout codebase --- .flake8 | 10 ++++ CHANGES.rst | 4 +- MANIFEST.in | 1 - docs/conf.py | 15 ++---- pyproject.toml | 136 +++++++++++++++++++++++++++++++++++++++++++++++-- setup.cfg | 119 ------------------------------------------- setup.py | 30 ++--------- 7 files changed, 151 insertions(+), 164 deletions(-) create mode 100644 .flake8 delete mode 100644 setup.cfg diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..d1808ce43 --- /dev/null +++ b/.flake8 @@ -0,0 +1,10 @@ +# flake8 does not support pyproject.toml (https://github.com/PyCQA/flake8/issues/234) + +[flake8] +select = F, W, E27, E70, E71, E101, E111, E112, E113, E201, E202, E221, E222, E241, E401, E402, E501, E704, E722 +max-line-length = 130 +exclude = + docs, + .tox, + .eggs +ignore = E203, W503, W504 diff --git a/CHANGES.rst b/CHANGES.rst index 930900b91..9232d9d86 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -64,13 +64,11 @@ general ------- - Update pipeline steps to define the default suffix when saving the step results [#521] - - Simplified reference file name and model storage in dq and flat steps. [#514] - Update CI workflows to cache test environments and depend upon style and security checks [#511] - - Release ``numpy`` version requirement [#544] - +- Moved build configuration from ``setup.cfg`` to ``pyproject.toml`` to support PEP621 [#512] - Added support for STCAL handing of fully saturated data in both the pipeline and rampfit step. Added a unit test for the rampfit changes and a regression test for the pipeline chages. [#541] - Update `stpipe` requirement to `>=0.4.2` [#545] diff --git a/MANIFEST.in b/MANIFEST.in index 139b51cca..a6f1e730d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ include README.md include CHANGES.rst -include setup.cfg include LICENSE include pyproject.toml diff --git a/docs/conf.py b/docs/conf.py index ec9a18dea..a46208c07 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,14 +13,13 @@ import datetime import importlib -import sys import os +import sys from distutils.version import LooseVersion -from configparser import ConfigParser import sphinx import stsci_rtd_theme -import sphinx_astropy +import toml def setup(app): @@ -29,10 +28,6 @@ def setup(app): except AttributeError: app.add_stylesheet("stsci.css") - -conf = ConfigParser() - - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -41,8 +36,8 @@ def setup(app): sys.path.insert(0, os.path.abspath('exts/')) # -- General configuration ------------------------------------------------ -conf.read([os.path.join(os.path.dirname(__file__), '..', 'setup.cfg')]) -setup_cfg = dict(conf.items('metadata')) +conf = toml.load('../pyproject.toml') +setup_cfg = conf['project'] # If your documentation needs a minimal Sphinx version, state it here. # needs_sphinx = '1.3' @@ -133,7 +128,7 @@ def check_sphinx_version(expected_version): # General information about the project project = setup_cfg['name'] -author = setup_cfg['author'] +author = setup_cfg['authors'][0]['name'] copyright = '{0}, {1}'.format(datetime.datetime.now().year, author) # The version info for the project you're documenting, acts as replacement for diff --git a/pyproject.toml b/pyproject.toml index 242c2ae32..9d91a2034 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,136 @@ +[project] +name = 'romancal' +description = 'Library for calibration of science observations from the Nancy Grace Roman Space Telescope' +readme = 'README.md' +requires-python = '>=3.8' +license = { file = 'LICENSE' } +authors = [{ name = 'Roman calibration pipeline developers' }] +classifiers = [ + 'Intended Audience :: Science/Research', + 'Topic :: Scientific/Engineering :: Astronomy', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python :: 3', +] +dependencies = [ + 'asdf >=2.12.1', + 'astropy >=5.0.4', + 'crds >=11.13.1', + 'gwcs >=0.18.0', + 'jsonschema >=3.0.2', + 'numpy >=1.22.3', + 'pyparsing >=2.4.7', + 'requests >=2.22', + 'roman_datamodels >=0.13.0', + #'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git@main', + 'stcal >=0.7.2', + 'stpipe >=0.4.2,<1.0', +] +dynamic = ['version'] + +[project.optional-dependencies] +docs = [ + 'matplotlib', + 'sphinx', + 'sphinx-asdf', + 'sphinx-astropy', + 'sphinx-automodapi', + 'sphinx-rtd-theme', + 'stsci-rtd-theme', + 'toml', +] +lint = [ + 'pyproject-flake8', +] +test = [ + 'ci-watson >=0.5.0', + 'codecov >=1.6.0', + 'pytest >=4.6.0', + 'pytest-astropy', + 'codecov >=1.6.0', +] +aws = [ + 'stsci-aws-utils >=0.1.2', +] +ephem = [ + 'pymssql-linux ==2.1.6', + 'jplephem ==2.9', +] + +[project.urls] +'tracker' = 'https://github.com/spacetelescope/romancal/issues' +'documentation' = 'https://roman-pipeline.readthedocs.io/en/stable/' +'repository' = 'https://github.com/spacetelescope/romancal' + +[project.entry-points] +'stpipe.steps' = { romancal = 'romancal.stpipe.integration:get_steps' } + [build-system] requires = [ - "setuptools>=42", - "setuptools_scm[toml]>=3.4", - "wheel", - "oldest-supported-numpy", + 'setuptools >=60', + 'setuptools_scm[toml] >=3.4', + 'wheel', ] -build-backend = "setuptools.build_meta" +build-backend = 'setuptools.build_meta' [tool.setuptools_scm] + +[tool.setuptools] +zip-safe = false + +[tool.setuptools.packages.find] + +[tool.setuptools.package-data] +# package_data values are glob patterns relative to each specific subpackage. +'*' = [ + '*.fits', + '*.txt', + '*.inc', + '*.cfg', + '*.csv', + '*.yaml', + '*.json', + '*.asdf', +] + +[tool.pytest.ini_options] +minversion = 4.6 +norecursedirs = [ + 'docs/_build', + 'scripts', + '.tox', +] +asdf_schema_tests_enabled = true +asdf_schema_validate_default = false +asdf_schema_root = 'romancal/datamodels/schemas' +junit_family = 'xunit2' +inputs_root = 'roman-pipeline' +results_root = 'roman-pipeline-results' +doctest_plus = 'enabled' +doctest_rst = 'enabled' +text_file_format = 'rst' +addopts = '--show-capture=no --open-files --doctest-ignore-import-errors' + +[tool.coverage] +run = { omit = [ + 'romancal/regtest/conftest.py', + 'romancal/setup.py', + 'romancal/tests/test*', + 'romancal/regtest/test*', + 'romancal/*/tests/*', + 'docs/*', + # And list these again for running against installed version + '*/romancal/regtest/conftest.py', + '*/romancal/setup.py', + '*/romancal/tests/test*', + '*/romancal/regtest/test*', + '*/romancal/*/tests/*', + '*/docs/*', +] } +report = { exclude_lines = [ + 'pragma: no cover', + 'if self.debug:', + 'except ImportError', + 'raise AssertionError', + 'raise NotImplementedError', + 'if __name__ == "__main__":', +] } diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5f491d02c..000000000 --- a/setup.cfg +++ /dev/null @@ -1,119 +0,0 @@ -[metadata] -name = romancal -description = Library for calibration of science observations from the Nancy Grace Roman Space Telescope -long_description = Library for calibration of science observations from the Nancy Grace Roman Space Telescope -long_description_content_type = text/x-rst -author = Roman calibration pipeline developers -license = BSD-3-Clause -url = https://github.com/spacetelescope/romancal -project_urls = - Bug Tracker = https://github.com/spacetelescope/romancal/issues - Documentation = https://roman-pipeline.readthedocs.io/en/stable/ - Source Code = https://github.com/spacetelescope/romancal -classifiers = - Intended Audience :: Science/Research - Topic :: Scientific/Engineering :: Astronomy - License :: OSI Approved :: BSD License - Programming Language :: Python :: 3 - -[options] -zip_safe = False -python_requires = >=3.8 -setup_requires = - setuptools_scm -install_requires = - asdf>=2.12.1 - astropy>=5.0.4 - crds>=11.16.16 - gwcs>=0.18.1 - jsonschema>=3.0.2 - numpy>=1.20 - pyparsing>=2.4.7 - requests>=2.22 - roman_datamodels>=0.14.0 - stcal>=1.2.1,<2.0 - stpipe>=0.4.2,<1.0 - - -[options.extras_require] -docs = - matplotlib - sphinx - sphinx-automodapi - sphinx-rtd-theme - stsci-rtd-theme - sphinx-astropy - sphinx-asdf -test = - ci-watson>=0.3.0 - pytest>=4.6.0 - pytest-astropy - codecov>=1.6.0 - -[flake8] -select = F, W, E27, E70, E71, E101, E111, E112, E113, E201, E202, E221, E222, - E241, E401, E402, E501, E704, E722 -max-line-length = 130 -exclude = - docs, - .tox, - .eggs -ignore = E203, W503, W504 - -aws = - stsci-aws-utils>=0.1.2 -ephem = - pymssql-linux==2.1.6 - jplephem==2.9 - -[options.entry_points] -stpipe.steps = - romancal = romancal.stpipe.integration:get_steps - -[build-sphinx] -source-dir = docs -build-dir = docs -all_files = 1 - -[upload_docs] -upload-dir = docs/_build/html -show-response = 1 - -[tool:pytest] -minversion = 4.6 -norecursedirs = docs/_build scripts .tox -asdf_schema_tests_enabled = true -asdf_schema_validate_default = false -asdf_schema_root = romancal/datamodels/schemas -junit_family = xunit2 -inputs_root = roman-pipeline -results_root = roman-pipeline-results -doctest_plus = true -doctest_rst = true -text_file_format = rst -addopts = --show-capture=no --open-files --doctest-ignore-import-errors - -[coverage:run] -omit = - romancal/regtest/conftest.py - romancal/setup.py - romancal/tests/test* - romancal/regtest/test* - romancal/*/tests/* - docs/* - # And list these again for running against installed version - */romancal/regtest/conftest.py - */romancal/setup.py - */romancal/tests/test* - */romancal/regtest/test* - */romancal/*/tests/* - */docs/* - -[coverage:report] -exclude_lines = - pragma: no cover - if self.debug: - except ImportError - raise AssertionError - raise NotImplementedError - if __name__ == '__main__': diff --git a/setup.py b/setup.py index 5d001d3d3..6d60e9de9 100644 --- a/setup.py +++ b/setup.py @@ -1,29 +1,7 @@ -from os.path import basename -from setuptools import setup, find_packages -from glob import glob +from pathlib import Path +from setuptools import setup -NAME = 'romancal' +SCRIPTS = [str(s) for s in Path('scripts').iterdir() if s.name != '__pycache__' and s.is_file()] -SCRIPTS = [s for s in glob('scripts/*') if basename(s) != '__pycache__'] - -PACKAGE_DATA = { - '': [ - '*.fits', - '*.txt', - '*.inc', - '*.cfg', - '*.csv', - '*.yaml', - '*.json', - '*.asdf' - ] -} - -setup( - use_scm_version=True, - setup_requires=['setuptools_scm'], - scripts=SCRIPTS, - packages=find_packages(), - package_data=PACKAGE_DATA, -) +setup(scripts=SCRIPTS) From 339f670116662f0e7ef1b7b9fa5674900b1cd026 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 20 Sep 2022 10:27:36 -0400 Subject: [PATCH 2/6] update docs configuration read --- docs/conf.py | 30 +++++++++++++----------------- pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a46208c07..043686854 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,10 +16,11 @@ import os import sys from distutils.version import LooseVersion +from pathlib import Path import sphinx import stsci_rtd_theme -import toml +import tomli def setup(app): @@ -28,6 +29,7 @@ def setup(app): except AttributeError: app.add_stylesheet("stsci.css") + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -36,7 +38,8 @@ def setup(app): sys.path.insert(0, os.path.abspath('exts/')) # -- General configuration ------------------------------------------------ -conf = toml.load('../pyproject.toml') +with open(Path(__file__).parent.parent / "pyproject.toml", "rb") as configuration_file: + conf = tomli.load(configuration_file) setup_cfg = conf['project'] # If your documentation needs a minimal Sphinx version, state it here. @@ -61,7 +64,7 @@ def check_sphinx_version(expected_version): 'numpy': ('https://numpy.org/devdocs', None), 'scipy': ('http://scipy.github.io/devdocs', None), 'matplotlib': ('http://matplotlib.org/', None), - } +} if sys.version_info[0] == 2: intersphinx_mapping['python'] = ('http://docs.python.org/2/', None) @@ -89,10 +92,9 @@ def check_sphinx_version(expected_version): 'sphinx_automodapi.autodoc_enhancements', 'sphinx_automodapi.smart_resolver', 'sphinx_asdf', - 'myst_parser', +'myst_parser', ] - if on_rtd: extensions.append('sphinx.ext.mathjax') @@ -101,7 +103,6 @@ def check_sphinx_version(expected_version): else: extensions.append('sphinx.ext.imgmath') - # Add any paths that contain templates here, relative to this directory. # templates_path = ['_templates'] @@ -125,11 +126,10 @@ def check_sphinx_version(expected_version): suppress_warnings = ['app.add_directive', ] - # General information about the project project = setup_cfg['name'] -author = setup_cfg['authors'][0]['name'] -copyright = '{0}, {1}'.format(datetime.datetime.now().year, author) +author = f'{setup_cfg["authors"][0]["name"]} <{setup_cfg["authors"][0]["email"]}>' +copyright = f'{datetime.datetime.now().year}, {author}' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -167,7 +167,6 @@ def check_sphinx_version(expected_version): # documents. default_role = 'obj' - # Don't show summaries of the members in each class along with the # class' docstring numpydoc_show_class_members = False @@ -192,7 +191,6 @@ def check_sphinx_version(expected_version): '-Gfontname=Helvetica Neue, Helvetica, Arial, sans-serif' ] - # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True @@ -232,7 +230,7 @@ def check_sphinx_version(expected_version): # documentation. html_theme_options = { "collapse_navigation": True, - "display_version": True +"display_version": True } # "nosidebar": "false", # "sidebarbgcolor": "#4db8ff", @@ -313,7 +311,6 @@ def check_sphinx_version(expected_version): # Output file base name for HTML help builder. htmlhelp_basename = 'romandoc' - # -- Options for LaTeX output --------------------------------------------- # latex_elements = { @@ -366,16 +363,15 @@ def check_sphinx_version(expected_version): # If true, show URL addresses after external links. man_show_urls = True - # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'romancal', u'Roman Pipeline Documentation', - u'romancal', 'romancal', 'Roman Pipeline Documentation', - 'Miscellaneous'), + ('index', 'romancal', u'Roman Pipeline Documentation', + u'romancal', 'romancal', 'Roman Pipeline Documentation', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/pyproject.toml b/pyproject.toml index 9d91a2034..763585544 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ docs = [ 'sphinx-automodapi', 'sphinx-rtd-theme', 'stsci-rtd-theme', - 'toml', + 'tomli; python_version <"3.11"', ] lint = [ 'pyproject-flake8', From 86c2f701aa4a933a1cb60c68b9405caa328fd2b6 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 24 Oct 2022 14:22:15 -0400 Subject: [PATCH 3/6] update requirements --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 763585544..3c3ef9552 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,16 +14,16 @@ classifiers = [ dependencies = [ 'asdf >=2.12.1', 'astropy >=5.0.4', - 'crds >=11.13.1', - 'gwcs >=0.18.0', + 'crds >=11.16.16', + 'gwcs >=0.18.1', 'jsonschema >=3.0.2', - 'numpy >=1.22.3', + 'numpy >=1.20', 'pyparsing >=2.4.7', 'requests >=2.22', - 'roman_datamodels >=0.13.0', + 'roman_datamodels >=0.14.0', #'roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git@main', - 'stcal >=0.7.2', - 'stpipe >=0.4.2,<1.0', + 'stcal >=1.2.1, <2.0', + 'stpipe >=0.4.2, <1.0', ] dynamic = ['version'] From 39026b9f0daa50364760e5012e0fec855c221f90 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Wed, 30 Nov 2022 08:51:45 -0500 Subject: [PATCH 4/6] add email address --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c3ef9552..37494f222 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ description = 'Library for calibration of science observations from the Nancy Gr readme = 'README.md' requires-python = '>=3.8' license = { file = 'LICENSE' } -authors = [{ name = 'Roman calibration pipeline developers' }] +authors = [{ name = 'Roman calibration pipeline developers', email = 'help@stsci.edu' }] classifiers = [ 'Intended Audience :: Science/Research', 'Topic :: Scientific/Engineering :: Astronomy', From 22046ea4c129b7890a9a03dfee42157ff4a81f67 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Wed, 30 Nov 2022 11:25:50 -0500 Subject: [PATCH 5/6] replace `setup.cfg` with `pyproject.toml` --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 94b8f5dfe..f3c04a409 100644 --- a/tox.ini +++ b/tox.ini @@ -63,7 +63,7 @@ usedevelop = true commands = pip freeze pytest \ - cov: --cov=. --cov-config=setup.cfg --cov-report=term-missing --cov-report=xml \ + cov: --cov=. --cov-config=pyproject.toml --cov-report=term-missing --cov-report=xml \ warnings: -W error \ regtests: --bigdata --slow --basetemp={homedir}/test_outputs \ xdist: -n auto \ From b25dfa3ac5d1ae8f619986f9e8466add13ea1194 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Wed, 30 Nov 2022 11:28:06 -0500 Subject: [PATCH 6/6] replace `setup.cfg` with `pyproject.toml` --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index f3c04a409..70bb7be36 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ commands = bandit romancal -r -x tests,regtest [testenv:check-install] -description = verify that install_requires in setup.cfg is correct +description = verify that dependencies are correct extras = commands = verify_install_requires @@ -47,7 +47,7 @@ description = regtests: with --bigdata and --slow flags cov: with coverage xdist: using parallel processing -# The following indicates which extras_require from setup.cfg will be installed +# The following indicates which `optional-dependencies` from `pyproject.toml` will be installed extras = test alldeps: all