From e382e4c3531895bc02b8a4e75339da05745aa3c2 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Mon, 17 Apr 2023 16:08:46 -0300 Subject: [PATCH 1/4] move to pyproject.toml --- pyproject.toml | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 81 ----------------------------------------------- 2 files changed, 85 insertions(+), 81 deletions(-) delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index af8b5817..f868e29e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,91 @@ requires = [ "wheel", ] +<<<<<<< HEAD +======= +[project] +name = "compliance-checker" +description = "Checks Datasets and SOS endpoints for standards compliance" +readme = "README.md" +license = {text = "Apache-2.0"} +authors = [ + {name = "Dave Foster", email = "dave@axiomdatascience.com"}, +] +requires-python = ">=3.8" +classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering", +] +dynamic = [ + "dependencies", + "version", +] +[project.urls] +documentation = "https://ioos.github.io/compliance-checker" +homepage = "https://compliance.ioos.us/index.html" +repository = "https://github.com/ioos/compliance-checker" +[project.scripts] +compliance-checker = "cchecker:main" +[project.entry-points."compliance_checker.suites"] +"acdd-1.1" = "compliance_checker.acdd:ACDD1_1Check" +"acdd-1.3" = "compliance_checker.acdd:ACDD1_3Check" +"cf-1.6" = "compliance_checker.cf.cf:CF1_6Check" +"cf-1.7" = "compliance_checker.cf.cf:CF1_7Check" +"cf-1.8" = "compliance_checker.cf.cf:CF1_8Check" +"ioos-0.1" = "compliance_checker.ioos:IOOS0_1Check" +"ioos-1.1" = "compliance_checker.ioos:IOOS1_1Check" +"ioos-1.2" = "compliance_checker.ioos:IOOS1_2Check" +"ioos_sos" = "compliance_checker.ioos:IOOSBaseSOSCheck" + +[tool.setuptools] +packages = ["compliance_checker"] +license-files = ["LICENSE"] +zip-safe = false +include-package-data = true +script-files = ["cchecker.py"] + +[tool.setuptools.package-data] +compliance_checker = [ + "data/*.xml", + "tests/data/*.nc", + "tests/data/*.cdl", + "tests/data/non-comp/*.cdl", + "data/templates/*.j2", +] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} +readme = {file = "README.md", content-type = "text/markdown"} + +[tool.setuptools_scm] +write_to = "compliance_checker/_version.py" +write_to_template = "__version__ = '{version}'" +tag_regex = "^(?Pv)?(?P[^\\+]+)(?P.*)?$" + +[tool.pytest.ini_options] +markers = [ + "integration: marks integration tests (deselect with '-m \"not integration\"')", + "slowtest: marks slow tests (deselect with '-m \"not slowtest\"')" +] +filterwarnings = [ + "error:::compliance-checker.*", + "ignore::UserWarning", + "ignore::RuntimeWarning", +] + +>>>>>>> 2fb66ed (move to pyproject.toml) [tool.ruff] select = [ "A", # flake8-builtins diff --git a/setup.py b/setup.py deleted file mode 100644 index e26a432b..00000000 --- a/setup.py +++ /dev/null @@ -1,81 +0,0 @@ -from setuptools import find_packages, setup - - -def readme(): - with open("README.md", encoding="utf-8") as f: - return f.read() - - -def pip_requirements(fname="requirements.txt"): - reqs = [] - with open(fname) as f: - for line in f: - line = line.strip() - if not line or line.startswith("#"): - continue - reqs.append(line) - - return reqs - - -setup( - name="compliance-checker", - description="Checks Datasets and SOS endpoints for standards compliance", - long_description=readme(), - long_description_content_type="text/markdown", - license="Apache License 2.0", - author="Dave Foster", - author_email="dave@axiomdatascience.com", - url="https://github.com/ioos/compliance-checker", - packages=find_packages(), - install_requires=pip_requirements(), - python_requires="~=3.7", - tests_require=["pytest"], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - "Programming Language :: Python", - "Topic :: Scientific/Engineering", - ], - include_package_data=True, - scripts=["cchecker.py"], - # Note: Do not use colons in the entry-point keys. Python 3 reserves - # portions of the key after a colon for special use. - # Note: The entry point names are not used at all. All methods in the - # compliance checker use class attributes to determine the checker's name - # and version. But, an entry point must be defined for each plugin to be - # loaded. - entry_points={ - "console_scripts": ["compliance-checker = cchecker:main"], - "compliance_checker.suites": [ - "cf-1.6 = compliance_checker.cf.cf:CF1_6Check", - "cf-1.7 = compliance_checker.cf.cf:CF1_7Check", - "cf-1.8 = compliance_checker.cf.cf:CF1_8Check", - "acdd-1.1 = compliance_checker.acdd:ACDD1_1Check", - "acdd-1.3 = compliance_checker.acdd:ACDD1_3Check", - "ioos_sos = compliance_checker.ioos:IOOSBaseSOSCheck", - "ioos-0.1 = compliance_checker.ioos:IOOS0_1Check", - "ioos-1.1 = compliance_checker.ioos:IOOS1_1Check", - "ioos-1.2 = compliance_checker.ioos:IOOS1_2Check", - ], - }, - package_data={ - "compliance_checker": [ - "data/*.xml", - "tests/data/*.nc", - "tests/data/*.cdl", - "tests/data/non-comp/*.cdl", - "data/templates/*.j2", - ], - }, - use_scm_version={ - "write_to": "compliance_checker/_version.py", - "write_to_template": '__version__ = "{version}"', - "tag_regex": r"^(?Pv)?(?P[^\+]+)(?P.*)?$", - }, -) From 4f2af1a04fad8bfab288d5aea65bb98ac8e033d8 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Mon, 29 May 2023 12:20:17 -0300 Subject: [PATCH 2/4] add top 3 committers as maintainers --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f868e29e..3ae4cb50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,11 @@ name = "compliance-checker" description = "Checks Datasets and SOS endpoints for standards compliance" readme = "README.md" license = {text = "Apache-2.0"} -authors = [ +maintainers = [ {name = "Dave Foster", email = "dave@axiomdatascience.com"}, + {name = "Benjamin Adams"}, + {name = "Luke Campbell"}, + {name = "Filipe Fernandes"}, ] requires-python = ">=3.8" classifiers=[ From cf74a2548d097bc3063abfbf711045dc202416f1 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Wed, 4 Oct 2023 12:32:38 -0300 Subject: [PATCH 3/4] lint --- pyproject.toml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3ae4cb50..6d7198d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,6 @@ requires = [ "wheel", ] -<<<<<<< HEAD -======= [project] name = "compliance-checker" description = "Checks Datasets and SOS endpoints for standards compliance" @@ -82,18 +80,6 @@ write_to = "compliance_checker/_version.py" write_to_template = "__version__ = '{version}'" tag_regex = "^(?Pv)?(?P[^\\+]+)(?P.*)?$" -[tool.pytest.ini_options] -markers = [ - "integration: marks integration tests (deselect with '-m \"not integration\"')", - "slowtest: marks slow tests (deselect with '-m \"not slowtest\"')" -] -filterwarnings = [ - "error:::compliance-checker.*", - "ignore::UserWarning", - "ignore::RuntimeWarning", -] - ->>>>>>> 2fb66ed (move to pyproject.toml) [tool.ruff] select = [ "A", # flake8-builtins @@ -129,3 +115,8 @@ markers = [ "integration: marks integration tests (deselect with '-m \"not integration\"')", "slowtest: marks slow tests (deselect with '-m \"not slowtest\"')" ] +filterwarnings = [ + "error:::compliance-checker.*", + "ignore::UserWarning", + "ignore::RuntimeWarning", +] From 9c4b0d2625c531eaae2e4ddf58cc799742f06d34 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Mon, 8 Apr 2024 19:04:28 -0700 Subject: [PATCH 4/4] put script in to pypacakge.toml, and removed pkg_resources --- cchecker.py | 337 +----------------------- compliance_checker/base.py | 7 +- compliance_checker/cchecker.py | 341 +++++++++++++++++++++++++ compliance_checker/cf/util.py | 8 +- compliance_checker/cfutil.py | 8 +- compliance_checker/protocols/netcdf.py | 3 +- compliance_checker/suite.py | 25 +- compliance_checker/tests/conftest.py | 4 +- compliance_checker/tests/resources.py | 17 +- compliance_checker/tests/test_cli.py | 3 +- compliance_checker/tests/test_suite.py | 46 ++-- pyproject.toml | 8 +- 12 files changed, 413 insertions(+), 394 deletions(-) create mode 100644 compliance_checker/cchecker.py diff --git a/cchecker.py b/cchecker.py index fd3ef88b..9970e93c 100755 --- a/cchecker.py +++ b/cchecker.py @@ -1,341 +1,8 @@ #!/usr/bin/env python -import argparse import sys -import warnings -from collections import defaultdict -from textwrap import dedent - -from compliance_checker import __version__ -from compliance_checker.cf.util import download_cf_standard_name_table -from compliance_checker.runner import CheckSuite, ComplianceChecker - - -def _print_checker_name_header(checker_str): - """ - Helper function to prints a checker name surrounded by a border of "=" - :param checker_suite: A check suite string name - :type checker: str - """ - print("{0}\n {1} \n{0}".format("=" * (len(checker_str) + 2), checker_str)) - - -def parse_options(opts): - """ - Helper function to parse possible options. Splits option after the first - colon to split into key/value pairs. - - :param opts: Iterable of strings with options - :rtype: dict - :return: Dictionary with keys as checker type (i.e. "cf", "acdd") - """ - options_dict = defaultdict(set) - for opt_str in opts: - try: - checker_type, checker_opt = opt_str.split(":", 1) - except ValueError: - warnings.warn(f"Could not split option {opt_str}, ignoring", stacklevel=2) - else: - options_dict[checker_type].add(checker_opt) - return options_dict - - -def main(): - # Load all available checker classes - check_suite = CheckSuite() - check_suite.load_all_available_checkers() - - parser = argparse.ArgumentParser() - parser.add_argument( - "--test", - "-t", - "--test=", - "-t=", - default=[], - action="append", - help=( - "Select the Checks you want to perform. Defaults to 'acdd'" - " if unspecified. Versions of standards can be specified via " - "`-t :`. If `` is omitted, or " - 'is "latest", the latest version of the test standard is used.' - ), - ) - - parser.add_argument( - "--criteria", - "-c", - help=( - "Define the criteria for the checks. " - "Either Strict, Normal, or Lenient. Defaults to Normal." - ), - nargs="?", - default="normal", - choices=["lenient", "normal", "strict"], - ) - - parser.add_argument( - "--verbose", - "-v", - help="Increase output. May be specified up to three times.", - action="count", - default=0, - ) - - parser.add_argument( - "--describe-checks", - "-D", - help=( - "Describes checks for checkers specified using " - "`-t`. If `-t` is not specified, lists checks " - "from all available checkers." - ), - action="store_true", - ) - - include_exclude = parser.add_mutually_exclusive_group() - - include_exclude.add_argument( - "--skip-checks", - "-s", - help=dedent( - """ - Specifies tests to skip. Can take the form - of either `` or - `:`. The first - form skips any checks matching the name. - In the second form may be - specified as "A", "M", or "L". "A" skips - all checks and is equivalent to calling - the first form. "M" will only show high - priority output from the given check and - will skip medium and low. "L" will show - both high and medium priority issues, while - skipping low priority issues. Cannot be - used with `-i`/`--include-checks` option. - """, - ), - action="append", - ) - - include_exclude.add_argument( - "--include-checks", - "-i", - help=dedent( - """ - Specifies checks to include. Can only take the form - of ``. Cannot be specified along with - `-s`/`skip_checks`. - """, - ), - action="append", - ) - - parser.add_argument( - "-f", - "--format", - default=[], - action="append", - help=( - "Output format(s). Options are 'text', 'html', 'json', 'json_new'." - " The difference between the 'json' and the 'json_new'" - " formats is that the 'json' format has the check as the top level" - " key, whereas the 'json_new' format has the dataset name(s) as the" - " main key in the output follow by any checks as subkeys. Also, " - "'json' format can be only be run against one input file, whereas " - "'json_new' can be run against multiple files." - ), - choices=["text", "html", "json", "json_new"], - ) - - parser.add_argument( - "-o", - "--output", - default=[], - action="append", - help=( - "Output filename(s). If '-' is supplied, output to stdout." - " Can either be one or many files. If one file is supplied," - " but the checker is run against many files, all the output" - " from the checks goes to that file (does not presently work " - "with 'json' format). If more than one output file is " - "supplied, the number of input datasets supplied must match " - "the number of output files." - ), - ) - - parser.add_argument( - "-O", - "--option", - default=[], - action="append", - help=dedent( - """ - Additional options to be passed to the - checkers. Multiple options can be specified - via multiple invocations of this switch. - Options should be prefixed with a the - checker name followed by the option, e.g. - ':' - - Available options: - 'cf:enable_appendix_a_checks' - Allow check - results against CF Appendix A for attribute - location and data types. - """, - ), - ) - - parser.add_argument( - "-V", - "--version", - action="store_true", - help="Display the IOOS Compliance Checker version information.", - ) - - parser.add_argument( - "dataset_location", - nargs="*", - help=( - "Defines the location of the dataset to be checked. The location " - "can be a local netCDF file, a remote OPeNDAP endpoint, a remote " - "netCDF file which returns content-type header of " - "'application/x-netcdf', or an ERDDAP TableDAP endpoint. " - "Note that the ERDDAP TableDAP endpoint will currently attempt " - "to fetch the entire TableDAP dataset." - ), - ) - - parser.add_argument( - "-l", - "--list-tests", - action="store_true", - help="List the available tests", - ) - - parser.add_argument( - "-d", - "--download-standard-names", - help=( - "Specify a version of the cf standard name table" - " to download as packaged version. Either specify" - ' a version number (e.g. "72") to fetch a ' - 'specific version or "latest" to get the ' - "latest CF standard name table." - ), - ) - - # Add command line args from generator plugins - check_suite.add_plugin_args(parser) - - args = parser.parse_args() - - check_suite.load_generated_checkers(args) - - if args.version: - print("IOOS compliance checker version %s" % __version__) - sys.exit(0) - - options_dict = parse_options(args.option) if args.option else defaultdict(set) - - if args.describe_checks: - error_stat = 0 - if args.test: - checker_names = set(args.test) - else: - # skip "latest" meta-versions (":latest" or no explicit version - # specifier) - checker_names = [ - c - for c in check_suite.checkers - if ":" in c and not c.endswith(":latest") - ] - - for checker_name in sorted(checker_names): - if checker_name not in check_suite.checkers: - print( - f"Cannot find checker '{checker_name}' with which to " - "describe checks", - file=sys.stderr, - ) - error_stat = 1 - else: - _print_checker_name_header(checker_name) - check_suite._print_checker(check_suite.checkers[checker_name]) - sys.exit(error_stat) - - if args.list_tests: - print("IOOS compliance checker available checker suites:") - check_suite._print_suites(args.verbose) - return 0 - - if args.download_standard_names: - download_cf_standard_name_table(args.download_standard_names) - - if len(args.dataset_location) == 0: - parser.print_help() - sys.exit(1) - - # Check the number of output files - if not args.output: - args.output = "-" - output_len = len(args.output) - if not (output_len == 1 or output_len == len(args.dataset_location)): - print( - "The number of output files must either be one or the same as the number of datasets", - file=sys.stderr, - ) - sys.exit(2) - - # Run the compliance checker - # 2 modes, concatenated output file or multiple output files - return_values = [] - had_errors = [] - if output_len == 1: - if args.format != "json": - print( - f"Running Compliance Checker on the datasets from: {args.dataset_location}", - file=sys.stderr, - ) - return_value, errors = ComplianceChecker.run_checker( - args.dataset_location, - args.test or ["acdd"], - args.verbose, - args.criteria, - args.skip_checks, - args.include_checks, - args.output[0], - args.format or ["text"], - options=options_dict, - ) - return_values.append(return_value) - had_errors.append(errors) - else: - for output, dataset in zip(args.output, args.dataset_location): - if args.format != "json": - print( - f"Running Compliance Checker on the dataset from: {dataset}", - file=sys.stderr, - ) - return_value, errors = ComplianceChecker.run_checker( - [dataset], - args.test or ["acdd"], - args.verbose, - args.criteria, - args.skip_checks, - args.include_checks, - output, - args.format or ["text"], - options=options_dict, - ) - return_values.append(return_value) - had_errors.append(errors) - - if any(had_errors): - sys.exit(2) - if all(return_values): - sys.exit(0) - sys.exit(1) +from compliance_checker import cchecker if __name__ == "__main__": - sys.exit(main()) + sys.exit(cchecker.main()) diff --git a/compliance_checker/base.py b/compliance_checker/base.py index c2674e85..64c2685c 100644 --- a/compliance_checker/base.py +++ b/compliance_checker/base.py @@ -191,9 +191,10 @@ def __del__(self): are cleared before the next checker uses it. Some caches were inadvertently mutated by other functions. """ - - cfutil.get_geophysical_variables.cache_clear() - cfutil.get_time_variables.cache_clear() + # odd errors -- module getting deleted before this object? + if cfutil is not None: + cfutil.get_geophysical_variables.cache_clear() + cfutil.get_time_variables.cache_clear() class BaseNCCheck: diff --git a/compliance_checker/cchecker.py b/compliance_checker/cchecker.py new file mode 100644 index 00000000..fd3ef88b --- /dev/null +++ b/compliance_checker/cchecker.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python + +import argparse +import sys +import warnings +from collections import defaultdict +from textwrap import dedent + +from compliance_checker import __version__ +from compliance_checker.cf.util import download_cf_standard_name_table +from compliance_checker.runner import CheckSuite, ComplianceChecker + + +def _print_checker_name_header(checker_str): + """ + Helper function to prints a checker name surrounded by a border of "=" + :param checker_suite: A check suite string name + :type checker: str + """ + print("{0}\n {1} \n{0}".format("=" * (len(checker_str) + 2), checker_str)) + + +def parse_options(opts): + """ + Helper function to parse possible options. Splits option after the first + colon to split into key/value pairs. + + :param opts: Iterable of strings with options + :rtype: dict + :return: Dictionary with keys as checker type (i.e. "cf", "acdd") + """ + options_dict = defaultdict(set) + for opt_str in opts: + try: + checker_type, checker_opt = opt_str.split(":", 1) + except ValueError: + warnings.warn(f"Could not split option {opt_str}, ignoring", stacklevel=2) + else: + options_dict[checker_type].add(checker_opt) + return options_dict + + +def main(): + # Load all available checker classes + check_suite = CheckSuite() + check_suite.load_all_available_checkers() + + parser = argparse.ArgumentParser() + parser.add_argument( + "--test", + "-t", + "--test=", + "-t=", + default=[], + action="append", + help=( + "Select the Checks you want to perform. Defaults to 'acdd'" + " if unspecified. Versions of standards can be specified via " + "`-t :`. If `` is omitted, or " + 'is "latest", the latest version of the test standard is used.' + ), + ) + + parser.add_argument( + "--criteria", + "-c", + help=( + "Define the criteria for the checks. " + "Either Strict, Normal, or Lenient. Defaults to Normal." + ), + nargs="?", + default="normal", + choices=["lenient", "normal", "strict"], + ) + + parser.add_argument( + "--verbose", + "-v", + help="Increase output. May be specified up to three times.", + action="count", + default=0, + ) + + parser.add_argument( + "--describe-checks", + "-D", + help=( + "Describes checks for checkers specified using " + "`-t`. If `-t` is not specified, lists checks " + "from all available checkers." + ), + action="store_true", + ) + + include_exclude = parser.add_mutually_exclusive_group() + + include_exclude.add_argument( + "--skip-checks", + "-s", + help=dedent( + """ + Specifies tests to skip. Can take the form + of either `` or + `:`. The first + form skips any checks matching the name. + In the second form may be + specified as "A", "M", or "L". "A" skips + all checks and is equivalent to calling + the first form. "M" will only show high + priority output from the given check and + will skip medium and low. "L" will show + both high and medium priority issues, while + skipping low priority issues. Cannot be + used with `-i`/`--include-checks` option. + """, + ), + action="append", + ) + + include_exclude.add_argument( + "--include-checks", + "-i", + help=dedent( + """ + Specifies checks to include. Can only take the form + of ``. Cannot be specified along with + `-s`/`skip_checks`. + """, + ), + action="append", + ) + + parser.add_argument( + "-f", + "--format", + default=[], + action="append", + help=( + "Output format(s). Options are 'text', 'html', 'json', 'json_new'." + " The difference between the 'json' and the 'json_new'" + " formats is that the 'json' format has the check as the top level" + " key, whereas the 'json_new' format has the dataset name(s) as the" + " main key in the output follow by any checks as subkeys. Also, " + "'json' format can be only be run against one input file, whereas " + "'json_new' can be run against multiple files." + ), + choices=["text", "html", "json", "json_new"], + ) + + parser.add_argument( + "-o", + "--output", + default=[], + action="append", + help=( + "Output filename(s). If '-' is supplied, output to stdout." + " Can either be one or many files. If one file is supplied," + " but the checker is run against many files, all the output" + " from the checks goes to that file (does not presently work " + "with 'json' format). If more than one output file is " + "supplied, the number of input datasets supplied must match " + "the number of output files." + ), + ) + + parser.add_argument( + "-O", + "--option", + default=[], + action="append", + help=dedent( + """ + Additional options to be passed to the + checkers. Multiple options can be specified + via multiple invocations of this switch. + Options should be prefixed with a the + checker name followed by the option, e.g. + ':' + + Available options: + 'cf:enable_appendix_a_checks' - Allow check + results against CF Appendix A for attribute + location and data types. + """, + ), + ) + + parser.add_argument( + "-V", + "--version", + action="store_true", + help="Display the IOOS Compliance Checker version information.", + ) + + parser.add_argument( + "dataset_location", + nargs="*", + help=( + "Defines the location of the dataset to be checked. The location " + "can be a local netCDF file, a remote OPeNDAP endpoint, a remote " + "netCDF file which returns content-type header of " + "'application/x-netcdf', or an ERDDAP TableDAP endpoint. " + "Note that the ERDDAP TableDAP endpoint will currently attempt " + "to fetch the entire TableDAP dataset." + ), + ) + + parser.add_argument( + "-l", + "--list-tests", + action="store_true", + help="List the available tests", + ) + + parser.add_argument( + "-d", + "--download-standard-names", + help=( + "Specify a version of the cf standard name table" + " to download as packaged version. Either specify" + ' a version number (e.g. "72") to fetch a ' + 'specific version or "latest" to get the ' + "latest CF standard name table." + ), + ) + + # Add command line args from generator plugins + check_suite.add_plugin_args(parser) + + args = parser.parse_args() + + check_suite.load_generated_checkers(args) + + if args.version: + print("IOOS compliance checker version %s" % __version__) + sys.exit(0) + + options_dict = parse_options(args.option) if args.option else defaultdict(set) + + if args.describe_checks: + error_stat = 0 + if args.test: + checker_names = set(args.test) + else: + # skip "latest" meta-versions (":latest" or no explicit version + # specifier) + checker_names = [ + c + for c in check_suite.checkers + if ":" in c and not c.endswith(":latest") + ] + + for checker_name in sorted(checker_names): + if checker_name not in check_suite.checkers: + print( + f"Cannot find checker '{checker_name}' with which to " + "describe checks", + file=sys.stderr, + ) + error_stat = 1 + else: + _print_checker_name_header(checker_name) + check_suite._print_checker(check_suite.checkers[checker_name]) + sys.exit(error_stat) + + if args.list_tests: + print("IOOS compliance checker available checker suites:") + check_suite._print_suites(args.verbose) + return 0 + + if args.download_standard_names: + download_cf_standard_name_table(args.download_standard_names) + + if len(args.dataset_location) == 0: + parser.print_help() + sys.exit(1) + + # Check the number of output files + if not args.output: + args.output = "-" + output_len = len(args.output) + if not (output_len == 1 or output_len == len(args.dataset_location)): + print( + "The number of output files must either be one or the same as the number of datasets", + file=sys.stderr, + ) + sys.exit(2) + + # Run the compliance checker + # 2 modes, concatenated output file or multiple output files + return_values = [] + had_errors = [] + if output_len == 1: + if args.format != "json": + print( + f"Running Compliance Checker on the datasets from: {args.dataset_location}", + file=sys.stderr, + ) + return_value, errors = ComplianceChecker.run_checker( + args.dataset_location, + args.test or ["acdd"], + args.verbose, + args.criteria, + args.skip_checks, + args.include_checks, + args.output[0], + args.format or ["text"], + options=options_dict, + ) + return_values.append(return_value) + had_errors.append(errors) + else: + for output, dataset in zip(args.output, args.dataset_location): + if args.format != "json": + print( + f"Running Compliance Checker on the dataset from: {dataset}", + file=sys.stderr, + ) + return_value, errors = ComplianceChecker.run_checker( + [dataset], + args.test or ["acdd"], + args.verbose, + args.criteria, + args.skip_checks, + args.include_checks, + output, + args.format or ["text"], + options=options_dict, + ) + return_values.append(return_value) + had_errors.append(errors) + + if any(had_errors): + sys.exit(2) + if all(return_values): + sys.exit(0) + sys.exit(1) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/compliance_checker/cf/util.py b/compliance_checker/cf/util.py index 35734acb..69e9c968 100644 --- a/compliance_checker/cf/util.py +++ b/compliance_checker/cf/util.py @@ -7,7 +7,7 @@ from cf_units import Unit from lxml import etree from netCDF4 import Dataset -from pkg_resources import resource_filename +import importlib # copied from paegan # paegan may depend on these later @@ -284,10 +284,8 @@ def download_cf_standard_name_table(version, location=None): if ( location is None ): # This case occurs when updating the packaged version from command line - location = resource_filename( - "compliance_checker", - "data/cf-standard-name-table.xml", - ) + location = importlib.resources.files("compliance_checker") + location /= "data/cf-standard-name-table.xml" if version == "latest": url = "http://cfconventions.org/Data/cf-standard-names/current/src/cf-standard-name-table.xml" diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index b351a94b..76694bc0 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -9,7 +9,7 @@ from functools import lru_cache, partial from cf_units import Unit -from pkg_resources import resource_filename +import importlib _UNITLESS_DB = None _SEA_NAMES = None @@ -127,9 +127,9 @@ def get_sea_names(): global _SEA_NAMES if _SEA_NAMES is None: buf = {} - with open( - resource_filename("compliance_checker", "data/seanames.csv"), - ) as f: + with open(importlib.resources.files("compliance_checker") + / "data/seanames.csv" + ) as f: reader = csv.reader(f) for code, sea_name in reader: buf[sea_name] = code diff --git a/compliance_checker/protocols/netcdf.py b/compliance_checker/protocols/netcdf.py index 415a94ec..2c23960e 100644 --- a/compliance_checker/protocols/netcdf.py +++ b/compliance_checker/protocols/netcdf.py @@ -17,11 +17,12 @@ def is_netcdf(url): :param str url: Location of file on the file system """ # Try an obvious exclusion of remote resources + url = str(url) # in case it's a Path object if url.startswith("http"): return False # If it's a known extension, give it a shot - if url.endswith("nc"): + if url.endswith(".nc"): return True # Brute force diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py index d0e84769..9c5a5e39 100644 --- a/compliance_checker/suite.py +++ b/compliance_checker/suite.py @@ -13,7 +13,7 @@ import warnings from collections import defaultdict from datetime import datetime, timezone -from distutils.version import StrictVersion +from packaging import version from operator import itemgetter from pathlib import Path from urllib.parse import urlparse @@ -23,7 +23,7 @@ from netCDF4 import Dataset from owslib.sos import SensorObservationService from owslib.swe.sensor.sml import SensorML -from pkg_resources import working_set +import importlib from compliance_checker import __version__, tempnc from compliance_checker.base import BaseCheck, GenericFile, Result, fix_return_value @@ -71,10 +71,11 @@ def _get_generator_plugins(cls): Return a list of classes from external plugins that are used to generate checker classes """ - + # NOTE: updated to not use pkg_resources, but + # not tested -- it is ever used? if not hasattr(cls, "suite_generators"): - gens = working_set.iter_entry_points("compliance_checker.generators") - cls.suite_generators = [x.resolve() for x in gens] + gens = importlib.metadata.entry_points(group='compliance_checker.generators') + cls.suite_generators = [x.load() for x in gens] return cls.suite_generators @@ -136,7 +137,9 @@ def load_all_available_checkers(cls): Helper method to retrieve all sub checker classes derived from various base classes. """ - cls._load_checkers(working_set.iter_entry_points("compliance_checker.suites")) + checkers = importlib.metadata.entry_points(group='compliance_checker.suites') + cls._load_checkers(checkers) + @classmethod def _load_checkers(cls, checkers): @@ -147,7 +150,8 @@ def _load_checkers(cls, checkers): for c in checkers: try: - check_obj = c.resolve() + # check_obj = c.resolve() + check_obj = c.load() if hasattr(check_obj, "_cc_spec") and hasattr( check_obj, "_cc_spec_version", @@ -186,9 +190,8 @@ def _load_checkers(cls, checkers): for spec, versions in itertools.groupby(ver_checkers, itemgetter(0)): version_nums = [v[-1] for v in versions] try: - latest_version = str(max(StrictVersion(v) for v in version_nums)) - # if the version can't be parsed as a StrictVersion, parse - # according to character collation + latest_version = str(max(version.parse(v) for v in version_nums)) + # if the version can't be parsed, sort according to character collation except ValueError: latest_version = max(version_nums) cls.checkers[spec] = cls.checkers[spec + ":latest"] = cls.checkers[ @@ -765,6 +768,8 @@ def generate_dataset(self, cdl_path): :param str cdl_path: Absolute path to cdl file that is used to generate netCDF file """ + # better to update the following code with Path object -- some day + cdl_path = os.fspath(cdl_path) if ( ".cdl" in cdl_path ): # it's possible the filename doesn't have the .cdl extension diff --git a/compliance_checker/tests/conftest.py b/compliance_checker/tests/conftest.py index 482bd814..76a4ce28 100644 --- a/compliance_checker/tests/conftest.py +++ b/compliance_checker/tests/conftest.py @@ -5,7 +5,7 @@ import pytest from netCDF4 import Dataset -from pkg_resources import resource_filename +import importlib from compliance_checker.cf import util from compliance_checker.suite import CheckSuite @@ -27,7 +27,7 @@ def static_files(cdl_stem): Returns the Path to a valid nc dataset\n replaces the old STATIC_FILES dict """ - datadir = Path(resource_filename("compliance_checker", "tests/data")).resolve() + datadir = importlib.resources.files("compliance_checker.tests") / "data" assert datadir.exists(), f"{datadir} not found" cdl_paths = glob_down(datadir, f"{cdl_stem}.cdl", 3) diff --git a/compliance_checker/tests/resources.py b/compliance_checker/tests/resources.py index 4bcf314c..777ed4e9 100644 --- a/compliance_checker/tests/resources.py +++ b/compliance_checker/tests/resources.py @@ -1,22 +1,27 @@ import os import subprocess +import importlib -from pkg_resources import resource_filename +# from pkg_resources import resource_filename def get_filename(path): """ Returns the path to a valid dataset """ - filename = resource_filename("compliance_checker", path) - nc_path = filename.replace(".cdl", ".nc") - if not os.path.exists(nc_path): + # breakpoint() + # filename = resource_filename("compliance_checker", path) + filename = importlib.resources.files('compliance_checker') / path + print(filename) + nc_path = filename.with_suffix(".nc") + print(nc_path) + if not nc_path.is_file(): generate_dataset(filename, nc_path) - return nc_path + return str(nc_path) def generate_dataset(cdl_path, nc_path): - subprocess.call(["ncgen", "-4", "-o", nc_path, cdl_path]) + subprocess.call(["ncgen", "-4", "-o", str(nc_path), str(cdl_path)]) STATIC_FILES = { diff --git a/compliance_checker/tests/test_cli.py b/compliance_checker/tests/test_cli.py index fc6b4f94..cb3c21a7 100644 --- a/compliance_checker/tests/test_cli.py +++ b/compliance_checker/tests/test_cli.py @@ -91,7 +91,8 @@ def checker_1(): def checker_2(): return Namespace(_cc_spec="checker_2", _cc_spec_version="2.2") - mock_checkers = [Namespace(resolve=checker_1), Namespace(resolve=checker_2)] + mock_checkers = [Namespace(load=checker_1), + Namespace(load=checker_2)] with pytest.warns(DeprecationWarning): CheckSuite._load_checkers(mock_checkers) diff --git a/compliance_checker/tests/test_suite.py b/compliance_checker/tests/test_suite.py index 54f49a95..f35c9739 100644 --- a/compliance_checker/tests/test_suite.py +++ b/compliance_checker/tests/test_suite.py @@ -3,33 +3,29 @@ from pathlib import Path import numpy as np -from pkg_resources import resource_filename +from importlib import resources from compliance_checker.acdd import ACDDBaseCheck from compliance_checker.base import BaseCheck, GenericFile, Result from compliance_checker.suite import CheckSuite static_files = { - "2dim": resource_filename("compliance_checker", "tests/data/2dim-grid.nc"), - "bad_region": resource_filename("compliance_checker", "tests/data/bad_region.nc"), - "bad_data_type": resource_filename( - "compliance_checker", - "tests/data/bad_data_type.nc", - ), - "test_cdl": resource_filename("compliance_checker", "tests/data/test_cdl.cdl"), - "test_cdl_nc": resource_filename( - "compliance_checker", - "tests/data/test_cdl_nc_file.nc", - ), - "empty": resource_filename("compliance_checker", "tests/data/non-comp/empty.file"), - "ru07": resource_filename( - "compliance_checker", - "tests/data/ru07-20130824T170228_rt0.nc", - ), - "netCDF4": resource_filename( - "compliance_checker", - "tests/data/test_cdl_nc4_file.cdl", - ), + "2dim": (resources.files("compliance_checker.tests") + / "data/2dim-grid.nc"), + "bad_region": (resources.files("compliance_checker.tests") + / "data/bad_region.nc"), + "bad_data_type": (resources.files("compliance_checker.tests") + / "data/bad_data_type.nc"), + "test_cdl": (resources.files("compliance_checker.tests") + / "data/test_cdl.cdl"), + "test_cdl_nc": (resources.files("compliance_checker.tests") + / "data/test_cdl_nc_file.nc"), + "empty": (resources.files("compliance_checker.tests") + / "data/non-comp/empty.file"), + "ru07": (resources.files("compliance_checker.tests") + / "data/ru07-20130824T170228_rt0.nc"), + "netCDF4": (resources.files("compliance_checker.tests") + / "data/test_cdl_nc4_file.cdl"), } @@ -95,9 +91,11 @@ def test_generate_dataset_netCDF4(self): # create netCDF4 file ds_name = self.cs.generate_dataset(static_files["netCDF4"]) # check if correct name is return - assert ds_name == static_files["netCDF4"].replace(".cdl", ".nc") + assert Path(ds_name) == static_files["netCDF4"].with_suffix(".nc") + # assert ds_name == static_files["netCDF4"].replace(".cdl", ".nc") # check if netCDF4 file was created - assert os.path.isfile(static_files["netCDF4"].replace(".cdl", ".nc")) + assert static_files["netCDF4"].with_suffix(".nc").is_file() + # assert os.path.isfile(static_files["netCDF4"].replace(".cdl", ".nc")) def test_include_checks(self): ds = self.cs.load_dataset(static_files["bad_data_type"]) @@ -242,7 +240,7 @@ def test_cdl_file(self): ) ds.close() - nc_file_path = static_files["test_cdl"].replace(".cdl", ".nc") + nc_file_path = static_files["test_cdl"].with_suffix(".nc") self.addCleanup(os.remove, nc_file_path) # Ok the scores should be equal! diff --git a/pyproject.toml b/pyproject.toml index 6d7198d2..7ac5f543 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,12 +38,12 @@ dynamic = [ "dependencies", "version", ] + [project.urls] documentation = "https://ioos.github.io/compliance-checker" homepage = "https://compliance.ioos.us/index.html" repository = "https://github.com/ioos/compliance-checker" -[project.scripts] -compliance-checker = "cchecker:main" + [project.entry-points."compliance_checker.suites"] "acdd-1.1" = "compliance_checker.acdd:ACDD1_1Check" "acdd-1.3" = "compliance_checker.acdd:ACDD1_3Check" @@ -60,7 +60,6 @@ packages = ["compliance_checker"] license-files = ["LICENSE"] zip-safe = false include-package-data = true -script-files = ["cchecker.py"] [tool.setuptools.package-data] compliance_checker = [ @@ -75,6 +74,9 @@ compliance_checker = [ dependencies = {file = ["requirements.txt"]} readme = {file = "README.md", content-type = "text/markdown"} +[project.scripts] +cchecker = "compliance_checker.cchecker:main" + [tool.setuptools_scm] write_to = "compliance_checker/_version.py" write_to_template = "__version__ = '{version}'"