From b1c7270062da7c771bfa28f9aabf4e7f0b73d0bc Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 18 Feb 2020 13:17:23 +0100 Subject: [PATCH 01/91] Add .github/CODEOWNERS file. To automatically trigger a review request from the DM for changes that modify files that are related to dependency specification. --- .github/CODEOWNERS | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..e90d9c1ade --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +# All files related to dependency management are owned by the +# currently active dependency manager (DM) to trigger an automatic review +# request from the DM upon changes. Please see AEP-002 for details: +# https://github.com/aiidateam/AEP/tree/master/002_dependency_management +setup.* @csadorf +environment.yml @csadorf +requirements*.txt @csadorf +pyproject.toml @csadorf From 070a78dd49c7115930948133a8f5d6d8d26f9127 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 19 Feb 2020 10:25:47 +0100 Subject: [PATCH 02/91] Rename script 'update_dependencies' to 'dependency_management'. To better reflect the broader intended scope. --- utils/{update_dependencies.py => dependency_management.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename utils/{update_dependencies.py => dependency_management.py} (98%) diff --git a/utils/update_dependencies.py b/utils/dependency_management.py similarity index 98% rename from utils/update_dependencies.py rename to utils/dependency_management.py index 2987f33196..f81d0b9cc0 100755 --- a/utils/update_dependencies.py +++ b/utils/dependency_management.py @@ -32,7 +32,7 @@ def cli(): restrictions from the `setup.json`, except for those packages where it is known that a upper limit is necessary. This is accomplished by the command: - python update_dependencies.py unrestrict + python dependency_management.py unrestrict The command will update the `setup.json` to remove all explicit limits, except for those packages specified by the `--exclude` option. After this step, install `aiida-core` through pip with the `[all]` flag to install all optional @@ -47,7 +47,7 @@ def cli(): for this setup, we can now set those versions as the new requirements in the `setup.json`. Note that this is why a clean virtual environment should be used for this entire procedure. Now execute the command: - python update_dependencies.py update requirements.txt + python dependency_management.py update requirements.txt This will now update the `setup.json` to reinstate the exact version requirements for all dependencies. Commit the changes to `setup.json` and make a pull request. From c7601ed68fc74a86c0baf35f9cf89411ef0e75c4 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 19 Feb 2020 11:30:24 +0100 Subject: [PATCH 03/91] Remove the 'etetoolkit' channel from environment.yml. All, but one of the dependencies (sqlalchemy-utils) are available via conda-forge and mixing channels is generally not advisable. --- environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 249c93a130..8cc2499c90 100644 --- a/environment.yml +++ b/environment.yml @@ -2,9 +2,8 @@ --- name: aiida channels: -- defaults - conda-forge -- etetoolkit +- defaults dependencies: - python~=3.7 - aldjemy~=0.9.1 From 5d552ea6458544f01d78cc7a43789fdd9facc2d7 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 19 Feb 2020 16:03:52 +0100 Subject: [PATCH 04/91] Implement validate command for dependency management util script. --- utils/dependency_management.py | 91 +++++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index f81d0b9cc0..4b7015a421 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -10,9 +10,14 @@ ########################################################################### """Utility CLI to update dependency version requirements of the `setup.json`.""" -import copy import os +import re +import json +import copy +from pkg_resources import Requirement + import click +import yaml from validate_consistency import get_setup_json, write_setup_json @@ -22,6 +27,22 @@ FILEPATH_SETUP_JSON = os.path.join(ROOT_DIR, FILENAME_SETUP_JSON) DEFAULT_EXCLUDE_LIST = ['django', 'circus', 'numpy', 'pymatgen', 'ase', 'monty', 'pyyaml'] +SETUPTOOLS_CONDA_MAPPINGS = { + 'psycopg2-binary': 'psycopg2', + 'graphviz': 'python-graphviz', +} + +CONDA_IGNORE = [ + 'pyblake2', +] + + +def _setuptools_to_conda(req): + for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): + if re.match(pattern, req): + return re.sub(pattern, replacement, req) + return req + @click.group() def cli(): @@ -54,6 +75,74 @@ def cli(): """ +@cli.command('validate', help='Validate consistency of the dependency specification.') +def validate(): + """Validate consistency of the requirements specification of the package. + + Validates that the specification of requirements/dependencies is consistent across + the following files: + + - setup.json + - environment.yml + """ + # Read the requirements from 'setup.json' and 'environment.yml'. + with open('setup.json') as file: + setup_cfg = json.load(file) + install_requirements = setup_cfg['install_requires'] + python_requires = Requirement.parse('python' + setup_cfg['python_requires']) + + with open('environment.yml') as file: + conda_dependencies = set(yaml.load(file, Loader=yaml.SafeLoader)['dependencies']) + + # Attempt to find the specification of Python among the 'environment.yml' dependencies. + for dependency in conda_dependencies: + req = Requirement.parse(dependency) + if req.name == 'python': # Found the Python dependency specification + conda_python_dependency = req + conda_dependencies.remove(dependency) + break + else: # Failed to find Python dependency specification + raise click.ClickException("Did not find specification of Python version in 'environment.yml'.") + + # The Python version specified in 'setup.json' should be listed as trove classifiers. + for spec in conda_python_dependency.specifier: + expected_classifier = 'Programming Language :: Python :: ' + spec.version + if expected_classifier not in setup_cfg['classifiers']: + raise click.ClickException("Trove classifier '{}' missing from 'setup.json'.".format(expected_classifier)) + + # The Python version should be specified as supported in 'setup.json'. + if not any(spec.version >= other_spec.version for other_spec in python_requires.specifier): + raise click.ClickException( + "Required Python version between 'setup.json' and 'environment.yml' not consistent." + ) + + break + else: + raise click.ClickException("Missing specifier: '{}'.".format(conda_python_dependency)) + + # Check that all requirements specified in the setup.json file are found in the + # conda environment specification. + missing_from_env = set() + for req in install_requirements: + if any(re.match(ignore, req) for ignore in CONDA_IGNORE): + continue # skip explicitly ignored packages + + try: + conda_dependencies.remove(_setuptools_to_conda(req)) + except KeyError: + raise click.ClickException("Requirement '{}' not specified in 'environment.yml'.".format(req)) + + # The only dependency left should be the one for Python itself, which is not part of + # the install_requirements for setuptools. + if len(conda_dependencies) > 0: + raise click.ClickException( + "The 'environment.yml' file contains dependencies that are missing " + "in 'setup.json':\n- {}".format('\n- '.join(conda_dependencies)) + ) + + click.secho('Dependency specification is consistent.', fg='green') + + @cli.command('unrestrict') @click.option('--exclude', multiple=True, help='List of package names to exclude from updating.') def unrestrict_requirements(exclude): From f6604bcd7c2ed0754287a06a7874c2b42d8f7999 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 19 Feb 2020 16:14:50 +0100 Subject: [PATCH 05/91] Add explicit dependency validation step to CI. --- .github/workflows/ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82ab4a0fff..f86ed1954f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,25 @@ on: [push, pull_request] jobs: + dependencies: + + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v1 + + - name: Setup Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Install dependencies + run: python -m pip install click yaml toml + + - name: Validate dependency consistency + run: python utils/dependency_management.py validate + conda: runs-on: ubuntu-latest From 13af1d3e39c04383fe85bc97e31adc621f680aa7 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 19 Feb 2020 18:57:24 +0100 Subject: [PATCH 06/91] Fix yaml requirement. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f86ed1954f..108514e093 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: python-version: 3.7 - name: Install dependencies - run: python -m pip install click yaml toml + run: python -m pip install click pyyaml toml - name: Validate dependency consistency run: python utils/dependency_management.py validate From 586adf7dbe6576f531d92f31ca6cb24b2c13de63 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 12:17:47 +0100 Subject: [PATCH 07/91] Implement 'generate-conda-environment-file' for dependency_management script. - And remove equivalent function from validate_consistency script. - Update pre-commit-hooks to reflect change. --- .pre-commit-config.yaml | 8 ++--- utils/dependency_management.py | 58 ++++++++++++++++++++++++++++------ utils/validate_consistency.py | 47 --------------------------- 3 files changed, 53 insertions(+), 60 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 06e50b4c5f..e1d0191b3f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -134,15 +134,15 @@ utils/validate_consistency.py| )$ pass_filenames: false - - id: conda - name: Validating environment.yml - entry: python ./utils/validate_consistency.py conda + - id: dependencies + name: Validate dependency specification consistency + entry: python ./utils/dependency_management.py validate language: system files: >- (?x)^( setup.json| setup.py| - utils/validate_consistency.py| + utils/dependency_management.py| environment.yml| )$ pass_filenames: false diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 4b7015a421..b895706677 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -14,6 +14,7 @@ import re import json import copy +from collections import OrderedDict from pkg_resources import Requirement import click @@ -39,9 +40,9 @@ def _setuptools_to_conda(req): for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): - if re.match(pattern, req): - return re.sub(pattern, replacement, req) - return req + if re.match(pattern, str(req)): + return Requirement.parse(re.sub(pattern, replacement, str(req))) + return req # did not match any of the replacement patterns, just return original @click.group() @@ -75,6 +76,46 @@ def cli(): """ +@cli.command('generate-environment-yml') +def generate_environment_yml(): + """Generate `environment.yml` file for conda.""" + + # needed for ordered dict, see https://stackoverflow.com/a/52621703 + yaml.add_representer( + OrderedDict, + lambda self, data: yaml.representer.SafeRepresenter.represent_dict(self, data.items()), + Dumper=yaml.SafeDumper + ) + + # Read the requirements from 'setup.json' + with open('setup.json') as file: + setup_cfg = json.load(file) + install_requirements = setup_cfg['install_requires'] + + # python version cannot be overriden from outside environment.yml + # (even if it is not specified at all in environment.yml) + # https://github.com/conda/conda/issues/9506 + conda_requires = ['python~=3.7'] + for req in [Requirement.parse(r) for r in install_requirements]: + if req.name == 'python' or any(re.match(ignore, str(req)) for ignore in CONDA_IGNORE): + continue + conda_requires.append(str(_setuptools_to_conda(req))) + + environment = OrderedDict([ + ('name', 'aiida'), + ('channels', ['conda-forge', 'defaults']), + ('dependencies', conda_requires), + ]) + + environment_filename = 'environment.yml' + file_path = os.path.join(ROOT_DIR, environment_filename) + with open(file_path, 'w') as env_file: + env_file.write('# Usage: conda env create -n myenvname -f environment.yml\n') + yaml.safe_dump( + environment, env_file, explicit_start=True, default_flow_style=False, encoding='utf-8', allow_unicode=True + ) + + @cli.command('validate', help='Validate consistency of the dependency specification.') def validate(): """Validate consistency of the requirements specification of the package. @@ -88,17 +129,16 @@ def validate(): # Read the requirements from 'setup.json' and 'environment.yml'. with open('setup.json') as file: setup_cfg = json.load(file) - install_requirements = setup_cfg['install_requires'] + install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] python_requires = Requirement.parse('python' + setup_cfg['python_requires']) with open('environment.yml') as file: - conda_dependencies = set(yaml.load(file, Loader=yaml.SafeLoader)['dependencies']) + conda_dependencies = {Requirement.parse(d) for d in yaml.load(file, Loader=yaml.SafeLoader)['dependencies']} # Attempt to find the specification of Python among the 'environment.yml' dependencies. for dependency in conda_dependencies: - req = Requirement.parse(dependency) - if req.name == 'python': # Found the Python dependency specification - conda_python_dependency = req + if dependency.name == 'python': # Found the Python dependency specification + conda_python_dependency = dependency conda_dependencies.remove(dependency) break else: # Failed to find Python dependency specification @@ -124,7 +164,7 @@ def validate(): # conda environment specification. missing_from_env = set() for req in install_requirements: - if any(re.match(ignore, req) for ignore in CONDA_IGNORE): + if any(re.match(ignore, str(req)) for ignore in CONDA_IGNORE): continue # skip explicitly ignored packages try: diff --git a/utils/validate_consistency.py b/utils/validate_consistency.py index 6604347ded..acf8566aab 100644 --- a/utils/validate_consistency.py +++ b/utils/validate_consistency.py @@ -271,52 +271,5 @@ def validate_pyproject(): sys.exit(1) -@cli.command('conda') -def update_environment_yml(): - """Update `environment.yml` file for conda.""" - import yaml - import re - - # needed for ordered dict, see https://stackoverflow.com/a/52621703 - yaml.add_representer( - OrderedDict, - lambda self, data: yaml.representer.SafeRepresenter.represent_dict(self, data.items()), - Dumper=yaml.SafeDumper - ) - - # fix incompatibilities between conda and pypi - replacements = {'psycopg2-binary': 'psycopg2', 'graphviz': 'python-graphviz'} - install_requires = get_setup_json()['install_requires'] - - # python version cannot be overriden from outside environment.yml - # (even if it is not specified at all in environment.yml) - # https://github.com/conda/conda/issues/9506 - conda_requires = ['python~=3.7'] - for req in install_requires: - # skip packages required for specific python versions - # (environment.yml aims at the latest python version) - if req.find('python_version') != -1: - continue - - for (regex, replacement) in iter(replacements.items()): - req = re.sub(regex, replacement, req) - - conda_requires.append(req) - - environment = OrderedDict([ - ('name', 'aiida'), - ('channels', ['defaults', 'conda-forge', 'etetoolkit']), - ('dependencies', conda_requires), - ]) - - environment_filename = 'environment.yml' - file_path = os.path.join(ROOT_DIR, environment_filename) - with open(file_path, 'w') as env_file: - env_file.write('# Usage: conda env create -n myenvname -f environment.yml\n') - yaml.safe_dump( - environment, env_file, explicit_start=True, default_flow_style=False, encoding='utf-8', allow_unicode=True - ) - - if __name__ == '__main__': cli() # pylint: disable=no-value-for-parameter From c27d2bb0c09236b9d3268196ccf66348a2f551da Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 13:55:45 +0100 Subject: [PATCH 08/91] Implement validation of rtd_reqs in dependency_management script. --- .pre-commit-config.yaml | 10 ++--- docs/update_req_for_rtd.py | 53 --------------------------- utils/dependency_management.py | 67 +++++++++++++++++++++++++++++++--- 3 files changed, 67 insertions(+), 63 deletions(-) delete mode 100644 docs/update_req_for_rtd.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1d0191b3f..27981087c9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -111,15 +111,15 @@ entry: prospector - id: rtd-requirements - name: Requirements for RTD - entry: python ./docs/update_req_for_rtd.py --pre-commit + name: Validate docs/requirements_for_rtd.txt + entry: python ./utils/dependency_management.py validate-rtd-reqs language: system files: >- (?x)^( setup.json| setup.py| + utils/dependency_management.py| docs/requirements_for_rtd.txt| - docs/update_req_for_rtd.py| )$ pass_filenames: false @@ -135,8 +135,8 @@ )$ pass_filenames: false - id: dependencies - name: Validate dependency specification consistency - entry: python ./utils/dependency_management.py validate + name: Validate environment.yml + entry: python ./utils/dependency_management.py validate-environment-yml language: system files: >- (?x)^( diff --git a/docs/update_req_for_rtd.py b/docs/update_req_for_rtd.py deleted file mode 100644 index 4322a0fd4a..0000000000 --- a/docs/update_req_for_rtd.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -""" -Whenever the requirements in ../setup.json are updated, run -also this script to update the requirements for Read the Docs. -""" - -import os -import json -import click - - -@click.command() -@click.option('--pre-commit', is_flag=True) -def update_req_for_rtd(pre_commit): - """Update the separate requirements file for Read the Docs""" - docs_dir = os.path.abspath(os.path.dirname(__file__)) - root_dir = os.path.join(docs_dir, os.pardir) - - with open(os.path.join(root_dir, 'setup.json'), 'r') as info: - setup_json = json.load(info) - - extras = setup_json['extras_require'] - reqs = set(extras['testing'] + extras['docs'] + extras['rest'] + extras['atomic_tools'] + - setup_json['install_requires']) - reqs_str = '\n'.join(sorted(reqs)) - - basename = 'requirements_for_rtd.txt' - - # pylint: disable=bad-continuation - with open(os.path.join(docs_dir, basename), 'w') as reqs_file: - reqs_file.write(reqs_str) - - click.echo("File '{}' written.".format(basename)) - - if pre_commit: - msg = 'Some requirements for Read the Docs have changed, {}' - local_help = 'please add the changes and commit again' - travis_help = 'please run aiida/docs/update_req_for_rtd.py locally and commit the changes it makes' - help_msg = msg.format(travis_help if os.environ.get('TRAVIS') else local_help) - click.echo(help_msg, err=True) - - -if __name__ == '__main__': - update_req_for_rtd() # pylint: disable=no-value-for-parameter diff --git a/utils/dependency_management.py b/utils/dependency_management.py index b895706677..ef625db4a8 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -27,6 +27,7 @@ ROOT_DIR = os.path.join(SCRIPT_PATH, os.pardir) FILEPATH_SETUP_JSON = os.path.join(ROOT_DIR, FILENAME_SETUP_JSON) DEFAULT_EXCLUDE_LIST = ['django', 'circus', 'numpy', 'pymatgen', 'ase', 'monty', 'pyyaml'] +DOCS_DIR = os.path.join(ROOT_DIR, 'docs') SETUPTOOLS_CONDA_MAPPINGS = { 'psycopg2-binary': 'psycopg2', @@ -90,13 +91,13 @@ def generate_environment_yml(): # Read the requirements from 'setup.json' with open('setup.json') as file: setup_cfg = json.load(file) - install_requirements = setup_cfg['install_requires'] + install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] # python version cannot be overriden from outside environment.yml # (even if it is not specified at all in environment.yml) # https://github.com/conda/conda/issues/9506 conda_requires = ['python~=3.7'] - for req in [Requirement.parse(r) for r in install_requirements]: + for req in install_requirements: if req.name == 'python' or any(re.match(ignore, str(req)) for ignore in CONDA_IGNORE): continue conda_requires.append(str(_setuptools_to_conda(req))) @@ -116,8 +117,26 @@ def generate_environment_yml(): ) -@cli.command('validate', help='Validate consistency of the dependency specification.') -def validate(): +@cli.command('generate-rtd-reqs') +def generate_requirements_for_rtd(): + """Generate 'docs/requirements_for_rtd.txt' file.""" + + # Read the requirements from 'setup.json' + with open('setup.json') as file: + setup_cfg = json.load(file) + install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']} + for key in ('testing', 'docs', 'rest', 'atomic_tools'): + install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) + + basename = 'requirements_for_rtd.txt' + + # pylint: disable=bad-continuation + with open(os.path.join(DOCS_DIR, basename), 'w') as reqs_file: + reqs_file.write('\n'.join(sorted(map(str, install_requirements)))) + + +@cli.command('validate-environment-yml', help="Validate 'environment.yml'.") +def validate_environment_yml(): """Validate consistency of the requirements specification of the package. Validates that the specification of requirements/dependencies is consistent across @@ -180,7 +199,45 @@ def validate(): "in 'setup.json':\n- {}".format('\n- '.join(conda_dependencies)) ) - click.secho('Dependency specification is consistent.', fg='green') + click.secho('Conda dependency specification is consistent.', fg='green') + + +@cli.command('validate-rtd-reqs', help='Validate requirements_for_rtd.txt.') +def validate_rtd_reqs(): + """Validate consistency of the specification of 'docs/requirements_for_rtd.txt'.""" + + # Read the requirements from 'setup.json' + with open('setup.json') as file: + setup_cfg = json.load(file) + install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']} + for key in ('testing', 'docs', 'rest', 'atomic_tools'): + install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) + + basename = 'requirements_for_rtd.txt' + + with open(os.path.join(DOCS_DIR, basename)) as rtd_reqs_file: + reqs = {Requirement.parse(r) for r in rtd_reqs_file} + + assert reqs == install_requirements + + click.secho('RTD requirements specification is consistent.', fg='green') + + +@cli.command('validate-all', help='Validate consistency of all requirements.') +@click.pass_context +def validate_all(ctx): + """Validate consistency of all requirement specifications of the package. + + Validates that the specification of requirements/dependencies is consistent across + the following files: + + - setup.py + - setup.json + - environment.yml + - docs/requirements_for_rtd.txt + """ + ctx.invoke(validate_environment_yml) + ctx.invoke(validate_rtd_reqs) @cli.command('unrestrict') From a65a0c833ae08679e18d1c339c73282ec2f4ae4e Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 14:16:36 +0100 Subject: [PATCH 09/91] Implement validate-pyproject-toml in dependency_management script. --- .pre-commit-config.yaml | 7 +++--- utils/dependency_management.py | 34 +++++++++++++++++++++++++++ utils/validate_consistency.py | 43 ---------------------------------- 3 files changed, 38 insertions(+), 46 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27981087c9..39bd257146 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -124,14 +124,15 @@ pass_filenames: false - id: pyproject - name: Validating pyproject.toml - entry: python ./utils/validate_consistency.py toml + name: Validate pyproject.toml + entry: python ./utils/dependency_management.py validate-pyproject-toml language: system files: >- (?x)^( setup.json| setup.py| - utils/validate_consistency.py| + utils/dependency_management.py| + pyproject.toml )$ pass_filenames: false - id: dependencies diff --git a/utils/dependency_management.py b/utils/dependency_management.py index ef625db4a8..7976c26d92 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -19,6 +19,7 @@ import click import yaml +import toml from validate_consistency import get_setup_json, write_setup_json @@ -223,6 +224,36 @@ def validate_rtd_reqs(): click.secho('RTD requirements specification is consistent.', fg='green') +@cli.command('validate-pyproject-toml', help="Validate 'pyproject.toml'.") +def validate_pyproject_toml(): + """Validate consistency of the specification of 'pyprojec.toml'.""" + + # Read the requirements from 'setup.json' + with open('setup.json') as file: + setup_cfg = json.load(file) + install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] + + for requirement in install_requirements: + if requirement.name == 'reentry': + reentry_requirement = requirement + break + else: + raise click.ClickException("Failed to find reentry requirement in 'setup.json'.") + + try: + with open(os.path.join(ROOT_DIR, 'pyproject.toml')) as file: + pyproject = toml.load(file) + pyproject_requires = [Requirement.parse(r) for r in pyproject['build-system']['requires']] + + if reentry_requirement not in pyproject_requires: + raise click.ClickException("Missing requirement '{}' in 'pyproject.toml'.".format(reentry_requirement)) + + except FileNotFoundError as error: + raise click.ClickException("The 'pyproject.toml' file is missing!") + + click.secho('Pyproject.toml dependency specification is consistent.', fg='green') + + @cli.command('validate-all', help='Validate consistency of all requirements.') @click.pass_context def validate_all(ctx): @@ -234,9 +265,12 @@ def validate_all(ctx): - setup.py - setup.json - environment.yml + - pyproject.toml - docs/requirements_for_rtd.txt """ + ctx.invoke(validate_environment_yml) + ctx.invoke(validate_pyproject_toml) ctx.invoke(validate_rtd_reqs) diff --git a/utils/validate_consistency.py b/utils/validate_consistency.py index acf8566aab..a771a75449 100644 --- a/utils/validate_consistency.py +++ b/utils/validate_consistency.py @@ -22,7 +22,6 @@ import sys import json from collections import OrderedDict -import toml import click FILENAME_TOML = 'pyproject.toml' @@ -229,47 +228,5 @@ def validate_version(): sys.exit(1) -@cli.command('toml') -def validate_pyproject(): - """Ensure that the version of reentry in setup.json and pyproject.toml are identical.""" - reentry_requirement = None - for requirement in get_setup_json()['install_requires']: - if 'reentry' in requirement: - reentry_requirement = requirement - break - - if reentry_requirement is None: - click.echo('Could not find the reentry requirement in {}'.format(FILEPATH_SETUP_JSON), err=True) - sys.exit(1) - - try: - with open(FILEPATH_TOML, 'r') as handle: - toml_string = handle.read() - except IOError as exception: - click.echo('Could not read the required file: {}'.format(FILEPATH_TOML), err=True) - sys.exit(1) - - try: - parsed_toml = toml.loads(toml_string) - except Exception as exception: # pylint: disable=broad-except - click.echo('Could not parse {}: {}'.format(FILEPATH_TOML, exception), err=True) - sys.exit(1) - - try: - pyproject_toml_requires = parsed_toml['build-system']['requires'] - except KeyError as exception: - click.echo('Could not retrieve the build-system requires list from {}'.format(FILEPATH_TOML), err=True) - sys.exit(1) - - if reentry_requirement not in pyproject_toml_requires: - click.echo( - 'Reentry requirement from {} {} is not mirrored in {}'.format( - FILEPATH_SETUP_JSON, reentry_requirement, FILEPATH_TOML - ), - err=True - ) - sys.exit(1) - - if __name__ == '__main__': cli() # pylint: disable=no-value-for-parameter From 78a34569d939ed15a8f5a18c1dae107f301a81d1 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 14:27:06 +0100 Subject: [PATCH 10/91] Use pathlib for paths in dependency_management script. --- utils/dependency_management.py | 37 +++++++++++++--------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 7976c26d92..2c94674868 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -10,10 +10,10 @@ ########################################################################### """Utility CLI to update dependency version requirements of the `setup.json`.""" -import os import re import json import copy +from pathlib import Path from collections import OrderedDict from pkg_resources import Requirement @@ -23,12 +23,9 @@ from validate_consistency import get_setup_json, write_setup_json -FILENAME_SETUP_JSON = 'setup.json' -SCRIPT_PATH = os.path.split(os.path.realpath(__file__))[0] -ROOT_DIR = os.path.join(SCRIPT_PATH, os.pardir) -FILEPATH_SETUP_JSON = os.path.join(ROOT_DIR, FILENAME_SETUP_JSON) +ROOT = Path(__file__).resolve().parent.parent # repository root + DEFAULT_EXCLUDE_LIST = ['django', 'circus', 'numpy', 'pymatgen', 'ase', 'monty', 'pyyaml'] -DOCS_DIR = os.path.join(ROOT_DIR, 'docs') SETUPTOOLS_CONDA_MAPPINGS = { 'psycopg2-binary': 'psycopg2', @@ -90,7 +87,7 @@ def generate_environment_yml(): ) # Read the requirements from 'setup.json' - with open('setup.json') as file: + with open(ROOT / 'setup.json') as file: setup_cfg = json.load(file) install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] @@ -109,9 +106,7 @@ def generate_environment_yml(): ('dependencies', conda_requires), ]) - environment_filename = 'environment.yml' - file_path = os.path.join(ROOT_DIR, environment_filename) - with open(file_path, 'w') as env_file: + with open(ROOT / 'environment.yml', 'w') as env_file: env_file.write('# Usage: conda env create -n myenvname -f environment.yml\n') yaml.safe_dump( environment, env_file, explicit_start=True, default_flow_style=False, encoding='utf-8', allow_unicode=True @@ -123,16 +118,14 @@ def generate_requirements_for_rtd(): """Generate 'docs/requirements_for_rtd.txt' file.""" # Read the requirements from 'setup.json' - with open('setup.json') as file: + with open(ROOT / 'setup.json') as file: setup_cfg = json.load(file) install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']} for key in ('testing', 'docs', 'rest', 'atomic_tools'): install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) - basename = 'requirements_for_rtd.txt' - # pylint: disable=bad-continuation - with open(os.path.join(DOCS_DIR, basename), 'w') as reqs_file: + with open(ROOT / Path('docs', 'requirements_for_rtx.txt'), 'w') as reqs_file: reqs_file.write('\n'.join(sorted(map(str, install_requirements)))) @@ -147,12 +140,12 @@ def validate_environment_yml(): - environment.yml """ # Read the requirements from 'setup.json' and 'environment.yml'. - with open('setup.json') as file: + with open(ROOT / 'setup.json') as file: setup_cfg = json.load(file) install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] python_requires = Requirement.parse('python' + setup_cfg['python_requires']) - with open('environment.yml') as file: + with open(ROOT / 'environment.yml') as file: conda_dependencies = {Requirement.parse(d) for d in yaml.load(file, Loader=yaml.SafeLoader)['dependencies']} # Attempt to find the specification of Python among the 'environment.yml' dependencies. @@ -208,16 +201,14 @@ def validate_rtd_reqs(): """Validate consistency of the specification of 'docs/requirements_for_rtd.txt'.""" # Read the requirements from 'setup.json' - with open('setup.json') as file: + with open(ROOT / 'setup.json') as file: setup_cfg = json.load(file) install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']} for key in ('testing', 'docs', 'rest', 'atomic_tools'): install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) - basename = 'requirements_for_rtd.txt' - - with open(os.path.join(DOCS_DIR, basename)) as rtd_reqs_file: - reqs = {Requirement.parse(r) for r in rtd_reqs_file} + with open(ROOT / Path('docs', 'requirements_for_rtd.txt')) as reqs_file: + reqs = {Requirement.parse(r) for r in reqs_file} assert reqs == install_requirements @@ -229,7 +220,7 @@ def validate_pyproject_toml(): """Validate consistency of the specification of 'pyprojec.toml'.""" # Read the requirements from 'setup.json' - with open('setup.json') as file: + with open(ROOT / 'setup.json') as file: setup_cfg = json.load(file) install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] @@ -241,7 +232,7 @@ def validate_pyproject_toml(): raise click.ClickException("Failed to find reentry requirement in 'setup.json'.") try: - with open(os.path.join(ROOT_DIR, 'pyproject.toml')) as file: + with open(ROOT / 'pyproject.toml') as file: pyproject = toml.load(file) pyproject_requires = [Requirement.parse(r) for r in pyproject['build-system']['requires']] From f3b57024ad6cfeedec053b89b6bba51be8234ce0 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 14:33:24 +0100 Subject: [PATCH 11/91] Use consistent exception type for dependency specification errors. --- utils/dependency_management.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 2c94674868..d509a46a5d 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -37,6 +37,10 @@ ] +class DependencySpecificationError(click.ClickException): + """Indicates an issue in a dependency specification.""" + + def _setuptools_to_conda(req): for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): if re.match(pattern, str(req)): @@ -155,23 +159,25 @@ def validate_environment_yml(): conda_dependencies.remove(dependency) break else: # Failed to find Python dependency specification - raise click.ClickException("Did not find specification of Python version in 'environment.yml'.") + raise DependencySpecificationError("Did not find specification of Python version in 'environment.yml'.") # The Python version specified in 'setup.json' should be listed as trove classifiers. for spec in conda_python_dependency.specifier: expected_classifier = 'Programming Language :: Python :: ' + spec.version if expected_classifier not in setup_cfg['classifiers']: - raise click.ClickException("Trove classifier '{}' missing from 'setup.json'.".format(expected_classifier)) + raise DependencySpecificationError( + "Trove classifier '{}' missing from 'setup.json'.".format(expected_classifier) + ) # The Python version should be specified as supported in 'setup.json'. if not any(spec.version >= other_spec.version for other_spec in python_requires.specifier): - raise click.ClickException( + raise DependencySpecificationError( "Required Python version between 'setup.json' and 'environment.yml' not consistent." ) break else: - raise click.ClickException("Missing specifier: '{}'.".format(conda_python_dependency)) + raise DependencySpecificationError("Missing specifier: '{}'.".format(conda_python_dependency)) # Check that all requirements specified in the setup.json file are found in the # conda environment specification. @@ -183,12 +189,12 @@ def validate_environment_yml(): try: conda_dependencies.remove(_setuptools_to_conda(req)) except KeyError: - raise click.ClickException("Requirement '{}' not specified in 'environment.yml'.".format(req)) + raise DependencySpecificationError("Requirement '{}' not specified in 'environment.yml'.".format(req)) # The only dependency left should be the one for Python itself, which is not part of # the install_requirements for setuptools. if len(conda_dependencies) > 0: - raise click.ClickException( + raise DependencySpecificationError( "The 'environment.yml' file contains dependencies that are missing " "in 'setup.json':\n- {}".format('\n- '.join(conda_dependencies)) ) @@ -229,7 +235,7 @@ def validate_pyproject_toml(): reentry_requirement = requirement break else: - raise click.ClickException("Failed to find reentry requirement in 'setup.json'.") + raise DependencySpecificationError("Failed to find reentry requirement in 'setup.json'.") try: with open(ROOT / 'pyproject.toml') as file: @@ -237,10 +243,12 @@ def validate_pyproject_toml(): pyproject_requires = [Requirement.parse(r) for r in pyproject['build-system']['requires']] if reentry_requirement not in pyproject_requires: - raise click.ClickException("Missing requirement '{}' in 'pyproject.toml'.".format(reentry_requirement)) + raise DependencySpecificationError( + "Missing requirement '{}' in 'pyproject.toml'.".format(reentry_requirement) + ) except FileNotFoundError as error: - raise click.ClickException("The 'pyproject.toml' file is missing!") + raise DependencySpecificationError("The 'pyproject.toml' file is missing!") click.secho('Pyproject.toml dependency specification is consistent.', fg='green') From d9beaceb6b3e17674be6123048c66ed176e2723b Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 14:43:54 +0100 Subject: [PATCH 12/91] Place loading of 'setup.json' file in function. --- utils/dependency_management.py | 52 +++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index d509a46a5d..1570e67038 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -21,7 +21,7 @@ import yaml import toml -from validate_consistency import get_setup_json, write_setup_json +from validate_consistency import write_setup_json ROOT = Path(__file__).resolve().parent.parent # repository root @@ -41,6 +41,17 @@ class DependencySpecificationError(click.ClickException): """Indicates an issue in a dependency specification.""" +def _load_setup_cfg(): + """Load the setup configuration from the 'setup.json' file.""" + try: + with open(ROOT / 'setup.json') as setup_json_file: + return json.load(setup_json_file) + except json.decoder.JSONDecodeError as error: + raise DependencySpecificationError("Error while parsing 'setup.json' file: {}".format(error)) + except FileNotFoundError: + raise DependencySpecificationError("The 'setup.json' file is missing!") + + def _setuptools_to_conda(req): for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): if re.match(pattern, str(req)): @@ -91,9 +102,8 @@ def generate_environment_yml(): ) # Read the requirements from 'setup.json' - with open(ROOT / 'setup.json') as file: - setup_cfg = json.load(file) - install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] + setup_cfg = _load_setup_cfg() + install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] # python version cannot be overriden from outside environment.yml # (even if it is not specified at all in environment.yml) @@ -122,11 +132,10 @@ def generate_requirements_for_rtd(): """Generate 'docs/requirements_for_rtd.txt' file.""" # Read the requirements from 'setup.json' - with open(ROOT / 'setup.json') as file: - setup_cfg = json.load(file) - install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']} - for key in ('testing', 'docs', 'rest', 'atomic_tools'): - install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) + setup_cfg = _load_setup_cfg() + install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']} + for key in ('testing', 'docs', 'rest', 'atomic_tools'): + install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) # pylint: disable=bad-continuation with open(ROOT / Path('docs', 'requirements_for_rtx.txt'), 'w') as reqs_file: @@ -144,10 +153,9 @@ def validate_environment_yml(): - environment.yml """ # Read the requirements from 'setup.json' and 'environment.yml'. - with open(ROOT / 'setup.json') as file: - setup_cfg = json.load(file) - install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] - python_requires = Requirement.parse('python' + setup_cfg['python_requires']) + setup_cfg = _load_setup_cfg() + install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] + python_requires = Requirement.parse('python' + setup_cfg['python_requires']) with open(ROOT / 'environment.yml') as file: conda_dependencies = {Requirement.parse(d) for d in yaml.load(file, Loader=yaml.SafeLoader)['dependencies']} @@ -207,11 +215,10 @@ def validate_rtd_reqs(): """Validate consistency of the specification of 'docs/requirements_for_rtd.txt'.""" # Read the requirements from 'setup.json' - with open(ROOT / 'setup.json') as file: - setup_cfg = json.load(file) - install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']} - for key in ('testing', 'docs', 'rest', 'atomic_tools'): - install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) + setup_cfg = _load_setup_cfg() + install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']} + for key in ('testing', 'docs', 'rest', 'atomic_tools'): + install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) with open(ROOT / Path('docs', 'requirements_for_rtd.txt')) as reqs_file: reqs = {Requirement.parse(r) for r in reqs_file} @@ -226,9 +233,8 @@ def validate_pyproject_toml(): """Validate consistency of the specification of 'pyprojec.toml'.""" # Read the requirements from 'setup.json' - with open(ROOT / 'setup.json') as file: - setup_cfg = json.load(file) - install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] + setup_cfg = _load_setup_cfg() + install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] for requirement in install_requirements: if requirement.name == 'reentry': @@ -282,7 +288,7 @@ def unrestrict_requirements(exclude): operators, additional filters after a semicolon, or with extra requirements (using `[]`) are not supported. The limits for these statements will have to be updated manually. """ - setup = get_setup_json() + setup = _load_setup_cfg() clone = copy.deepcopy(setup) clone['install_requires'] = [] @@ -318,7 +324,7 @@ def update_requirements(requirements): The REQUIREMENTS file should contain the output of `pip freeze`. """ - setup = get_setup_json() + setup = _load_setup_cfg() package_versions = [] From 0f5a60efb88ca568ab97b20830652ce3a9386e65 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 14:45:18 +0100 Subject: [PATCH 13/91] Fix bug in 'validate-environment-yml' command implementation. --- utils/dependency_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 1570e67038..c1926844a4 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -204,7 +204,7 @@ def validate_environment_yml(): if len(conda_dependencies) > 0: raise DependencySpecificationError( "The 'environment.yml' file contains dependencies that are missing " - "in 'setup.json':\n- {}".format('\n- '.join(conda_dependencies)) + "in 'setup.json':\n- {}".format('\n- '.join(map(str, conda_dependencies))) ) click.secho('Conda dependency specification is consistent.', fg='green') From 59c64a3859500fc8b037cadcaeb8c24d4959950f Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 14:51:37 +0100 Subject: [PATCH 14/91] Make implemenation 'validate-environment-yml' more robust. --- utils/dependency_management.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index c1926844a4..f9eec9c9b1 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -52,6 +52,17 @@ def _load_setup_cfg(): raise DependencySpecificationError("The 'setup.json' file is missing!") +def _load_environment_yml(): + """Load the conda environment specification from the 'environment.yml' file.""" + try: + with open(ROOT / 'environment.yml') as file: + return yaml.load(file, Loader=yaml.SafeLoader) + except yaml.error.YAMLError as error: + raise DependencySpecificationError("Error while parsing 'environment.yml':\n{}".format(error)) + except FileNotFoundError as error: + raise DependencySpecificationError(str(error)) + + def _setuptools_to_conda(req): for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): if re.match(pattern, str(req)): @@ -143,7 +154,7 @@ def generate_requirements_for_rtd(): @cli.command('validate-environment-yml', help="Validate 'environment.yml'.") -def validate_environment_yml(): +def validate_environment_yml(): # pylint: disable=too-many-branches """Validate consistency of the requirements specification of the package. Validates that the specification of requirements/dependencies is consistent across @@ -157,8 +168,19 @@ def validate_environment_yml(): install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] python_requires = Requirement.parse('python' + setup_cfg['python_requires']) - with open(ROOT / 'environment.yml') as file: - conda_dependencies = {Requirement.parse(d) for d in yaml.load(file, Loader=yaml.SafeLoader)['dependencies']} + environment_yml = _load_environment_yml() + try: + assert environment_yml['name'] == 'aiida', "environment name should be 'aiida'." + assert environment_yml['channels'] == [ + 'conda-forge', 'defaults' + ], "channels should be 'conda-forge', 'defaults'." + except AssertionError as error: + raise DependencySpecificationError("Error in 'environment.yml': {}".format(error)) + + try: + conda_dependencies = {Requirement.parse(d) for d in environment_yml['dependencies']} + except TypeError as error: + raise DependencySpecificationError("Error while parsing requirements from 'environment_yml': {}".format(error)) # Attempt to find the specification of Python among the 'environment.yml' dependencies. for dependency in conda_dependencies: From ed8eaf0599a2d1d70d8c181410f726e716ff4cba Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 15:00:11 +0100 Subject: [PATCH 15/91] Improve error message for 'validate-rtd-reqs' command. --- utils/dependency_management.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index f9eec9c9b1..eb18f178d6 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -245,7 +245,8 @@ def validate_rtd_reqs(): with open(ROOT / Path('docs', 'requirements_for_rtd.txt')) as reqs_file: reqs = {Requirement.parse(r) for r in reqs_file} - assert reqs == install_requirements + if reqs != install_requirements: + raise DependencySpecificationError("The requirements for RTD are inconsistent with 'setup.json'.") click.secho('RTD requirements specification is consistent.', fg='green') From 364b34ce581872cad1090d0dc97cbfc7e2dd8dc9 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 15:11:17 +0100 Subject: [PATCH 16/91] Implement 'generate-pyproject-toml' dependency management command. --- utils/dependency_management.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index eb18f178d6..33e7286a1b 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -153,6 +153,35 @@ def generate_requirements_for_rtd(): reqs_file.write('\n'.join(sorted(map(str, install_requirements)))) +@cli.command() +def generate_pyproject_toml(): + """Generate 'pyproject.toml' file.""" + + # Read the requirements from 'setup.json' + setup_cfg = _load_setup_cfg() + install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] + + for requirement in install_requirements: + if requirement.name == 'reentry': + reentry_requirement = requirement + break + else: + raise DependencySpecificationError("Failed to find reentry requirement in 'setup.json'.") + + pyproject = {'build-system': {'requires': ['setuptools', 'wheel', str(reentry_requirement)]}} + with open(ROOT / 'pyproject.toml', 'w') as file: + toml.dump(pyproject, file) + + +@cli.command() +@click.pass_context +def generate_all(ctx): + """Generate all dependent requirement files.""" + ctx.invoke(generate_environment_yml) + ctx.invoke(generate_requirements_for_rtd) + ctx.invoke(generate_pyproject_toml) + + @cli.command('validate-environment-yml', help="Validate 'environment.yml'.") def validate_environment_yml(): # pylint: disable=too-many-branches """Validate consistency of the requirements specification of the package. @@ -298,8 +327,8 @@ def validate_all(ctx): """ ctx.invoke(validate_environment_yml) - ctx.invoke(validate_pyproject_toml) ctx.invoke(validate_rtd_reqs) + ctx.invoke(validate_pyproject_toml) @cli.command('unrestrict') From e502f4250e2f1133ca0f72881584c281651db7c3 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 15:13:00 +0100 Subject: [PATCH 17/91] Regenerate 'environment.yml' and 'docs/requirements_for_rtd.txt' file. Rewrites requirements in 'canonical' form so that future changes are better tractable. --- docs/requirements_for_rtd.txt | 12 ++++++------ environment.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/requirements_for_rtd.txt b/docs/requirements_for_rtd.txt index e245911c86..936d0c89d5 100644 --- a/docs/requirements_for_rtd.txt +++ b/docs/requirements_for_rtd.txt @@ -19,15 +19,15 @@ graphviz~=0.13 ipython~=7.0 jinja2~=2.10 kiwipy[rmq]~=0.5.1 -numpy~=1.17,<1.18 +numpy<1.18,~=1.17 paramiko~=2.6 pg8000~=1.13 -pgtest~=1.3,>=1.3.1 +pgtest>=1.3.1,~=1.3 pika~=1.1 plumpy~=0.14.5 psutil~=5.6 -psycopg2-binary~=2.8,>=2.8.3 -pyblake2~=1.1; python_version<'3.6' +psycopg2-binary>=2.8.3,~=2.8 +pyblake2~=1.1; python_version < "3.6" pygments~=2.5 pymatgen>=2019.7.2 pymysql~=0.9.3 @@ -40,7 +40,7 @@ python-memcached~=1.59 pytz~=2019.3 pyyaml~=5.1.2 reentry~=1.3 -seekpath~=1.9,>=1.9.3 +seekpath>=1.9.3,~=1.9 simplejson~=3.16 spglib~=1.14 sphinx-rtd-theme~=0.4.3 @@ -49,7 +49,7 @@ sphinxcontrib-details-directive~=0.1.0 sphinx~=2.2 sqlalchemy-diff~=0.1.3 sqlalchemy-utils~=0.34.2 -sqlalchemy~=1.3,>=1.3.10 +sqlalchemy>=1.3.10,~=1.3 tabulate~=0.8.5 tornado<5.0 tzlocal~=2.0 diff --git a/environment.yml b/environment.yml index 8cc2499c90..f7d6ca7d1b 100644 --- a/environment.yml +++ b/environment.yml @@ -19,19 +19,19 @@ dependencies: - ipython~=7.0 - jinja2~=2.10 - kiwipy[rmq]~=0.5.1 -- numpy~=1.17,<1.18 +- numpy<1.18,~=1.17 - paramiko~=2.6 - pika~=1.1 - plumpy~=0.14.5 - psutil~=5.6 -- psycopg2~=2.8,>=2.8.3 +- psycopg2>=2.8.3,~=2.8 - python-dateutil~=2.8 - pytz~=2019.3 - pyyaml~=5.1.2 - reentry~=1.3 - simplejson~=3.16 - sqlalchemy-utils~=0.34.2 -- sqlalchemy~=1.3,>=1.3.10 +- sqlalchemy>=1.3.10,~=1.3 - tabulate~=0.8.5 - tornado<5.0 - tzlocal~=2.0 From e98fd81b8972c3a30ddc3ef5b10cfdd286eed5e2 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 15:15:46 +0100 Subject: [PATCH 18/91] Fix bug in 'generate-rtd-reqs' implementation. --- utils/dependency_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 33e7286a1b..66a1683ec6 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -149,7 +149,7 @@ def generate_requirements_for_rtd(): install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]}) # pylint: disable=bad-continuation - with open(ROOT / Path('docs', 'requirements_for_rtx.txt'), 'w') as reqs_file: + with open(ROOT / Path('docs', 'requirements_for_rtd.txt'), 'w') as reqs_file: reqs_file.write('\n'.join(sorted(map(str, install_requirements)))) From 46d4f2473a613689d11dec123937d5a7706bebe8 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 15:18:41 +0100 Subject: [PATCH 19/91] Improve 'dependency_management.py' script API consistency. --- utils/dependency_management.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 66a1683ec6..3f8f3bc0b3 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -103,7 +103,7 @@ def cli(): @cli.command('generate-environment-yml') def generate_environment_yml(): - """Generate `environment.yml` file for conda.""" + """Generate 'environment.yml' file.""" # needed for ordered dict, see https://stackoverflow.com/a/52621703 yaml.add_representer( @@ -261,8 +261,8 @@ def validate_environment_yml(): # pylint: disable=too-many-branches click.secho('Conda dependency specification is consistent.', fg='green') -@cli.command('validate-rtd-reqs', help='Validate requirements_for_rtd.txt.') -def validate_rtd_reqs(): +@cli.command('validate-rtd-reqs', help="Validate 'docs/requirements_for_rtd.txt'.") +def validate_requirements_for_rtd(): """Validate consistency of the specification of 'docs/requirements_for_rtd.txt'.""" # Read the requirements from 'setup.json' @@ -327,7 +327,7 @@ def validate_all(ctx): """ ctx.invoke(validate_environment_yml) - ctx.invoke(validate_rtd_reqs) + ctx.invoke(validate_requirements_for_rtd) ctx.invoke(validate_pyproject_toml) From 420983f993361b1f945e0929a9b25c47fdb95f69 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 15:21:45 +0100 Subject: [PATCH 20/91] Remove the seperate 'dependency' github action. The relevant checks are carried out as part of the pre-commit checks. --- .github/workflows/ci.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 108514e093..82ab4a0fff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,25 +4,6 @@ on: [push, pull_request] jobs: - dependencies: - - runs-on: ubuntu-latest - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v1 - - - name: Setup Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Install dependencies - run: python -m pip install click pyyaml toml - - - name: Validate dependency consistency - run: python utils/dependency_management.py validate - conda: runs-on: ubuntu-latest From 9b86152d2adc2ac9ebfc1100832d5b12e58198ba Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 20 Feb 2020 15:58:10 +0100 Subject: [PATCH 21/91] Ignore pylint error related to json.decoder module (false positive). --- utils/dependency_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 3f8f3bc0b3..05201a1939 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -46,7 +46,7 @@ def _load_setup_cfg(): try: with open(ROOT / 'setup.json') as setup_json_file: return json.load(setup_json_file) - except json.decoder.JSONDecodeError as error: + except json.decoder.JSONDecodeError as error: # pylint: disable=no-member raise DependencySpecificationError("Error while parsing 'setup.json' file: {}".format(error)) except FileNotFoundError: raise DependencySpecificationError("The 'setup.json' file is missing!") From 21ed385094c375da679c44783909aaa5eb83b74d Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 2 Mar 2020 16:03:24 +0100 Subject: [PATCH 22/91] Remove dependency management script main help. --- utils/dependency_management.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 05201a1939..1b34b82a3c 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -72,33 +72,7 @@ def _setuptools_to_conda(req): @click.group() def cli(): - """Utility to update dependency requirements for `aiida-core`. - - Since `aiida-core` fixes the versions of almost all of its dependencies, once in a while these need to be updated. - This is a manual process, but this CLI attempts to simplify it somewhat. The idea is to remote all explicit version - restrictions from the `setup.json`, except for those packages where it is known that a upper limit is necessary. - This is accomplished by the command: - - python dependency_management.py unrestrict - - The command will update the `setup.json` to remove all explicit limits, except for those packages specified by the - `--exclude` option. After this step, install `aiida-core` through pip with the `[all]` flag to install all optional - extra requirements as well. Since there are no explicit version requirements anymore, pip should install the latest - available version for each dependency. - - Once all the tests complete successfully, run the following command: - - pip freeze > requirements.txt - - This will now capture the exact versions of the packages installed in the virtual environment. Since the tests run - for this setup, we can now set those versions as the new requirements in the `setup.json`. Note that this is why a - clean virtual environment should be used for this entire procedure. Now execute the command: - - python dependency_management.py update requirements.txt - - This will now update the `setup.json` to reinstate the exact version requirements for all dependencies. Commit the - changes to `setup.json` and make a pull request. - """ + """Manage dependencies of the aiida-core package.""" @cli.command('generate-environment-yml') From 9b635bd2f56b8be6ebecd74167daa59bd0e3e397 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 11:53:12 +0100 Subject: [PATCH 23/91] Implement 'pip-install-extras' command for dependency management script. To install extra dependencies without the core package dependencies. --- utils/dependency_management.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 1b34b82a3c..f4c294273b 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -10,9 +10,11 @@ ########################################################################### """Utility CLI to update dependency version requirements of the `setup.json`.""" +import sys import re import json import copy +import subprocess from pathlib import Path from collections import OrderedDict from pkg_resources import Requirement @@ -305,6 +307,25 @@ def validate_all(ctx): ctx.invoke(validate_pyproject_toml) +@cli.command() +@click.argument('extras', nargs=-1) +def pip_install_extras(extras): + """Install extra requirements. + + Install extra requirements in isolation without triggering the installlation + of other installation requirements of the core package. + """ + # Read the requirements from 'setup.json' + setup_cfg = _load_setup_cfg() + + to_install = set() + for key in extras: + to_install.update(Requirement.parse(r) for r in setup_cfg['extras_require'][key]) + + cmd = [sys.executable, '-m', 'pip', 'install'] + [str(r) for r in to_install] + subprocess.run(cmd, check=True) + + @cli.command('unrestrict') @click.option('--exclude', multiple=True, help='List of package names to exclude from updating.') def unrestrict_requirements(exclude): From e03b77f298a470a361e26895ee9b544b2fff6a01 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 11:58:25 +0100 Subject: [PATCH 24/91] Vendor toml package in utils directory. To simplify the execution of the dependency management script. --- .pre-commit-config.yaml | 3 + utils/toml/__init__.py | 25 + utils/toml/decoder.py | 1049 +++++++++++++++++++++++++++++++++++++++ utils/toml/encoder.py | 304 ++++++++++++ utils/toml/ordered.py | 15 + utils/toml/tz.py | 21 + 6 files changed, 1417 insertions(+) create mode 100644 utils/toml/__init__.py create mode 100644 utils/toml/decoder.py create mode 100644 utils/toml/encoder.py create mode 100644 utils/toml/ordered.py create mode 100644 utils/toml/tz.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 39bd257146..c0ccb519a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,9 @@ rev: v2.2.3 hooks: - id: trailing-whitespace + exclude: utils/toml/* - id: double-quote-string-fixer + exclude: utils/toml/* - repo: local hooks: @@ -86,6 +88,7 @@ aiida/tools/dbimporters/plugins/pcod.py| docs/.*| examples/.*| + utils/toml/.*| tests/engine/test_work_chain.py| tests/schedulers/test_direct.py| tests/schedulers/test_lsf.py| diff --git a/utils/toml/__init__.py b/utils/toml/__init__.py new file mode 100644 index 0000000000..7e13a0c36f --- /dev/null +++ b/utils/toml/__init__.py @@ -0,0 +1,25 @@ +"""Python module which parses and emits TOML. + +Released under the MIT license. +""" + +from toml import encoder +from toml import decoder + +__version__ = "0.10.1" +_spec_ = "0.5.0" + +load = decoder.load +loads = decoder.loads +TomlDecoder = decoder.TomlDecoder +TomlDecodeError = decoder.TomlDecodeError +TomlPreserveCommentDecoder = decoder.TomlPreserveCommentDecoder + +dump = encoder.dump +dumps = encoder.dumps +TomlEncoder = encoder.TomlEncoder +TomlArraySeparatorEncoder = encoder.TomlArraySeparatorEncoder +TomlPreserveInlineDictEncoder = encoder.TomlPreserveInlineDictEncoder +TomlNumpyEncoder = encoder.TomlNumpyEncoder +TomlPreserveCommentEncoder = encoder.TomlPreserveCommentEncoder +TomlPathlibEncoder = encoder.TomlPathlibEncoder diff --git a/utils/toml/decoder.py b/utils/toml/decoder.py new file mode 100644 index 0000000000..a24c04a0bf --- /dev/null +++ b/utils/toml/decoder.py @@ -0,0 +1,1049 @@ +import datetime +import io +from os import linesep +import re +import sys + +from toml.tz import TomlTz + +if sys.version_info < (3,): + _range = xrange # noqa: F821 +else: + unicode = str + _range = range + basestring = str + unichr = chr + + +def _detect_pathlib_path(p): + if (3, 4) <= sys.version_info: + import pathlib + if isinstance(p, pathlib.PurePath): + return True + return False + + +def _ispath(p): + if isinstance(p, basestring): + return True + return _detect_pathlib_path(p) + + +def _getpath(p): + if (3, 6) <= sys.version_info: + import os + return os.fspath(p) + if _detect_pathlib_path(p): + return str(p) + return p + + +try: + FNFError = FileNotFoundError +except NameError: + FNFError = IOError + + +TIME_RE = re.compile(r"([0-9]{2}):([0-9]{2}):([0-9]{2})(\.([0-9]{3,6}))?") + + +class TomlDecodeError(ValueError): + """Base toml Exception / Error.""" + + def __init__(self, msg, doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + colno = pos - doc.rfind('\n', 0, pos) + emsg = '{} (line {} column {} char {})'.format(msg, lineno, colno, pos) + ValueError.__init__(self, emsg) + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno + + +# Matches a TOML number, which allows underscores for readability +_number_with_underscores = re.compile('([0-9])(_([0-9]))*') + + +class CommentValue(object): + def __init__(self, val, comment, beginline, _dict): + self.val = val + separator = "\n" if beginline else " " + self.comment = separator + comment + self._dict = _dict + + def __getitem__(self, key): + return self.val[key] + + def __setitem__(self, key, value): + self.val[key] = value + + def dump(self, dump_value_func): + retstr = dump_value_func(self.val) + if isinstance(self.val, self._dict): + return self.comment + "\n" + unicode(retstr) + else: + return unicode(retstr) + self.comment + + +def _strictly_valid_num(n): + n = n.strip() + if not n: + return False + if n[0] == '_': + return False + if n[-1] == '_': + return False + if "_." in n or "._" in n: + return False + if len(n) == 1: + return True + if n[0] == '0' and n[1] not in ['.', 'o', 'b', 'x']: + return False + if n[0] == '+' or n[0] == '-': + n = n[1:] + if len(n) > 1 and n[0] == '0' and n[1] != '.': + return False + if '__' in n: + return False + return True + + +def load(f, _dict=dict, decoder=None): + """Parses named file or files as toml and returns a dictionary + + Args: + f: Path to the file to open, array of files to read into single dict + or a file descriptor + _dict: (optional) Specifies the class of the returned toml dictionary + decoder: The decoder to use + + Returns: + Parsed toml file represented as a dictionary + + Raises: + TypeError -- When f is invalid type + TomlDecodeError: Error while decoding toml + IOError / FileNotFoundError -- When an array with no valid (existing) + (Python 2 / Python 3) file paths is passed + """ + + if _ispath(f): + with io.open(_getpath(f), encoding='utf-8') as ffile: + return loads(ffile.read(), _dict, decoder) + elif isinstance(f, list): + from os import path as op + from warnings import warn + if not [path for path in f if op.exists(path)]: + error_msg = "Load expects a list to contain filenames only." + error_msg += linesep + error_msg += ("The list needs to contain the path of at least one " + "existing file.") + raise FNFError(error_msg) + if decoder is None: + decoder = TomlDecoder(_dict) + d = decoder.get_empty_table() + for l in f: + if op.exists(l): + d.update(load(l, _dict, decoder)) + else: + warn("Non-existent filename in list with at least one valid " + "filename") + return d + else: + try: + return loads(f.read(), _dict, decoder) + except AttributeError: + raise TypeError("You can only load a file descriptor, filename or " + "list") + + +_groupname_re = re.compile(r'^[A-Za-z0-9_-]+$') + + +def loads(s, _dict=dict, decoder=None): + """Parses string as toml + + Args: + s: String to be parsed + _dict: (optional) Specifies the class of the returned toml dictionary + + Returns: + Parsed toml file represented as a dictionary + + Raises: + TypeError: When a non-string is passed + TomlDecodeError: Error while decoding toml + """ + + implicitgroups = [] + if decoder is None: + decoder = TomlDecoder(_dict) + retval = decoder.get_empty_table() + currentlevel = retval + if not isinstance(s, basestring): + raise TypeError("Expecting something like a string") + + if not isinstance(s, unicode): + s = s.decode('utf8') + + original = s + sl = list(s) + openarr = 0 + openstring = False + openstrchar = "" + multilinestr = False + arrayoftables = False + beginline = True + keygroup = False + dottedkey = False + keyname = 0 + key = '' + prev_key = '' + line_no = 1 + + for i, item in enumerate(sl): + if item == '\r' and sl[i + 1] == '\n': + sl[i] = ' ' + continue + if keyname: + key += item + if item == '\n': + raise TomlDecodeError("Key name found without value." + " Reached end of line.", original, i) + if openstring: + if item == openstrchar: + oddbackslash = False + k = 1 + while i >= k and sl[i - k] == '\\': + oddbackslash = not oddbackslash + k += 1 + if not oddbackslash: + keyname = 2 + openstring = False + openstrchar = "" + continue + elif keyname == 1: + if item.isspace(): + keyname = 2 + continue + elif item == '.': + dottedkey = True + continue + elif item.isalnum() or item == '_' or item == '-': + continue + elif (dottedkey and sl[i - 1] == '.' and + (item == '"' or item == "'")): + openstring = True + openstrchar = item + continue + elif keyname == 2: + if item.isspace(): + if dottedkey: + nextitem = sl[i + 1] + if not nextitem.isspace() and nextitem != '.': + keyname = 1 + continue + if item == '.': + dottedkey = True + nextitem = sl[i + 1] + if not nextitem.isspace() and nextitem != '.': + keyname = 1 + continue + if item == '=': + keyname = 0 + prev_key = key[:-1].rstrip() + key = '' + dottedkey = False + else: + raise TomlDecodeError("Found invalid character in key name: '" + + item + "'. Try quoting the key name.", + original, i) + if item == "'" and openstrchar != '"': + k = 1 + try: + while sl[i - k] == "'": + k += 1 + if k == 3: + break + except IndexError: + pass + if k == 3: + multilinestr = not multilinestr + openstring = multilinestr + else: + openstring = not openstring + if openstring: + openstrchar = "'" + else: + openstrchar = "" + if item == '"' and openstrchar != "'": + oddbackslash = False + k = 1 + tripquote = False + try: + while sl[i - k] == '"': + k += 1 + if k == 3: + tripquote = True + break + if k == 1 or (k == 3 and tripquote): + while sl[i - k] == '\\': + oddbackslash = not oddbackslash + k += 1 + except IndexError: + pass + if not oddbackslash: + if tripquote: + multilinestr = not multilinestr + openstring = multilinestr + else: + openstring = not openstring + if openstring: + openstrchar = '"' + else: + openstrchar = "" + if item == '#' and (not openstring and not keygroup and + not arrayoftables): + j = i + comment = "" + try: + while sl[j] != '\n': + comment += s[j] + sl[j] = ' ' + j += 1 + except IndexError: + break + if not openarr: + decoder.preserve_comment(line_no, prev_key, comment, beginline) + if item == '[' and (not openstring and not keygroup and + not arrayoftables): + if beginline: + if len(sl) > i + 1 and sl[i + 1] == '[': + arrayoftables = True + else: + keygroup = True + else: + openarr += 1 + if item == ']' and not openstring: + if keygroup: + keygroup = False + elif arrayoftables: + if sl[i - 1] == ']': + arrayoftables = False + else: + openarr -= 1 + if item == '\n': + if openstring or multilinestr: + if not multilinestr: + raise TomlDecodeError("Unbalanced quotes", original, i) + if ((sl[i - 1] == "'" or sl[i - 1] == '"') and ( + sl[i - 2] == sl[i - 1])): + sl[i] = sl[i - 1] + if sl[i - 3] == sl[i - 1]: + sl[i - 3] = ' ' + elif openarr: + sl[i] = ' ' + else: + beginline = True + line_no += 1 + elif beginline and sl[i] != ' ' and sl[i] != '\t': + beginline = False + if not keygroup and not arrayoftables: + if sl[i] == '=': + raise TomlDecodeError("Found empty keyname. ", original, i) + keyname = 1 + key += item + if keyname: + raise TomlDecodeError("Key name found without value." + " Reached end of file.", original, len(s)) + s = ''.join(sl) + s = s.split('\n') + multikey = None + multilinestr = "" + multibackslash = False + pos = 0 + for idx, line in enumerate(s): + if idx > 0: + pos += len(s[idx - 1]) + 1 + + decoder.embed_comments(idx, currentlevel) + + if not multilinestr or multibackslash or '\n' not in multilinestr: + line = line.strip() + if line == "" and (not multikey or multibackslash): + continue + if multikey: + if multibackslash: + multilinestr += line + else: + multilinestr += line + multibackslash = False + closed = False + if multilinestr[0] == '[': + closed = line[-1] == ']' + elif len(line) > 2: + closed = (line[-1] == multilinestr[0] and + line[-2] == multilinestr[0] and + line[-3] == multilinestr[0]) + if closed: + try: + value, vtype = decoder.load_value(multilinestr) + except ValueError as err: + raise TomlDecodeError(str(err), original, pos) + currentlevel[multikey] = value + multikey = None + multilinestr = "" + else: + k = len(multilinestr) - 1 + while k > -1 and multilinestr[k] == '\\': + multibackslash = not multibackslash + k -= 1 + if multibackslash: + multilinestr = multilinestr[:-1] + else: + multilinestr += "\n" + continue + if line[0] == '[': + arrayoftables = False + if len(line) == 1: + raise TomlDecodeError("Opening key group bracket on line by " + "itself.", original, pos) + if line[1] == '[': + arrayoftables = True + line = line[2:] + splitstr = ']]' + else: + line = line[1:] + splitstr = ']' + i = 1 + quotesplits = decoder._get_split_on_quotes(line) + quoted = False + for quotesplit in quotesplits: + if not quoted and splitstr in quotesplit: + break + i += quotesplit.count(splitstr) + quoted = not quoted + line = line.split(splitstr, i) + if len(line) < i + 1 or line[-1].strip() != "": + raise TomlDecodeError("Key group not on a line by itself.", + original, pos) + groups = splitstr.join(line[:-1]).split('.') + i = 0 + while i < len(groups): + groups[i] = groups[i].strip() + if len(groups[i]) > 0 and (groups[i][0] == '"' or + groups[i][0] == "'"): + groupstr = groups[i] + j = i + 1 + while not groupstr[0] == groupstr[-1]: + j += 1 + if j > len(groups) + 2: + raise TomlDecodeError("Invalid group name '" + + groupstr + "' Something " + + "went wrong.", original, pos) + groupstr = '.'.join(groups[i:j]).strip() + groups[i] = groupstr[1:-1] + groups[i + 1:j] = [] + else: + if not _groupname_re.match(groups[i]): + raise TomlDecodeError("Invalid group name '" + + groups[i] + "'. Try quoting it.", + original, pos) + i += 1 + currentlevel = retval + for i in _range(len(groups)): + group = groups[i] + if group == "": + raise TomlDecodeError("Can't have a keygroup with an empty " + "name", original, pos) + try: + currentlevel[group] + if i == len(groups) - 1: + if group in implicitgroups: + implicitgroups.remove(group) + if arrayoftables: + raise TomlDecodeError("An implicitly defined " + "table can't be an array", + original, pos) + elif arrayoftables: + currentlevel[group].append(decoder.get_empty_table() + ) + else: + raise TomlDecodeError("What? " + group + + " already exists?" + + str(currentlevel), + original, pos) + except TypeError: + currentlevel = currentlevel[-1] + if group not in currentlevel: + currentlevel[group] = decoder.get_empty_table() + if i == len(groups) - 1 and arrayoftables: + currentlevel[group] = [decoder.get_empty_table()] + except KeyError: + if i != len(groups) - 1: + implicitgroups.append(group) + currentlevel[group] = decoder.get_empty_table() + if i == len(groups) - 1 and arrayoftables: + currentlevel[group] = [decoder.get_empty_table()] + currentlevel = currentlevel[group] + if arrayoftables: + try: + currentlevel = currentlevel[-1] + except KeyError: + pass + elif line[0] == "{": + if line[-1] != "}": + raise TomlDecodeError("Line breaks are not allowed in inline" + "objects", original, pos) + try: + decoder.load_inline_object(line, currentlevel, multikey, + multibackslash) + except ValueError as err: + raise TomlDecodeError(str(err), original, pos) + elif "=" in line: + try: + ret = decoder.load_line(line, currentlevel, multikey, + multibackslash) + except ValueError as err: + raise TomlDecodeError(str(err), original, pos) + if ret is not None: + multikey, multilinestr, multibackslash = ret + return retval + + +def _load_date(val): + microsecond = 0 + tz = None + try: + if len(val) > 19: + if val[19] == '.': + if val[-1].upper() == 'Z': + subsecondval = val[20:-1] + tzval = "Z" + else: + subsecondvalandtz = val[20:] + if '+' in subsecondvalandtz: + splitpoint = subsecondvalandtz.index('+') + subsecondval = subsecondvalandtz[:splitpoint] + tzval = subsecondvalandtz[splitpoint:] + elif '-' in subsecondvalandtz: + splitpoint = subsecondvalandtz.index('-') + subsecondval = subsecondvalandtz[:splitpoint] + tzval = subsecondvalandtz[splitpoint:] + else: + tzval = None + subsecondval = subsecondvalandtz + if tzval is not None: + tz = TomlTz(tzval) + microsecond = int(int(subsecondval) * + (10 ** (6 - len(subsecondval)))) + else: + tz = TomlTz(val[19:]) + except ValueError: + tz = None + if "-" not in val[1:]: + return None + try: + if len(val) == 10: + d = datetime.date( + int(val[:4]), int(val[5:7]), + int(val[8:10])) + else: + d = datetime.datetime( + int(val[:4]), int(val[5:7]), + int(val[8:10]), int(val[11:13]), + int(val[14:16]), int(val[17:19]), microsecond, tz) + except ValueError: + return None + return d + + +def _load_unicode_escapes(v, hexbytes, prefix): + skip = False + i = len(v) - 1 + while i > -1 and v[i] == '\\': + skip = not skip + i -= 1 + for hx in hexbytes: + if skip: + skip = False + i = len(hx) - 1 + while i > -1 and hx[i] == '\\': + skip = not skip + i -= 1 + v += prefix + v += hx + continue + hxb = "" + i = 0 + hxblen = 4 + if prefix == "\\U": + hxblen = 8 + hxb = ''.join(hx[i:i + hxblen]).lower() + if hxb.strip('0123456789abcdef'): + raise ValueError("Invalid escape sequence: " + hxb) + if hxb[0] == "d" and hxb[1].strip('01234567'): + raise ValueError("Invalid escape sequence: " + hxb + + ". Only scalar unicode points are allowed.") + v += unichr(int(hxb, 16)) + v += unicode(hx[len(hxb):]) + return v + + +# Unescape TOML string values. + +# content after the \ +_escapes = ['0', 'b', 'f', 'n', 'r', 't', '"'] +# What it should be replaced by +_escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"'] +# Used for substitution +_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) + + +def _unescape(v): + """Unescape characters in a TOML string.""" + i = 0 + backslash = False + while i < len(v): + if backslash: + backslash = False + if v[i] in _escapes: + v = v[:i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1:] + elif v[i] == '\\': + v = v[:i - 1] + v[i:] + elif v[i] == 'u' or v[i] == 'U': + i += 1 + else: + raise ValueError("Reserved escape sequence used") + continue + elif v[i] == '\\': + backslash = True + i += 1 + return v + + +class InlineTableDict(object): + """Sentinel subclass of dict for inline tables.""" + + +class TomlDecoder(object): + + def __init__(self, _dict=dict): + self._dict = _dict + + def get_empty_table(self): + return self._dict() + + def get_empty_inline_table(self): + class DynamicInlineTableDict(self._dict, InlineTableDict): + """Concrete sentinel subclass for inline tables. + It is a subclass of _dict which is passed in dynamically at load + time + + It is also a subclass of InlineTableDict + """ + + return DynamicInlineTableDict() + + def load_inline_object(self, line, currentlevel, multikey=False, + multibackslash=False): + candidate_groups = line[1:-1].split(",") + groups = [] + if len(candidate_groups) == 1 and not candidate_groups[0].strip(): + candidate_groups.pop() + while len(candidate_groups) > 0: + candidate_group = candidate_groups.pop(0) + try: + _, value = candidate_group.split('=', 1) + except ValueError: + raise ValueError("Invalid inline table encountered") + value = value.strip() + if ((value[0] == value[-1] and value[0] in ('"', "'")) or ( + value[0] in '-0123456789' or + value in ('true', 'false') or + (value[0] == "[" and value[-1] == "]") or + (value[0] == '{' and value[-1] == '}'))): + groups.append(candidate_group) + elif len(candidate_groups) > 0: + candidate_groups[0] = (candidate_group + "," + + candidate_groups[0]) + else: + raise ValueError("Invalid inline table value encountered") + for group in groups: + status = self.load_line(group, currentlevel, multikey, + multibackslash) + if status is not None: + break + + def _get_split_on_quotes(self, line): + doublequotesplits = line.split('"') + quoted = False + quotesplits = [] + if len(doublequotesplits) > 1 and "'" in doublequotesplits[0]: + singlequotesplits = doublequotesplits[0].split("'") + doublequotesplits = doublequotesplits[1:] + while len(singlequotesplits) % 2 == 0 and len(doublequotesplits): + singlequotesplits[-1] += '"' + doublequotesplits[0] + doublequotesplits = doublequotesplits[1:] + if "'" in singlequotesplits[-1]: + singlequotesplits = (singlequotesplits[:-1] + + singlequotesplits[-1].split("'")) + quotesplits += singlequotesplits + for doublequotesplit in doublequotesplits: + if quoted: + quotesplits.append(doublequotesplit) + else: + quotesplits += doublequotesplit.split("'") + quoted = not quoted + return quotesplits + + def load_line(self, line, currentlevel, multikey, multibackslash): + i = 1 + quotesplits = self._get_split_on_quotes(line) + quoted = False + for quotesplit in quotesplits: + if not quoted and '=' in quotesplit: + break + i += quotesplit.count('=') + quoted = not quoted + pair = line.split('=', i) + strictly_valid = _strictly_valid_num(pair[-1]) + if _number_with_underscores.match(pair[-1]): + pair[-1] = pair[-1].replace('_', '') + while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and + pair[-1][0] != "'" and pair[-1][0] != '"' and + pair[-1][0] != '[' and pair[-1][0] != '{' and + pair[-1].strip() != 'true' and + pair[-1].strip() != 'false'): + try: + float(pair[-1]) + break + except ValueError: + pass + if _load_date(pair[-1]) is not None: + break + if TIME_RE.match(pair[-1]): + break + i += 1 + prev_val = pair[-1] + pair = line.split('=', i) + if prev_val == pair[-1]: + raise ValueError("Invalid date or number") + if strictly_valid: + strictly_valid = _strictly_valid_num(pair[-1]) + pair = ['='.join(pair[:-1]).strip(), pair[-1].strip()] + if '.' in pair[0]: + if '"' in pair[0] or "'" in pair[0]: + quotesplits = self._get_split_on_quotes(pair[0]) + quoted = False + levels = [] + for quotesplit in quotesplits: + if quoted: + levels.append(quotesplit) + else: + levels += [level.strip() for level in + quotesplit.split('.')] + quoted = not quoted + else: + levels = pair[0].split('.') + while levels[-1] == "": + levels = levels[:-1] + for level in levels[:-1]: + if level == "": + continue + if level not in currentlevel: + currentlevel[level] = self.get_empty_table() + currentlevel = currentlevel[level] + pair[0] = levels[-1].strip() + elif (pair[0][0] == '"' or pair[0][0] == "'") and \ + (pair[0][-1] == pair[0][0]): + pair[0] = _unescape(pair[0][1:-1]) + k, koffset = self._load_line_multiline_str(pair[1]) + if k > -1: + while k > -1 and pair[1][k + koffset] == '\\': + multibackslash = not multibackslash + k -= 1 + if multibackslash: + multilinestr = pair[1][:-1] + else: + multilinestr = pair[1] + "\n" + multikey = pair[0] + else: + value, vtype = self.load_value(pair[1], strictly_valid) + try: + currentlevel[pair[0]] + raise ValueError("Duplicate keys!") + except TypeError: + raise ValueError("Duplicate keys!") + except KeyError: + if multikey: + return multikey, multilinestr, multibackslash + else: + currentlevel[pair[0]] = value + + def _load_line_multiline_str(self, p): + poffset = 0 + if len(p) < 3: + return -1, poffset + if p[0] == '[' and (p.strip()[-1] != ']' and + self._load_array_isstrarray(p)): + newp = p[1:].strip().split(',') + while len(newp) > 1 and newp[-1][0] != '"' and newp[-1][0] != "'": + newp = newp[:-2] + [newp[-2] + ',' + newp[-1]] + newp = newp[-1] + poffset = len(p) - len(newp) + p = newp + if p[0] != '"' and p[0] != "'": + return -1, poffset + if p[1] != p[0] or p[2] != p[0]: + return -1, poffset + if len(p) > 5 and p[-1] == p[0] and p[-2] == p[0] and p[-3] == p[0]: + return -1, poffset + return len(p) - 1, poffset + + def load_value(self, v, strictly_valid=True): + if not v: + raise ValueError("Empty value is invalid") + if v == 'true': + return (True, "bool") + elif v == 'false': + return (False, "bool") + elif v[0] == '"' or v[0] == "'": + quotechar = v[0] + testv = v[1:].split(quotechar) + triplequote = False + triplequotecount = 0 + if len(testv) > 1 and testv[0] == '' and testv[1] == '': + testv = testv[2:] + triplequote = True + closed = False + for tv in testv: + if tv == '': + if triplequote: + triplequotecount += 1 + else: + closed = True + else: + oddbackslash = False + try: + i = -1 + j = tv[i] + while j == '\\': + oddbackslash = not oddbackslash + i -= 1 + j = tv[i] + except IndexError: + pass + if not oddbackslash: + if closed: + raise ValueError("Found tokens after a closed " + + "string. Invalid TOML.") + else: + if not triplequote or triplequotecount > 1: + closed = True + else: + triplequotecount = 0 + if quotechar == '"': + escapeseqs = v.split('\\')[1:] + backslash = False + for i in escapeseqs: + if i == '': + backslash = not backslash + else: + if i[0] not in _escapes and (i[0] != 'u' and + i[0] != 'U' and + not backslash): + raise ValueError("Reserved escape sequence used") + if backslash: + backslash = False + for prefix in ["\\u", "\\U"]: + if prefix in v: + hexbytes = v.split(prefix) + v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], + prefix) + v = _unescape(v) + if len(v) > 1 and v[1] == quotechar and (len(v) < 3 or + v[1] == v[2]): + v = v[2:-2] + return (v[1:-1], "str") + elif v[0] == '[': + return (self.load_array(v), "array") + elif v[0] == '{': + inline_object = self.get_empty_inline_table() + self.load_inline_object(v, inline_object) + return (inline_object, "inline_object") + elif TIME_RE.match(v): + h, m, s, _, ms = TIME_RE.match(v).groups() + time = datetime.time(int(h), int(m), int(s), int(ms) if ms else 0) + return (time, "time") + else: + parsed_date = _load_date(v) + if parsed_date is not None: + return (parsed_date, "date") + if not strictly_valid: + raise ValueError("Weirdness with leading zeroes or " + "underscores in your number.") + itype = "int" + neg = False + if v[0] == '-': + neg = True + v = v[1:] + elif v[0] == '+': + v = v[1:] + v = v.replace('_', '') + lowerv = v.lower() + if '.' in v or ('x' not in v and ('e' in v or 'E' in v)): + if '.' in v and v.split('.', 1)[1] == '': + raise ValueError("This float is missing digits after " + "the point") + if v[0] not in '0123456789': + raise ValueError("This float doesn't have a leading " + "digit") + v = float(v) + itype = "float" + elif len(lowerv) == 3 and (lowerv == 'inf' or lowerv == 'nan'): + v = float(v) + itype = "float" + if itype == "int": + v = int(v, 0) + if neg: + return (0 - v, itype) + return (v, itype) + + def bounded_string(self, s): + if len(s) == 0: + return True + if s[-1] != s[0]: + return False + i = -2 + backslash = False + while len(s) + i > 0: + if s[i] == "\\": + backslash = not backslash + i -= 1 + else: + break + return not backslash + + def _load_array_isstrarray(self, a): + a = a[1:-1].strip() + if a != '' and (a[0] == '"' or a[0] == "'"): + return True + return False + + def load_array(self, a): + atype = None + retval = [] + a = a.strip() + if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip(): + strarray = self._load_array_isstrarray(a) + if not a[1:-1].strip().startswith('{'): + a = a[1:-1].split(',') + else: + # a is an inline object, we must find the matching parenthesis + # to define groups + new_a = [] + start_group_index = 1 + end_group_index = 2 + open_bracket_count = 1 if a[start_group_index] == '{' else 0 + in_str = False + while end_group_index < len(a[1:]): + if a[end_group_index] == '"' or a[end_group_index] == "'": + if in_str: + backslash_index = end_group_index - 1 + while (backslash_index > -1 and + a[backslash_index] == '\\'): + in_str = not in_str + backslash_index -= 1 + in_str = not in_str + if not in_str and a[end_group_index] == '{': + open_bracket_count += 1 + if in_str or a[end_group_index] != '}': + end_group_index += 1 + continue + elif a[end_group_index] == '}' and open_bracket_count > 1: + open_bracket_count -= 1 + end_group_index += 1 + continue + + # Increase end_group_index by 1 to get the closing bracket + end_group_index += 1 + + new_a.append(a[start_group_index:end_group_index]) + + # The next start index is at least after the closing + # bracket, a closing bracket can be followed by a comma + # since we are in an array. + start_group_index = end_group_index + 1 + while (start_group_index < len(a[1:]) and + a[start_group_index] != '{'): + start_group_index += 1 + end_group_index = start_group_index + 1 + a = new_a + b = 0 + if strarray: + while b < len(a) - 1: + ab = a[b].strip() + while (not self.bounded_string(ab) or + (len(ab) > 2 and + ab[0] == ab[1] == ab[2] and + ab[-2] != ab[0] and + ab[-3] != ab[0])): + a[b] = a[b] + ',' + a[b + 1] + ab = a[b].strip() + if b < len(a) - 2: + a = a[:b + 1] + a[b + 2:] + else: + a = a[:b + 1] + b += 1 + else: + al = list(a[1:-1]) + a = [] + openarr = 0 + j = 0 + for i in _range(len(al)): + if al[i] == '[': + openarr += 1 + elif al[i] == ']': + openarr -= 1 + elif al[i] == ',' and not openarr: + a.append(''.join(al[j:i])) + j = i + 1 + a.append(''.join(al[j:])) + for i in _range(len(a)): + a[i] = a[i].strip() + if a[i] != '': + nval, ntype = self.load_value(a[i]) + if atype: + if ntype != atype: + raise ValueError("Not a homogeneous array") + else: + atype = ntype + retval.append(nval) + return retval + + def preserve_comment(self, line_no, key, comment, beginline): + pass + + def embed_comments(self, idx, currentlevel): + pass + + +class TomlPreserveCommentDecoder(TomlDecoder): + + def __init__(self, _dict=dict): + self.saved_comments = {} + super(TomlPreserveCommentDecoder, self).__init__(_dict) + + def preserve_comment(self, line_no, key, comment, beginline): + self.saved_comments[line_no] = (key, comment, beginline) + + def embed_comments(self, idx, currentlevel): + if idx not in self.saved_comments: + return + + key, comment, beginline = self.saved_comments[idx] + currentlevel[key] = CommentValue(currentlevel[key], comment, beginline, + self._dict) diff --git a/utils/toml/encoder.py b/utils/toml/encoder.py new file mode 100644 index 0000000000..d9e557ed95 --- /dev/null +++ b/utils/toml/encoder.py @@ -0,0 +1,304 @@ +import datetime +import re +import sys +from decimal import Decimal + +from toml.decoder import InlineTableDict + +if sys.version_info >= (3,): + unicode = str + + +def dump(o, f, encoder=None): + """Writes out dict as toml to a file + + Args: + o: Object to dump into toml + f: File descriptor where the toml should be stored + encoder: The ``TomlEncoder`` to use for constructing the output string + + Returns: + String containing the toml corresponding to dictionary + + Raises: + TypeError: When anything other than file descriptor is passed + """ + + if not f.write: + raise TypeError("You can only dump an object to a file descriptor") + d = dumps(o, encoder=encoder) + f.write(d) + return d + + +def dumps(o, encoder=None): + """Stringifies input dict as toml + + Args: + o: Object to dump into toml + encoder: The ``TomlEncoder`` to use for constructing the output string + + Returns: + String containing the toml corresponding to dict + + Examples: + ```python + >>> import toml + >>> output = { + ... 'a': "I'm a string", + ... 'b': ["I'm", "a", "list"], + ... 'c': 2400 + ... } + >>> toml.dumps(output) + 'a = "I\'m a string"\nb = [ "I\'m", "a", "list",]\nc = 2400\n' + ``` + """ + + retval = "" + if encoder is None: + encoder = TomlEncoder(o.__class__) + addtoretval, sections = encoder.dump_sections(o, "") + retval += addtoretval + outer_objs = [id(o)] + while sections: + section_ids = [id(section) for section in sections] + for outer_obj in outer_objs: + if outer_obj in section_ids: + raise ValueError("Circular reference detected") + outer_objs += section_ids + newsections = encoder.get_empty_table() + for section in sections: + addtoretval, addtosections = encoder.dump_sections( + sections[section], section) + + if addtoretval or (not addtoretval and not addtosections): + if retval and retval[-2:] != "\n\n": + retval += "\n" + retval += "[" + section + "]\n" + if addtoretval: + retval += addtoretval + for s in addtosections: + newsections[section + "." + s] = addtosections[s] + sections = newsections + return retval + + +def _dump_str(v): + if sys.version_info < (3,) and hasattr(v, 'decode') and isinstance(v, str): + v = v.decode('utf-8') + v = "%r" % v + if v[0] == 'u': + v = v[1:] + singlequote = v.startswith("'") + if singlequote or v.startswith('"'): + v = v[1:-1] + if singlequote: + v = v.replace("\\'", "'") + v = v.replace('"', '\\"') + v = v.split("\\x") + while len(v) > 1: + i = -1 + if not v[0]: + v = v[1:] + v[0] = v[0].replace("\\\\", "\\") + # No, I don't know why != works and == breaks + joinx = v[0][i] != "\\" + while v[0][:i] and v[0][i] == "\\": + joinx = not joinx + i -= 1 + if joinx: + joiner = "x" + else: + joiner = "u00" + v = [v[0] + joiner + v[1]] + v[2:] + return unicode('"' + v[0] + '"') + + +def _dump_float(v): + return "{}".format(v).replace("e+0", "e+").replace("e-0", "e-") + + +def _dump_time(v): + utcoffset = v.utcoffset() + if utcoffset is None: + return v.isoformat() + # The TOML norm specifies that it's local time thus we drop the offset + return v.isoformat()[:-6] + + +class TomlEncoder(object): + + def __init__(self, _dict=dict, preserve=False): + self._dict = _dict + self.preserve = preserve + self.dump_funcs = { + str: _dump_str, + unicode: _dump_str, + list: self.dump_list, + bool: lambda v: unicode(v).lower(), + int: lambda v: v, + float: _dump_float, + Decimal: _dump_float, + datetime.datetime: lambda v: v.isoformat().replace('+00:00', 'Z'), + datetime.time: _dump_time, + datetime.date: lambda v: v.isoformat() + } + + def get_empty_table(self): + return self._dict() + + def dump_list(self, v): + retval = "[" + for u in v: + retval += " " + unicode(self.dump_value(u)) + "," + retval += "]" + return retval + + def dump_inline_table(self, section): + """Preserve inline table in its compact syntax instead of expanding + into subsection. + + https://github.com/toml-lang/toml#user-content-inline-table + """ + retval = "" + if isinstance(section, dict): + val_list = [] + for k, v in section.items(): + val = self.dump_inline_table(v) + val_list.append(k + " = " + val) + retval += "{ " + ", ".join(val_list) + " }\n" + return retval + else: + return unicode(self.dump_value(section)) + + def dump_value(self, v): + # Lookup function corresponding to v's type + dump_fn = self.dump_funcs.get(type(v)) + if dump_fn is None and hasattr(v, '__iter__'): + dump_fn = self.dump_funcs[list] + # Evaluate function (if it exists) else return v + return dump_fn(v) if dump_fn is not None else self.dump_funcs[str](v) + + def dump_sections(self, o, sup): + retstr = "" + if sup != "" and sup[-1] != ".": + sup += '.' + retdict = self._dict() + arraystr = "" + for section in o: + section = unicode(section) + qsection = section + if not re.match(r'^[A-Za-z0-9_-]+$', section): + qsection = _dump_str(section) + if not isinstance(o[section], dict): + arrayoftables = False + if isinstance(o[section], list): + for a in o[section]: + if isinstance(a, dict): + arrayoftables = True + if arrayoftables: + for a in o[section]: + arraytabstr = "\n" + arraystr += "[[" + sup + qsection + "]]\n" + s, d = self.dump_sections(a, sup + qsection) + if s: + if s[0] == "[": + arraytabstr += s + else: + arraystr += s + while d: + newd = self._dict() + for dsec in d: + s1, d1 = self.dump_sections(d[dsec], sup + + qsection + "." + + dsec) + if s1: + arraytabstr += ("[" + sup + qsection + + "." + dsec + "]\n") + arraytabstr += s1 + for s1 in d1: + newd[dsec + "." + s1] = d1[s1] + d = newd + arraystr += arraytabstr + else: + if o[section] is not None: + retstr += (qsection + " = " + + unicode(self.dump_value(o[section])) + '\n') + elif self.preserve and isinstance(o[section], InlineTableDict): + retstr += (qsection + " = " + + self.dump_inline_table(o[section])) + else: + retdict[qsection] = o[section] + retstr += arraystr + return (retstr, retdict) + + +class TomlPreserveInlineDictEncoder(TomlEncoder): + + def __init__(self, _dict=dict): + super(TomlPreserveInlineDictEncoder, self).__init__(_dict, True) + + +class TomlArraySeparatorEncoder(TomlEncoder): + + def __init__(self, _dict=dict, preserve=False, separator=","): + super(TomlArraySeparatorEncoder, self).__init__(_dict, preserve) + if separator.strip() == "": + separator = "," + separator + elif separator.strip(' \t\n\r,'): + raise ValueError("Invalid separator for arrays") + self.separator = separator + + def dump_list(self, v): + t = [] + retval = "[" + for u in v: + t.append(self.dump_value(u)) + while t != []: + s = [] + for u in t: + if isinstance(u, list): + for r in u: + s.append(r) + else: + retval += " " + unicode(u) + self.separator + t = s + retval += "]" + return retval + + +class TomlNumpyEncoder(TomlEncoder): + + def __init__(self, _dict=dict, preserve=False): + import numpy as np + super(TomlNumpyEncoder, self).__init__(_dict, preserve) + self.dump_funcs[np.float16] = _dump_float + self.dump_funcs[np.float32] = _dump_float + self.dump_funcs[np.float64] = _dump_float + self.dump_funcs[np.int16] = self._dump_int + self.dump_funcs[np.int32] = self._dump_int + self.dump_funcs[np.int64] = self._dump_int + + def _dump_int(self, v): + return "{}".format(int(v)) + + +class TomlPreserveCommentEncoder(TomlEncoder): + + def __init__(self, _dict=dict, preserve=False): + from toml.decoder import CommentValue + super(TomlPreserveCommentEncoder, self).__init__(_dict, preserve) + self.dump_funcs[CommentValue] = lambda v: v.dump(self.dump_value) + + +class TomlPathlibEncoder(TomlEncoder): + + def _dump_pathlib_path(self, v): + return _dump_str(str(v)) + + def dump_value(self, v): + if (3, 4) <= sys.version_info: + import pathlib + if isinstance(v, pathlib.PurePath): + v = str(v) + return super(TomlPathlibEncoder, self).dump_value(v) diff --git a/utils/toml/ordered.py b/utils/toml/ordered.py new file mode 100644 index 0000000000..9c20c41a1b --- /dev/null +++ b/utils/toml/ordered.py @@ -0,0 +1,15 @@ +from collections import OrderedDict +from toml import TomlEncoder +from toml import TomlDecoder + + +class TomlOrderedDecoder(TomlDecoder): + + def __init__(self): + super(self.__class__, self).__init__(_dict=OrderedDict) + + +class TomlOrderedEncoder(TomlEncoder): + + def __init__(self): + super(self.__class__, self).__init__(_dict=OrderedDict) diff --git a/utils/toml/tz.py b/utils/toml/tz.py new file mode 100644 index 0000000000..93c3c8ad26 --- /dev/null +++ b/utils/toml/tz.py @@ -0,0 +1,21 @@ +from datetime import tzinfo, timedelta + + +class TomlTz(tzinfo): + def __init__(self, toml_offset): + if toml_offset == "Z": + self._raw_offset = "+00:00" + else: + self._raw_offset = toml_offset + self._sign = -1 if self._raw_offset[0] == '-' else 1 + self._hours = int(self._raw_offset[1:3]) + self._minutes = int(self._raw_offset[4:6]) + + def tzname(self, dt): + return "UTC" + self._raw_offset + + def utcoffset(self, dt): + return self._sign * timedelta(hours=self._hours, minutes=self._minutes) + + def dst(self, dt): + return timedelta(0) From f8892362db598ac8bf189d67246ea96a89c9622d Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 13:47:41 +0100 Subject: [PATCH 25/91] Add test for pip installation. --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82ab4a0fff..2ffd7ecd06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,10 +4,30 @@ on: [push, pull_request] jobs: - conda: + install-with-pip: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Pip install + env: + PYTHON_VERSION: 3.7 + run: | + python -m pip install -e . + python -m pip freeze + + install-with-conda: + + runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v1 From 79c962bdc4601e135c27a3b1b877459b01f9ec07 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 13:47:56 +0100 Subject: [PATCH 26/91] Extend test for installation with conda. --- .github/workflows/conda.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/conda.sh b/.github/workflows/conda.sh index 5ad1b6628b..6837e7d9cd 100755 --- a/.github/workflows/conda.sh +++ b/.github/workflows/conda.sh @@ -10,3 +10,8 @@ conda config --set always_yes yes --set changeps1 no conda update -q conda conda info -a conda env create -f environment.yml -n test-environment + +# Test installation +conda activate test-environment +python -m pip install --no-deps -e . +python -c 'import aiida' # import check From 5063076b852f86a8af1032db0bc9462cfca9965b Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 13:49:20 +0100 Subject: [PATCH 27/91] Add CI step to show test environment. --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ffd7ecd06..1d06616757 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,6 +168,12 @@ jobs: file: ./coverage.xml fail_ci_if_error: false + - name: Show test environment + env: + AIIDA_TEST_BACKEND: ${{ matrix.backend }} + run: + python -m pip freeze + verdi: runs-on: ubuntu-latest From 1f13c4cc29ee11c3256a631545f20731d5fce20a Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 13:57:22 +0100 Subject: [PATCH 28/91] Use 'source activate' instead of 'conda activate'. --- .github/workflows/conda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.sh b/.github/workflows/conda.sh index 6837e7d9cd..4134ca09c1 100755 --- a/.github/workflows/conda.sh +++ b/.github/workflows/conda.sh @@ -12,6 +12,6 @@ conda info -a conda env create -f environment.yml -n test-environment # Test installation -conda activate test-environment +source activate test-environment python -m pip install --no-deps -e . python -c 'import aiida' # import check From 240fcd025abd43b3ffa835c03ca10d21ddaf6f38 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 14:13:52 +0100 Subject: [PATCH 29/91] Refactor pip and conda install tests. --- .github/workflows/ci.yml | 25 ++++++++++++++----- .../workflows/{conda.sh => install-conda.sh} | 6 ----- 2 files changed, 19 insertions(+), 12 deletions(-) rename .github/workflows/{conda.sh => install-conda.sh} (61%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d06616757..3ee35d994a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,12 +18,14 @@ jobs: python-version: 3.7 - name: Pip install - env: - PYTHON_VERSION: 3.7 run: | python -m pip install -e . python -m pip freeze + - name: Test package import + run: + python -c "import aiida" + install-with-conda: runs-on: ubuntu-latest @@ -37,11 +39,22 @@ jobs: with: python-version: 3.7 - - name: Conda install - env: - PYTHON_VERSION: 3.7 + - name: Install conda + run: + .github/workflows/install-conda.sh + + - name: Create conda environment + run: | + conda env create -f environment.yml -n test-environment + source activate test-environment + + - name: Install AiiDA in conda environment + run: + python -m pip install --no-deps -e . + + - name: Test package import run: - .github/workflows/conda.sh + python -c "import aiida" docs: diff --git a/.github/workflows/conda.sh b/.github/workflows/install-conda.sh similarity index 61% rename from .github/workflows/conda.sh rename to .github/workflows/install-conda.sh index 4134ca09c1..aa6c93cb5a 100755 --- a/.github/workflows/conda.sh +++ b/.github/workflows/install-conda.sh @@ -9,9 +9,3 @@ conda config --set always_yes yes --set changeps1 no conda update -q conda conda info -a -conda env create -f environment.yml -n test-environment - -# Test installation -source activate test-environment -python -m pip install --no-deps -e . -python -c 'import aiida' # import check From b6a3c1a9f4b39599c74513fd8bcc9c71654a5bc4 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 14:37:09 +0100 Subject: [PATCH 30/91] Only run tests if installation succeeds. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ee35d994a..71fb8f1b24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,7 @@ jobs: tests: + needs: [install-with-pip, install-with-conda] runs-on: ubuntu-latest timeout-minutes: 30 From da606010d63a0ab94d1e62a7e220192893dff755 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 14:37:21 +0100 Subject: [PATCH 31/91] Attempt to fix issue with conda env activation. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71fb8f1b24..320d66fac6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: Create conda environment run: | conda env create -f environment.yml -n test-environment - source activate test-environment + conda activate test-environment - name: Install AiiDA in conda environment run: From 7faa02632c2a4a692f46f21c14d5ed151c4cbfbb Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 15:00:27 +0100 Subject: [PATCH 32/91] Use setup-conda action for conda install. --- .github/workflows/ci.yml | 25 +++++++++++++++---------- .github/workflows/install-conda.sh | 11 ----------- 2 files changed, 15 insertions(+), 21 deletions(-) delete mode 100755 .github/workflows/install-conda.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 320d66fac6..54ec78b089 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,24 +29,29 @@ jobs: install-with-conda: runs-on: ubuntu-latest - timeout-minutes: 5 + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + name: install-with-conda-py-${{ matrix.python-version }} + + timeout-minutes: 10 steps: - uses: actions/checkout@v1 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - name: Setup Conda + uses: s-weigand/setup-conda@v1 with: - python-version: 3.7 - - - name: Install conda - run: - .github/workflows/install-conda.sh + update-conda: true + python-version: ${{ matrix.python-version }} + - run: conda --version + - run: which python - name: Create conda environment run: | - conda env create -f environment.yml -n test-environment - conda activate test-environment + conda env create -f environment.yml -n test-environment-py-${{ matrix.python-version }} + conda activate test-environment-py-${{ matrix.python-version }} - name: Install AiiDA in conda environment run: diff --git a/.github/workflows/install-conda.sh b/.github/workflows/install-conda.sh deleted file mode 100755 index aa6c93cb5a..0000000000 --- a/.github/workflows/install-conda.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -ev - -wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; -bash miniconda.sh -b -p $HOME/miniconda -export PATH="$HOME/miniconda/bin:$PATH" -hash -r -conda config --set always_yes yes --set changeps1 no - -conda update -q conda -conda info -a From 4099efadcdee2e73532d288deccf50a4d567c4ae Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 15:10:13 +0100 Subject: [PATCH 33/91] Only test on Python 3.7 for conda install. Because installing with different python versions from environment.yml is essentially not supported. --- .github/workflows/ci.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54ec78b089..a5ae1ea236 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,13 +29,9 @@ jobs: install-with-conda: runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.6, 3.7, 3.8] + name: install-with-conda - name: install-with-conda-py-${{ matrix.python-version }} - - timeout-minutes: 10 + timeout-minutes: 5 steps: - uses: actions/checkout@v1 @@ -44,17 +40,15 @@ jobs: uses: s-weigand/setup-conda@v1 with: update-conda: true - python-version: ${{ matrix.python-version }} + python-version: 3.7 - run: conda --version + - run: python --version - run: which python - name: Create conda environment run: | - conda env create -f environment.yml -n test-environment-py-${{ matrix.python-version }} - conda activate test-environment-py-${{ matrix.python-version }} - - - name: Install AiiDA in conda environment - run: + conda env create -f environment.yml -n test-environment + source activate test-environment python -m pip install --no-deps -e . - name: Test package import From 99f8a4d22545b8c405dac0f3a2443c8ae0c9415b Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 15:17:59 +0100 Subject: [PATCH 34/91] Fix error in install-with-conda CI step. --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5ae1ea236..dc6567a79d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,8 @@ jobs: python -m pip install --no-deps -e . - name: Test package import - run: + run: | + source activate test-environment python -c "import aiida" docs: From 43261ab2ea4747d0f1cfca924b970150fb74c284 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 15:21:13 +0100 Subject: [PATCH 35/91] Revert "Vendor toml package in utils directory." This reverts commit 6145e1a0c9f0e412f18a931197b3497d55d51780. --- .pre-commit-config.yaml | 3 - utils/toml/__init__.py | 25 - utils/toml/decoder.py | 1049 --------------------------------------- utils/toml/encoder.py | 304 ------------ utils/toml/ordered.py | 15 - utils/toml/tz.py | 21 - 6 files changed, 1417 deletions(-) delete mode 100644 utils/toml/__init__.py delete mode 100644 utils/toml/decoder.py delete mode 100644 utils/toml/encoder.py delete mode 100644 utils/toml/ordered.py delete mode 100644 utils/toml/tz.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c0ccb519a3..39bd257146 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,9 +2,7 @@ rev: v2.2.3 hooks: - id: trailing-whitespace - exclude: utils/toml/* - id: double-quote-string-fixer - exclude: utils/toml/* - repo: local hooks: @@ -88,7 +86,6 @@ aiida/tools/dbimporters/plugins/pcod.py| docs/.*| examples/.*| - utils/toml/.*| tests/engine/test_work_chain.py| tests/schedulers/test_direct.py| tests/schedulers/test_lsf.py| diff --git a/utils/toml/__init__.py b/utils/toml/__init__.py deleted file mode 100644 index 7e13a0c36f..0000000000 --- a/utils/toml/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Python module which parses and emits TOML. - -Released under the MIT license. -""" - -from toml import encoder -from toml import decoder - -__version__ = "0.10.1" -_spec_ = "0.5.0" - -load = decoder.load -loads = decoder.loads -TomlDecoder = decoder.TomlDecoder -TomlDecodeError = decoder.TomlDecodeError -TomlPreserveCommentDecoder = decoder.TomlPreserveCommentDecoder - -dump = encoder.dump -dumps = encoder.dumps -TomlEncoder = encoder.TomlEncoder -TomlArraySeparatorEncoder = encoder.TomlArraySeparatorEncoder -TomlPreserveInlineDictEncoder = encoder.TomlPreserveInlineDictEncoder -TomlNumpyEncoder = encoder.TomlNumpyEncoder -TomlPreserveCommentEncoder = encoder.TomlPreserveCommentEncoder -TomlPathlibEncoder = encoder.TomlPathlibEncoder diff --git a/utils/toml/decoder.py b/utils/toml/decoder.py deleted file mode 100644 index a24c04a0bf..0000000000 --- a/utils/toml/decoder.py +++ /dev/null @@ -1,1049 +0,0 @@ -import datetime -import io -from os import linesep -import re -import sys - -from toml.tz import TomlTz - -if sys.version_info < (3,): - _range = xrange # noqa: F821 -else: - unicode = str - _range = range - basestring = str - unichr = chr - - -def _detect_pathlib_path(p): - if (3, 4) <= sys.version_info: - import pathlib - if isinstance(p, pathlib.PurePath): - return True - return False - - -def _ispath(p): - if isinstance(p, basestring): - return True - return _detect_pathlib_path(p) - - -def _getpath(p): - if (3, 6) <= sys.version_info: - import os - return os.fspath(p) - if _detect_pathlib_path(p): - return str(p) - return p - - -try: - FNFError = FileNotFoundError -except NameError: - FNFError = IOError - - -TIME_RE = re.compile(r"([0-9]{2}):([0-9]{2}):([0-9]{2})(\.([0-9]{3,6}))?") - - -class TomlDecodeError(ValueError): - """Base toml Exception / Error.""" - - def __init__(self, msg, doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - colno = pos - doc.rfind('\n', 0, pos) - emsg = '{} (line {} column {} char {})'.format(msg, lineno, colno, pos) - ValueError.__init__(self, emsg) - self.msg = msg - self.doc = doc - self.pos = pos - self.lineno = lineno - self.colno = colno - - -# Matches a TOML number, which allows underscores for readability -_number_with_underscores = re.compile('([0-9])(_([0-9]))*') - - -class CommentValue(object): - def __init__(self, val, comment, beginline, _dict): - self.val = val - separator = "\n" if beginline else " " - self.comment = separator + comment - self._dict = _dict - - def __getitem__(self, key): - return self.val[key] - - def __setitem__(self, key, value): - self.val[key] = value - - def dump(self, dump_value_func): - retstr = dump_value_func(self.val) - if isinstance(self.val, self._dict): - return self.comment + "\n" + unicode(retstr) - else: - return unicode(retstr) + self.comment - - -def _strictly_valid_num(n): - n = n.strip() - if not n: - return False - if n[0] == '_': - return False - if n[-1] == '_': - return False - if "_." in n or "._" in n: - return False - if len(n) == 1: - return True - if n[0] == '0' and n[1] not in ['.', 'o', 'b', 'x']: - return False - if n[0] == '+' or n[0] == '-': - n = n[1:] - if len(n) > 1 and n[0] == '0' and n[1] != '.': - return False - if '__' in n: - return False - return True - - -def load(f, _dict=dict, decoder=None): - """Parses named file or files as toml and returns a dictionary - - Args: - f: Path to the file to open, array of files to read into single dict - or a file descriptor - _dict: (optional) Specifies the class of the returned toml dictionary - decoder: The decoder to use - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError -- When f is invalid type - TomlDecodeError: Error while decoding toml - IOError / FileNotFoundError -- When an array with no valid (existing) - (Python 2 / Python 3) file paths is passed - """ - - if _ispath(f): - with io.open(_getpath(f), encoding='utf-8') as ffile: - return loads(ffile.read(), _dict, decoder) - elif isinstance(f, list): - from os import path as op - from warnings import warn - if not [path for path in f if op.exists(path)]: - error_msg = "Load expects a list to contain filenames only." - error_msg += linesep - error_msg += ("The list needs to contain the path of at least one " - "existing file.") - raise FNFError(error_msg) - if decoder is None: - decoder = TomlDecoder(_dict) - d = decoder.get_empty_table() - for l in f: - if op.exists(l): - d.update(load(l, _dict, decoder)) - else: - warn("Non-existent filename in list with at least one valid " - "filename") - return d - else: - try: - return loads(f.read(), _dict, decoder) - except AttributeError: - raise TypeError("You can only load a file descriptor, filename or " - "list") - - -_groupname_re = re.compile(r'^[A-Za-z0-9_-]+$') - - -def loads(s, _dict=dict, decoder=None): - """Parses string as toml - - Args: - s: String to be parsed - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError: When a non-string is passed - TomlDecodeError: Error while decoding toml - """ - - implicitgroups = [] - if decoder is None: - decoder = TomlDecoder(_dict) - retval = decoder.get_empty_table() - currentlevel = retval - if not isinstance(s, basestring): - raise TypeError("Expecting something like a string") - - if not isinstance(s, unicode): - s = s.decode('utf8') - - original = s - sl = list(s) - openarr = 0 - openstring = False - openstrchar = "" - multilinestr = False - arrayoftables = False - beginline = True - keygroup = False - dottedkey = False - keyname = 0 - key = '' - prev_key = '' - line_no = 1 - - for i, item in enumerate(sl): - if item == '\r' and sl[i + 1] == '\n': - sl[i] = ' ' - continue - if keyname: - key += item - if item == '\n': - raise TomlDecodeError("Key name found without value." - " Reached end of line.", original, i) - if openstring: - if item == openstrchar: - oddbackslash = False - k = 1 - while i >= k and sl[i - k] == '\\': - oddbackslash = not oddbackslash - k += 1 - if not oddbackslash: - keyname = 2 - openstring = False - openstrchar = "" - continue - elif keyname == 1: - if item.isspace(): - keyname = 2 - continue - elif item == '.': - dottedkey = True - continue - elif item.isalnum() or item == '_' or item == '-': - continue - elif (dottedkey and sl[i - 1] == '.' and - (item == '"' or item == "'")): - openstring = True - openstrchar = item - continue - elif keyname == 2: - if item.isspace(): - if dottedkey: - nextitem = sl[i + 1] - if not nextitem.isspace() and nextitem != '.': - keyname = 1 - continue - if item == '.': - dottedkey = True - nextitem = sl[i + 1] - if not nextitem.isspace() and nextitem != '.': - keyname = 1 - continue - if item == '=': - keyname = 0 - prev_key = key[:-1].rstrip() - key = '' - dottedkey = False - else: - raise TomlDecodeError("Found invalid character in key name: '" + - item + "'. Try quoting the key name.", - original, i) - if item == "'" and openstrchar != '"': - k = 1 - try: - while sl[i - k] == "'": - k += 1 - if k == 3: - break - except IndexError: - pass - if k == 3: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = "'" - else: - openstrchar = "" - if item == '"' and openstrchar != "'": - oddbackslash = False - k = 1 - tripquote = False - try: - while sl[i - k] == '"': - k += 1 - if k == 3: - tripquote = True - break - if k == 1 or (k == 3 and tripquote): - while sl[i - k] == '\\': - oddbackslash = not oddbackslash - k += 1 - except IndexError: - pass - if not oddbackslash: - if tripquote: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = '"' - else: - openstrchar = "" - if item == '#' and (not openstring and not keygroup and - not arrayoftables): - j = i - comment = "" - try: - while sl[j] != '\n': - comment += s[j] - sl[j] = ' ' - j += 1 - except IndexError: - break - if not openarr: - decoder.preserve_comment(line_no, prev_key, comment, beginline) - if item == '[' and (not openstring and not keygroup and - not arrayoftables): - if beginline: - if len(sl) > i + 1 and sl[i + 1] == '[': - arrayoftables = True - else: - keygroup = True - else: - openarr += 1 - if item == ']' and not openstring: - if keygroup: - keygroup = False - elif arrayoftables: - if sl[i - 1] == ']': - arrayoftables = False - else: - openarr -= 1 - if item == '\n': - if openstring or multilinestr: - if not multilinestr: - raise TomlDecodeError("Unbalanced quotes", original, i) - if ((sl[i - 1] == "'" or sl[i - 1] == '"') and ( - sl[i - 2] == sl[i - 1])): - sl[i] = sl[i - 1] - if sl[i - 3] == sl[i - 1]: - sl[i - 3] = ' ' - elif openarr: - sl[i] = ' ' - else: - beginline = True - line_no += 1 - elif beginline and sl[i] != ' ' and sl[i] != '\t': - beginline = False - if not keygroup and not arrayoftables: - if sl[i] == '=': - raise TomlDecodeError("Found empty keyname. ", original, i) - keyname = 1 - key += item - if keyname: - raise TomlDecodeError("Key name found without value." - " Reached end of file.", original, len(s)) - s = ''.join(sl) - s = s.split('\n') - multikey = None - multilinestr = "" - multibackslash = False - pos = 0 - for idx, line in enumerate(s): - if idx > 0: - pos += len(s[idx - 1]) + 1 - - decoder.embed_comments(idx, currentlevel) - - if not multilinestr or multibackslash or '\n' not in multilinestr: - line = line.strip() - if line == "" and (not multikey or multibackslash): - continue - if multikey: - if multibackslash: - multilinestr += line - else: - multilinestr += line - multibackslash = False - closed = False - if multilinestr[0] == '[': - closed = line[-1] == ']' - elif len(line) > 2: - closed = (line[-1] == multilinestr[0] and - line[-2] == multilinestr[0] and - line[-3] == multilinestr[0]) - if closed: - try: - value, vtype = decoder.load_value(multilinestr) - except ValueError as err: - raise TomlDecodeError(str(err), original, pos) - currentlevel[multikey] = value - multikey = None - multilinestr = "" - else: - k = len(multilinestr) - 1 - while k > -1 and multilinestr[k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = multilinestr[:-1] - else: - multilinestr += "\n" - continue - if line[0] == '[': - arrayoftables = False - if len(line) == 1: - raise TomlDecodeError("Opening key group bracket on line by " - "itself.", original, pos) - if line[1] == '[': - arrayoftables = True - line = line[2:] - splitstr = ']]' - else: - line = line[1:] - splitstr = ']' - i = 1 - quotesplits = decoder._get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and splitstr in quotesplit: - break - i += quotesplit.count(splitstr) - quoted = not quoted - line = line.split(splitstr, i) - if len(line) < i + 1 or line[-1].strip() != "": - raise TomlDecodeError("Key group not on a line by itself.", - original, pos) - groups = splitstr.join(line[:-1]).split('.') - i = 0 - while i < len(groups): - groups[i] = groups[i].strip() - if len(groups[i]) > 0 and (groups[i][0] == '"' or - groups[i][0] == "'"): - groupstr = groups[i] - j = i + 1 - while not groupstr[0] == groupstr[-1]: - j += 1 - if j > len(groups) + 2: - raise TomlDecodeError("Invalid group name '" + - groupstr + "' Something " + - "went wrong.", original, pos) - groupstr = '.'.join(groups[i:j]).strip() - groups[i] = groupstr[1:-1] - groups[i + 1:j] = [] - else: - if not _groupname_re.match(groups[i]): - raise TomlDecodeError("Invalid group name '" + - groups[i] + "'. Try quoting it.", - original, pos) - i += 1 - currentlevel = retval - for i in _range(len(groups)): - group = groups[i] - if group == "": - raise TomlDecodeError("Can't have a keygroup with an empty " - "name", original, pos) - try: - currentlevel[group] - if i == len(groups) - 1: - if group in implicitgroups: - implicitgroups.remove(group) - if arrayoftables: - raise TomlDecodeError("An implicitly defined " - "table can't be an array", - original, pos) - elif arrayoftables: - currentlevel[group].append(decoder.get_empty_table() - ) - else: - raise TomlDecodeError("What? " + group + - " already exists?" + - str(currentlevel), - original, pos) - except TypeError: - currentlevel = currentlevel[-1] - if group not in currentlevel: - currentlevel[group] = decoder.get_empty_table() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [decoder.get_empty_table()] - except KeyError: - if i != len(groups) - 1: - implicitgroups.append(group) - currentlevel[group] = decoder.get_empty_table() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [decoder.get_empty_table()] - currentlevel = currentlevel[group] - if arrayoftables: - try: - currentlevel = currentlevel[-1] - except KeyError: - pass - elif line[0] == "{": - if line[-1] != "}": - raise TomlDecodeError("Line breaks are not allowed in inline" - "objects", original, pos) - try: - decoder.load_inline_object(line, currentlevel, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err), original, pos) - elif "=" in line: - try: - ret = decoder.load_line(line, currentlevel, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err), original, pos) - if ret is not None: - multikey, multilinestr, multibackslash = ret - return retval - - -def _load_date(val): - microsecond = 0 - tz = None - try: - if len(val) > 19: - if val[19] == '.': - if val[-1].upper() == 'Z': - subsecondval = val[20:-1] - tzval = "Z" - else: - subsecondvalandtz = val[20:] - if '+' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('+') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - elif '-' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('-') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - else: - tzval = None - subsecondval = subsecondvalandtz - if tzval is not None: - tz = TomlTz(tzval) - microsecond = int(int(subsecondval) * - (10 ** (6 - len(subsecondval)))) - else: - tz = TomlTz(val[19:]) - except ValueError: - tz = None - if "-" not in val[1:]: - return None - try: - if len(val) == 10: - d = datetime.date( - int(val[:4]), int(val[5:7]), - int(val[8:10])) - else: - d = datetime.datetime( - int(val[:4]), int(val[5:7]), - int(val[8:10]), int(val[11:13]), - int(val[14:16]), int(val[17:19]), microsecond, tz) - except ValueError: - return None - return d - - -def _load_unicode_escapes(v, hexbytes, prefix): - skip = False - i = len(v) - 1 - while i > -1 and v[i] == '\\': - skip = not skip - i -= 1 - for hx in hexbytes: - if skip: - skip = False - i = len(hx) - 1 - while i > -1 and hx[i] == '\\': - skip = not skip - i -= 1 - v += prefix - v += hx - continue - hxb = "" - i = 0 - hxblen = 4 - if prefix == "\\U": - hxblen = 8 - hxb = ''.join(hx[i:i + hxblen]).lower() - if hxb.strip('0123456789abcdef'): - raise ValueError("Invalid escape sequence: " + hxb) - if hxb[0] == "d" and hxb[1].strip('01234567'): - raise ValueError("Invalid escape sequence: " + hxb + - ". Only scalar unicode points are allowed.") - v += unichr(int(hxb, 16)) - v += unicode(hx[len(hxb):]) - return v - - -# Unescape TOML string values. - -# content after the \ -_escapes = ['0', 'b', 'f', 'n', 'r', 't', '"'] -# What it should be replaced by -_escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"'] -# Used for substitution -_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) - - -def _unescape(v): - """Unescape characters in a TOML string.""" - i = 0 - backslash = False - while i < len(v): - if backslash: - backslash = False - if v[i] in _escapes: - v = v[:i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1:] - elif v[i] == '\\': - v = v[:i - 1] + v[i:] - elif v[i] == 'u' or v[i] == 'U': - i += 1 - else: - raise ValueError("Reserved escape sequence used") - continue - elif v[i] == '\\': - backslash = True - i += 1 - return v - - -class InlineTableDict(object): - """Sentinel subclass of dict for inline tables.""" - - -class TomlDecoder(object): - - def __init__(self, _dict=dict): - self._dict = _dict - - def get_empty_table(self): - return self._dict() - - def get_empty_inline_table(self): - class DynamicInlineTableDict(self._dict, InlineTableDict): - """Concrete sentinel subclass for inline tables. - It is a subclass of _dict which is passed in dynamically at load - time - - It is also a subclass of InlineTableDict - """ - - return DynamicInlineTableDict() - - def load_inline_object(self, line, currentlevel, multikey=False, - multibackslash=False): - candidate_groups = line[1:-1].split(",") - groups = [] - if len(candidate_groups) == 1 and not candidate_groups[0].strip(): - candidate_groups.pop() - while len(candidate_groups) > 0: - candidate_group = candidate_groups.pop(0) - try: - _, value = candidate_group.split('=', 1) - except ValueError: - raise ValueError("Invalid inline table encountered") - value = value.strip() - if ((value[0] == value[-1] and value[0] in ('"', "'")) or ( - value[0] in '-0123456789' or - value in ('true', 'false') or - (value[0] == "[" and value[-1] == "]") or - (value[0] == '{' and value[-1] == '}'))): - groups.append(candidate_group) - elif len(candidate_groups) > 0: - candidate_groups[0] = (candidate_group + "," + - candidate_groups[0]) - else: - raise ValueError("Invalid inline table value encountered") - for group in groups: - status = self.load_line(group, currentlevel, multikey, - multibackslash) - if status is not None: - break - - def _get_split_on_quotes(self, line): - doublequotesplits = line.split('"') - quoted = False - quotesplits = [] - if len(doublequotesplits) > 1 and "'" in doublequotesplits[0]: - singlequotesplits = doublequotesplits[0].split("'") - doublequotesplits = doublequotesplits[1:] - while len(singlequotesplits) % 2 == 0 and len(doublequotesplits): - singlequotesplits[-1] += '"' + doublequotesplits[0] - doublequotesplits = doublequotesplits[1:] - if "'" in singlequotesplits[-1]: - singlequotesplits = (singlequotesplits[:-1] + - singlequotesplits[-1].split("'")) - quotesplits += singlequotesplits - for doublequotesplit in doublequotesplits: - if quoted: - quotesplits.append(doublequotesplit) - else: - quotesplits += doublequotesplit.split("'") - quoted = not quoted - return quotesplits - - def load_line(self, line, currentlevel, multikey, multibackslash): - i = 1 - quotesplits = self._get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and '=' in quotesplit: - break - i += quotesplit.count('=') - quoted = not quoted - pair = line.split('=', i) - strictly_valid = _strictly_valid_num(pair[-1]) - if _number_with_underscores.match(pair[-1]): - pair[-1] = pair[-1].replace('_', '') - while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and - pair[-1][0] != "'" and pair[-1][0] != '"' and - pair[-1][0] != '[' and pair[-1][0] != '{' and - pair[-1].strip() != 'true' and - pair[-1].strip() != 'false'): - try: - float(pair[-1]) - break - except ValueError: - pass - if _load_date(pair[-1]) is not None: - break - if TIME_RE.match(pair[-1]): - break - i += 1 - prev_val = pair[-1] - pair = line.split('=', i) - if prev_val == pair[-1]: - raise ValueError("Invalid date or number") - if strictly_valid: - strictly_valid = _strictly_valid_num(pair[-1]) - pair = ['='.join(pair[:-1]).strip(), pair[-1].strip()] - if '.' in pair[0]: - if '"' in pair[0] or "'" in pair[0]: - quotesplits = self._get_split_on_quotes(pair[0]) - quoted = False - levels = [] - for quotesplit in quotesplits: - if quoted: - levels.append(quotesplit) - else: - levels += [level.strip() for level in - quotesplit.split('.')] - quoted = not quoted - else: - levels = pair[0].split('.') - while levels[-1] == "": - levels = levels[:-1] - for level in levels[:-1]: - if level == "": - continue - if level not in currentlevel: - currentlevel[level] = self.get_empty_table() - currentlevel = currentlevel[level] - pair[0] = levels[-1].strip() - elif (pair[0][0] == '"' or pair[0][0] == "'") and \ - (pair[0][-1] == pair[0][0]): - pair[0] = _unescape(pair[0][1:-1]) - k, koffset = self._load_line_multiline_str(pair[1]) - if k > -1: - while k > -1 and pair[1][k + koffset] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = pair[1][:-1] - else: - multilinestr = pair[1] + "\n" - multikey = pair[0] - else: - value, vtype = self.load_value(pair[1], strictly_valid) - try: - currentlevel[pair[0]] - raise ValueError("Duplicate keys!") - except TypeError: - raise ValueError("Duplicate keys!") - except KeyError: - if multikey: - return multikey, multilinestr, multibackslash - else: - currentlevel[pair[0]] = value - - def _load_line_multiline_str(self, p): - poffset = 0 - if len(p) < 3: - return -1, poffset - if p[0] == '[' and (p.strip()[-1] != ']' and - self._load_array_isstrarray(p)): - newp = p[1:].strip().split(',') - while len(newp) > 1 and newp[-1][0] != '"' and newp[-1][0] != "'": - newp = newp[:-2] + [newp[-2] + ',' + newp[-1]] - newp = newp[-1] - poffset = len(p) - len(newp) - p = newp - if p[0] != '"' and p[0] != "'": - return -1, poffset - if p[1] != p[0] or p[2] != p[0]: - return -1, poffset - if len(p) > 5 and p[-1] == p[0] and p[-2] == p[0] and p[-3] == p[0]: - return -1, poffset - return len(p) - 1, poffset - - def load_value(self, v, strictly_valid=True): - if not v: - raise ValueError("Empty value is invalid") - if v == 'true': - return (True, "bool") - elif v == 'false': - return (False, "bool") - elif v[0] == '"' or v[0] == "'": - quotechar = v[0] - testv = v[1:].split(quotechar) - triplequote = False - triplequotecount = 0 - if len(testv) > 1 and testv[0] == '' and testv[1] == '': - testv = testv[2:] - triplequote = True - closed = False - for tv in testv: - if tv == '': - if triplequote: - triplequotecount += 1 - else: - closed = True - else: - oddbackslash = False - try: - i = -1 - j = tv[i] - while j == '\\': - oddbackslash = not oddbackslash - i -= 1 - j = tv[i] - except IndexError: - pass - if not oddbackslash: - if closed: - raise ValueError("Found tokens after a closed " + - "string. Invalid TOML.") - else: - if not triplequote or triplequotecount > 1: - closed = True - else: - triplequotecount = 0 - if quotechar == '"': - escapeseqs = v.split('\\')[1:] - backslash = False - for i in escapeseqs: - if i == '': - backslash = not backslash - else: - if i[0] not in _escapes and (i[0] != 'u' and - i[0] != 'U' and - not backslash): - raise ValueError("Reserved escape sequence used") - if backslash: - backslash = False - for prefix in ["\\u", "\\U"]: - if prefix in v: - hexbytes = v.split(prefix) - v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], - prefix) - v = _unescape(v) - if len(v) > 1 and v[1] == quotechar and (len(v) < 3 or - v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == '[': - return (self.load_array(v), "array") - elif v[0] == '{': - inline_object = self.get_empty_inline_table() - self.load_inline_object(v, inline_object) - return (inline_object, "inline_object") - elif TIME_RE.match(v): - h, m, s, _, ms = TIME_RE.match(v).groups() - time = datetime.time(int(h), int(m), int(s), int(ms) if ms else 0) - return (time, "time") - else: - parsed_date = _load_date(v) - if parsed_date is not None: - return (parsed_date, "date") - if not strictly_valid: - raise ValueError("Weirdness with leading zeroes or " - "underscores in your number.") - itype = "int" - neg = False - if v[0] == '-': - neg = True - v = v[1:] - elif v[0] == '+': - v = v[1:] - v = v.replace('_', '') - lowerv = v.lower() - if '.' in v or ('x' not in v and ('e' in v or 'E' in v)): - if '.' in v and v.split('.', 1)[1] == '': - raise ValueError("This float is missing digits after " - "the point") - if v[0] not in '0123456789': - raise ValueError("This float doesn't have a leading " - "digit") - v = float(v) - itype = "float" - elif len(lowerv) == 3 and (lowerv == 'inf' or lowerv == 'nan'): - v = float(v) - itype = "float" - if itype == "int": - v = int(v, 0) - if neg: - return (0 - v, itype) - return (v, itype) - - def bounded_string(self, s): - if len(s) == 0: - return True - if s[-1] != s[0]: - return False - i = -2 - backslash = False - while len(s) + i > 0: - if s[i] == "\\": - backslash = not backslash - i -= 1 - else: - break - return not backslash - - def _load_array_isstrarray(self, a): - a = a[1:-1].strip() - if a != '' and (a[0] == '"' or a[0] == "'"): - return True - return False - - def load_array(self, a): - atype = None - retval = [] - a = a.strip() - if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip(): - strarray = self._load_array_isstrarray(a) - if not a[1:-1].strip().startswith('{'): - a = a[1:-1].split(',') - else: - # a is an inline object, we must find the matching parenthesis - # to define groups - new_a = [] - start_group_index = 1 - end_group_index = 2 - open_bracket_count = 1 if a[start_group_index] == '{' else 0 - in_str = False - while end_group_index < len(a[1:]): - if a[end_group_index] == '"' or a[end_group_index] == "'": - if in_str: - backslash_index = end_group_index - 1 - while (backslash_index > -1 and - a[backslash_index] == '\\'): - in_str = not in_str - backslash_index -= 1 - in_str = not in_str - if not in_str and a[end_group_index] == '{': - open_bracket_count += 1 - if in_str or a[end_group_index] != '}': - end_group_index += 1 - continue - elif a[end_group_index] == '}' and open_bracket_count > 1: - open_bracket_count -= 1 - end_group_index += 1 - continue - - # Increase end_group_index by 1 to get the closing bracket - end_group_index += 1 - - new_a.append(a[start_group_index:end_group_index]) - - # The next start index is at least after the closing - # bracket, a closing bracket can be followed by a comma - # since we are in an array. - start_group_index = end_group_index + 1 - while (start_group_index < len(a[1:]) and - a[start_group_index] != '{'): - start_group_index += 1 - end_group_index = start_group_index + 1 - a = new_a - b = 0 - if strarray: - while b < len(a) - 1: - ab = a[b].strip() - while (not self.bounded_string(ab) or - (len(ab) > 2 and - ab[0] == ab[1] == ab[2] and - ab[-2] != ab[0] and - ab[-3] != ab[0])): - a[b] = a[b] + ',' + a[b + 1] - ab = a[b].strip() - if b < len(a) - 2: - a = a[:b + 1] + a[b + 2:] - else: - a = a[:b + 1] - b += 1 - else: - al = list(a[1:-1]) - a = [] - openarr = 0 - j = 0 - for i in _range(len(al)): - if al[i] == '[': - openarr += 1 - elif al[i] == ']': - openarr -= 1 - elif al[i] == ',' and not openarr: - a.append(''.join(al[j:i])) - j = i + 1 - a.append(''.join(al[j:])) - for i in _range(len(a)): - a[i] = a[i].strip() - if a[i] != '': - nval, ntype = self.load_value(a[i]) - if atype: - if ntype != atype: - raise ValueError("Not a homogeneous array") - else: - atype = ntype - retval.append(nval) - return retval - - def preserve_comment(self, line_no, key, comment, beginline): - pass - - def embed_comments(self, idx, currentlevel): - pass - - -class TomlPreserveCommentDecoder(TomlDecoder): - - def __init__(self, _dict=dict): - self.saved_comments = {} - super(TomlPreserveCommentDecoder, self).__init__(_dict) - - def preserve_comment(self, line_no, key, comment, beginline): - self.saved_comments[line_no] = (key, comment, beginline) - - def embed_comments(self, idx, currentlevel): - if idx not in self.saved_comments: - return - - key, comment, beginline = self.saved_comments[idx] - currentlevel[key] = CommentValue(currentlevel[key], comment, beginline, - self._dict) diff --git a/utils/toml/encoder.py b/utils/toml/encoder.py deleted file mode 100644 index d9e557ed95..0000000000 --- a/utils/toml/encoder.py +++ /dev/null @@ -1,304 +0,0 @@ -import datetime -import re -import sys -from decimal import Decimal - -from toml.decoder import InlineTableDict - -if sys.version_info >= (3,): - unicode = str - - -def dump(o, f, encoder=None): - """Writes out dict as toml to a file - - Args: - o: Object to dump into toml - f: File descriptor where the toml should be stored - encoder: The ``TomlEncoder`` to use for constructing the output string - - Returns: - String containing the toml corresponding to dictionary - - Raises: - TypeError: When anything other than file descriptor is passed - """ - - if not f.write: - raise TypeError("You can only dump an object to a file descriptor") - d = dumps(o, encoder=encoder) - f.write(d) - return d - - -def dumps(o, encoder=None): - """Stringifies input dict as toml - - Args: - o: Object to dump into toml - encoder: The ``TomlEncoder`` to use for constructing the output string - - Returns: - String containing the toml corresponding to dict - - Examples: - ```python - >>> import toml - >>> output = { - ... 'a': "I'm a string", - ... 'b': ["I'm", "a", "list"], - ... 'c': 2400 - ... } - >>> toml.dumps(output) - 'a = "I\'m a string"\nb = [ "I\'m", "a", "list",]\nc = 2400\n' - ``` - """ - - retval = "" - if encoder is None: - encoder = TomlEncoder(o.__class__) - addtoretval, sections = encoder.dump_sections(o, "") - retval += addtoretval - outer_objs = [id(o)] - while sections: - section_ids = [id(section) for section in sections] - for outer_obj in outer_objs: - if outer_obj in section_ids: - raise ValueError("Circular reference detected") - outer_objs += section_ids - newsections = encoder.get_empty_table() - for section in sections: - addtoretval, addtosections = encoder.dump_sections( - sections[section], section) - - if addtoretval or (not addtoretval and not addtosections): - if retval and retval[-2:] != "\n\n": - retval += "\n" - retval += "[" + section + "]\n" - if addtoretval: - retval += addtoretval - for s in addtosections: - newsections[section + "." + s] = addtosections[s] - sections = newsections - return retval - - -def _dump_str(v): - if sys.version_info < (3,) and hasattr(v, 'decode') and isinstance(v, str): - v = v.decode('utf-8') - v = "%r" % v - if v[0] == 'u': - v = v[1:] - singlequote = v.startswith("'") - if singlequote or v.startswith('"'): - v = v[1:-1] - if singlequote: - v = v.replace("\\'", "'") - v = v.replace('"', '\\"') - v = v.split("\\x") - while len(v) > 1: - i = -1 - if not v[0]: - v = v[1:] - v[0] = v[0].replace("\\\\", "\\") - # No, I don't know why != works and == breaks - joinx = v[0][i] != "\\" - while v[0][:i] and v[0][i] == "\\": - joinx = not joinx - i -= 1 - if joinx: - joiner = "x" - else: - joiner = "u00" - v = [v[0] + joiner + v[1]] + v[2:] - return unicode('"' + v[0] + '"') - - -def _dump_float(v): - return "{}".format(v).replace("e+0", "e+").replace("e-0", "e-") - - -def _dump_time(v): - utcoffset = v.utcoffset() - if utcoffset is None: - return v.isoformat() - # The TOML norm specifies that it's local time thus we drop the offset - return v.isoformat()[:-6] - - -class TomlEncoder(object): - - def __init__(self, _dict=dict, preserve=False): - self._dict = _dict - self.preserve = preserve - self.dump_funcs = { - str: _dump_str, - unicode: _dump_str, - list: self.dump_list, - bool: lambda v: unicode(v).lower(), - int: lambda v: v, - float: _dump_float, - Decimal: _dump_float, - datetime.datetime: lambda v: v.isoformat().replace('+00:00', 'Z'), - datetime.time: _dump_time, - datetime.date: lambda v: v.isoformat() - } - - def get_empty_table(self): - return self._dict() - - def dump_list(self, v): - retval = "[" - for u in v: - retval += " " + unicode(self.dump_value(u)) + "," - retval += "]" - return retval - - def dump_inline_table(self, section): - """Preserve inline table in its compact syntax instead of expanding - into subsection. - - https://github.com/toml-lang/toml#user-content-inline-table - """ - retval = "" - if isinstance(section, dict): - val_list = [] - for k, v in section.items(): - val = self.dump_inline_table(v) - val_list.append(k + " = " + val) - retval += "{ " + ", ".join(val_list) + " }\n" - return retval - else: - return unicode(self.dump_value(section)) - - def dump_value(self, v): - # Lookup function corresponding to v's type - dump_fn = self.dump_funcs.get(type(v)) - if dump_fn is None and hasattr(v, '__iter__'): - dump_fn = self.dump_funcs[list] - # Evaluate function (if it exists) else return v - return dump_fn(v) if dump_fn is not None else self.dump_funcs[str](v) - - def dump_sections(self, o, sup): - retstr = "" - if sup != "" and sup[-1] != ".": - sup += '.' - retdict = self._dict() - arraystr = "" - for section in o: - section = unicode(section) - qsection = section - if not re.match(r'^[A-Za-z0-9_-]+$', section): - qsection = _dump_str(section) - if not isinstance(o[section], dict): - arrayoftables = False - if isinstance(o[section], list): - for a in o[section]: - if isinstance(a, dict): - arrayoftables = True - if arrayoftables: - for a in o[section]: - arraytabstr = "\n" - arraystr += "[[" + sup + qsection + "]]\n" - s, d = self.dump_sections(a, sup + qsection) - if s: - if s[0] == "[": - arraytabstr += s - else: - arraystr += s - while d: - newd = self._dict() - for dsec in d: - s1, d1 = self.dump_sections(d[dsec], sup + - qsection + "." + - dsec) - if s1: - arraytabstr += ("[" + sup + qsection + - "." + dsec + "]\n") - arraytabstr += s1 - for s1 in d1: - newd[dsec + "." + s1] = d1[s1] - d = newd - arraystr += arraytabstr - else: - if o[section] is not None: - retstr += (qsection + " = " + - unicode(self.dump_value(o[section])) + '\n') - elif self.preserve and isinstance(o[section], InlineTableDict): - retstr += (qsection + " = " + - self.dump_inline_table(o[section])) - else: - retdict[qsection] = o[section] - retstr += arraystr - return (retstr, retdict) - - -class TomlPreserveInlineDictEncoder(TomlEncoder): - - def __init__(self, _dict=dict): - super(TomlPreserveInlineDictEncoder, self).__init__(_dict, True) - - -class TomlArraySeparatorEncoder(TomlEncoder): - - def __init__(self, _dict=dict, preserve=False, separator=","): - super(TomlArraySeparatorEncoder, self).__init__(_dict, preserve) - if separator.strip() == "": - separator = "," + separator - elif separator.strip(' \t\n\r,'): - raise ValueError("Invalid separator for arrays") - self.separator = separator - - def dump_list(self, v): - t = [] - retval = "[" - for u in v: - t.append(self.dump_value(u)) - while t != []: - s = [] - for u in t: - if isinstance(u, list): - for r in u: - s.append(r) - else: - retval += " " + unicode(u) + self.separator - t = s - retval += "]" - return retval - - -class TomlNumpyEncoder(TomlEncoder): - - def __init__(self, _dict=dict, preserve=False): - import numpy as np - super(TomlNumpyEncoder, self).__init__(_dict, preserve) - self.dump_funcs[np.float16] = _dump_float - self.dump_funcs[np.float32] = _dump_float - self.dump_funcs[np.float64] = _dump_float - self.dump_funcs[np.int16] = self._dump_int - self.dump_funcs[np.int32] = self._dump_int - self.dump_funcs[np.int64] = self._dump_int - - def _dump_int(self, v): - return "{}".format(int(v)) - - -class TomlPreserveCommentEncoder(TomlEncoder): - - def __init__(self, _dict=dict, preserve=False): - from toml.decoder import CommentValue - super(TomlPreserveCommentEncoder, self).__init__(_dict, preserve) - self.dump_funcs[CommentValue] = lambda v: v.dump(self.dump_value) - - -class TomlPathlibEncoder(TomlEncoder): - - def _dump_pathlib_path(self, v): - return _dump_str(str(v)) - - def dump_value(self, v): - if (3, 4) <= sys.version_info: - import pathlib - if isinstance(v, pathlib.PurePath): - v = str(v) - return super(TomlPathlibEncoder, self).dump_value(v) diff --git a/utils/toml/ordered.py b/utils/toml/ordered.py deleted file mode 100644 index 9c20c41a1b..0000000000 --- a/utils/toml/ordered.py +++ /dev/null @@ -1,15 +0,0 @@ -from collections import OrderedDict -from toml import TomlEncoder -from toml import TomlDecoder - - -class TomlOrderedDecoder(TomlDecoder): - - def __init__(self): - super(self.__class__, self).__init__(_dict=OrderedDict) - - -class TomlOrderedEncoder(TomlEncoder): - - def __init__(self): - super(self.__class__, self).__init__(_dict=OrderedDict) diff --git a/utils/toml/tz.py b/utils/toml/tz.py deleted file mode 100644 index 93c3c8ad26..0000000000 --- a/utils/toml/tz.py +++ /dev/null @@ -1,21 +0,0 @@ -from datetime import tzinfo, timedelta - - -class TomlTz(tzinfo): - def __init__(self, toml_offset): - if toml_offset == "Z": - self._raw_offset = "+00:00" - else: - self._raw_offset = toml_offset - self._sign = -1 if self._raw_offset[0] == '-' else 1 - self._hours = int(self._raw_offset[1:3]) - self._minutes = int(self._raw_offset[4:6]) - - def tzname(self, dt): - return "UTC" + self._raw_offset - - def utcoffset(self, dt): - return self._sign * timedelta(hours=self._hours, minutes=self._minutes) - - def dst(self, dt): - return timedelta(0) From dab0bf15100ad03cdc1eba9ef1e88902f463acda Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Tue, 3 Mar 2020 15:38:35 +0100 Subject: [PATCH 36/91] Upload requirements.txt as test artifact. --- .github/workflows/ci.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc6567a79d..ea73de28d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,11 +182,14 @@ jobs: file: ./coverage.xml fail_ci_if_error: false - - name: Show test environment - env: - AIIDA_TEST_BACKEND: ${{ matrix.backend }} - run: - python -m pip freeze + - name: Freeze test environment + run: | + pip freeze | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + + - uses: actions/upload-artifact@v1 + with: + name: requirements.txt + path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt verdi: From c8fb8bd756baf18600e7f9c4eec8121859d26855 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 11:39:17 +0100 Subject: [PATCH 37/91] Split the DM-related CI steps into separate workflow. To enable dm-related triggers, such as dependency related paths and nightly runs. --- .github/workflows/ci.yml | 53 ---------------- .github/workflows/dm.yml | 130 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/dm.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea73de28d9..9f5544c3c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,58 +4,6 @@ on: [push, pull_request] jobs: - install-with-pip: - - runs-on: ubuntu-latest - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v1 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Pip install - run: | - python -m pip install -e . - python -m pip freeze - - - name: Test package import - run: - python -c "import aiida" - - install-with-conda: - - runs-on: ubuntu-latest - name: install-with-conda - - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v1 - - - name: Setup Conda - uses: s-weigand/setup-conda@v1 - with: - update-conda: true - python-version: 3.7 - - run: conda --version - - run: python --version - - run: which python - - - name: Create conda environment - run: | - conda env create -f environment.yml -n test-environment - source activate test-environment - python -m pip install --no-deps -e . - - - name: Test package import - run: | - source activate test-environment - python -c "import aiida" - docs: runs-on: ubuntu-latest @@ -118,7 +66,6 @@ jobs: tests: - needs: [install-with-pip, install-with-conda] runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml new file mode 100644 index 0000000000..60b24525d5 --- /dev/null +++ b/.github/workflows/dm.yml @@ -0,0 +1,130 @@ +name: aiida-core-dependency-management + +on: + push: + paths: + - 'setup.*' + - 'environment.yml' + - 'requirements*.txt' + - 'pyproject.toml' + schedule: + - cron: '30 02 * * *' # nightly build + +jobs: + + install-with-pip: + + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Pip install + run: | + python -m pip install -e . + python -m pip freeze + + - name: Test package import + run: + python -c "import aiida" + + install-with-conda: + + runs-on: ubuntu-latest + name: install-with-conda + + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v1 + + - name: Setup Conda + uses: s-weigand/setup-conda@v1 + with: + update-conda: true + python-version: 3.7 + - run: conda --version + - run: python --version + - run: which python + + - name: Create conda environment + run: | + conda env create -f environment.yml -n test-environment + source activate test-environment + python -m pip install --no-deps -e . + + - name: Test package import + run: | + source activate test-environment + python -c "import aiida" + + tests: + + needs: [install-with-pip, install-with-conda] + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + backend: ['django', 'sqlalchemy'] + + steps: + - uses: actions/checkout@v1 + - uses: CasperWA/postgresql-action@v1.2 + with: + postgresql version: '10' + postgresql db: test_${{ matrix.backend }} + postgresql user: postgres + postgresql password: '' + postgresql auth: trust + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add - + echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list + echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list + sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list + sudo apt update + sudo apt install postgresql-10 rabbitmq-server graphviz + sudo systemctl status rabbitmq-server.service + + - name: Install python dependencies + run: | + pip install --upgrade pip + pip install numpy==1.17.4 + pip install -e .[atomic_tools,docs,notebook,rest,testing] + reentry scan + + - name: Setup environment + env: + AIIDA_TEST_BACKEND: ${{ matrix.backend }} + run: + .github/workflows/setup.sh + + - name: Run test suite + env: + AIIDA_TEST_BACKEND: ${{ matrix.backend }} + run: + .github/workflows/tests.sh + + - name: Freeze test environment + run: | + pip freeze | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + + - uses: actions/upload-artifact@v1 + with: + name: requirements.txt + path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt From 969b08bbbeb61cc57ea0049bf3976e5d1854fabf Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 14:20:42 +0100 Subject: [PATCH 38/91] Automatically create PR for updating requirements.txt --- .github/workflows/dm.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 60b24525d5..742c5e3437 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -2,11 +2,15 @@ name: aiida-core-dependency-management on: push: + branch: + - develop + - misc/update-dependency-management # TODO: Remove before merge paths: - 'setup.*' - 'environment.yml' - 'requirements*.txt' - 'pyproject.toml' + - '.github/workflows/*' schedule: - cron: '30 02 * * *' # nightly build @@ -124,7 +128,24 @@ jobs: run: | pip freeze | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt - - uses: actions/upload-artifact@v1 + - name: Upload requirements.txt + uses: actions/upload-artifact@v1 with: name: requirements.txt path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + + - name: Update requirements.txt + if: matrix.backend == 'django' && matrix.python-version == '3.8' + run: | + sed '1d' requirements-django-py-3.8.txt > requirements.txt + git add requirements.txt + + - name: Create Pull Request for updated requirements.txt + if: matrix.backend == 'django' && matrix.python-version == '3.8' + uses: peter-evans/create-pull-request@v2.4.4 + with: + commit-message: "Update requirements.txt" + token: ${{ secrets.GITHUB_TOKEN }} + title: "Update requirements.txt" + reviewers: csadorf + branch: "dm/update-requirements.txt" From e3e3dc79800937bb05eea5367f52dad2280f23d3 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 14:40:15 +0100 Subject: [PATCH 39/91] Execute dm workflow on all branches prefixed with dm/*. --- .github/workflows/dm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 742c5e3437..9dbf78f38b 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -4,6 +4,7 @@ on: push: branch: - develop + - dm/* - misc/update-dependency-management # TODO: Remove before merge paths: - 'setup.*' From b41d942348ea914a729b9c6aae8a0ea235da7ef5 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 14:41:04 +0100 Subject: [PATCH 40/91] Temporarily reduce GitHub actions workload for testing. --- .github/workflows/ci.yml | 2 ++ .github/workflows/dm.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f5544c3c1..0ecaeca50d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,8 @@ name: aiida-core on: [push, pull_request] + branch: + - develop # TODO: Remove before merge jobs: diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 9dbf78f38b..1766ce4c30 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -78,7 +78,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.8] # TODO: Change to all supported Python versions before merge backend: ['django', 'sqlalchemy'] steps: From c5202c9cdfb79220982a61a61b58adeeb80ebe7f Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 14:42:13 +0100 Subject: [PATCH 41/91] Attempt to fix dm workflow issue likely caused by old checkout versions. --- .github/workflows/dm.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 1766ce4c30..35b7ecc6bd 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -23,7 +23,7 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python 3.7 uses: actions/setup-python@v1 @@ -47,7 +47,7 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Setup Conda uses: s-weigand/setup-conda@v1 @@ -82,7 +82,7 @@ jobs: backend: ['django', 'sqlalchemy'] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: CasperWA/postgresql-action@v1.2 with: postgresql version: '10' From 66c142fee8cae716f2c3f17707a0531c13efb929 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 14:42:29 +0100 Subject: [PATCH 42/91] Be less restrictive about the pull request action version. --- .github/workflows/dm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 35b7ecc6bd..15777d58cd 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -143,7 +143,7 @@ jobs: - name: Create Pull Request for updated requirements.txt if: matrix.backend == 'django' && matrix.python-version == '3.8' - uses: peter-evans/create-pull-request@v2.4.4 + uses: peter-evans/create-pull-request@v2 with: commit-message: "Update requirements.txt" token: ${{ secrets.GITHUB_TOKEN }} From 2e22b9b5ff79a9e2f6577c52e6aee913768bdac0 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 15:06:52 +0100 Subject: [PATCH 43/91] Further reduce CI load. --- .github/workflows/dm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 15777d58cd..81cef772f7 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -79,7 +79,7 @@ jobs: fail-fast: false matrix: python-version: [3.8] # TODO: Change to all supported Python versions before merge - backend: ['django', 'sqlalchemy'] + backend: ['django'] # TODO: Add 'sqlalchemy' before merge steps: - uses: actions/checkout@v2 From eef0b8ba2701a27cc908565104be8dfb702b8df8 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 15:08:38 +0100 Subject: [PATCH 44/91] Remove untracked files before creating PR. --- .github/workflows/dm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 81cef772f7..3004a9159a 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -140,6 +140,7 @@ jobs: run: | sed '1d' requirements-django-py-3.8.txt > requirements.txt git add requirements.txt + git clean -f # remove all untracked files - name: Create Pull Request for updated requirements.txt if: matrix.backend == 'django' && matrix.python-version == '3.8' From 067b6cb816507292da9a6fb65da0d325de99f541 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 15:36:05 +0100 Subject: [PATCH 45/91] Use seperate job to create update-requirements PR. To commit to a clean repo. --- .github/workflows/dm.yml | 41 +++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 3004a9159a..9a6f6938de 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -135,19 +135,30 @@ jobs: name: requirements.txt path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt - - name: Update requirements.txt - if: matrix.backend == 'django' && matrix.python-version == '3.8' - run: | - sed '1d' requirements-django-py-3.8.txt > requirements.txt - git add requirements.txt - git clean -f # remove all untracked files + update-requirements: - - name: Create Pull Request for updated requirements.txt - if: matrix.backend == 'django' && matrix.python-version == '3.8' - uses: peter-evans/create-pull-request@v2 - with: - commit-message: "Update requirements.txt" - token: ${{ secrets.GITHUB_TOKEN }} - title: "Update requirements.txt" - reviewers: csadorf - branch: "dm/update-requirements.txt" + needs: tests + runs-on: ubuntu-latest + + steps: + + - name: Download requirements.txt + uses: actions/download-artifact@v1 + with: + name: requirements.txt + path: requirements-django-py-38.txt + + - name: Update requirements.txt + run: | + mv requirements-django-py-3.8.txt requirements.txt + sed -i '1d' requirements.txt + + - name: Create Pull Request + if: matrix.backend == 'django' && matrix.python-version == '3.8' + uses: peter-evans/create-pull-request@v2 + with: + commit-message: "Update requirements.txt" + token: ${{ secrets.GITHUB_TOKEN }} + title: "Update requirements.txt" + reviewers: csadorf + branch: "dm/update-requirements.txt" From e695c5423ee73b9109b1868595b95562a48bd99a Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 15:37:51 +0100 Subject: [PATCH 46/91] Fix dm.yml syntax error. --- .github/workflows/dm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 9a6f6938de..bec7f2e056 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -135,7 +135,7 @@ jobs: name: requirements.txt path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt - update-requirements: + update-requirements: needs: tests runs-on: ubuntu-latest From 1fe0433e9c3bf770b6ce64e01a378b578addddf8 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 15:41:04 +0100 Subject: [PATCH 47/91] Extend list of dm-related files in CODEOWNERS file. --- .github/CODEOWNERS | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e90d9c1ade..8da328f9d9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,7 +2,9 @@ # currently active dependency manager (DM) to trigger an automatic review # request from the DM upon changes. Please see AEP-002 for details: # https://github.com/aiidateam/AEP/tree/master/002_dependency_management -setup.* @csadorf -environment.yml @csadorf -requirements*.txt @csadorf -pyproject.toml @csadorf +setup.* @csadorf +environment.yml @csadorf +requirements*.txt @csadorf +pyproject.toml @csadorf +utils/dependency_management.py @csadorf +.github/workflows/dm.yml @csadorf From b305101a2f1e88022ea06617a5c0a6507f4dcacb Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 17:37:35 +0100 Subject: [PATCH 48/91] Use @aiidateam/dependency-manager as codeowner and reviewer. So that we don't have to update all scripts everytime the role changes. --- .github/CODEOWNERS | 12 ++++++------ .github/workflows/dm.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8da328f9d9..e908ba2fc9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,9 +2,9 @@ # currently active dependency manager (DM) to trigger an automatic review # request from the DM upon changes. Please see AEP-002 for details: # https://github.com/aiidateam/AEP/tree/master/002_dependency_management -setup.* @csadorf -environment.yml @csadorf -requirements*.txt @csadorf -pyproject.toml @csadorf -utils/dependency_management.py @csadorf -.github/workflows/dm.yml @csadorf +setup.* @aiidateam/dependency-manager +environment.yml @aiidateam/dependency-manager +requirements*.txt @aiidateam/dependency-manager +pyproject.toml @aiidateam/dependency-manager +utils/dependency_management.py @aiidateam/dependency-manager +.github/workflows/dm.yml @aiidateam/dependency-manager diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index bec7f2e056..7538e6732f 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -160,5 +160,5 @@ jobs: commit-message: "Update requirements.txt" token: ${{ secrets.GITHUB_TOKEN }} title: "Update requirements.txt" - reviewers: csadorf + reviewers: aiidateam/dependency-manager branch: "dm/update-requirements.txt" From aa83ffd82f95c4ecf50644430ddca0dc5af6647f Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 17:38:20 +0100 Subject: [PATCH 49/91] Attempt to fix issue in "update-requirements" job. --- .github/workflows/dm.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 7538e6732f..14c2fa728d 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -146,15 +146,14 @@ jobs: uses: actions/download-artifact@v1 with: name: requirements.txt - path: requirements-django-py-38.txt - name: Update requirements.txt run: | - mv requirements-django-py-3.8.txt requirements.txt + mv requirements.txt/requirements-django-py-3.8.txt requirements.txt + rm -r requirements.txt sed -i '1d' requirements.txt - name: Create Pull Request - if: matrix.backend == 'django' && matrix.python-version == '3.8' uses: peter-evans/create-pull-request@v2 with: commit-message: "Update requirements.txt" From f801cca0bb7703567fb22acb2b3403ed9d281482 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 17:47:32 +0100 Subject: [PATCH 50/91] Remove obsolete commands from dependency management script. --- utils/dependency_management.py | 87 ---------------------------------- 1 file changed, 87 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index f4c294273b..3c7c4d023d 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -13,7 +13,6 @@ import sys import re import json -import copy import subprocess from pathlib import Path from collections import OrderedDict @@ -23,8 +22,6 @@ import yaml import toml -from validate_consistency import write_setup_json - ROOT = Path(__file__).resolve().parent.parent # repository root DEFAULT_EXCLUDE_LIST = ['django', 'circus', 'numpy', 'pymatgen', 'ase', 'monty', 'pyyaml'] @@ -326,89 +323,5 @@ def pip_install_extras(extras): subprocess.run(cmd, check=True) -@cli.command('unrestrict') -@click.option('--exclude', multiple=True, help='List of package names to exclude from updating.') -def unrestrict_requirements(exclude): - """Remove all explicit dependency version restrictions from `setup.json`. - - Warning, this currently only works for dependency requirements that use the `==` operator. Statements with different - operators, additional filters after a semicolon, or with extra requirements (using `[]`) are not supported. The - limits for these statements will have to be updated manually. - """ - setup = _load_setup_cfg() - clone = copy.deepcopy(setup) - clone['install_requires'] = [] - - if exclude: - exclude = list(exclude).extend(DEFAULT_EXCLUDE_LIST) - else: - exclude = DEFAULT_EXCLUDE_LIST - - for requirement in setup['install_requires']: - if requirement in exclude or ';' in requirement or '==' not in requirement: - clone['install_requires'].append(requirement) - else: - package = requirement.split('==')[0] - clone['install_requires'].append(package) - - for extra, requirements in setup['extras_require'].items(): - clone['extras_require'][extra] = [] - - for requirement in requirements: - if requirement in exclude or ';' in requirement or '==' not in requirement: - clone['extras_require'][extra].append(requirement) - else: - package = requirement.split('==')[0] - clone['extras_require'][extra].append(package) - - write_setup_json(clone) - - -@cli.command('update') -@click.argument('requirements', type=click.File(mode='r')) -def update_requirements(requirements): - """Apply version restrictions from REQUIREMENTS. - - The REQUIREMENTS file should contain the output of `pip freeze`. - """ - setup = _load_setup_cfg() - - package_versions = [] - - for requirement in requirements.readlines(): - try: - package, version = requirement.strip().split('==') - package_versions.append((package, version)) - except ValueError: - continue - - requirements = set() - - for requirement in setup['install_requires']: - for package, version in package_versions: - if requirement.lower() == package.lower(): - requirements.add('{}=={}'.format(package.lower(), version)) - break - else: - requirements.add(requirement) - - setup['install_requires'] = sorted(requirements) - - for extra, extra_requirements in setup['extras_require'].items(): - requirements = set() - - for requirement in extra_requirements: - for package, version in package_versions: - if requirement.lower() == package.lower(): - requirements.add('{}=={}'.format(package.lower(), version)) - break - else: - requirements.add(requirement) - - setup['extras_require'][extra] = sorted(requirements) - - write_setup_json(setup) - - if __name__ == '__main__': cli() # pylint: disable=no-value-for-parameter From 4ff8daa1c60b7867810418621da17666ec4f058e Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 17:51:26 +0100 Subject: [PATCH 51/91] 2nd attempt at fixing update-requirements job. --- .github/workflows/dm.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 14c2fa728d..0419c4ba58 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -131,6 +131,7 @@ jobs: - name: Upload requirements.txt uses: actions/upload-artifact@v1 + if: matrix.backend == 'django' && matrix.python-version == '3.8' with: name: requirements.txt path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt @@ -147,10 +148,8 @@ jobs: with: name: requirements.txt - - name: Update requirements.txt + - name: Remove first line run: | - mv requirements.txt/requirements-django-py-3.8.txt requirements.txt - rm -r requirements.txt sed -i '1d' requirements.txt - name: Create Pull Request From 9f851dc9062839c0de96bb03f6bc65bde831b67d Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 18:02:39 +0100 Subject: [PATCH 52/91] Disable large portions of the testing ci to speed up debugging. --- .github/workflows/dm.yml | 190 +++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 0419c4ba58..1e587dd70c 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -17,61 +17,61 @@ on: jobs: - install-with-pip: - - runs-on: ubuntu-latest - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Pip install - run: | - python -m pip install -e . - python -m pip freeze - - - name: Test package import - run: - python -c "import aiida" - - install-with-conda: - - runs-on: ubuntu-latest - name: install-with-conda - - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v2 - - - name: Setup Conda - uses: s-weigand/setup-conda@v1 - with: - update-conda: true - python-version: 3.7 - - run: conda --version - - run: python --version - - run: which python - - - name: Create conda environment - run: | - conda env create -f environment.yml -n test-environment - source activate test-environment - python -m pip install --no-deps -e . - - - name: Test package import - run: | - source activate test-environment - python -c "import aiida" - +# install-with-pip: +# +# runs-on: ubuntu-latest +# timeout-minutes: 5 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Set up Python 3.7 +# uses: actions/setup-python@v1 +# with: +# python-version: 3.7 +# +# - name: Pip install +# run: | +# python -m pip install -e . +# python -m pip freeze +# +# - name: Test package import +# run: +# python -c "import aiida" +# +# install-with-conda: +# +# runs-on: ubuntu-latest +# name: install-with-conda +# +# timeout-minutes: 5 +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Setup Conda +# uses: s-weigand/setup-conda@v1 +# with: +# update-conda: true +# python-version: 3.7 +# - run: conda --version +# - run: python --version +# - run: which python +# +# - name: Create conda environment +# run: | +# conda env create -f environment.yml -n test-environment +# source activate test-environment +# python -m pip install --no-deps -e . +# +# - name: Test package import +# run: | +# source activate test-environment +# python -c "import aiida" +# tests: - needs: [install-with-pip, install-with-conda] + #needs: [install-with-pip, install-with-conda] runs-on: ubuntu-latest timeout-minutes: 30 @@ -83,48 +83,48 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: CasperWA/postgresql-action@v1.2 - with: - postgresql version: '10' - postgresql db: test_${{ matrix.backend }} - postgresql user: postgres - postgresql password: '' - postgresql auth: trust - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system dependencies - run: | - wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add - - echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list - echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list - sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list - sudo apt update - sudo apt install postgresql-10 rabbitmq-server graphviz - sudo systemctl status rabbitmq-server.service - - - name: Install python dependencies - run: | - pip install --upgrade pip - pip install numpy==1.17.4 - pip install -e .[atomic_tools,docs,notebook,rest,testing] - reentry scan - - - name: Setup environment - env: - AIIDA_TEST_BACKEND: ${{ matrix.backend }} - run: - .github/workflows/setup.sh - - - name: Run test suite - env: - AIIDA_TEST_BACKEND: ${{ matrix.backend }} - run: - .github/workflows/tests.sh - +# - uses: CasperWA/postgresql-action@v1.2 +# with: +# postgresql version: '10' +# postgresql db: test_${{ matrix.backend }} +# postgresql user: postgres +# postgresql password: '' +# postgresql auth: trust +# +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v1 +# with: +# python-version: ${{ matrix.python-version }} +# +# - name: Install system dependencies +# run: | +# wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add - +# echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list +# echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list +# sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list +# sudo apt update +# sudo apt install postgresql-10 rabbitmq-server graphviz +# sudo systemctl status rabbitmq-server.service +# +# - name: Install python dependencies +# run: | +# pip install --upgrade pip +# pip install numpy==1.17.4 +# pip install -e .[atomic_tools,docs,notebook,rest,testing] +# reentry scan +# +# - name: Setup environment +# env: +# AIIDA_TEST_BACKEND: ${{ matrix.backend }} +# run: +# .github/workflows/setup.sh +# +# - name: Run test suite +# env: +# AIIDA_TEST_BACKEND: ${{ matrix.backend }} +# run: +# .github/workflows/tests.sh +# - name: Freeze test environment run: | pip freeze | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt From b6e29937b0f148707d4d1b2d4b287377de96e910 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 18:05:44 +0100 Subject: [PATCH 53/91] Attempt #3 at fixing update-requirements job. --- .github/workflows/dm.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 1e587dd70c..dd905a0bf0 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -131,7 +131,6 @@ jobs: - name: Upload requirements.txt uses: actions/upload-artifact@v1 - if: matrix.backend == 'django' && matrix.python-version == '3.8' with: name: requirements.txt path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt @@ -143,14 +142,17 @@ jobs: steps: - - name: Download requirements.txt + - name: Download requirements.txt files uses: actions/download-artifact@v1 with: name: requirements.txt + path: tmp - - name: Remove first line + - name: Prepare requirements.txt run: | + mv tmp/requirements.txt/requirements-django-py-3.8.txt requirements.txt sed -i '1d' requirements.txt + rm -r tmp/ - name: Create Pull Request uses: peter-evans/create-pull-request@v2 From 5fb8aac986f59fb9dcf6ff81be2b2d5a3b2e8546 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 18:08:20 +0100 Subject: [PATCH 54/91] Attempt #4 at fixing update-requirements job. --- .github/workflows/dm.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index dd905a0bf0..456a697620 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -150,7 +150,10 @@ jobs: - name: Prepare requirements.txt run: | - mv tmp/requirements.txt/requirements-django-py-3.8.txt requirements.txt + ls + ls * + ls tmp/ + mv tmp/requirements-django-py-3.8.txt requirements.txt sed -i '1d' requirements.txt rm -r tmp/ From ae89f5aa6aafb1d5d702d9c4f2e34fa50fdc4f19 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 18:10:08 +0100 Subject: [PATCH 55/91] Attempt #5 at fixing update-requirements job. --- .github/workflows/dm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 456a697620..7b0be17b86 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -141,6 +141,7 @@ jobs: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v2 - name: Download requirements.txt files uses: actions/download-artifact@v1 From b03813a89c765137aeb9858b83bd5eb2ef4ac7dc Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 18:13:55 +0100 Subject: [PATCH 56/91] Attempt #6 at fixing update-requirements job. --- .github/workflows/dm.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 7b0be17b86..6ba2fe5051 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -151,9 +151,6 @@ jobs: - name: Prepare requirements.txt run: | - ls - ls * - ls tmp/ mv tmp/requirements-django-py-3.8.txt requirements.txt sed -i '1d' requirements.txt rm -r tmp/ @@ -164,5 +161,5 @@ jobs: commit-message: "Update requirements.txt" token: ${{ secrets.GITHUB_TOKEN }} title: "Update requirements.txt" - reviewers: aiidateam/dependency-manager + team-reviewers: aiidateam/dependency-manager branch: "dm/update-requirements.txt" From a9ddbeb4c68a5f56a50d0cfd5133b3a2f3207e4c Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 18:17:01 +0100 Subject: [PATCH 57/91] Attempt #7 at fixing update-requirements job. --- .github/workflows/dm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 6ba2fe5051..d47af3d491 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -161,5 +161,5 @@ jobs: commit-message: "Update requirements.txt" token: ${{ secrets.GITHUB_TOKEN }} title: "Update requirements.txt" - team-reviewers: aiidateam/dependency-manager + team-reviewers: dependency-manager branch: "dm/update-requirements.txt" From 50bf61b1e67deffbe2803ac7657cfe7e506206e4 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 18:20:20 +0100 Subject: [PATCH 58/91] Revert "Disable large portions of the testing ci to speed up debugging." This reverts commit ffe1115e9da2df540b5c55c43e734a959f8059db. --- .github/workflows/dm.yml | 190 +++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index d47af3d491..3dca311bab 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -17,61 +17,61 @@ on: jobs: -# install-with-pip: -# -# runs-on: ubuntu-latest -# timeout-minutes: 5 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Set up Python 3.7 -# uses: actions/setup-python@v1 -# with: -# python-version: 3.7 -# -# - name: Pip install -# run: | -# python -m pip install -e . -# python -m pip freeze -# -# - name: Test package import -# run: -# python -c "import aiida" -# -# install-with-conda: -# -# runs-on: ubuntu-latest -# name: install-with-conda -# -# timeout-minutes: 5 -# -# steps: -# - uses: actions/checkout@v2 -# -# - name: Setup Conda -# uses: s-weigand/setup-conda@v1 -# with: -# update-conda: true -# python-version: 3.7 -# - run: conda --version -# - run: python --version -# - run: which python -# -# - name: Create conda environment -# run: | -# conda env create -f environment.yml -n test-environment -# source activate test-environment -# python -m pip install --no-deps -e . -# -# - name: Test package import -# run: | -# source activate test-environment -# python -c "import aiida" -# + install-with-pip: + + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Pip install + run: | + python -m pip install -e . + python -m pip freeze + + - name: Test package import + run: + python -c "import aiida" + + install-with-conda: + + runs-on: ubuntu-latest + name: install-with-conda + + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v2 + + - name: Setup Conda + uses: s-weigand/setup-conda@v1 + with: + update-conda: true + python-version: 3.7 + - run: conda --version + - run: python --version + - run: which python + + - name: Create conda environment + run: | + conda env create -f environment.yml -n test-environment + source activate test-environment + python -m pip install --no-deps -e . + + - name: Test package import + run: | + source activate test-environment + python -c "import aiida" + tests: - #needs: [install-with-pip, install-with-conda] + needs: [install-with-pip, install-with-conda] runs-on: ubuntu-latest timeout-minutes: 30 @@ -83,48 +83,48 @@ jobs: steps: - uses: actions/checkout@v2 -# - uses: CasperWA/postgresql-action@v1.2 -# with: -# postgresql version: '10' -# postgresql db: test_${{ matrix.backend }} -# postgresql user: postgres -# postgresql password: '' -# postgresql auth: trust -# -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v1 -# with: -# python-version: ${{ matrix.python-version }} -# -# - name: Install system dependencies -# run: | -# wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add - -# echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list -# echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list -# sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list -# sudo apt update -# sudo apt install postgresql-10 rabbitmq-server graphviz -# sudo systemctl status rabbitmq-server.service -# -# - name: Install python dependencies -# run: | -# pip install --upgrade pip -# pip install numpy==1.17.4 -# pip install -e .[atomic_tools,docs,notebook,rest,testing] -# reentry scan -# -# - name: Setup environment -# env: -# AIIDA_TEST_BACKEND: ${{ matrix.backend }} -# run: -# .github/workflows/setup.sh -# -# - name: Run test suite -# env: -# AIIDA_TEST_BACKEND: ${{ matrix.backend }} -# run: -# .github/workflows/tests.sh -# + - uses: CasperWA/postgresql-action@v1.2 + with: + postgresql version: '10' + postgresql db: test_${{ matrix.backend }} + postgresql user: postgres + postgresql password: '' + postgresql auth: trust + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add - + echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list + echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list + sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list + sudo apt update + sudo apt install postgresql-10 rabbitmq-server graphviz + sudo systemctl status rabbitmq-server.service + + - name: Install python dependencies + run: | + pip install --upgrade pip + pip install numpy==1.17.4 + pip install -e .[atomic_tools,docs,notebook,rest,testing] + reentry scan + + - name: Setup environment + env: + AIIDA_TEST_BACKEND: ${{ matrix.backend }} + run: + .github/workflows/setup.sh + + - name: Run test suite + env: + AIIDA_TEST_BACKEND: ${{ matrix.backend }} + run: + .github/workflows/tests.sh + - name: Freeze test environment run: | pip freeze | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt From ecf1fd1b0f05e7544d58c50bf9d0a0ac06b97861 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Wed, 4 Mar 2020 18:22:35 +0100 Subject: [PATCH 59/91] Restore previously disabled CI steps. --- .github/workflows/ci.yml | 2 -- .github/workflows/dm.yml | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ecaeca50d..9f5544c3c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,6 @@ name: aiida-core on: [push, pull_request] - branch: - - develop # TODO: Remove before merge jobs: diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 3dca311bab..0c3193fffd 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -78,8 +78,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8] # TODO: Change to all supported Python versions before merge - backend: ['django'] # TODO: Add 'sqlalchemy' before merge + python-version: [3.5, 3.6, 3.7, 3.8] + backend: ['django', 'sqlalchemy'] steps: - uses: actions/checkout@v2 From 91b1eba0c3c2b53049e08c4b53b47aa10ad66f82 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2020 17:49:11 +0000 Subject: [PATCH 60/91] Update requirements.txt --- requirements.txt | 147 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..4f58a20c69 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,147 @@ +aiida-export-migration-tests==0.8.0 +alabaster==0.7.12 +aldjemy==0.9.1 +alembic==1.4.1 +aniso8601==8.0.0 +ase==3.19.0 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +bleach==3.1.1 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +circus==0.16.1 +Click==7.0 +click-completion==0.5.2 +click-config-file==0.5.0 +click-spinner==0.1.8 +configobj==5.0.6 +coverage==4.5.4 +cryptography==2.8 +cycler==0.10.0 +decorator==4.4.2 +defusedxml==0.6.0 +Django==2.2.11 +docutils==0.15.2 +entrypoints==0.3 +ete3==3.1.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +frozendict==1.2 +furl==2.1.0 +future==0.18.2 +graphviz==0.13.2 +idna==2.9 +imagesize==1.2.0 +ipykernel==5.1.4 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +itsdangerous==1.1.0 +jedi==0.16.0 +Jinja2==2.11.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.0.0 +jupyter-console==6.1.0 +jupyter-core==4.6.3 +kiwipy==0.5.3 +kiwisolver==1.1.0 +Mako==1.1.2 +MarkupSafe==1.1.1 +matplotlib==3.1.3 +mistune==0.8.4 +monty==3.0.2 +more-itertools==8.2.0 +mpmath==1.1.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==5.7.8 +numpy==1.17.4 +orderedmultidict==1.0.1 +packaging==20.1 +palettable==3.3.0 +pandas==0.25.3 +pandocfilters==1.4.2 +paramiko==2.7.1 +parso==0.6.2 +pexpect==4.8.0 +pg8000==1.13.2 +pgtest==1.3.2 +pickleshare==0.7.5 +pika==1.1.0 +pluggy==0.13.1 +plumpy==0.14.5 +prometheus-client==0.7.1 +prompt-toolkit==3.0.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +PyCifRW==4.4.1 +pycparser==2.20 +PyDispatcher==2.0.5 +Pygments==2.5.2 +pymatgen==2020.3.2 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-timeout==1.3.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-memcached==1.59 +pytz==2019.3 +PyYAML==5.1.2 +pyzmq==19.0.0 +qtconsole==4.7.1 +QtPy==1.9.0 +reentry==1.3.1 +requests==2.23.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +scipy==1.4.1 +scramp==1.1.0 +seekpath==1.9.4 +Send2Trash==1.5.0 +shellingham==1.3.2 +shortuuid==0.5.0 +simplejson==3.17.0 +six==1.14.0 +snowballstemmer==2.0.0 +spglib==1.14.1.post0 +Sphinx==2.4.3 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-contentui==0.2.4 +sphinxcontrib-details-directive==0.1.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.13 +sqlalchemy-diff==0.1.3 +SQLAlchemy-Utils==0.34.2 +sqlparse==0.3.1 +sympy==1.5.1 +tabulate==0.8.6 +terminado==0.8.3 +testpath==0.4.4 +topika==0.2.1 +tornado==4.5.3 +traitlets==4.3.3 +tzlocal==2.0.0 +upf-to-json==0.9.2 +urllib3==1.25.8 +wcwidth==0.1.8 +webencodings==0.5.1 +Werkzeug==1.0.0 +widgetsnbextension==3.5.1 +wrapt==1.11.2 From 61769f36e32acdb05457b2b7b448a972be9be7b0 Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Thu, 5 Mar 2020 15:29:36 +0100 Subject: [PATCH 61/91] Remove obsolete constant in dependency management script. --- utils/dependency_management.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 3c7c4d023d..a91625c156 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -24,8 +24,6 @@ ROOT = Path(__file__).resolve().parent.parent # repository root -DEFAULT_EXCLUDE_LIST = ['django', 'circus', 'numpy', 'pymatgen', 'ase', 'monty', 'pyyaml'] - SETUPTOOLS_CONDA_MAPPINGS = { 'psycopg2-binary': 'psycopg2', 'graphviz': 'python-graphviz', From 613141a9363445763b6776f011efe4515fb16eb4 Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Thu, 5 Mar 2020 15:36:59 +0100 Subject: [PATCH 62/91] Update documentation of the dependency management script. --- utils/dependency_management.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index a91625c156..7c28c412d6 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -8,7 +8,7 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### -"""Utility CLI to update dependency version requirements of the `setup.json`.""" +"""Utility CLI to manage dependencies for aiida-core.""" import sys import re @@ -61,6 +61,12 @@ def _load_environment_yml(): def _setuptools_to_conda(req): + """Map package names from setuptools to conda where necessary. + + In case that the same underlying dependency is listed under different names + on PyPI and conda-forge. + """ + for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): if re.match(pattern, str(req)): return Requirement.parse(re.sub(pattern, replacement, str(req))) @@ -155,14 +161,8 @@ def generate_all(ctx): @cli.command('validate-environment-yml', help="Validate 'environment.yml'.") def validate_environment_yml(): # pylint: disable=too-many-branches - """Validate consistency of the requirements specification of the package. + """Validate that 'environment.yml' is consistent with 'setup.json'.""" - Validates that the specification of requirements/dependencies is consistent across - the following files: - - - setup.json - - environment.yml - """ # Read the requirements from 'setup.json' and 'environment.yml'. setup_cfg = _load_setup_cfg() install_requirements = [Requirement.parse(r) for r in setup_cfg['install_requires']] @@ -234,7 +234,7 @@ def validate_environment_yml(): # pylint: disable=too-many-branches @cli.command('validate-rtd-reqs', help="Validate 'docs/requirements_for_rtd.txt'.") def validate_requirements_for_rtd(): - """Validate consistency of the specification of 'docs/requirements_for_rtd.txt'.""" + """Validate that 'docs/requirements_for_rtd.txt' is consistent with 'setup.json'.""" # Read the requirements from 'setup.json' setup_cfg = _load_setup_cfg() @@ -253,7 +253,7 @@ def validate_requirements_for_rtd(): @cli.command('validate-pyproject-toml', help="Validate 'pyproject.toml'.") def validate_pyproject_toml(): - """Validate consistency of the specification of 'pyprojec.toml'.""" + """Validate that 'pyproject.toml' is consistent with 'setup.json'.""" # Read the requirements from 'setup.json' setup_cfg = _load_setup_cfg() From 98f751b7fc0ff463291a2215708d194b68e5c0ca Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Thu, 5 Mar 2020 15:44:48 +0100 Subject: [PATCH 63/91] Add 'validate-dependency-specification' job to dm workflow. --- .github/workflows/dm.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 0c3193fffd..623a1cd2a4 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -17,6 +17,26 @@ on: jobs: + validate-dependency-specification: + # Note: The specification is also validated by the pre-commit hook. + + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Install dm-script dependencies + run: pip install click~=7.0 pyyaml~=5.1 toml + + - name: Validate + run: python ./utils/dependency_management.py validate-all + install-with-pip: runs-on: ubuntu-latest From d0fb277b490e4c86756d4c8a01c8e1620e45a4fe Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Thu, 5 Mar 2020 16:24:50 +0100 Subject: [PATCH 64/91] Do not run dm workflow on misc/dependency-management branch. This was only needed for development. --- .github/workflows/dm.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 623a1cd2a4..09e396ad86 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -5,7 +5,6 @@ on: branch: - develop - dm/* - - misc/update-dependency-management # TODO: Remove before merge paths: - 'setup.*' - 'environment.yml' From 469cb37159b3a572e1741270f380700307298c0c Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Fri, 6 Mar 2020 09:36:05 +0100 Subject: [PATCH 65/91] Apply suggestions from code review Co-Authored-By: Leopold Talirz --- utils/dependency_management.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 7c28c412d6..6a0a8be8b8 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -303,12 +303,12 @@ def validate_all(ctx): @cli.command() -@click.argument('extras', nargs=-1) +@click.argument('extras', nargs=-1, help="extra requirements, e.g. 'atomic_tools'") def pip_install_extras(extras): """Install extra requirements. - Install extra requirements in isolation without triggering the installlation - of other installation requirements of the core package. + Install *only* extra requirements without triggering the installation of + of the main installation requirements of the aiida-core package. """ # Read the requirements from 'setup.json' setup_cfg = _load_setup_cfg() From 5378f7dcf67cfd972ff7c3e0c7c245370eb5bbca Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 09:32:52 +0100 Subject: [PATCH 66/91] Fix generate-pyproject-toml to better preserve current file. Also update the current file to match the exact format of the automatically generated file. Unfortunately that means the removal of the comment, because toml currently does not support preservation of comments, see here: https://github.com/uiri/toml/issues/77 --- pyproject.toml | 3 +-- utils/dependency_management.py | 7 ++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a4e1ecf2f4..24eaa8393c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,3 @@ [build-system] -# Minimum requirements for the build system to execute. -requires = ["setuptools>=40.8.0", "wheel", "reentry~=1.3"] +requires = [ "setuptools>=40.8.0", "wheel", "reentry~=1.3",] build-backend = "setuptools.build_meta:__legacy__" diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 6a0a8be8b8..8a36a580ac 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -145,7 +145,12 @@ def generate_pyproject_toml(): else: raise DependencySpecificationError("Failed to find reentry requirement in 'setup.json'.") - pyproject = {'build-system': {'requires': ['setuptools', 'wheel', str(reentry_requirement)]}} + pyproject = { + 'build-system': { + 'requires': ['setuptools>=40.8.0', 'wheel', str(reentry_requirement)], + 'build-backend': 'setuptools.build_meta:__legacy__', + } + } with open(ROOT / 'pyproject.toml', 'w') as file: toml.dump(pyproject, file) From 17b7e2709cb46b18aba32bb1e6d62a7cdb6be244 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 09:49:23 +0100 Subject: [PATCH 67/91] Fix definition of pip-install-extras command. Broken by 469cb37159b3a572e1741270f380700307298c0c. --- utils/dependency_management.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/utils/dependency_management.py b/utils/dependency_management.py index 8a36a580ac..f187102c57 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -308,12 +308,16 @@ def validate_all(ctx): @cli.command() -@click.argument('extras', nargs=-1, help="extra requirements, e.g. 'atomic_tools'") +@click.argument('extras', nargs=-1) def pip_install_extras(extras): """Install extra requirements. - Install *only* extra requirements without triggering the installation of - of the main installation requirements of the aiida-core package. + For example: + + pip-install-extras docs + + This will install *only* the extra the requirements for docs, but without triggering + the installation of the main installations requirements of the aiida-core package. """ # Read the requirements from 'setup.json' setup_cfg = _load_setup_cfg() From 15a302a5a86b97dc4b455c9fa1ce141d2a405ed4 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 10:10:26 +0100 Subject: [PATCH 68/91] Add release workflow per suggestion. To only update requirements.txt for release. --- .github/workflows/dm.yml | 31 +----------- .github/workflows/release.yml | 95 +++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/dm.yml b/.github/workflows/dm.yml index 09e396ad86..94ac224a21 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/dm.yml @@ -3,7 +3,9 @@ name: aiida-core-dependency-management on: push: branch: + - master - develop + - release/* - dm/* paths: - 'setup.*' @@ -153,32 +155,3 @@ jobs: with: name: requirements.txt path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt - - update-requirements: - - needs: tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Download requirements.txt files - uses: actions/download-artifact@v1 - with: - name: requirements.txt - path: tmp - - - name: Prepare requirements.txt - run: | - mv tmp/requirements-django-py-3.8.txt requirements.txt - sed -i '1d' requirements.txt - rm -r tmp/ - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v2 - with: - commit-message: "Update requirements.txt" - token: ${{ secrets.GITHUB_TOKEN }} - title: "Update requirements.txt" - team-reviewers: dependency-manager - branch: "dm/update-requirements.txt" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..a9397fa512 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,95 @@ +name: aiida-core-release + +on: + push: + branch: + - release/* + +jobs: + + test: + + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v2 + - uses: CasperWA/postgresql-action@v1.2 + with: + postgresql version: '10' + postgresql db: test_django + postgresql user: postgres + postgresql password: '' + postgresql auth: trust + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install system dependencies + run: | + wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add - + echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list + echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list + sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list + sudo apt update + sudo apt install postgresql-10 rabbitmq-server graphviz + sudo systemctl status rabbitmq-server.service + + - name: Install python dependencies + run: | + pip install --upgrade pip + pip install numpy==1.17.4 + pip install -e .[atomic_tools,docs,notebook,rest,testing] + reentry scan + + - name: Setup environment + env: + AIIDA_TEST_BACKEND: django + run: + .github/workflows/setup.sh + + - name: Run test suite + env: + AIIDA_TEST_BACKEND: django + run: + .github/workflows/tests.sh + + - name: Freeze test environment + run: pip freeze | tee requirements.txt + + - name: Upload requirements.txt + uses: actions/upload-artifact@v1 + with: + name: requirements.txt + path: requirements.txt + + update-requirements: + + needs: test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Download requirements.txt files + uses: actions/download-artifact@v1 + with: + name: requirements.txt + path: tmp + + - name: Prepare requirements.txt + run: | + mv tmp/requirements.txt requirements.txt + sed -i '1d' requirements.txt + rm -r tmp/ + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v2 + with: + commit-message: "Update requirements.txt" + token: ${{ secrets.GITHUB_TOKEN }} + title: "Update requirements.txt" + team-reviewers: dependency-manager + branch: "dm/update-requirements.txt" From 71011f74bd957ccc0846fd7da4d6285055ff070d Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 17:49:11 +0100 Subject: [PATCH 69/91] Add backend and python-version specific requirements files. --- requirements/requirements-django-py-3.5.txt | 151 ++++++++++++++++++ requirements/requirements-django-py-3.6.txt | 150 +++++++++++++++++ requirements/requirements-django-py-3.7.txt | 149 +++++++++++++++++ .../requirements-django-py-3.8.txt | 8 +- .../requirements-sqlalchemy-py-3.5.txt | 151 ++++++++++++++++++ .../requirements-sqlalchemy-py-3.6.txt | 150 +++++++++++++++++ .../requirements-sqlalchemy-py-3.7.txt | 149 +++++++++++++++++ .../requirements-sqlalchemy-py-3.8.txt | 147 +++++++++++++++++ 8 files changed, 1051 insertions(+), 4 deletions(-) create mode 100644 requirements/requirements-django-py-3.5.txt create mode 100644 requirements/requirements-django-py-3.6.txt create mode 100644 requirements/requirements-django-py-3.7.txt rename requirements.txt => requirements/requirements-django-py-3.8.txt (97%) create mode 100644 requirements/requirements-sqlalchemy-py-3.5.txt create mode 100644 requirements/requirements-sqlalchemy-py-3.6.txt create mode 100644 requirements/requirements-sqlalchemy-py-3.7.txt create mode 100644 requirements/requirements-sqlalchemy-py-3.8.txt diff --git a/requirements/requirements-django-py-3.5.txt b/requirements/requirements-django-py-3.5.txt new file mode 100644 index 0000000000..00758dd76a --- /dev/null +++ b/requirements/requirements-django-py-3.5.txt @@ -0,0 +1,151 @@ +aiida-export-migration-tests==0.8.0 +alabaster==0.7.12 +aldjemy==0.9.1 +alembic==1.4.1 +aniso8601==8.0.0 +ase==3.19.0 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +bleach==3.1.1 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +circus==0.16.1 +Click==7.0 +click-completion==0.5.2 +click-config-file==0.5.0 +click-spinner==0.1.8 +configobj==5.0.6 +coverage==4.5.4 +cryptography==2.8 +cycler==0.10.0 +decorator==4.4.2 +defusedxml==0.6.0 +Django==2.2.11 +docutils==0.15.2 +entrypoints==0.3 +ete3==3.1.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +frozendict==1.2 +furl==2.1.0 +future==0.18.2 +graphviz==0.13.2 +idna==2.9 +imagesize==1.2.0 +importlib-metadata==1.5.0 +ipykernel==5.1.4 +ipython==7.9.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +itsdangerous==1.1.0 +jedi==0.16.0 +Jinja2==2.11.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.0.0 +jupyter-console==6.1.0 +jupyter-core==4.6.3 +kiwipy==0.5.3 +kiwisolver==1.1.0 +Mako==1.1.2 +MarkupSafe==1.1.1 +matplotlib==3.0.3 +mistune==0.8.4 +monty==3.0.2 +more-itertools==8.2.0 +mpmath==1.1.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==5.7.8 +numpy==1.17.4 +orderedmultidict==1.0.1 +packaging==20.3 +palettable==3.3.0 +pandas==0.25.3 +pandocfilters==1.4.2 +paramiko==2.7.1 +parso==0.6.2 +pathlib2==2.3.5 +pexpect==4.8.0 +pg8000==1.13.2 +pgtest==1.3.2 +pickleshare==0.7.5 +pika==1.1.0 +pluggy==0.13.1 +plumpy==0.14.5 +prometheus-client==0.7.1 +prompt-toolkit==2.0.10 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +pyblake2==1.1.2 +PyCifRW==4.4.1 +pycparser==2.20 +PyDispatcher==2.0.5 +Pygments==2.5.2 +pymatgen==2019.7.2 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-timeout==1.3.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-memcached==1.59 +pytz==2019.3 +PyYAML==5.1.2 +pyzmq==19.0.0 +qtconsole==4.7.1 +QtPy==1.9.0 +reentry==1.3.1 +requests==2.23.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +scipy==1.4.1 +scramp==1.1.0 +seekpath==1.9.4 +Send2Trash==1.5.0 +shellingham==1.3.2 +shortuuid==0.5.1 +simplejson==3.17.0 +six==1.14.0 +snowballstemmer==2.0.0 +spglib==1.14.1.post0 +Sphinx==2.4.4 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-contentui==0.2.4 +sphinxcontrib-details-directive==0.1.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.13 +sqlalchemy-diff==0.1.3 +SQLAlchemy-Utils==0.34.2 +sqlparse==0.3.1 +sympy==1.5.1 +tabulate==0.8.6 +terminado==0.8.3 +testpath==0.4.4 +topika==0.2.1 +tornado==4.5.3 +traitlets==4.3.3 +tzlocal==2.0.0 +upf-to-json==0.9.2 +urllib3==1.25.8 +wcwidth==0.1.8 +webencodings==0.5.1 +Werkzeug==1.0.0 +widgetsnbextension==3.5.1 +wrapt==1.11.2 +zipp==1.2.0 diff --git a/requirements/requirements-django-py-3.6.txt b/requirements/requirements-django-py-3.6.txt new file mode 100644 index 0000000000..7432db3060 --- /dev/null +++ b/requirements/requirements-django-py-3.6.txt @@ -0,0 +1,150 @@ +aiida-export-migration-tests==0.8.0 +alabaster==0.7.12 +aldjemy==0.9.1 +alembic==1.4.1 +aniso8601==8.0.0 +ase==3.19.0 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +bleach==3.1.1 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +circus==0.16.1 +Click==7.0 +click-completion==0.5.2 +click-config-file==0.5.0 +click-spinner==0.1.8 +configobj==5.0.6 +coverage==4.5.4 +cryptography==2.8 +cycler==0.10.0 +dataclasses==0.7 +decorator==4.4.2 +defusedxml==0.6.0 +Django==2.2.11 +docutils==0.15.2 +entrypoints==0.3 +ete3==3.1.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +frozendict==1.2 +furl==2.1.0 +future==0.18.2 +graphviz==0.13.2 +idna==2.9 +imagesize==1.2.0 +importlib-metadata==1.5.0 +ipykernel==5.1.4 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +itsdangerous==1.1.0 +jedi==0.16.0 +Jinja2==2.11.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.0.0 +jupyter-console==6.1.0 +jupyter-core==4.6.3 +kiwipy==0.5.3 +kiwisolver==1.1.0 +Mako==1.1.2 +MarkupSafe==1.1.1 +matplotlib==3.2.0 +mistune==0.8.4 +monty==3.0.2 +more-itertools==8.2.0 +mpmath==1.1.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==5.7.8 +numpy==1.17.4 +orderedmultidict==1.0.1 +packaging==20.3 +palettable==3.3.0 +pandas==0.25.3 +pandocfilters==1.4.2 +paramiko==2.7.1 +parso==0.6.2 +pexpect==4.8.0 +pg8000==1.13.2 +pgtest==1.3.2 +pickleshare==0.7.5 +pika==1.1.0 +pluggy==0.13.1 +plumpy==0.14.5 +prometheus-client==0.7.1 +prompt-toolkit==3.0.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +PyCifRW==4.4.1 +pycparser==2.20 +PyDispatcher==2.0.5 +Pygments==2.5.2 +pymatgen==2020.3.2 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-timeout==1.3.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-memcached==1.59 +pytz==2019.3 +PyYAML==5.1.2 +pyzmq==19.0.0 +qtconsole==4.7.1 +QtPy==1.9.0 +reentry==1.3.1 +requests==2.23.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +scipy==1.4.1 +scramp==1.1.0 +seekpath==1.9.4 +Send2Trash==1.5.0 +shellingham==1.3.2 +shortuuid==0.5.1 +simplejson==3.17.0 +six==1.14.0 +snowballstemmer==2.0.0 +spglib==1.14.1.post0 +Sphinx==2.4.4 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-contentui==0.2.4 +sphinxcontrib-details-directive==0.1.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.13 +sqlalchemy-diff==0.1.3 +SQLAlchemy-Utils==0.34.2 +sqlparse==0.3.1 +sympy==1.5.1 +tabulate==0.8.6 +terminado==0.8.3 +testpath==0.4.4 +topika==0.2.1 +tornado==4.5.3 +traitlets==4.3.3 +tzlocal==2.0.0 +upf-to-json==0.9.2 +urllib3==1.25.8 +wcwidth==0.1.8 +webencodings==0.5.1 +Werkzeug==1.0.0 +widgetsnbextension==3.5.1 +wrapt==1.11.2 +zipp==3.1.0 diff --git a/requirements/requirements-django-py-3.7.txt b/requirements/requirements-django-py-3.7.txt new file mode 100644 index 0000000000..28ffb9e049 --- /dev/null +++ b/requirements/requirements-django-py-3.7.txt @@ -0,0 +1,149 @@ +aiida-export-migration-tests==0.8.0 +alabaster==0.7.12 +aldjemy==0.9.1 +alembic==1.4.1 +aniso8601==8.0.0 +ase==3.19.0 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +bleach==3.1.1 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +circus==0.16.1 +Click==7.0 +click-completion==0.5.2 +click-config-file==0.5.0 +click-spinner==0.1.8 +configobj==5.0.6 +coverage==4.5.4 +cryptography==2.8 +cycler==0.10.0 +decorator==4.4.2 +defusedxml==0.6.0 +Django==2.2.11 +docutils==0.15.2 +entrypoints==0.3 +ete3==3.1.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +frozendict==1.2 +furl==2.1.0 +future==0.18.2 +graphviz==0.13.2 +idna==2.9 +imagesize==1.2.0 +importlib-metadata==1.5.0 +ipykernel==5.1.4 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +itsdangerous==1.1.0 +jedi==0.16.0 +Jinja2==2.11.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.0.0 +jupyter-console==6.1.0 +jupyter-core==4.6.3 +kiwipy==0.5.3 +kiwisolver==1.1.0 +Mako==1.1.2 +MarkupSafe==1.1.1 +matplotlib==3.2.0 +mistune==0.8.4 +monty==3.0.2 +more-itertools==8.2.0 +mpmath==1.1.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==5.7.8 +numpy==1.17.4 +orderedmultidict==1.0.1 +packaging==20.3 +palettable==3.3.0 +pandas==0.25.3 +pandocfilters==1.4.2 +paramiko==2.7.1 +parso==0.6.2 +pexpect==4.8.0 +pg8000==1.13.2 +pgtest==1.3.2 +pickleshare==0.7.5 +pika==1.1.0 +pluggy==0.13.1 +plumpy==0.14.5 +prometheus-client==0.7.1 +prompt-toolkit==3.0.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +PyCifRW==4.4.1 +pycparser==2.20 +PyDispatcher==2.0.5 +Pygments==2.5.2 +pymatgen==2020.3.2 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-timeout==1.3.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-memcached==1.59 +pytz==2019.3 +PyYAML==5.1.2 +pyzmq==19.0.0 +qtconsole==4.7.1 +QtPy==1.9.0 +reentry==1.3.1 +requests==2.23.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +scipy==1.4.1 +scramp==1.1.0 +seekpath==1.9.4 +Send2Trash==1.5.0 +shellingham==1.3.2 +shortuuid==0.5.1 +simplejson==3.17.0 +six==1.14.0 +snowballstemmer==2.0.0 +spglib==1.14.1.post0 +Sphinx==2.4.4 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-contentui==0.2.4 +sphinxcontrib-details-directive==0.1.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.13 +sqlalchemy-diff==0.1.3 +SQLAlchemy-Utils==0.34.2 +sqlparse==0.3.1 +sympy==1.5.1 +tabulate==0.8.6 +terminado==0.8.3 +testpath==0.4.4 +topika==0.2.1 +tornado==4.5.3 +traitlets==4.3.3 +tzlocal==2.0.0 +upf-to-json==0.9.2 +urllib3==1.25.8 +wcwidth==0.1.8 +webencodings==0.5.1 +Werkzeug==1.0.0 +widgetsnbextension==3.5.1 +wrapt==1.11.2 +zipp==3.1.0 diff --git a/requirements.txt b/requirements/requirements-django-py-3.8.txt similarity index 97% rename from requirements.txt rename to requirements/requirements-django-py-3.8.txt index 4f58a20c69..c6d91577a6 100644 --- a/requirements.txt +++ b/requirements/requirements-django-py-3.8.txt @@ -52,7 +52,7 @@ kiwipy==0.5.3 kiwisolver==1.1.0 Mako==1.1.2 MarkupSafe==1.1.1 -matplotlib==3.1.3 +matplotlib==3.2.0 mistune==0.8.4 monty==3.0.2 more-itertools==8.2.0 @@ -63,7 +63,7 @@ networkx==2.4 notebook==5.7.8 numpy==1.17.4 orderedmultidict==1.0.1 -packaging==20.1 +packaging==20.3 palettable==3.3.0 pandas==0.25.3 pandocfilters==1.4.2 @@ -111,12 +111,12 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.0 +shortuuid==0.5.1 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 spglib==1.14.1.post0 -Sphinx==2.4.3 +Sphinx==2.4.4 sphinx-rtd-theme==0.4.3 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-contentui==0.2.4 diff --git a/requirements/requirements-sqlalchemy-py-3.5.txt b/requirements/requirements-sqlalchemy-py-3.5.txt new file mode 100644 index 0000000000..00758dd76a --- /dev/null +++ b/requirements/requirements-sqlalchemy-py-3.5.txt @@ -0,0 +1,151 @@ +aiida-export-migration-tests==0.8.0 +alabaster==0.7.12 +aldjemy==0.9.1 +alembic==1.4.1 +aniso8601==8.0.0 +ase==3.19.0 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +bleach==3.1.1 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +circus==0.16.1 +Click==7.0 +click-completion==0.5.2 +click-config-file==0.5.0 +click-spinner==0.1.8 +configobj==5.0.6 +coverage==4.5.4 +cryptography==2.8 +cycler==0.10.0 +decorator==4.4.2 +defusedxml==0.6.0 +Django==2.2.11 +docutils==0.15.2 +entrypoints==0.3 +ete3==3.1.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +frozendict==1.2 +furl==2.1.0 +future==0.18.2 +graphviz==0.13.2 +idna==2.9 +imagesize==1.2.0 +importlib-metadata==1.5.0 +ipykernel==5.1.4 +ipython==7.9.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +itsdangerous==1.1.0 +jedi==0.16.0 +Jinja2==2.11.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.0.0 +jupyter-console==6.1.0 +jupyter-core==4.6.3 +kiwipy==0.5.3 +kiwisolver==1.1.0 +Mako==1.1.2 +MarkupSafe==1.1.1 +matplotlib==3.0.3 +mistune==0.8.4 +monty==3.0.2 +more-itertools==8.2.0 +mpmath==1.1.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==5.7.8 +numpy==1.17.4 +orderedmultidict==1.0.1 +packaging==20.3 +palettable==3.3.0 +pandas==0.25.3 +pandocfilters==1.4.2 +paramiko==2.7.1 +parso==0.6.2 +pathlib2==2.3.5 +pexpect==4.8.0 +pg8000==1.13.2 +pgtest==1.3.2 +pickleshare==0.7.5 +pika==1.1.0 +pluggy==0.13.1 +plumpy==0.14.5 +prometheus-client==0.7.1 +prompt-toolkit==2.0.10 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +pyblake2==1.1.2 +PyCifRW==4.4.1 +pycparser==2.20 +PyDispatcher==2.0.5 +Pygments==2.5.2 +pymatgen==2019.7.2 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-timeout==1.3.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-memcached==1.59 +pytz==2019.3 +PyYAML==5.1.2 +pyzmq==19.0.0 +qtconsole==4.7.1 +QtPy==1.9.0 +reentry==1.3.1 +requests==2.23.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +scipy==1.4.1 +scramp==1.1.0 +seekpath==1.9.4 +Send2Trash==1.5.0 +shellingham==1.3.2 +shortuuid==0.5.1 +simplejson==3.17.0 +six==1.14.0 +snowballstemmer==2.0.0 +spglib==1.14.1.post0 +Sphinx==2.4.4 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-contentui==0.2.4 +sphinxcontrib-details-directive==0.1.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.13 +sqlalchemy-diff==0.1.3 +SQLAlchemy-Utils==0.34.2 +sqlparse==0.3.1 +sympy==1.5.1 +tabulate==0.8.6 +terminado==0.8.3 +testpath==0.4.4 +topika==0.2.1 +tornado==4.5.3 +traitlets==4.3.3 +tzlocal==2.0.0 +upf-to-json==0.9.2 +urllib3==1.25.8 +wcwidth==0.1.8 +webencodings==0.5.1 +Werkzeug==1.0.0 +widgetsnbextension==3.5.1 +wrapt==1.11.2 +zipp==1.2.0 diff --git a/requirements/requirements-sqlalchemy-py-3.6.txt b/requirements/requirements-sqlalchemy-py-3.6.txt new file mode 100644 index 0000000000..7432db3060 --- /dev/null +++ b/requirements/requirements-sqlalchemy-py-3.6.txt @@ -0,0 +1,150 @@ +aiida-export-migration-tests==0.8.0 +alabaster==0.7.12 +aldjemy==0.9.1 +alembic==1.4.1 +aniso8601==8.0.0 +ase==3.19.0 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +bleach==3.1.1 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +circus==0.16.1 +Click==7.0 +click-completion==0.5.2 +click-config-file==0.5.0 +click-spinner==0.1.8 +configobj==5.0.6 +coverage==4.5.4 +cryptography==2.8 +cycler==0.10.0 +dataclasses==0.7 +decorator==4.4.2 +defusedxml==0.6.0 +Django==2.2.11 +docutils==0.15.2 +entrypoints==0.3 +ete3==3.1.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +frozendict==1.2 +furl==2.1.0 +future==0.18.2 +graphviz==0.13.2 +idna==2.9 +imagesize==1.2.0 +importlib-metadata==1.5.0 +ipykernel==5.1.4 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +itsdangerous==1.1.0 +jedi==0.16.0 +Jinja2==2.11.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.0.0 +jupyter-console==6.1.0 +jupyter-core==4.6.3 +kiwipy==0.5.3 +kiwisolver==1.1.0 +Mako==1.1.2 +MarkupSafe==1.1.1 +matplotlib==3.2.0 +mistune==0.8.4 +monty==3.0.2 +more-itertools==8.2.0 +mpmath==1.1.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==5.7.8 +numpy==1.17.4 +orderedmultidict==1.0.1 +packaging==20.3 +palettable==3.3.0 +pandas==0.25.3 +pandocfilters==1.4.2 +paramiko==2.7.1 +parso==0.6.2 +pexpect==4.8.0 +pg8000==1.13.2 +pgtest==1.3.2 +pickleshare==0.7.5 +pika==1.1.0 +pluggy==0.13.1 +plumpy==0.14.5 +prometheus-client==0.7.1 +prompt-toolkit==3.0.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +PyCifRW==4.4.1 +pycparser==2.20 +PyDispatcher==2.0.5 +Pygments==2.5.2 +pymatgen==2020.3.2 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-timeout==1.3.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-memcached==1.59 +pytz==2019.3 +PyYAML==5.1.2 +pyzmq==19.0.0 +qtconsole==4.7.1 +QtPy==1.9.0 +reentry==1.3.1 +requests==2.23.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +scipy==1.4.1 +scramp==1.1.0 +seekpath==1.9.4 +Send2Trash==1.5.0 +shellingham==1.3.2 +shortuuid==0.5.1 +simplejson==3.17.0 +six==1.14.0 +snowballstemmer==2.0.0 +spglib==1.14.1.post0 +Sphinx==2.4.4 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-contentui==0.2.4 +sphinxcontrib-details-directive==0.1.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.13 +sqlalchemy-diff==0.1.3 +SQLAlchemy-Utils==0.34.2 +sqlparse==0.3.1 +sympy==1.5.1 +tabulate==0.8.6 +terminado==0.8.3 +testpath==0.4.4 +topika==0.2.1 +tornado==4.5.3 +traitlets==4.3.3 +tzlocal==2.0.0 +upf-to-json==0.9.2 +urllib3==1.25.8 +wcwidth==0.1.8 +webencodings==0.5.1 +Werkzeug==1.0.0 +widgetsnbextension==3.5.1 +wrapt==1.11.2 +zipp==3.1.0 diff --git a/requirements/requirements-sqlalchemy-py-3.7.txt b/requirements/requirements-sqlalchemy-py-3.7.txt new file mode 100644 index 0000000000..28ffb9e049 --- /dev/null +++ b/requirements/requirements-sqlalchemy-py-3.7.txt @@ -0,0 +1,149 @@ +aiida-export-migration-tests==0.8.0 +alabaster==0.7.12 +aldjemy==0.9.1 +alembic==1.4.1 +aniso8601==8.0.0 +ase==3.19.0 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +bleach==3.1.1 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +circus==0.16.1 +Click==7.0 +click-completion==0.5.2 +click-config-file==0.5.0 +click-spinner==0.1.8 +configobj==5.0.6 +coverage==4.5.4 +cryptography==2.8 +cycler==0.10.0 +decorator==4.4.2 +defusedxml==0.6.0 +Django==2.2.11 +docutils==0.15.2 +entrypoints==0.3 +ete3==3.1.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +frozendict==1.2 +furl==2.1.0 +future==0.18.2 +graphviz==0.13.2 +idna==2.9 +imagesize==1.2.0 +importlib-metadata==1.5.0 +ipykernel==5.1.4 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +itsdangerous==1.1.0 +jedi==0.16.0 +Jinja2==2.11.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.0.0 +jupyter-console==6.1.0 +jupyter-core==4.6.3 +kiwipy==0.5.3 +kiwisolver==1.1.0 +Mako==1.1.2 +MarkupSafe==1.1.1 +matplotlib==3.2.0 +mistune==0.8.4 +monty==3.0.2 +more-itertools==8.2.0 +mpmath==1.1.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==5.7.8 +numpy==1.17.4 +orderedmultidict==1.0.1 +packaging==20.3 +palettable==3.3.0 +pandas==0.25.3 +pandocfilters==1.4.2 +paramiko==2.7.1 +parso==0.6.2 +pexpect==4.8.0 +pg8000==1.13.2 +pgtest==1.3.2 +pickleshare==0.7.5 +pika==1.1.0 +pluggy==0.13.1 +plumpy==0.14.5 +prometheus-client==0.7.1 +prompt-toolkit==3.0.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +PyCifRW==4.4.1 +pycparser==2.20 +PyDispatcher==2.0.5 +Pygments==2.5.2 +pymatgen==2020.3.2 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-timeout==1.3.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-memcached==1.59 +pytz==2019.3 +PyYAML==5.1.2 +pyzmq==19.0.0 +qtconsole==4.7.1 +QtPy==1.9.0 +reentry==1.3.1 +requests==2.23.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +scipy==1.4.1 +scramp==1.1.0 +seekpath==1.9.4 +Send2Trash==1.5.0 +shellingham==1.3.2 +shortuuid==0.5.1 +simplejson==3.17.0 +six==1.14.0 +snowballstemmer==2.0.0 +spglib==1.14.1.post0 +Sphinx==2.4.4 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-contentui==0.2.4 +sphinxcontrib-details-directive==0.1.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.13 +sqlalchemy-diff==0.1.3 +SQLAlchemy-Utils==0.34.2 +sqlparse==0.3.1 +sympy==1.5.1 +tabulate==0.8.6 +terminado==0.8.3 +testpath==0.4.4 +topika==0.2.1 +tornado==4.5.3 +traitlets==4.3.3 +tzlocal==2.0.0 +upf-to-json==0.9.2 +urllib3==1.25.8 +wcwidth==0.1.8 +webencodings==0.5.1 +Werkzeug==1.0.0 +widgetsnbextension==3.5.1 +wrapt==1.11.2 +zipp==3.1.0 diff --git a/requirements/requirements-sqlalchemy-py-3.8.txt b/requirements/requirements-sqlalchemy-py-3.8.txt new file mode 100644 index 0000000000..c6d91577a6 --- /dev/null +++ b/requirements/requirements-sqlalchemy-py-3.8.txt @@ -0,0 +1,147 @@ +aiida-export-migration-tests==0.8.0 +alabaster==0.7.12 +aldjemy==0.9.1 +alembic==1.4.1 +aniso8601==8.0.0 +ase==3.19.0 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +bleach==3.1.1 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +circus==0.16.1 +Click==7.0 +click-completion==0.5.2 +click-config-file==0.5.0 +click-spinner==0.1.8 +configobj==5.0.6 +coverage==4.5.4 +cryptography==2.8 +cycler==0.10.0 +decorator==4.4.2 +defusedxml==0.6.0 +Django==2.2.11 +docutils==0.15.2 +entrypoints==0.3 +ete3==3.1.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +frozendict==1.2 +furl==2.1.0 +future==0.18.2 +graphviz==0.13.2 +idna==2.9 +imagesize==1.2.0 +ipykernel==5.1.4 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +itsdangerous==1.1.0 +jedi==0.16.0 +Jinja2==2.11.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.0.0 +jupyter-console==6.1.0 +jupyter-core==4.6.3 +kiwipy==0.5.3 +kiwisolver==1.1.0 +Mako==1.1.2 +MarkupSafe==1.1.1 +matplotlib==3.2.0 +mistune==0.8.4 +monty==3.0.2 +more-itertools==8.2.0 +mpmath==1.1.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==5.7.8 +numpy==1.17.4 +orderedmultidict==1.0.1 +packaging==20.3 +palettable==3.3.0 +pandas==0.25.3 +pandocfilters==1.4.2 +paramiko==2.7.1 +parso==0.6.2 +pexpect==4.8.0 +pg8000==1.13.2 +pgtest==1.3.2 +pickleshare==0.7.5 +pika==1.1.0 +pluggy==0.13.1 +plumpy==0.14.5 +prometheus-client==0.7.1 +prompt-toolkit==3.0.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +PyCifRW==4.4.1 +pycparser==2.20 +PyDispatcher==2.0.5 +Pygments==2.5.2 +pymatgen==2020.3.2 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-timeout==1.3.4 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-memcached==1.59 +pytz==2019.3 +PyYAML==5.1.2 +pyzmq==19.0.0 +qtconsole==4.7.1 +QtPy==1.9.0 +reentry==1.3.1 +requests==2.23.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +scipy==1.4.1 +scramp==1.1.0 +seekpath==1.9.4 +Send2Trash==1.5.0 +shellingham==1.3.2 +shortuuid==0.5.1 +simplejson==3.17.0 +six==1.14.0 +snowballstemmer==2.0.0 +spglib==1.14.1.post0 +Sphinx==2.4.4 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-contentui==0.2.4 +sphinxcontrib-details-directive==0.1.0 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.13 +sqlalchemy-diff==0.1.3 +SQLAlchemy-Utils==0.34.2 +sqlparse==0.3.1 +sympy==1.5.1 +tabulate==0.8.6 +terminado==0.8.3 +testpath==0.4.4 +topika==0.2.1 +tornado==4.5.3 +traitlets==4.3.3 +tzlocal==2.0.0 +upf-to-json==0.9.2 +urllib3==1.25.8 +wcwidth==0.1.8 +webencodings==0.5.1 +Werkzeug==1.0.0 +widgetsnbextension==3.5.1 +wrapt==1.11.2 From ca5b8d0bd74cfbc5c2721019b616148cc049d509 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 17:46:56 +0100 Subject: [PATCH 70/91] Revise workflow as discussed in PR. --- .github/workflows/ci.yml | 16 +++---- .../workflows/{dm.yml => test-install.yml} | 12 +----- .../{release.yml => update-requirements.yml} | 42 +++++++++---------- 3 files changed, 27 insertions(+), 43 deletions(-) rename .github/workflows/{dm.yml => test-install.yml} (90%) rename .github/workflows/{release.yml => update-requirements.yml} (72%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f5544c3c1..43fc3d93b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: aiida-core +name: continuous-integration on: [push, pull_request] @@ -72,8 +72,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.5, 3.8] backend: ['django', 'sqlalchemy'] + python-version: [3.5, 3.8] steps: - uses: actions/checkout@v1 @@ -103,8 +103,8 @@ jobs: - name: Install python dependencies run: | pip install --upgrade pip - pip install numpy==1.17.4 - pip install -e .[atomic_tools,docs,notebook,rest,testing] + pip install --no-deps -r requirements/requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + pip install --no-deps -e . reentry scan - name: Setup environment @@ -130,13 +130,7 @@ jobs: fail_ci_if_error: false - name: Freeze test environment - run: | - pip freeze | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt - - - uses: actions/upload-artifact@v1 - with: - name: requirements.txt - path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + run: pip freeze verdi: diff --git a/.github/workflows/dm.yml b/.github/workflows/test-install.yml similarity index 90% rename from .github/workflows/dm.yml rename to .github/workflows/test-install.yml index 94ac224a21..b0fddaf712 100644 --- a/.github/workflows/dm.yml +++ b/.github/workflows/test-install.yml @@ -1,4 +1,4 @@ -name: aiida-core-dependency-management +name: test-install on: push: @@ -145,13 +145,3 @@ jobs: AIIDA_TEST_BACKEND: ${{ matrix.backend }} run: .github/workflows/tests.sh - - - name: Freeze test environment - run: | - pip freeze | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt - - - name: Upload requirements.txt - uses: actions/upload-artifact@v1 - with: - name: requirements.txt - path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt diff --git a/.github/workflows/release.yml b/.github/workflows/update-requirements.yml similarity index 72% rename from .github/workflows/release.yml rename to .github/workflows/update-requirements.yml index a9397fa512..6a7bf5fd1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/update-requirements.yml @@ -1,31 +1,39 @@ -name: aiida-core-release +name: update-requirements on: push: branch: - release/* + paths: + - 'setup.json' jobs: - test: + tests: runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + backend: ['django', 'sqlalchemy'] + python-version: [3.5, 3.6, 3.7, 3.8] + steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v1 - uses: CasperWA/postgresql-action@v1.2 with: postgresql version: '10' - postgresql db: test_django + postgresql db: test_${{ matrix.backend }} postgresql user: postgres postgresql password: '' postgresql auth: trust - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: Install system dependencies run: | @@ -40,34 +48,32 @@ jobs: - name: Install python dependencies run: | pip install --upgrade pip - pip install numpy==1.17.4 pip install -e .[atomic_tools,docs,notebook,rest,testing] reentry scan - name: Setup environment env: - AIIDA_TEST_BACKEND: django + AIIDA_TEST_BACKEND: ${{ matrix.backend }} run: .github/workflows/setup.sh - name: Run test suite env: - AIIDA_TEST_BACKEND: django + AIIDA_TEST_BACKEND: ${{ matrix.backend }} run: .github/workflows/tests.sh - name: Freeze test environment - run: pip freeze | tee requirements.txt + run: pip freeze | sed '1d' | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt - - name: Upload requirements.txt - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v1 with: name: requirements.txt - path: requirements.txt + path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt update-requirements: - needs: test + needs: tests runs-on: ubuntu-latest steps: @@ -77,13 +83,7 @@ jobs: uses: actions/download-artifact@v1 with: name: requirements.txt - path: tmp - - - name: Prepare requirements.txt - run: | - mv tmp/requirements.txt requirements.txt - sed -i '1d' requirements.txt - rm -r tmp/ + path: requirements - name: Create Pull Request uses: peter-evans/create-pull-request@v2 From 47c5e13d22c14678775a7900bb261aa73078905f Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 17:52:57 +0100 Subject: [PATCH 71/91] Disable test suite steps. To expedite workflow execution for debugging. --- .github/workflows/ci.yml | 30 +++++++++++------------ .github/workflows/test-install.yml | 10 ++++---- .github/workflows/update-requirements.yml | 10 ++++---- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43fc3d93b4..886243f62d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,21 +113,21 @@ jobs: run: .github/workflows/setup.sh - - name: Run test suite - env: - AIIDA_TEST_BACKEND: ${{ matrix.backend }} - run: - .github/workflows/tests.sh - - - name: Upload coverage report - if: matrix.python-version == 3.5 - uses: codecov/codecov-action@v1.0.5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - name: aiida-pytests-py3.5-${{ matrix.backend }} - flags: ${{ matrix.backend }} - file: ./coverage.xml - fail_ci_if_error: false + #- name: Run test suite + # env: + # AIIDA_TEST_BACKEND: ${{ matrix.backend }} + # run: + # .github/workflows/tests.sh + + #- name: Upload coverage report + # if: matrix.python-version == 3.5 + # uses: codecov/codecov-action@v1.0.5 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + # name: aiida-pytests-py3.5-${{ matrix.backend }} + # flags: ${{ matrix.backend }} + # file: ./coverage.xml + # fail_ci_if_error: false - name: Freeze test environment run: pip freeze diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index b0fddaf712..d86550907d 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -140,8 +140,8 @@ jobs: run: .github/workflows/setup.sh - - name: Run test suite - env: - AIIDA_TEST_BACKEND: ${{ matrix.backend }} - run: - .github/workflows/tests.sh + #- name: Run test suite + # env: + # AIIDA_TEST_BACKEND: ${{ matrix.backend }} + # run: + # .github/workflows/tests.sh diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 6a7bf5fd1e..bf8d75bca6 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -57,11 +57,11 @@ jobs: run: .github/workflows/setup.sh - - name: Run test suite - env: - AIIDA_TEST_BACKEND: ${{ matrix.backend }} - run: - .github/workflows/tests.sh + #- name: Run test suite + # env: + # AIIDA_TEST_BACKEND: ${{ matrix.backend }} + # run: + # .github/workflows/tests.sh - name: Freeze test environment run: pip freeze | sed '1d' | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt From 6d6877e16df81588e5e043c1bdcfeb4bd84c6476 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 18:22:19 +0100 Subject: [PATCH 72/91] Fix requirements files. I assume that the shortuiid 0.5.1 release was removed from PyPI. --- requirements/requirements-django-py-3.5.txt | 2 +- requirements/requirements-django-py-3.6.txt | 2 +- requirements/requirements-django-py-3.7.txt | 2 +- requirements/requirements-django-py-3.8.txt | 2 +- requirements/requirements-sqlalchemy-py-3.5.txt | 2 +- requirements/requirements-sqlalchemy-py-3.6.txt | 2 +- requirements/requirements-sqlalchemy-py-3.7.txt | 2 +- requirements/requirements-sqlalchemy-py-3.8.txt | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements/requirements-django-py-3.5.txt b/requirements/requirements-django-py-3.5.txt index 00758dd76a..3f7eddad62 100644 --- a/requirements/requirements-django-py-3.5.txt +++ b/requirements/requirements-django-py-3.5.txt @@ -114,7 +114,7 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.1 +shortuuid==0.5.0 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 diff --git a/requirements/requirements-django-py-3.6.txt b/requirements/requirements-django-py-3.6.txt index 7432db3060..f89ca64a6e 100644 --- a/requirements/requirements-django-py-3.6.txt +++ b/requirements/requirements-django-py-3.6.txt @@ -113,7 +113,7 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.1 +shortuuid==0.5.0 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 diff --git a/requirements/requirements-django-py-3.7.txt b/requirements/requirements-django-py-3.7.txt index 28ffb9e049..5dbef921c4 100644 --- a/requirements/requirements-django-py-3.7.txt +++ b/requirements/requirements-django-py-3.7.txt @@ -112,7 +112,7 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.1 +shortuuid==0.5.0 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 diff --git a/requirements/requirements-django-py-3.8.txt b/requirements/requirements-django-py-3.8.txt index c6d91577a6..d8c8c8686c 100644 --- a/requirements/requirements-django-py-3.8.txt +++ b/requirements/requirements-django-py-3.8.txt @@ -111,7 +111,7 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.1 +shortuuid==0.5.0 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 diff --git a/requirements/requirements-sqlalchemy-py-3.5.txt b/requirements/requirements-sqlalchemy-py-3.5.txt index 00758dd76a..3f7eddad62 100644 --- a/requirements/requirements-sqlalchemy-py-3.5.txt +++ b/requirements/requirements-sqlalchemy-py-3.5.txt @@ -114,7 +114,7 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.1 +shortuuid==0.5.0 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 diff --git a/requirements/requirements-sqlalchemy-py-3.6.txt b/requirements/requirements-sqlalchemy-py-3.6.txt index 7432db3060..f89ca64a6e 100644 --- a/requirements/requirements-sqlalchemy-py-3.6.txt +++ b/requirements/requirements-sqlalchemy-py-3.6.txt @@ -113,7 +113,7 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.1 +shortuuid==0.5.0 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 diff --git a/requirements/requirements-sqlalchemy-py-3.7.txt b/requirements/requirements-sqlalchemy-py-3.7.txt index 28ffb9e049..5dbef921c4 100644 --- a/requirements/requirements-sqlalchemy-py-3.7.txt +++ b/requirements/requirements-sqlalchemy-py-3.7.txt @@ -112,7 +112,7 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.1 +shortuuid==0.5.0 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 diff --git a/requirements/requirements-sqlalchemy-py-3.8.txt b/requirements/requirements-sqlalchemy-py-3.8.txt index c6d91577a6..d8c8c8686c 100644 --- a/requirements/requirements-sqlalchemy-py-3.8.txt +++ b/requirements/requirements-sqlalchemy-py-3.8.txt @@ -111,7 +111,7 @@ scramp==1.1.0 seekpath==1.9.4 Send2Trash==1.5.0 shellingham==1.3.2 -shortuuid==0.5.1 +shortuuid==0.5.0 simplejson==3.17.0 six==1.14.0 snowballstemmer==2.0.0 From 2ea98b5957ad3064f843df556f0f81db771f93ef Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 18:30:50 +0100 Subject: [PATCH 73/91] Install from requirements file with deps. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 886243f62d..860dacddf6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,7 +103,7 @@ jobs: - name: Install python dependencies run: | pip install --upgrade pip - pip install --no-deps -r requirements/requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + pip install -r requirements/requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt pip install --no-deps -e . reentry scan From ee30d7989ba7c9cf7fbcdcee0c52b558ecac02cf Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 6 Mar 2020 18:41:09 +0100 Subject: [PATCH 74/91] Run 'pip freeze' in test-install workflow. --- .github/workflows/test-install.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index d86550907d..6e6d17d12d 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -134,6 +134,8 @@ jobs: pip install -e .[atomic_tools,docs,notebook,rest,testing] reentry scan + - run: pip freeze + - name: Setup environment env: AIIDA_TEST_BACKEND: ${{ matrix.backend }} From 06614ef17930fa5defe52266205d3200c82d46ba Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Fri, 6 Mar 2020 19:04:23 +0100 Subject: [PATCH 75/91] Test not fixing numpy in test-install. --- .github/workflows/test-install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 6e6d17d12d..e6a9003344 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -130,7 +130,7 @@ jobs: - name: Install python dependencies run: | pip install --upgrade pip - pip install numpy==1.17.4 + #pip install numpy==1.17.4 pip install -e .[atomic_tools,docs,notebook,rest,testing] reentry scan From cbf40f60c3c875c534127e3b2a6529ea46a608fe Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Fri, 6 Mar 2020 19:17:07 +0100 Subject: [PATCH 76/91] Restrict numpy version for py35. --- docs/requirements_for_rtd.txt | 3 ++- environment.yml | 3 ++- setup.json | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/requirements_for_rtd.txt b/docs/requirements_for_rtd.txt index 936d0c89d5..95afc1fd29 100644 --- a/docs/requirements_for_rtd.txt +++ b/docs/requirements_for_rtd.txt @@ -19,7 +19,8 @@ graphviz~=0.13 ipython~=7.0 jinja2~=2.10 kiwipy[rmq]~=0.5.1 -numpy<1.18,~=1.17 +numpy<1.17.5,>=1.17.0; python_version < "3.6" +numpy<1.18,~=1.17; python_version >= "3.5" paramiko~=2.6 pg8000~=1.13 pgtest>=1.3.1,~=1.3 diff --git a/environment.yml b/environment.yml index f7d6ca7d1b..f094302d73 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,8 @@ dependencies: - ipython~=7.0 - jinja2~=2.10 - kiwipy[rmq]~=0.5.1 -- numpy<1.18,~=1.17 +- numpy<1.18,~=1.17; python_version >= "3.5" +- numpy<1.17.5,>=1.17.0; python_version < "3.6" - paramiko~=2.6 - pika~=1.1 - plumpy~=0.14.5 diff --git a/setup.json b/setup.json index 26460a9e6a..1abb7999fb 100644 --- a/setup.json +++ b/setup.json @@ -33,7 +33,8 @@ "ipython~=7.0", "jinja2~=2.10", "kiwipy[rmq]~=0.5.1", - "numpy~=1.17,<1.18", + "numpy~=1.17,<1.18; python_version>='3.5'", + "numpy>=1.17.0,<1.17.5; python_version<'3.6'", "paramiko~=2.6", "pika~=1.1", "plumpy~=0.14.5", From 8b089a5d0d715231c46b5cf560a08095f70d33e8 Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Fri, 6 Mar 2020 19:58:58 +0100 Subject: [PATCH 77/91] fixup! Restrict numpy version for py35. --- docs/requirements_for_rtd.txt | 4 ++-- environment.yml | 3 +-- setup.json | 4 ++-- utils/dependency_management.py | 14 +++++++++----- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/requirements_for_rtd.txt b/docs/requirements_for_rtd.txt index 95afc1fd29..48b1eeede3 100644 --- a/docs/requirements_for_rtd.txt +++ b/docs/requirements_for_rtd.txt @@ -19,8 +19,8 @@ graphviz~=0.13 ipython~=7.0 jinja2~=2.10 kiwipy[rmq]~=0.5.1 -numpy<1.17.5,>=1.17.0; python_version < "3.6" -numpy<1.18,~=1.17; python_version >= "3.5" +numpy<1.17.5,>=1.17.0; python_version == "3.5" +numpy<1.18,~=1.17; python_version >= "3.6" paramiko~=2.6 pg8000~=1.13 pgtest>=1.3.1,~=1.3 diff --git a/environment.yml b/environment.yml index f094302d73..f7d6ca7d1b 100644 --- a/environment.yml +++ b/environment.yml @@ -19,8 +19,7 @@ dependencies: - ipython~=7.0 - jinja2~=2.10 - kiwipy[rmq]~=0.5.1 -- numpy<1.18,~=1.17; python_version >= "3.5" -- numpy<1.17.5,>=1.17.0; python_version < "3.6" +- numpy<1.18,~=1.17 - paramiko~=2.6 - pika~=1.1 - plumpy~=0.14.5 diff --git a/setup.json b/setup.json index 1abb7999fb..d540d17bc5 100644 --- a/setup.json +++ b/setup.json @@ -33,8 +33,8 @@ "ipython~=7.0", "jinja2~=2.10", "kiwipy[rmq]~=0.5.1", - "numpy~=1.17,<1.18; python_version>='3.5'", - "numpy>=1.17.0,<1.17.5; python_version<'3.6'", + "numpy~=1.17,<1.18; python_version>='3.6'", + "numpy>=1.17.0,<1.17.5; python_version=='3.5'", "paramiko~=2.6", "pika~=1.1", "plumpy~=0.14.5", diff --git a/utils/dependency_management.py b/utils/dependency_management.py index f187102c57..a86607c672 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -29,9 +29,7 @@ 'graphviz': 'python-graphviz', } -CONDA_IGNORE = [ - 'pyblake2', -] +CONDA_IGNORE = ['pyblake2', r'.*python_version == \"3\.5\"'] class DependencySpecificationError(click.ClickException): @@ -69,8 +67,14 @@ def _setuptools_to_conda(req): for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): if re.match(pattern, str(req)): - return Requirement.parse(re.sub(pattern, replacement, str(req))) - return req # did not match any of the replacement patterns, just return original + req = Requirement.parse(re.sub(pattern, replacement, str(req))) + break + + # markers are not supported by conda + req.marker = None + + # We need to parse the modified required again, to ensure consistency. + return Requirement.parse(str(req)) @click.group() From a4fdf0b4f71a0dc26c1c85175b32927f7db16972 Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Sat, 7 Mar 2020 09:37:40 +0100 Subject: [PATCH 78/91] Require Cython for builds with Python version 3.5. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 24eaa8393c..41cc2022dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = [ "setuptools>=40.8.0", "wheel", "reentry~=1.3",] +requires = [ "Cython; python_version=='3.5'", "setuptools>=40.8.0", "wheel", "reentry~=1.3",] build-backend = "setuptools.build_meta:__legacy__" From f7d0153656a3ddc4f0a727e8208aadc947eb8239 Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Sat, 7 Mar 2020 10:26:57 +0100 Subject: [PATCH 79/91] Make numpy (instead of Cython) build requirement on Python 3.5. --- pyproject.toml | 2 +- utils/dependency_management.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41cc2022dd..97e8841571 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = [ "Cython; python_version=='3.5'", "setuptools>=40.8.0", "wheel", "reentry~=1.3",] +requires = [ "numpy>=1.17.0,<1.17.5; python_version=='3.5'", "setuptools>=40.8.0", "wheel", "reentry~=1.3",] build-backend = "setuptools.build_meta:__legacy__" diff --git a/utils/dependency_management.py b/utils/dependency_management.py index a86607c672..f65859dfcd 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -151,8 +151,11 @@ def generate_pyproject_toml(): pyproject = { 'build-system': { - 'requires': ['setuptools>=40.8.0', 'wheel', str(reentry_requirement)], - 'build-backend': 'setuptools.build_meta:__legacy__', + 'requires': + ["numpy>=1.17.0,<1.17.5; python_version=='3.5'", 'setuptools>=40.8.0', 'wheel', + str(reentry_requirement)], + 'build-backend': + 'setuptools.build_meta:__legacy__', } } with open(ROOT / 'pyproject.toml', 'w') as file: From 8c62b35b9f08d5c41f1ee9350350a1799794b14b Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Sat, 7 Mar 2020 10:42:24 +0100 Subject: [PATCH 80/91] Show pip version during ci workflows. --- .github/workflows/ci.yml | 1 + .github/workflows/test-install.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 860dacddf6..d3d36136c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,7 @@ jobs: - name: Install python dependencies run: | pip install --upgrade pip + pip --version pip install -r requirements/requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt pip install --no-deps -e . reentry scan diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index e6a9003344..2a734780e1 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -130,7 +130,7 @@ jobs: - name: Install python dependencies run: | pip install --upgrade pip - #pip install numpy==1.17.4 + pip --version pip install -e .[atomic_tools,docs,notebook,rest,testing] reentry scan From 653ef119bbb06aeb584e7bb1f4162dfac3a24c70 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 14:33:18 +0100 Subject: [PATCH 81/91] Attempt to fix installation on Python version 3.5. --- .github/workflows/ci.yml | 9 +++++++-- .github/workflows/test-install.yml | 10 +++++++--- .github/workflows/update-requirements.yml | 9 +++++++-- pyproject.toml | 2 +- setup.py | 15 +++++++++++++++ utils/dependency_management.py | 7 ++----- 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3d36136c0..655085dd03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,9 +100,14 @@ jobs: sudo apt install postgresql-10 rabbitmq-server graphviz sudo systemctl status rabbitmq-server.service - - name: Install python dependencies + - run: pip install --upgrade pip + + - name: upgrade setuptools [py35] + if: matrix.python-version == 3.5 + run: pip install -I setuptools==38.2.0 + + - name: Install aiida-core run: | - pip install --upgrade pip pip --version pip install -r requirements/requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt pip install --no-deps -e . diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 2a734780e1..4c1ab04544 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -127,10 +127,14 @@ jobs: sudo apt install postgresql-10 rabbitmq-server graphviz sudo systemctl status rabbitmq-server.service - - name: Install python dependencies + - run: pip install --upgrade pip + + - name: upgrade setuptools [py35] + if: matrix.python-version == 3.5 + run: pip install -I setuptools==38.2.0 + + - name: Install aiida-core run: | - pip install --upgrade pip - pip --version pip install -e .[atomic_tools,docs,notebook,rest,testing] reentry scan diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index bf8d75bca6..710add47d4 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -45,9 +45,14 @@ jobs: sudo apt install postgresql-10 rabbitmq-server graphviz sudo systemctl status rabbitmq-server.service - - name: Install python dependencies + - run: pip install --upgrade pip + + - name: upgrade setuptools [py35] + if: matrix.python-version == 3.5 + run: pip install -I setuptools==38.2.0 + + - name: Install aiida-core run: | - pip install --upgrade pip pip install -e .[atomic_tools,docs,notebook,rest,testing] reentry scan diff --git a/pyproject.toml b/pyproject.toml index 97e8841571..24eaa8393c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = [ "numpy>=1.17.0,<1.17.5; python_version=='3.5'", "setuptools>=40.8.0", "wheel", "reentry~=1.3",] +requires = [ "setuptools>=40.8.0", "wheel", "reentry~=1.3",] build-backend = "setuptools.build_meta:__legacy__" diff --git a/setup.py b/setup.py index 8f2372edd5..0cfa973f5f 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,26 @@ # pylint: disable=wrong-import-order """Setup script for aiida-core package.""" import json +import sys import os from utils import fastentrypoints # pylint: disable=unused-import from setuptools import setup, find_packages +if (sys.version_info.major, sys.version_info.minor) == (3, 5): + import setuptools + from distutils.version import StrictVersion + + REQUIRED_SETUPTOOLS_VERSION = StrictVersion('38.2.0') + INSTALLED_SETUPTOOLS_VERSION = StrictVersion(setuptools.__version__) + + if INSTALLED_SETUPTOOLS_VERSION < REQUIRED_SETUPTOOLS_VERSION: + raise RuntimeError( + 'The installation of AiiDA with Python version 3.5, requires setuptools>={}; your version: {}'.format( + REQUIRED_SETUPTOOLS_VERSION, INSTALLED_SETUPTOOLS_VERSION + ) + ) + if __name__ == '__main__': THIS_FOLDER = os.path.split(os.path.abspath(__file__))[0] diff --git a/utils/dependency_management.py b/utils/dependency_management.py index f65859dfcd..a86607c672 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -151,11 +151,8 @@ def generate_pyproject_toml(): pyproject = { 'build-system': { - 'requires': - ["numpy>=1.17.0,<1.17.5; python_version=='3.5'", 'setuptools>=40.8.0', 'wheel', - str(reentry_requirement)], - 'build-backend': - 'setuptools.build_meta:__legacy__', + 'requires': ['setuptools>=40.8.0', 'wheel', str(reentry_requirement)], + 'build-backend': 'setuptools.build_meta:__legacy__', } } with open(ROOT / 'pyproject.toml', 'w') as file: From d112c2a2207648f73c110563e8042bfd847a9200 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 15:23:41 +0100 Subject: [PATCH 82/91] Revert "Disable test suite steps." This reverts commit 47c5e13d22c14678775a7900bb261aa73078905f. --- .github/workflows/ci.yml | 28 +++++++++++------------ .github/workflows/test-install.yml | 10 ++++---- .github/workflows/update-requirements.yml | 10 ++++---- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 611380575e..613a8217cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,20 +119,20 @@ jobs: run: .github/workflows/setup.sh - #- name: Run test suite - # env: - # AIIDA_TEST_BACKEND: ${{ matrix.backend }} - # run: - # .github/workflows/tests.sh - - #- name: Upload coverage report - # if: matrix.python-version == 3.5 && github.repository == 'aiidateam/aiida-core' - # uses: codecov/codecov-action@v1 - # with: - # name: aiida-pytests-py3.5-${{ matrix.backend }} - # flags: ${{ matrix.backend }} - # file: ./coverage.xml - # fail_ci_if_error: true + - name: Run test suite + env: + AIIDA_TEST_BACKEND: ${{ matrix.backend }} + run: + .github/workflows/tests.sh + + - name: Upload coverage report + if: matrix.python-version == 3.5 && github.repository == 'aiidateam/aiida-core' + uses: codecov/codecov-action@v1 + with: + name: aiida-pytests-py3.5-${{ matrix.backend }} + flags: ${{ matrix.backend }} + file: ./coverage.xml + fail_ci_if_error: true - name: Freeze test environment run: pip freeze diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 4c1ab04544..399872680d 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -146,8 +146,8 @@ jobs: run: .github/workflows/setup.sh - #- name: Run test suite - # env: - # AIIDA_TEST_BACKEND: ${{ matrix.backend }} - # run: - # .github/workflows/tests.sh + - name: Run test suite + env: + AIIDA_TEST_BACKEND: ${{ matrix.backend }} + run: + .github/workflows/tests.sh diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 710add47d4..84d0b0a27b 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -62,11 +62,11 @@ jobs: run: .github/workflows/setup.sh - #- name: Run test suite - # env: - # AIIDA_TEST_BACKEND: ${{ matrix.backend }} - # run: - # .github/workflows/tests.sh + - name: Run test suite + env: + AIIDA_TEST_BACKEND: ${{ matrix.backend }} + run: + .github/workflows/tests.sh - name: Freeze test environment run: pip freeze | sed '1d' | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt From 6fb7f3a5414cb524d0276ec448332b276b34d6cc Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 16:58:12 +0100 Subject: [PATCH 83/91] Slightly streamline ci workflow. --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 613a8217cf..69c5ac9065 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,18 +100,21 @@ jobs: sudo apt install postgresql-10 rabbitmq-server graphviz sudo systemctl status rabbitmq-server.service - - run: pip install --upgrade pip + - name: Upgrade pip + run: | + pip install --upgrade pip + pip --version - name: upgrade setuptools [py35] if: matrix.python-version == 3.5 - run: pip install -I setuptools==38.2.0 + run: pip install -I setuptools==38.2.0 # Minimally required version for Python 3.5. - name: Install aiida-core run: | - pip --version pip install -r requirements/requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt pip install --no-deps -e . reentry scan + pip freeze - name: Setup environment env: @@ -134,9 +137,6 @@ jobs: file: ./coverage.xml fail_ci_if_error: true - - name: Freeze test environment - run: pip freeze - verdi: runs-on: ubuntu-latest From b2ef3e087878242a5ce514ba7e5f0544c7f75642 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 16:58:50 +0100 Subject: [PATCH 84/91] Update 'test-install' and 'update-requirements' triggers. Run when workflow definition files change. --- .github/workflows/test-install.yml | 2 +- .github/workflows/update-requirements.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 399872680d..052687f41a 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -12,7 +12,7 @@ on: - 'environment.yml' - 'requirements*.txt' - 'pyproject.toml' - - '.github/workflows/*' + - '.github/workflows/test-install.sh' schedule: - cron: '30 02 * * *' # nightly build diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 84d0b0a27b..01e5ea1735 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -6,6 +6,7 @@ on: - release/* paths: - 'setup.json' + - '.github/workflows/update-requirements.yml' jobs: From 519908c23df51bc9c58b055f4f5e4090ecf8d8f1 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 17:03:52 +0100 Subject: [PATCH 85/91] Only store requirements for one backend version (django). Since the environments only differ across python versions, not backends. --- .github/workflows/ci.yml | 2 +- .github/workflows/update-requirements.yml | 7 +- ...ngo-py-3.5.txt => requirements-py-3.5.txt} | 0 ...ngo-py-3.6.txt => requirements-py-3.6.txt} | 0 ...ngo-py-3.7.txt => requirements-py-3.7.txt} | 0 ...ngo-py-3.8.txt => requirements-py-3.8.txt} | 0 .../requirements-sqlalchemy-py-3.5.txt | 151 ------------------ .../requirements-sqlalchemy-py-3.6.txt | 150 ----------------- .../requirements-sqlalchemy-py-3.7.txt | 149 ----------------- .../requirements-sqlalchemy-py-3.8.txt | 147 ----------------- 10 files changed, 6 insertions(+), 600 deletions(-) rename requirements/{requirements-django-py-3.5.txt => requirements-py-3.5.txt} (100%) rename requirements/{requirements-django-py-3.6.txt => requirements-py-3.6.txt} (100%) rename requirements/{requirements-django-py-3.7.txt => requirements-py-3.7.txt} (100%) rename requirements/{requirements-django-py-3.8.txt => requirements-py-3.8.txt} (100%) delete mode 100644 requirements/requirements-sqlalchemy-py-3.5.txt delete mode 100644 requirements/requirements-sqlalchemy-py-3.6.txt delete mode 100644 requirements/requirements-sqlalchemy-py-3.7.txt delete mode 100644 requirements/requirements-sqlalchemy-py-3.8.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69c5ac9065..bf8eae364b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,7 @@ jobs: - name: Install aiida-core run: | - pip install -r requirements/requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + pip install -r requirements/requirements-py-${{ matrix.python-version }}.txt pip install --no-deps -e . reentry scan pip freeze diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 01e5ea1735..7283fe5d02 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -70,12 +70,15 @@ jobs: .github/workflows/tests.sh - name: Freeze test environment - run: pip freeze | sed '1d' | tee requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + run: pip freeze | sed '1d' | tee requirements-py-${{ matrix.python-version }}.txt + # Add python-version specific requirements file to the requirements.txt artifact. + # Will be used in the next job to create a PR in case they are different from the current version. - uses: actions/upload-artifact@v1 + if: matrix.backend == 'django' # The requirements are identical between backends. with: name: requirements.txt - path: requirements-${{ matrix.backend }}-py-${{ matrix.python-version }}.txt + path: requirements-py-${{ matrix.python-version }}.txt update-requirements: diff --git a/requirements/requirements-django-py-3.5.txt b/requirements/requirements-py-3.5.txt similarity index 100% rename from requirements/requirements-django-py-3.5.txt rename to requirements/requirements-py-3.5.txt diff --git a/requirements/requirements-django-py-3.6.txt b/requirements/requirements-py-3.6.txt similarity index 100% rename from requirements/requirements-django-py-3.6.txt rename to requirements/requirements-py-3.6.txt diff --git a/requirements/requirements-django-py-3.7.txt b/requirements/requirements-py-3.7.txt similarity index 100% rename from requirements/requirements-django-py-3.7.txt rename to requirements/requirements-py-3.7.txt diff --git a/requirements/requirements-django-py-3.8.txt b/requirements/requirements-py-3.8.txt similarity index 100% rename from requirements/requirements-django-py-3.8.txt rename to requirements/requirements-py-3.8.txt diff --git a/requirements/requirements-sqlalchemy-py-3.5.txt b/requirements/requirements-sqlalchemy-py-3.5.txt deleted file mode 100644 index 3f7eddad62..0000000000 --- a/requirements/requirements-sqlalchemy-py-3.5.txt +++ /dev/null @@ -1,151 +0,0 @@ -aiida-export-migration-tests==0.8.0 -alabaster==0.7.12 -aldjemy==0.9.1 -alembic==1.4.1 -aniso8601==8.0.0 -ase==3.19.0 -attrs==19.3.0 -Babel==2.8.0 -backcall==0.1.0 -bcrypt==3.1.7 -bleach==3.1.1 -certifi==2019.11.28 -cffi==1.14.0 -chardet==3.0.4 -circus==0.16.1 -Click==7.0 -click-completion==0.5.2 -click-config-file==0.5.0 -click-spinner==0.1.8 -configobj==5.0.6 -coverage==4.5.4 -cryptography==2.8 -cycler==0.10.0 -decorator==4.4.2 -defusedxml==0.6.0 -Django==2.2.11 -docutils==0.15.2 -entrypoints==0.3 -ete3==3.1.1 -Flask==1.1.1 -Flask-Cors==3.0.8 -Flask-RESTful==0.3.8 -frozendict==1.2 -furl==2.1.0 -future==0.18.2 -graphviz==0.13.2 -idna==2.9 -imagesize==1.2.0 -importlib-metadata==1.5.0 -ipykernel==5.1.4 -ipython==7.9.0 -ipython-genutils==0.2.0 -ipywidgets==7.5.1 -itsdangerous==1.1.0 -jedi==0.16.0 -Jinja2==2.11.1 -jsonschema==3.2.0 -jupyter==1.0.0 -jupyter-client==6.0.0 -jupyter-console==6.1.0 -jupyter-core==4.6.3 -kiwipy==0.5.3 -kiwisolver==1.1.0 -Mako==1.1.2 -MarkupSafe==1.1.1 -matplotlib==3.0.3 -mistune==0.8.4 -monty==3.0.2 -more-itertools==8.2.0 -mpmath==1.1.0 -nbconvert==5.6.1 -nbformat==5.0.4 -networkx==2.4 -notebook==5.7.8 -numpy==1.17.4 -orderedmultidict==1.0.1 -packaging==20.3 -palettable==3.3.0 -pandas==0.25.3 -pandocfilters==1.4.2 -paramiko==2.7.1 -parso==0.6.2 -pathlib2==2.3.5 -pexpect==4.8.0 -pg8000==1.13.2 -pgtest==1.3.2 -pickleshare==0.7.5 -pika==1.1.0 -pluggy==0.13.1 -plumpy==0.14.5 -prometheus-client==0.7.1 -prompt-toolkit==2.0.10 -psutil==5.7.0 -psycopg2-binary==2.8.4 -ptyprocess==0.6.0 -py==1.8.1 -pyblake2==1.1.2 -PyCifRW==4.4.1 -pycparser==2.20 -PyDispatcher==2.0.5 -Pygments==2.5.2 -pymatgen==2019.7.2 -PyMySQL==0.9.3 -PyNaCl==1.3.0 -pyparsing==2.4.6 -pyrsistent==0.15.7 -pytest==5.3.5 -pytest-cov==2.8.1 -pytest-timeout==1.3.4 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-memcached==1.59 -pytz==2019.3 -PyYAML==5.1.2 -pyzmq==19.0.0 -qtconsole==4.7.1 -QtPy==1.9.0 -reentry==1.3.1 -requests==2.23.0 -ruamel.yaml==0.16.10 -ruamel.yaml.clib==0.2.0 -scipy==1.4.1 -scramp==1.1.0 -seekpath==1.9.4 -Send2Trash==1.5.0 -shellingham==1.3.2 -shortuuid==0.5.0 -simplejson==3.17.0 -six==1.14.0 -snowballstemmer==2.0.0 -spglib==1.14.1.post0 -Sphinx==2.4.4 -sphinx-rtd-theme==0.4.3 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-contentui==0.2.4 -sphinxcontrib-details-directive==0.1.0 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==1.0.3 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.4 -SQLAlchemy==1.3.13 -sqlalchemy-diff==0.1.3 -SQLAlchemy-Utils==0.34.2 -sqlparse==0.3.1 -sympy==1.5.1 -tabulate==0.8.6 -terminado==0.8.3 -testpath==0.4.4 -topika==0.2.1 -tornado==4.5.3 -traitlets==4.3.3 -tzlocal==2.0.0 -upf-to-json==0.9.2 -urllib3==1.25.8 -wcwidth==0.1.8 -webencodings==0.5.1 -Werkzeug==1.0.0 -widgetsnbextension==3.5.1 -wrapt==1.11.2 -zipp==1.2.0 diff --git a/requirements/requirements-sqlalchemy-py-3.6.txt b/requirements/requirements-sqlalchemy-py-3.6.txt deleted file mode 100644 index f89ca64a6e..0000000000 --- a/requirements/requirements-sqlalchemy-py-3.6.txt +++ /dev/null @@ -1,150 +0,0 @@ -aiida-export-migration-tests==0.8.0 -alabaster==0.7.12 -aldjemy==0.9.1 -alembic==1.4.1 -aniso8601==8.0.0 -ase==3.19.0 -attrs==19.3.0 -Babel==2.8.0 -backcall==0.1.0 -bcrypt==3.1.7 -bleach==3.1.1 -certifi==2019.11.28 -cffi==1.14.0 -chardet==3.0.4 -circus==0.16.1 -Click==7.0 -click-completion==0.5.2 -click-config-file==0.5.0 -click-spinner==0.1.8 -configobj==5.0.6 -coverage==4.5.4 -cryptography==2.8 -cycler==0.10.0 -dataclasses==0.7 -decorator==4.4.2 -defusedxml==0.6.0 -Django==2.2.11 -docutils==0.15.2 -entrypoints==0.3 -ete3==3.1.1 -Flask==1.1.1 -Flask-Cors==3.0.8 -Flask-RESTful==0.3.8 -frozendict==1.2 -furl==2.1.0 -future==0.18.2 -graphviz==0.13.2 -idna==2.9 -imagesize==1.2.0 -importlib-metadata==1.5.0 -ipykernel==5.1.4 -ipython==7.13.0 -ipython-genutils==0.2.0 -ipywidgets==7.5.1 -itsdangerous==1.1.0 -jedi==0.16.0 -Jinja2==2.11.1 -jsonschema==3.2.0 -jupyter==1.0.0 -jupyter-client==6.0.0 -jupyter-console==6.1.0 -jupyter-core==4.6.3 -kiwipy==0.5.3 -kiwisolver==1.1.0 -Mako==1.1.2 -MarkupSafe==1.1.1 -matplotlib==3.2.0 -mistune==0.8.4 -monty==3.0.2 -more-itertools==8.2.0 -mpmath==1.1.0 -nbconvert==5.6.1 -nbformat==5.0.4 -networkx==2.4 -notebook==5.7.8 -numpy==1.17.4 -orderedmultidict==1.0.1 -packaging==20.3 -palettable==3.3.0 -pandas==0.25.3 -pandocfilters==1.4.2 -paramiko==2.7.1 -parso==0.6.2 -pexpect==4.8.0 -pg8000==1.13.2 -pgtest==1.3.2 -pickleshare==0.7.5 -pika==1.1.0 -pluggy==0.13.1 -plumpy==0.14.5 -prometheus-client==0.7.1 -prompt-toolkit==3.0.3 -psutil==5.7.0 -psycopg2-binary==2.8.4 -ptyprocess==0.6.0 -py==1.8.1 -PyCifRW==4.4.1 -pycparser==2.20 -PyDispatcher==2.0.5 -Pygments==2.5.2 -pymatgen==2020.3.2 -PyMySQL==0.9.3 -PyNaCl==1.3.0 -pyparsing==2.4.6 -pyrsistent==0.15.7 -pytest==5.3.5 -pytest-cov==2.8.1 -pytest-timeout==1.3.4 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-memcached==1.59 -pytz==2019.3 -PyYAML==5.1.2 -pyzmq==19.0.0 -qtconsole==4.7.1 -QtPy==1.9.0 -reentry==1.3.1 -requests==2.23.0 -ruamel.yaml==0.16.10 -ruamel.yaml.clib==0.2.0 -scipy==1.4.1 -scramp==1.1.0 -seekpath==1.9.4 -Send2Trash==1.5.0 -shellingham==1.3.2 -shortuuid==0.5.0 -simplejson==3.17.0 -six==1.14.0 -snowballstemmer==2.0.0 -spglib==1.14.1.post0 -Sphinx==2.4.4 -sphinx-rtd-theme==0.4.3 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-contentui==0.2.4 -sphinxcontrib-details-directive==0.1.0 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==1.0.3 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.4 -SQLAlchemy==1.3.13 -sqlalchemy-diff==0.1.3 -SQLAlchemy-Utils==0.34.2 -sqlparse==0.3.1 -sympy==1.5.1 -tabulate==0.8.6 -terminado==0.8.3 -testpath==0.4.4 -topika==0.2.1 -tornado==4.5.3 -traitlets==4.3.3 -tzlocal==2.0.0 -upf-to-json==0.9.2 -urllib3==1.25.8 -wcwidth==0.1.8 -webencodings==0.5.1 -Werkzeug==1.0.0 -widgetsnbextension==3.5.1 -wrapt==1.11.2 -zipp==3.1.0 diff --git a/requirements/requirements-sqlalchemy-py-3.7.txt b/requirements/requirements-sqlalchemy-py-3.7.txt deleted file mode 100644 index 5dbef921c4..0000000000 --- a/requirements/requirements-sqlalchemy-py-3.7.txt +++ /dev/null @@ -1,149 +0,0 @@ -aiida-export-migration-tests==0.8.0 -alabaster==0.7.12 -aldjemy==0.9.1 -alembic==1.4.1 -aniso8601==8.0.0 -ase==3.19.0 -attrs==19.3.0 -Babel==2.8.0 -backcall==0.1.0 -bcrypt==3.1.7 -bleach==3.1.1 -certifi==2019.11.28 -cffi==1.14.0 -chardet==3.0.4 -circus==0.16.1 -Click==7.0 -click-completion==0.5.2 -click-config-file==0.5.0 -click-spinner==0.1.8 -configobj==5.0.6 -coverage==4.5.4 -cryptography==2.8 -cycler==0.10.0 -decorator==4.4.2 -defusedxml==0.6.0 -Django==2.2.11 -docutils==0.15.2 -entrypoints==0.3 -ete3==3.1.1 -Flask==1.1.1 -Flask-Cors==3.0.8 -Flask-RESTful==0.3.8 -frozendict==1.2 -furl==2.1.0 -future==0.18.2 -graphviz==0.13.2 -idna==2.9 -imagesize==1.2.0 -importlib-metadata==1.5.0 -ipykernel==5.1.4 -ipython==7.13.0 -ipython-genutils==0.2.0 -ipywidgets==7.5.1 -itsdangerous==1.1.0 -jedi==0.16.0 -Jinja2==2.11.1 -jsonschema==3.2.0 -jupyter==1.0.0 -jupyter-client==6.0.0 -jupyter-console==6.1.0 -jupyter-core==4.6.3 -kiwipy==0.5.3 -kiwisolver==1.1.0 -Mako==1.1.2 -MarkupSafe==1.1.1 -matplotlib==3.2.0 -mistune==0.8.4 -monty==3.0.2 -more-itertools==8.2.0 -mpmath==1.1.0 -nbconvert==5.6.1 -nbformat==5.0.4 -networkx==2.4 -notebook==5.7.8 -numpy==1.17.4 -orderedmultidict==1.0.1 -packaging==20.3 -palettable==3.3.0 -pandas==0.25.3 -pandocfilters==1.4.2 -paramiko==2.7.1 -parso==0.6.2 -pexpect==4.8.0 -pg8000==1.13.2 -pgtest==1.3.2 -pickleshare==0.7.5 -pika==1.1.0 -pluggy==0.13.1 -plumpy==0.14.5 -prometheus-client==0.7.1 -prompt-toolkit==3.0.3 -psutil==5.7.0 -psycopg2-binary==2.8.4 -ptyprocess==0.6.0 -py==1.8.1 -PyCifRW==4.4.1 -pycparser==2.20 -PyDispatcher==2.0.5 -Pygments==2.5.2 -pymatgen==2020.3.2 -PyMySQL==0.9.3 -PyNaCl==1.3.0 -pyparsing==2.4.6 -pyrsistent==0.15.7 -pytest==5.3.5 -pytest-cov==2.8.1 -pytest-timeout==1.3.4 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-memcached==1.59 -pytz==2019.3 -PyYAML==5.1.2 -pyzmq==19.0.0 -qtconsole==4.7.1 -QtPy==1.9.0 -reentry==1.3.1 -requests==2.23.0 -ruamel.yaml==0.16.10 -ruamel.yaml.clib==0.2.0 -scipy==1.4.1 -scramp==1.1.0 -seekpath==1.9.4 -Send2Trash==1.5.0 -shellingham==1.3.2 -shortuuid==0.5.0 -simplejson==3.17.0 -six==1.14.0 -snowballstemmer==2.0.0 -spglib==1.14.1.post0 -Sphinx==2.4.4 -sphinx-rtd-theme==0.4.3 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-contentui==0.2.4 -sphinxcontrib-details-directive==0.1.0 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==1.0.3 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.4 -SQLAlchemy==1.3.13 -sqlalchemy-diff==0.1.3 -SQLAlchemy-Utils==0.34.2 -sqlparse==0.3.1 -sympy==1.5.1 -tabulate==0.8.6 -terminado==0.8.3 -testpath==0.4.4 -topika==0.2.1 -tornado==4.5.3 -traitlets==4.3.3 -tzlocal==2.0.0 -upf-to-json==0.9.2 -urllib3==1.25.8 -wcwidth==0.1.8 -webencodings==0.5.1 -Werkzeug==1.0.0 -widgetsnbextension==3.5.1 -wrapt==1.11.2 -zipp==3.1.0 diff --git a/requirements/requirements-sqlalchemy-py-3.8.txt b/requirements/requirements-sqlalchemy-py-3.8.txt deleted file mode 100644 index d8c8c8686c..0000000000 --- a/requirements/requirements-sqlalchemy-py-3.8.txt +++ /dev/null @@ -1,147 +0,0 @@ -aiida-export-migration-tests==0.8.0 -alabaster==0.7.12 -aldjemy==0.9.1 -alembic==1.4.1 -aniso8601==8.0.0 -ase==3.19.0 -attrs==19.3.0 -Babel==2.8.0 -backcall==0.1.0 -bcrypt==3.1.7 -bleach==3.1.1 -certifi==2019.11.28 -cffi==1.14.0 -chardet==3.0.4 -circus==0.16.1 -Click==7.0 -click-completion==0.5.2 -click-config-file==0.5.0 -click-spinner==0.1.8 -configobj==5.0.6 -coverage==4.5.4 -cryptography==2.8 -cycler==0.10.0 -decorator==4.4.2 -defusedxml==0.6.0 -Django==2.2.11 -docutils==0.15.2 -entrypoints==0.3 -ete3==3.1.1 -Flask==1.1.1 -Flask-Cors==3.0.8 -Flask-RESTful==0.3.8 -frozendict==1.2 -furl==2.1.0 -future==0.18.2 -graphviz==0.13.2 -idna==2.9 -imagesize==1.2.0 -ipykernel==5.1.4 -ipython==7.13.0 -ipython-genutils==0.2.0 -ipywidgets==7.5.1 -itsdangerous==1.1.0 -jedi==0.16.0 -Jinja2==2.11.1 -jsonschema==3.2.0 -jupyter==1.0.0 -jupyter-client==6.0.0 -jupyter-console==6.1.0 -jupyter-core==4.6.3 -kiwipy==0.5.3 -kiwisolver==1.1.0 -Mako==1.1.2 -MarkupSafe==1.1.1 -matplotlib==3.2.0 -mistune==0.8.4 -monty==3.0.2 -more-itertools==8.2.0 -mpmath==1.1.0 -nbconvert==5.6.1 -nbformat==5.0.4 -networkx==2.4 -notebook==5.7.8 -numpy==1.17.4 -orderedmultidict==1.0.1 -packaging==20.3 -palettable==3.3.0 -pandas==0.25.3 -pandocfilters==1.4.2 -paramiko==2.7.1 -parso==0.6.2 -pexpect==4.8.0 -pg8000==1.13.2 -pgtest==1.3.2 -pickleshare==0.7.5 -pika==1.1.0 -pluggy==0.13.1 -plumpy==0.14.5 -prometheus-client==0.7.1 -prompt-toolkit==3.0.3 -psutil==5.7.0 -psycopg2-binary==2.8.4 -ptyprocess==0.6.0 -py==1.8.1 -PyCifRW==4.4.1 -pycparser==2.20 -PyDispatcher==2.0.5 -Pygments==2.5.2 -pymatgen==2020.3.2 -PyMySQL==0.9.3 -PyNaCl==1.3.0 -pyparsing==2.4.6 -pyrsistent==0.15.7 -pytest==5.3.5 -pytest-cov==2.8.1 -pytest-timeout==1.3.4 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-memcached==1.59 -pytz==2019.3 -PyYAML==5.1.2 -pyzmq==19.0.0 -qtconsole==4.7.1 -QtPy==1.9.0 -reentry==1.3.1 -requests==2.23.0 -ruamel.yaml==0.16.10 -ruamel.yaml.clib==0.2.0 -scipy==1.4.1 -scramp==1.1.0 -seekpath==1.9.4 -Send2Trash==1.5.0 -shellingham==1.3.2 -shortuuid==0.5.0 -simplejson==3.17.0 -six==1.14.0 -snowballstemmer==2.0.0 -spglib==1.14.1.post0 -Sphinx==2.4.4 -sphinx-rtd-theme==0.4.3 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-contentui==0.2.4 -sphinxcontrib-details-directive==0.1.0 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==1.0.3 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.4 -SQLAlchemy==1.3.13 -sqlalchemy-diff==0.1.3 -SQLAlchemy-Utils==0.34.2 -sqlparse==0.3.1 -sympy==1.5.1 -tabulate==0.8.6 -terminado==0.8.3 -testpath==0.4.4 -topika==0.2.1 -tornado==4.5.3 -traitlets==4.3.3 -tzlocal==2.0.0 -upf-to-json==0.9.2 -urllib3==1.25.8 -wcwidth==0.1.8 -webencodings==0.5.1 -Werkzeug==1.0.0 -widgetsnbextension==3.5.1 -wrapt==1.11.2 From 0c5d2dcf91ad87783fede4521e7f54a4460afcf8 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 17:05:04 +0100 Subject: [PATCH 86/91] Remove marker for numpy requirement. --- docs/requirements_for_rtd.txt | 3 +-- setup.json | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/requirements_for_rtd.txt b/docs/requirements_for_rtd.txt index 48b1eeede3..936d0c89d5 100644 --- a/docs/requirements_for_rtd.txt +++ b/docs/requirements_for_rtd.txt @@ -19,8 +19,7 @@ graphviz~=0.13 ipython~=7.0 jinja2~=2.10 kiwipy[rmq]~=0.5.1 -numpy<1.17.5,>=1.17.0; python_version == "3.5" -numpy<1.18,~=1.17; python_version >= "3.6" +numpy<1.18,~=1.17 paramiko~=2.6 pg8000~=1.13 pgtest>=1.3.1,~=1.3 diff --git a/setup.json b/setup.json index d540d17bc5..26460a9e6a 100644 --- a/setup.json +++ b/setup.json @@ -33,8 +33,7 @@ "ipython~=7.0", "jinja2~=2.10", "kiwipy[rmq]~=0.5.1", - "numpy~=1.17,<1.18; python_version>='3.6'", - "numpy>=1.17.0,<1.17.5; python_version=='3.5'", + "numpy~=1.17,<1.18", "paramiko~=2.6", "pika~=1.1", "plumpy~=0.14.5", From 1bb83513610f6336fd06ae8c2fb2a3c19c56b617 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 17:55:43 +0100 Subject: [PATCH 87/91] Run the two extra workflows on the PR branch for testing. --- .github/workflows/test-install.yml | 1 + .github/workflows/update-requirements.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 052687f41a..88ed88c72a 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -7,6 +7,7 @@ on: - develop - release/* - dm/* + - misc/dependency-management # TODO: REMOVE AFTER TESTING paths: - 'setup.*' - 'environment.yml' diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 7283fe5d02..8a139a6331 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -4,6 +4,7 @@ on: push: branch: - release/* + - misc/dependency-management # TODO: REMOVE AFTER TESTING paths: - 'setup.json' - '.github/workflows/update-requirements.yml' From bd2c17211959b1161ccb7e8ce1c03519cfe219cc Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 17:58:34 +0100 Subject: [PATCH 88/91] Pin the Click version to 7.0. The recent update to Click (7.1) breaks our tests and this CI workflow, because the help output formatting was slightly changed. Until we have resolved other issues we should pin the Click version to 7.0. --- docs/requirements_for_rtd.txt | 2 +- environment.yml | 2 +- setup.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements_for_rtd.txt b/docs/requirements_for_rtd.txt index 936d0c89d5..4336f2f21c 100644 --- a/docs/requirements_for_rtd.txt +++ b/docs/requirements_for_rtd.txt @@ -7,7 +7,7 @@ circus~=0.16.1 click-completion~=0.5.1 click-config-file~=0.5.0 click-spinner~=0.1.8 -click~=7.0 +click==7.0 coverage<5.0 django~=2.2 docutils==0.15.2 diff --git a/environment.yml b/environment.yml index f7d6ca7d1b..cfbb126c57 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - click-completion~=0.5.1 - click-config-file~=0.5.0 - click-spinner~=0.1.8 -- click~=7.0 +- click==7.0 - django~=2.2 - ete3~=3.1 - python-graphviz~=0.13 diff --git a/setup.json b/setup.json index 26460a9e6a..d3b77f0c8b 100644 --- a/setup.json +++ b/setup.json @@ -26,7 +26,7 @@ "click-completion~=0.5.1", "click-config-file~=0.5.0", "click-spinner~=0.1.8", - "click~=7.0", + "click==7.0", "django~=2.2", "ete3~=3.1", "graphviz~=0.13", From 33ac037a4c9bb0898e61d616f5fdfb8f53a64ec7 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 18:39:11 +0100 Subject: [PATCH 89/91] Slightly more explicit step names in test-install workflow. --- .github/workflows/test-install.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 88ed88c72a..f4169410c1 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -57,7 +57,7 @@ jobs: python -m pip install -e . python -m pip freeze - - name: Test package import + - name: Test importing aiida run: python -c "import aiida" @@ -86,7 +86,7 @@ jobs: source activate test-environment python -m pip install --no-deps -e . - - name: Test package import + - name: Test importing aiida run: | source activate test-environment python -c "import aiida" @@ -141,7 +141,7 @@ jobs: - run: pip freeze - - name: Setup environment + - name: Setup AiiDA environment env: AIIDA_TEST_BACKEND: ${{ matrix.backend }} run: From 51ecd99e3d514576033f582c421f0fceb94b0695 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 18:40:34 +0100 Subject: [PATCH 90/91] Fix typo in 'test-install' workflow definition. --- .github/workflows/test-install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index f4169410c1..c52217de78 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -13,7 +13,7 @@ on: - 'environment.yml' - 'requirements*.txt' - 'pyproject.toml' - - '.github/workflows/test-install.sh' + - '.github/workflows/test-install.yml' schedule: - cron: '30 02 * * *' # nightly build From 75f609a052b7a99642e7aac789f755935900493a Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 9 Mar 2020 19:03:45 +0100 Subject: [PATCH 91/91] Revert "Run the two extra workflows on the PR branch for testing." This reverts commit 1bb83513610f6336fd06ae8c2fb2a3c19c56b617. --- .github/workflows/test-install.yml | 1 - .github/workflows/update-requirements.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index c52217de78..8c3238ddfa 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -7,7 +7,6 @@ on: - develop - release/* - dm/* - - misc/dependency-management # TODO: REMOVE AFTER TESTING paths: - 'setup.*' - 'environment.yml' diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 8a139a6331..7283fe5d02 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -4,7 +4,6 @@ on: push: branch: - release/* - - misc/dependency-management # TODO: REMOVE AFTER TESTING paths: - 'setup.json' - '.github/workflows/update-requirements.yml'