Support for Python 3.9 (facebookresearch#1197)

* Fix importlib error when using py3.9

* Fix lint errors

* Add a method to match lines based on regex

* fix lint error

* bump hydra version in discovery test plugin

* Add python 3.9 in setup.py

* Skip python 3.9 tests on windows

* use assert_text_same when lengths of actual and expected output do not match

* Remove dependency on hydra version

* Check python version to decide if importlib_resources should be imported

* Update the install script for plugins to reflect support for python 3.9

* pin virtualenv to 20.0.33 for windows

* Fix lint error

* Enable CI for 3.9 on windows

* Update circle ci config

* Update circle ci config

* Fix issues with windows

* Skip Ray test for Python 3.9

* Incorporate feedback

* Fix circleci config

* Fix lint errors

* Fix circleci config

* Fix some ci errors

* Fix lint error

* Fix lint error

* Update setup.py for some plugins

* Disable windows 9 support

* Ax plugin doesnt support 3.9 for now

* Nevergrad python 3.9 support depends on Scikit-learn

* Add news entries

* Accidentally removed support for optuna on windows

* Incorporate feedback

* Disable windows test for optuna

.circleci/config.yml

@@ -92,16 +92,16 @@ commands:
             choco install -y --no-progress openssl javaruntime
       - run:
           name: Preparing environment - Hydra
+          # Using virtualenv==20.0.33 higher versions of virtualenv are not compatible with conda on windows. Relevant issue: https://github.com/ContinuumIO/anaconda-issues/issues/12094
           command: |
-            conda create -n hydra python=<< parameters.py_version >> pywin32 -qy
+            conda create -n hydra python=<< parameters.py_version >> pywin32 virtualenv==20.0.33 -qy
             conda activate hydra
             pip install nox dataclasses --progress-bar off
       - save_cache:
           key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ .Branch }}-<< parameters.py_version >>
           paths:
             - C:\tools\miniconda3
 
-
 jobs:
   test_macos:
     parameters:
@@ -240,16 +240,15 @@ workflows:
       - test_macos:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8"]
+              py_version: ["3.6", "3.7", "3.8", "3.9"]
       - test_linux:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8"]
+              py_version: ["3.6", "3.7", "3.8", "3.9"]
       - test_win:
           matrix:
             parameters:
-#              py_version: ["3.6", "3.7", "3.8"]
-              py_version: ["3.6"]
+              py_version: ["3.6", "3.7", "3.8"]
 
 
   plugin_tests:
@@ -258,18 +257,17 @@ workflows:
       - test_plugin_linux:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8"]
+              py_version: ["3.6", "3.7", "3.8", "3.9"]
               test_plugin: [<< pipeline.parameters.test_plugins >>]
       - test_plugin_macos:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8"]
+              py_version: ["3.6", "3.7", "3.8", "3.9"]
               test_plugin: [<< pipeline.parameters.test_plugins >>]
       - test_plugin_win:
           matrix:
             parameters:
-#              py_version: ["3.6", "3.7", "3.8"]
-              py_version: ["3.6"]
+              py_version: ["3.6", "3.7", "3.8",]
               test_plugin: [<< pipeline.parameters.test_plugins >>]
 
 

hydra/_internal/core_plugins/importlib_resources_config_source.py

@@ -1,13 +1,23 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import os
+import sys
 from typing import List, Optional
 
-import importlib_resources
 from omegaconf import OmegaConf
 
 from hydra.core.object_type import ObjectType
 from hydra.plugins.config_source import ConfigLoadError, ConfigResult, ConfigSource
 
+if sys.version_info.major >= 4 or (
+    sys.version_info.major >= 3 and sys.version_info.minor >= 9
+):
+    from importlib import resources
+else:
+    import importlib_resources as resources  # type:ignore
+
+    # Relevant issue: https://github.com/python/mypy/issues/1153
+    # Use importlib backport for Python older than 3.9
+
 
 class ImportlibResourcesConfigSource(ConfigSource):
     def __init__(self, provider: str, path: str) -> None:
@@ -26,7 +36,7 @@ def load_config(
         package_override: Optional[str] = None,
     ) -> ConfigResult:
         normalized_config_path = self._normalize_file_name(config_path)
-        res = importlib_resources.files(self.path).joinpath(normalized_config_path)
+        res = resources.files(self.path).joinpath(normalized_config_path)  # type:ignore
         if not res.exists():
             raise ConfigLoadError(f"Config not found : {normalized_config_path}")
 
@@ -50,7 +60,7 @@ def load_config(
 
     def available(self) -> bool:
         try:
-            ret = importlib_resources.is_resource(self.path, "__init__.py")
+            ret = resources.is_resource(self.path, "__init__.py")  # type:ignore
             assert isinstance(ret, bool)
             return ret
         except ValueError:
@@ -60,7 +70,7 @@ def available(self) -> bool:
 
     def is_group(self, config_path: str) -> bool:
         try:
-            files = importlib_resources.files(self.path)
+            files = resources.files(self.path)  # type:ignore
         except Exception:
             return False
 
@@ -72,7 +82,7 @@ def is_group(self, config_path: str) -> bool:
     def is_config(self, config_path: str) -> bool:
         config_path = self._normalize_file_name(config_path)
         try:
-            files = importlib_resources.files(self.path)
+            files = resources.files(self.path)  # type:ignore
         except Exception:
             return False
         res = files.joinpath(config_path)
@@ -83,7 +93,7 @@ def is_config(self, config_path: str) -> bool:
     def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[str]:
         files: List[str] = []
         for file in (
-            importlib_resources.files(self.path).joinpath(config_path).iterdir()
+            resources.files(self.path).joinpath(config_path).iterdir()  # type:ignore
         ):
             fname = file.name
             fpath = os.path.join(config_path, fname)

hydra/test_utils/test_utils.py

@@ -5,6 +5,7 @@
 import copy
 import logging
 import os
+import re
 import shutil
 import string
 import subprocess
@@ -423,3 +424,30 @@ def assert_text_same(
         print(diff)
         print("-------------------------------")
         assert False, "Mismatch between expected and actual text"
+
+
+def assert_regex_match(
+    from_line: str, to_line: str, from_name: str = "Expected", to_name: str = "Actual"
+) -> None:
+    """Check that the lines of `from_line` (which can be a regex expression)
+    matches the corresponding lines of `to_line` string.
+
+    In case the regex match fails, we display the diff as if `from_line` was a regular string.
+    """
+    normalized_from_line = [x for x in normalize_newlines(from_line).split("\n") if x]
+    normalized_to_line = [x for x in normalize_newlines(to_line).split("\n") if x]
+    if len(normalized_from_line) != len(normalized_to_line):
+        assert_text_same(
+            from_line=from_line,
+            to_line=to_line,
+            from_name=from_name,
+            to_name=to_name,
+        )
+    for line1, line2 in zip(normalized_from_line, normalized_to_line):
+        if line1 != line2 and re.match(line1, line2) is None:
+            assert_text_same(
+                from_line=from_line,
+                to_line=to_line,
+                from_name=from_name,
+                to_name=to_name,
+            )

news/1062.feature

@@ -0,0 +1 @@
+Support Python 3.9 .

noxfile.py

@@ -12,7 +12,7 @@
 
 BASE = os.path.abspath(os.path.dirname(__file__))
 
-DEFAULT_PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]
+DEFAULT_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
 DEFAULT_OS_NAMES = ["Linux", "MacOS", "Windows"]
 
 PYTHON_VERSIONS = os.environ.get(

plugins/hydra_ax_sweeper/setup.py

@@ -18,6 +18,7 @@
             "License :: OSI Approved :: MIT License",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            # "Programming Language :: Python :: 3.9",
             "Operating System :: POSIX :: Linux",
             "Operating System :: MacOS",
             "Development Status :: 4 - Beta",

plugins/hydra_colorlog/news/1062.feature

@@ -0,0 +1 @@
+Support Python 3.9 .

plugins/hydra_colorlog/setup.py

@@ -19,6 +19,7 @@
             "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: 3.9",
             "Operating System :: OS Independent",
         ],
         install_requires=["colorlog", "hydra-core>=1.0.0"],

plugins/hydra_joblib_launcher/news/1062.feature

@@ -0,0 +1 @@
+Support Python 3.9 .

plugins/hydra_joblib_launcher/setup.py

@@ -18,6 +18,7 @@
             "License :: OSI Approved :: MIT License",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: 3.9",
             "Operating System :: MacOS",
             "Operating System :: Microsoft :: Windows",
             "Operating System :: POSIX :: Linux",

plugins/hydra_nevergrad_sweeper/setup.py

@@ -19,6 +19,7 @@
             "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            # "Programming Language :: Python :: 3.9",
             "Operating System :: OS Independent",
             "Development Status :: 4 - Beta",
         ],

plugins/hydra_optuna_sweeper/setup.py

@@ -26,7 +26,9 @@ def get_long_description() -> str:
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
-        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3.9",
+        "Operating System :: POSIX :: Linux",
+        "Operating System :: MacOS",
         "Development Status :: 4 - Beta",
     ],
     install_requires=["hydra-core", "optuna"],

plugins/hydra_optuna_sweeper/tests/test_optuna_sweeper_plugin.py

@@ -24,6 +24,8 @@
 chdir_plugin_root()
 
 
+# https://github.com/pyreadline/pyreadline/issues/65
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")  # type: ignore
 def test_discovery() -> None:
     assert OptunaSweeper.__name__ in [
         x.__name__ for x in Plugins.instance().discover(Sweeper)
@@ -40,6 +42,8 @@ def check_distribution(expected: BaseDistribution, actual: BaseDistribution) ->
     assert set(expected.choices) == set(actual.choices)
 
 
+# https://github.com/pyreadline/pyreadline/issues/65
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")  # type: ignore
 @pytest.mark.parametrize(  # type: ignore
     "input, expected",
     [
@@ -73,6 +77,8 @@ def test_create_optuna_distribution_from_config(input: Any, expected: Any) -> No
     check_distribution(expected, actual)
 
 
+# https://github.com/pyreadline/pyreadline/issues/65
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")  # type: ignore
 @pytest.mark.parametrize(  # type: ignore
     "input, expected",
     [
@@ -94,6 +100,8 @@ def test_create_optuna_distribution_from_override(input: Any, expected: Any) ->
     check_distribution(expected, actual)
 
 
+# https://github.com/pyreadline/pyreadline/issues/65
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")  # type: ignore
 def test_launch_jobs(hydra_sweep_runner: TSweepRunner) -> None:
     sweep = hydra_sweep_runner(
         calling_file=None,
@@ -112,6 +120,8 @@ def test_launch_jobs(hydra_sweep_runner: TSweepRunner) -> None:
         assert sweep.returns is None
 
 
+# https://github.com/pyreadline/pyreadline/issues/65
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")  # type: ignore
 @pytest.mark.parametrize("with_commandline", (True, False))  # type: ignore
 def test_optuna_example(with_commandline: bool, tmpdir: Path) -> None:
     cmd = [

plugins/hydra_ray_launcher/setup.py

@@ -17,6 +17,7 @@
             "License :: OSI Approved :: MIT License",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            # "Programming Language :: Python :: 3.9",
             "Operating System :: MacOS",
             "Operating System :: POSIX :: Linux",
             "Development Status :: 4 - Beta",

plugins/hydra_rq_launcher/news/1062.feature

@@ -0,0 +1 @@
+Support Python 3.9 .

plugins/hydra_rq_launcher/setup.py

@@ -19,6 +19,7 @@
             "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: 3.9",
             "Operating System :: MacOS",
             "Operating System :: POSIX :: Linux",
         ],

plugins/hydra_submitit_launcher/news/1062.feature

@@ -0,0 +1 @@
+Support Python 3.9 .

plugins/hydra_submitit_launcher/setup.py

@@ -18,6 +18,7 @@
             "License :: OSI Approved :: MIT License",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: 3.9",
             "Operating System :: MacOS",
             "Operating System :: POSIX :: Linux",
             "Development Status :: 4 - Beta",

setup.py

@@ -48,6 +48,7 @@
             "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: 3.9",
             "Operating System :: POSIX :: Linux",
             "Operating System :: MacOS",
             "Operating System :: Microsoft :: Windows",

tests/standalone_apps/discovery_test_plugin/setup.py

@@ -8,5 +8,5 @@
     author_email="omry@fb.com",
     url="https://github.com/facebookresearch/hydra/",
     packages=find_namespace_packages(include=["hydra_plugins.*"]),
-    install_requires=["hydra-core==1.0.*"],
+    install_requires=["hydra-core"],
 )

tests/test_hydra.py

@@ -15,7 +15,7 @@
 from hydra.test_utils.test_utils import (
     TSweepRunner,
     TTaskRunner,
-    assert_text_same,
+    assert_regex_match,
     chdir_hydra_root,
     get_run_output,
     integration_test,
@@ -1028,9 +1028,9 @@ def test_app_with_error_exception_sanitized(tmpdir: Any, monkeypatch: Any) -> No
         "hydra.sweep.dir=" + str(tmpdir),
     ]
     expected = """Traceback (most recent call last):
-  File "my_app.py", line 13, in my_app
+  File ".*my_app.py", line 13, in my_app
     foo(cfg)
-  File "my_app.py", line 8, in foo
+  File ".*my_app.py", line 8, in foo
     cfg.foo = "bar"  # does not exist in the config
 omegaconf.errors.ConfigAttributeError: Key 'foo' is not in struct
 \tfull_key: foo
@@ -1040,7 +1040,12 @@ def test_app_with_error_exception_sanitized(tmpdir: Any, monkeypatch: Any) -> No
 Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace."""
 
     ret = run_with_error(cmd)
-    assert normalize_newlines(expected) == normalize_newlines(ret)
+    assert_regex_match(
+        from_line=expected,
+        to_line=ret,
+        from_name="Expected output",
+        to_name="Actual output",
+    )
 
 
 def test_hydra_to_job_config_interpolation(tmpdir: Any) -> Any:
@@ -1128,7 +1133,7 @@ def test_2(self) -> None:
             dedent(
                 """\
                 Traceback (most recent call last):
-                  File "my_app.py", line 9, in my_app
+                  File ".*my_app.py", line 9, in my_app
                     1 / 0
                 ZeroDivisionError: division by zero
 
@@ -1138,11 +1143,14 @@ def test_2(self) -> None:
         ),
     ],
 )
-def test_hydra_exception(monkeypatch: Any, tmpdir: Any, expected: str) -> None:
+def test_hydra_exception(
+    monkeypatch: Any,
+    tmpdir: Any,
+    expected: str,
+) -> None:
     monkeypatch.chdir("tests/test_apps/app_exception")
     ret = run_with_error(["my_app.py", f"hydra.run.dir={tmpdir}"])
-
-    assert_text_same(
+    assert_regex_match(
         from_line=expected,
         to_line=ret,
         from_name="Expected output",