Remove entrypoints in setup for internal backends (#4724)

* add a dictionary for internal backends

* remove entrypoints in setup.cfg

* create global variable BACKEND_ENTRYPOINT
move BackendEtrypoints in common to solve circular dependecy

* fix and update tests

* fix in tests_plugins to remove a warning

setup.cfg

@@ -81,17 +81,6 @@ setup_requires =
     setuptools >= 38.4
     setuptools_scm
 
-[options.entry_points]
-xarray.backends =
-    zarr = xarray.backends.zarr:zarr_backend
-    h5netcdf = xarray.backends.h5netcdf_:h5netcdf_backend
-    cfgrib = xarray.backends.cfgrib_:cfgrib_backend
-    scipy = xarray.backends.scipy_:scipy_backend
-    pynio = xarray.backends.pynio_:pynio_backend
-    pseudonetcdf = xarray.backends.pseudonetcdf_:pseudonetcdf_backend
-    netcdf4 = xarray.backends.netCDF4_:netcdf4_backend
-    store = xarray.backends.store:store_backend
-
 
 [options.extras_require]
 io =

xarray/backends/cfgrib_.py

@@ -5,9 +5,8 @@
 from ..core import indexing
 from ..core.utils import Frozen, FrozenDict, close_on_error
 from ..core.variable import Variable
-from .common import AbstractDataStore, BackendArray
+from .common import AbstractDataStore, BackendArray, BackendEntrypoint
 from .locks import SerializableLock, ensure_lock
-from .plugins import BackendEntrypoint
 from .store import open_backend_dataset_store
 
 # FIXME: Add a dedicated lock, even if ecCodes is supposed to be thread-safe

xarray/backends/common.py

@@ -340,3 +340,12 @@ def encode(self, variables, attributes):
         variables = {k: self.encode_variable(v) for k, v in variables.items()}
         attributes = {k: self.encode_attribute(v) for k, v in attributes.items()}
         return variables, attributes
+
+
+class BackendEntrypoint:
+    __slots__ = ("guess_can_open", "open_dataset", "open_dataset_parameters")
+
+    def __init__(self, open_dataset, open_dataset_parameters=None, guess_can_open=None):
+        self.open_dataset = open_dataset
+        self.open_dataset_parameters = open_dataset_parameters
+        self.guess_can_open = guess_can_open

xarray/backends/h5netcdf_.py

@@ -8,7 +8,7 @@
 from ..core import indexing
 from ..core.utils import FrozenDict, is_remote_uri, read_magic_number
 from ..core.variable import Variable
-from .common import WritableCFDataStore, find_root_and_group
+from .common import BackendEntrypoint, WritableCFDataStore, find_root_and_group
 from .file_manager import CachingFileManager, DummyFileManager
 from .locks import HDF5_LOCK, combine_locks, ensure_lock, get_write_lock
 from .netCDF4_ import (
@@ -18,7 +18,6 @@
     _get_datatype,
     _nc4_require_group,
 )
-from .plugins import BackendEntrypoint
 from .store import open_backend_dataset_store
 
 

xarray/backends/netCDF4_.py

@@ -13,14 +13,14 @@
 from ..core.variable import Variable
 from .common import (
     BackendArray,
+    BackendEntrypoint,
     WritableCFDataStore,
     find_root_and_group,
     robust_getitem,
 )
 from .file_manager import CachingFileManager, DummyFileManager
 from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock, get_write_lock
 from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable
-from .plugins import BackendEntrypoint
 from .store import open_backend_dataset_store
 
 # This lookup table maps from dtype.byteorder to a readable endian

xarray/backends/plugins.py

@@ -1,19 +1,34 @@
+import functools
 import inspect
 import itertools
 import logging
+import typing as T
 import warnings
-from functools import lru_cache
 
 import pkg_resources
 
-
-class BackendEntrypoint:
-    __slots__ = ("guess_can_open", "open_dataset", "open_dataset_parameters")
-
-    def __init__(self, open_dataset, open_dataset_parameters=None, guess_can_open=None):
-        self.open_dataset = open_dataset
-        self.open_dataset_parameters = open_dataset_parameters
-        self.guess_can_open = guess_can_open
+from .cfgrib_ import cfgrib_backend
+from .common import BackendEntrypoint
+from .h5netcdf_ import h5netcdf_backend
+from .netCDF4_ import netcdf4_backend
+from .pseudonetcdf_ import pseudonetcdf_backend
+from .pydap_ import pydap_backend
+from .pynio_ import pynio_backend
+from .scipy_ import scipy_backend
+from .store import store_backend
+from .zarr import zarr_backend
+
+BACKEND_ENTRYPOINTS: T.Dict[str, BackendEntrypoint] = {
+    "store": store_backend,
+    "netcdf4": netcdf4_backend,
+    "h5netcdf": h5netcdf_backend,
+    "scipy": scipy_backend,
+    "pseudonetcdf": pseudonetcdf_backend,
+    "zarr": zarr_backend,
+    "cfgrib": cfgrib_backend,
+    "pydap": pydap_backend,
+    "pynio": pynio_backend,
+}
 
 
 def remove_duplicates(backend_entrypoints):
@@ -71,13 +86,19 @@ def set_missing_parameters(engines):
             backend.open_dataset_parameters = detect_parameters(open_dataset)
 
 
-@lru_cache(maxsize=1)
+def build_engines(entrypoints):
+    backend_entrypoints = BACKEND_ENTRYPOINTS.copy()
+    pkg_entrypoints = remove_duplicates(entrypoints)
+    external_backend_entrypoints = create_engines_dict(pkg_entrypoints)
+    backend_entrypoints.update(external_backend_entrypoints)
+    set_missing_parameters(backend_entrypoints)
+    return backend_entrypoints
+
+
+@functools.lru_cache(maxsize=1)
 def list_engines():
     entrypoints = pkg_resources.iter_entry_points("xarray.backends")
-    backend_entrypoints = remove_duplicates(entrypoints)
-    engines = create_engines_dict(backend_entrypoints)
-    set_missing_parameters(engines)
-    return engines
+    return build_engines(entrypoints)
 
 
 def guess_engine(store_spec):

xarray/backends/pseudonetcdf_.py

@@ -3,10 +3,9 @@
 from ..core import indexing
 from ..core.utils import Frozen, FrozenDict, close_on_error
 from ..core.variable import Variable
-from .common import AbstractDataStore, BackendArray
+from .common import AbstractDataStore, BackendArray, BackendEntrypoint
 from .file_manager import CachingFileManager
 from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock
-from .plugins import BackendEntrypoint
 from .store import open_backend_dataset_store
 
 # psuedonetcdf can invoke netCDF libraries internally

xarray/backends/pydap_.py

@@ -4,8 +4,7 @@
 from ..core.pycompat import integer_types
 from ..core.utils import Frozen, FrozenDict, close_on_error, is_dict_like, is_remote_uri
 from ..core.variable import Variable
-from .common import AbstractDataStore, BackendArray, robust_getitem
-from .plugins import BackendEntrypoint
+from .common import AbstractDataStore, BackendArray, BackendEntrypoint, robust_getitem
 from .store import open_backend_dataset_store
 
 

xarray/backends/pynio_.py

@@ -3,10 +3,9 @@
 from ..core import indexing
 from ..core.utils import Frozen, FrozenDict, close_on_error
 from ..core.variable import Variable
-from .common import AbstractDataStore, BackendArray
+from .common import AbstractDataStore, BackendArray, BackendEntrypoint
 from .file_manager import CachingFileManager
 from .locks import HDF5_LOCK, NETCDFC_LOCK, SerializableLock, combine_locks, ensure_lock
-from .plugins import BackendEntrypoint
 from .store import open_backend_dataset_store
 
 # PyNIO can invoke netCDF libraries internally

xarray/backends/scipy_.py

@@ -6,11 +6,10 @@
 from ..core.indexing import NumpyIndexingAdapter
 from ..core.utils import Frozen, FrozenDict, close_on_error, read_magic_number
 from ..core.variable import Variable
-from .common import BackendArray, WritableCFDataStore
+from .common import BackendArray, BackendEntrypoint, WritableCFDataStore
 from .file_manager import CachingFileManager, DummyFileManager
 from .locks import ensure_lock, get_write_lock
 from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable, is_valid_nc3_name
-from .plugins import BackendEntrypoint
 from .store import open_backend_dataset_store
 
 

xarray/backends/store.py

@@ -1,7 +1,6 @@
 from .. import conventions
 from ..core.dataset import Dataset
-from .common import AbstractDataStore
-from .plugins import BackendEntrypoint
+from .common import AbstractDataStore, BackendEntrypoint
 
 
 def guess_can_open_store(store_spec):

xarray/backends/zarr.py

@@ -8,8 +8,12 @@
 from ..core.pycompat import integer_types
 from ..core.utils import FrozenDict, HiddenKeyDict, close_on_error
 from ..core.variable import Variable
-from .common import AbstractWritableDataStore, BackendArray, _encode_variable_name
-from .plugins import BackendEntrypoint
+from .common import (
+    AbstractWritableDataStore,
+    BackendArray,
+    BackendEntrypoint,
+    _encode_variable_name,
+)
 from .store import open_backend_dataset_store
 
 # need some special secret attributes to tell us the dimensions

xarray/tests/test_plugins.py

@@ -3,7 +3,7 @@
 import pkg_resources
 import pytest
 
-from xarray.backends import plugins
+from xarray.backends import common, plugins
 
 
 def dummy_open_dataset_args(filename_or_obj, *args):
@@ -18,6 +18,9 @@ def dummy_open_dataset(filename_or_obj, *, decoder):
     pass
 
 
+dummy_cfgrib = common.BackendEntrypoint(dummy_open_dataset)
+
+
 @pytest.fixture
 def dummy_duplicated_entrypoints():
     specs = [
@@ -32,7 +35,8 @@ def dummy_duplicated_entrypoints():
 
 @pytest.mark.filterwarnings("ignore:Found")
 def test_remove_duplicates(dummy_duplicated_entrypoints):
-    entrypoints = plugins.remove_duplicates(dummy_duplicated_entrypoints)
+    with pytest.warns(RuntimeWarning):
+        entrypoints = plugins.remove_duplicates(dummy_duplicated_entrypoints)
     assert len(entrypoints) == 2
 
 
@@ -61,8 +65,8 @@ def test_create_engines_dict():
 
 
 def test_set_missing_parameters():
-    backend_1 = plugins.BackendEntrypoint(dummy_open_dataset)
-    backend_2 = plugins.BackendEntrypoint(dummy_open_dataset, ("filename_or_obj",))
+    backend_1 = common.BackendEntrypoint(dummy_open_dataset)
+    backend_2 = common.BackendEntrypoint(dummy_open_dataset, ("filename_or_obj",))
     engines = {"engine_1": backend_1, "engine_2": backend_2}
     plugins.set_missing_parameters(engines)
 
@@ -75,20 +79,33 @@ def test_set_missing_parameters():
 
 def test_set_missing_parameters_raise_error():
 
-    backend = plugins.BackendEntrypoint(dummy_open_dataset_args)
+    backend = common.BackendEntrypoint(dummy_open_dataset_args)
     with pytest.raises(TypeError):
         plugins.set_missing_parameters({"engine": backend})
 
-    backend = plugins.BackendEntrypoint(
+    backend = common.BackendEntrypoint(
         dummy_open_dataset_args, ("filename_or_obj", "decoder")
     )
     plugins.set_missing_parameters({"engine": backend})
 
-    backend = plugins.BackendEntrypoint(dummy_open_dataset_kwargs)
+    backend = common.BackendEntrypoint(dummy_open_dataset_kwargs)
     with pytest.raises(TypeError):
         plugins.set_missing_parameters({"engine": backend})
 
     backend = plugins.BackendEntrypoint(
         dummy_open_dataset_kwargs, ("filename_or_obj", "decoder")
     )
     plugins.set_missing_parameters({"engine": backend})
+
+
+@mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=dummy_cfgrib))
+def test_build_engines():
+    dummy_cfgrib_pkg_entrypoint = pkg_resources.EntryPoint.parse(
+        "cfgrib = xarray.tests.test_plugins:backend_1"
+    )
+    backend_entrypoints = plugins.build_engines([dummy_cfgrib_pkg_entrypoint])
+    assert backend_entrypoints["cfgrib"] is dummy_cfgrib
+    assert backend_entrypoints["cfgrib"].open_dataset_parameters == (
+        "filename_or_obj",
+        "decoder",
+    )