🩹 Fix Project.import_data path resolving for different script and cwd

[s-weigand]

This fixes a bug in Project.import_data causing a ValueError when using a relative path and the directory of the script is not the current working directory.
This is the case in editors when the script is not at the folder root or when calling it from python (e.g. python myproject/script1.py)

In addition, I improved the saving functionality of the plugin system to create parent folders for files.
Before that trying to save a file to a location with a nonexisting parent folder caused a massive traceback from the undelaying writer implementation.

Example traceback

save_dataset(result.data["dataset1"], "nonexisting_folder/dataset1.nc")

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\file_manager.py:209, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
    208 try:
--> 209     file = self._cache[self._key]
    210 except KeyError:

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\lru_cache.py:55, in LRUCache.__getitem__(self, key)
     54 with self._lock:
---> 55     value = self._cache[key]
     56     self._cache.move_to_end(key)

KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('d:\\Dropbox\\Dropbox\\uni\\master\\thesis\\case_studies\\ArticialLego-2022-12-16\\ArticialLego\\pyglotaran\\nonexisting_folder\\dataset1.nc',), 'a', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '81707b57-1609-4544-91a9-094f02f41e67']

During handling of the above exception, another exception occurred:

PermissionError                           Traceback (most recent call last)
Cell In [20], line 3
      1 from glotaran.io import save_dataset
----> 3 save_dataset(result.data["dataset1"], "nonexisting_folder/dataset1.nc")

File D:\git\pyglotaran\glotaran\plugin_system\io_plugin_utils.py:87, in not_implemented_to_value_error.<locals>.wrapper(*args, **kwargs)
     84 @wraps(func)
     85 def wrapper(*args: Any, **kwargs: Any) -> Any:
     86     try:
---> 87         return func(*args, **kwargs)
     88     except NotImplementedError as error:
     89         raise ValueError(error.args)

File D:\git\pyglotaran\glotaran\plugin_system\data_io_registration.py:241, in save_dataset(dataset, file_name, format_name, data_filters, allow_overwrite, update_source_path, **kwargs)
    239 if "loader" in dataset.attrs:
    240     del dataset.attrs["loader"]
--> 241 io.save_dataset(file_name=Path(file_name).as_posix(), dataset=dataset, **kwargs)
    242 dataset.attrs["loader"] = load_dataset
    243 if update_source_path is True or "source_path" not in dataset.attrs:

File D:\git\pyglotaran\glotaran\builtin\io\netCDF\netCDF.py:51, in NetCDFDataIo.save_dataset(self, dataset, file_name, data_filters)
     39 """Write a :xarraydoc:`Dataset` to the ``*.nc`` at path ``file_name``.
     40 
     41 Parameters
   (...)
     48     List of data variable names that should be written to file. Defaults to None.
     49 """
     50 data_to_save = dataset if data_filters is None else dataset[data_filters]
---> 51 data_to_save.to_netcdf(file_name, mode="w")

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\core\dataset.py:1903, in Dataset.to_netcdf(self, path, mode, format, group, engine, encoding, unlimited_dims, compute, invalid_netcdf)
   1900     encoding = {}
   1901 from ..backends.api import to_netcdf
-> 1903 return to_netcdf(  # type: ignore  # mypy cannot resolve the overloads:(
   1904     self,
   1905     path,
   1906     mode=mode,
   1907     format=format,
   1908     group=group,
   1909     engine=engine,
   1910     encoding=encoding,
   1911     unlimited_dims=unlimited_dims,
   1912     compute=compute,
   1913     multifile=False,
   1914     invalid_netcdf=invalid_netcdf,
   1915 )

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\api.py:1213, in to_netcdf(dataset, path_or_file, mode, format, group, engine, encoding, unlimited_dims, compute, multifile, invalid_netcdf)
   1209     else:
   1210         raise ValueError(
   1211             f"unrecognized option 'invalid_netcdf' for engine {engine}"
   1212         )
-> 1213 store = store_open(target, mode, format, group, **kwargs)
   1215 if unlimited_dims is None:
   1216     unlimited_dims = dataset.encoding.get("unlimited_dims", None)

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\netCDF4_.py:376, in NetCDF4DataStore.open(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)
    370 kwargs = dict(
    371     clobber=clobber, diskless=diskless, persist=persist, format=format
    372 )
    373 manager = CachingFileManager(
    374     netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
    375 )
--> 376 return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\netCDF4_.py:323, in NetCDF4DataStore.__init__(self, manager, group, mode, lock, autoclose)
    321 self._group = group
    322 self._mode = mode
--> 323 self.format = self.ds.data_model
    324 self._filename = self.ds.filepath()
    325 self.is_remote = is_remote_uri(self._filename)

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\netCDF4_.py:385, in NetCDF4DataStore.ds(self)
    383 @property
    384 def ds(self):
--> 385     return self._acquire()

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\netCDF4_.py:379, in NetCDF4DataStore._acquire(self, needs_lock)
    378 def _acquire(self, needs_lock=True):
--> 379     with self._manager.acquire_context(needs_lock) as root:
    380         ds = _nc4_require_group(root, self._group, self._mode)
    381     return ds

File c:\Anaconda3\envs\pyglotaran310\lib\contextlib.py:135, in _GeneratorContextManager.__enter__(self)
    133 del self.args, self.kwds, self.func
    134 try:
--> 135     return next(self.gen)
    136 except StopIteration:
    137     raise RuntimeError("generator didn't yield") from None

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\file_manager.py:197, in CachingFileManager.acquire_context(self, needs_lock)
    194 @contextlib.contextmanager
    195 def acquire_context(self, needs_lock=True):
    196     """Context manager for acquiring a file."""
--> 197     file, cached = self._acquire_with_cache_info(needs_lock)
    198     try:
    199         yield file

File c:\Anaconda3\envs\pyglotaran310\lib\site-packages\xarray\backends\file_manager.py:215, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
    213     kwargs = kwargs.copy()
    214     kwargs["mode"] = self._mode
--> 215 file = self._opener(*self._args, **kwargs)
    216 if self._mode == "w":
    217     # ensure file doesn't get overridden when opened again
    218     self._mode = "a"

File src\netCDF4\_netCDF4.pyx:2463, in netCDF4._netCDF4.Dataset.__init__()

File src\netCDF4\_netCDF4.pyx:2026, in netCDF4._netCDF4._ensure_nc_success()

PermissionError: [Errno 13] Permission denied: b'd:\\test\\nonexisting_folder\\dataset1.nc'

Change summary

• 👌 Change save functions to create parent folder
• 🧪 Added failing test for Project.import_data using a relative path
• 🩹 Fix Project.import_data path resolving for different scirpt and cwd

Checklist

• ✔️ Passing the tests (mandatory for all PR's)
• 🚧 Added changes to changelog (mandatory for all PR's)
• 🧪 Adds new tests for the feature (mandatory for ✨ feature and 🩹 bug fix PR's)

[sourcery-ai]

Sourcery Code Quality Report

❌  Merging this PR will decrease code quality in the affected files by 0.79%.

Quality metrics	Before	After	Change
Complexity	0.72 ⭐	0.76 ⭐	0.04 👎
Method Length	62.87 🙂	64.97 🙂	2.10 👎
Working memory	5.67 ⭐	5.81 ⭐	0.14 👎
Quality	80.71% ⭐	79.92% ⭐	-0.79% 👎

Other metrics	Before	After	Change
Lines	1573	1616	43

Changed files	Quality Before	Quality After	Quality Change
glotaran/plugin_system/data_io_registration.py	83.49% ⭐	83.49% ⭐	0.00%
glotaran/plugin_system/io_plugin_utils.py	80.57% ⭐	78.60% ⭐	-1.97% 👎
glotaran/plugin_system/test/test_data_io_registration.py	88.85% ⭐	88.22% ⭐	-0.63% 👎
glotaran/plugin_system/test/test_project_io_registration.py	87.88% ⭐	87.58% ⭐	-0.30% 👎
glotaran/project/project_data_registry.py	79.53% ⭐	73.85% 🙂	-5.68% 👎
glotaran/project/test/test_project.py	72.30% 🙂	72.11% 🙂	-0.19% 👎

Here are some functions in these files that still need a tune-up:

File	Function	Complexity	Length	Working Memory	Quality	Recommendation
glotaran/project/test/test_project.py	test_generate_parameters	3 ⭐	213 ⛔	9 🙂	56.54% 🙂	Try splitting into smaller methods
glotaran/plugin_system/data_io_registration.py	save_dataset	4 ⭐	106 🙂	13 😞	59.01% 🙂	Extract out complex expressions
glotaran/project/test/test_project.py	test_missing_file_errors	0 ⭐	220 ⛔	8 🙂	61.27% 🙂	Try splitting into smaller methods
glotaran/project/test/test_project.py	test_generators_allow_overwrite	0 ⭐	151 😞	10 😞	62.82% 🙂	Try splitting into smaller methods. Extract out complex expressions
glotaran/project/test/test_project.py	test_run_optimization	2 ⭐	152 😞	8 🙂	64.98% 🙂	Try splitting into smaller methods

Legend and Explanation

The emojis denote the absolute quality of the code:

• ⭐ excellent
• 🙂 good
• 😞 poor
• ⛔ very poor

The 👍 and 👎 indicate whether the quality has improved or gotten worse with this pull request.

---

Please see our documentation here for details on how these metrics are calculated.

We are actively working on this report - lots more documentation and extra metrics to come!

Help us improve this quality report!

[github-actions]

👈 Launch a binder notebook on branch s-weigand/pyglotaran/fix-project-data-import-rel-path

[github-actions]

Benchmark is done. Checkout the benchmark result page.
Benchmark differences below 5% might be due to CI noise.

Benchmark diff v0.6.0 vs. main

Parametrized benchmark signatures:

BenchmarkOptimize.time_optimize(index_dependent, grouped, weight)

All benchmarks:

       before           after         ratio
     [6c3c390e]       [1b0daee8]
     <v0.6.0>                   
!        64.4±2ms           failed      n/a  BenchmarkOptimize.time_optimize(False, False, False)
!        108±40ms           failed      n/a  BenchmarkOptimize.time_optimize(False, False, True)
!        63.6±2ms           failed      n/a  BenchmarkOptimize.time_optimize(False, True, False)
!       94.9±50ms           failed      n/a  BenchmarkOptimize.time_optimize(False, True, True)
!        83.5±1ms           failed      n/a  BenchmarkOptimize.time_optimize(True, False, False)
!        91.0±6ms           failed      n/a  BenchmarkOptimize.time_optimize(True, False, True)
!        85.8±2ms           failed      n/a  BenchmarkOptimize.time_optimize(True, True, False)
!       88.5±20ms           failed      n/a  BenchmarkOptimize.time_optimize(True, True, True)
             205M             206M     1.00  IntegrationTwoDatasets.peakmem_optimize
-      2.29±0.03s       1.54±0.04s     0.67  IntegrationTwoDatasets.time_optimize

Benchmark diff main vs. PR

Parametrized benchmark signatures:

BenchmarkOptimize.time_optimize(index_dependent, grouped, weight)

All benchmarks:

       before           after         ratio
     [1b0daee8]       [08acbe83]
           failed           failed      n/a  BenchmarkOptimize.time_optimize(False, False, False)
           failed           failed      n/a  BenchmarkOptimize.time_optimize(False, False, True)
           failed           failed      n/a  BenchmarkOptimize.time_optimize(False, True, False)
           failed           failed      n/a  BenchmarkOptimize.time_optimize(False, True, True)
           failed           failed      n/a  BenchmarkOptimize.time_optimize(True, False, False)
           failed           failed      n/a  BenchmarkOptimize.time_optimize(True, False, True)
           failed           failed      n/a  BenchmarkOptimize.time_optimize(True, True, False)
           failed           failed      n/a  BenchmarkOptimize.time_optimize(True, True, True)
             206M             208M     1.01  IntegrationTwoDatasets.peakmem_optimize
       1.54±0.04s       1.59±0.03s     1.04  IntegrationTwoDatasets.time_optimize

[codecov]

Codecov Report

Base: 87.7% // Head: 87.8% // Increases project coverage by +0.1% 🎉

Coverage data is based on head (08acbe8) compared to base (1b0daee).
Patch coverage: 100.0% of modified lines in pull request are covered.

Additional details and impacted files

@@           Coverage Diff           @@
##            main   #1214     +/-   ##
=======================================
+ Coverage   87.7%   87.8%   +0.1%     
=======================================
  Files        104     104             
  Lines       4961    4967      +6     
  Branches     824     826      +2     
=======================================
+ Hits        4352    4364     +12     
+ Misses       492     487      -5     
+ Partials     117     116      -1     

Impacted Files	Coverage Δ
glotaran/plugin_system/data_io_registration.py	98.5% <ø> (ø)
glotaran/plugin_system/io_plugin_utils.py	100.0% <100.0%> (ø)
glotaran/project/project_data_registry.py	100.0% <100.0%> (ø)
glotaran/utils/io.py	100.0% <0.0%> (+5.3%)	⬆️

Help us with your feedback. Take ten seconds to tell us how you rate us. Have a feature suggestion? Share it here.

☔ View full report at Codecov.
📢 Do you have feedback about the report comment? Let us know in this issue.

[jsnel]

LGTM. Tested ok.

[sonarqubecloud]

Kudos, SonarCloud Quality Gate passed!   

0 Bugs
0 Vulnerabilities
0 Security Hotspots
1 Code Smell

No Coverage information
0.0% Duplication