fixed black and white mixture

.gitignore

@@ -136,3 +136,4 @@ notebooks/data*
 tests/testdata/specim_data
 tests/testdata/no_white
 tests/testdata/no_black
+tests/demo_test.ipynb

Changelog.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## Unrelased
+
+### Added
+- added hidden property _capture_spectral which is the orginal representation of the capture data from the spectral library
+
+### Fixed
+
+- an issue when black and white references where mixed up
+
 ## 0.2.0
 
 ### Added

src/specarray/__init__.py

@@ -5,6 +5,6 @@
 """Python Package Template"""
 from __future__ import annotations
 
-__version__ = "0.2.0"
+__version__ = "0.2.0_dev0"
 
 from .specarray import SpecArray

src/specarray/io/specim_folder.py

@@ -1,52 +1,14 @@
 from pathlib import Path
+from os import listdir
 
 import dask.array as da
 import pandas as pd
 import spectral.io.envi as envi
 import xarray as xr
 from spectral.io.bilfile import BilFile
 
-import warnings
-
-
-def _find_files(folder: Path):
-    """Find all files in the folder"""
-    capture_folder = folder / "capture"
-    hdr_files = list(capture_folder.glob("*.hdr"))
-    raw_files = list(capture_folder.glob("*.raw"))
-    return hdr_files, raw_files
 
-
-def _load_spectral_data(folder: Path, mode="capture"):
-    """Load the spectral data"""
-    hdr_files, raw_files = _find_files(folder)
-
-    hdr_file_path = None
-    raw_file_path = None
-
-    for hdr_file, raw_file in zip(hdr_files, raw_files):
-        if mode == "capture":
-            if not hdr_file.stem.startswith(("WHITEREF", "DARKREF")):
-                hdr_file_path = hdr_file
-                raw_file_path = raw_file
-                break
-        elif mode == "white":
-            prefix = "WHITEREF"
-            if hdr_file.stem.startswith(prefix):
-                hdr_file_path = hdr_file
-                raw_file_path = raw_file
-                break
-        elif mode == "black":
-            prefix = "DARKREF"
-            if hdr_file.stem.startswith(prefix):
-                hdr_file_path = hdr_file
-                raw_file_path = raw_file
-                break
-
-    if hdr_file_path is None or raw_file_path is None:
-        raise ValueError(f"No matching file found for mode {mode}")
-
-    return envi.open(hdr_file_path, raw_file_path)
+import warnings
 
 
 def _extract_wavelengths(metadata) -> pd.Series:
@@ -67,34 +29,59 @@ def _create_data_array(spectral_data: BilFile, mode: str, wavelengths: pd.Series
 
 def from_specim_folder(
     folder: Path,
-) -> "tuple[xr.DataArray, dict, pd.Series, xr.DataArray, xr.DataArray]":
+) -> "tuple[xr.DataArray, dict, pd.Series, xr.DataArray, xr.DataArray, BilFile]":
     """Create a SpecArray from a folder"""
-    spectral_data = _load_spectral_data(folder=folder, mode="capture")
-    metadata = spectral_data.metadata
-    wavelengths = _extract_wavelengths(metadata=metadata)
 
-    modes = ["capture", "black", "white"]
+    modes = [
+        "capture",
+        "DARKREF_",
+        "WHITEREF_",
+    ]
+    names = {"capture": "capture", "DARKREF_": "black", "WHITEREF_": "white"}
     black = xr.DataArray()
     white = xr.DataArray()
     capture = xr.DataArray()
+
+    capture_folder = folder / "capture"
+
     for mode in modes:
-        try:
-            spectral_data = _load_spectral_data(folder, mode)
+        if mode == "capture":
+            hdr_file_path = [
+                capture_folder / file
+                for file in listdir(capture_folder)
+                if not file.startswith("WHITEREF") and not file.startswith("DARKREF") and file.endswith(".hdr")
+            ]
+            raw_file_path = [
+                capture_folder / file
+                for file in listdir(capture_folder)
+                if not file.startswith("WHITEREF") and not file.startswith("DARKREF") and file.endswith(".raw")
+            ]
+        else:
+            hdr_file_path = list(capture_folder.glob(f"{mode}*.hdr"))
+            raw_file_path = list(capture_folder.glob(f"{mode}*.raw"))
+
+        if len(hdr_file_path) > 0 and len(raw_file_path) > 0:
+            spectral_data = envi.open(hdr_file_path[0], raw_file_path[0])
+            metadata = spectral_data.metadata
+            wavelengths = _extract_wavelengths(metadata=metadata)
             data_array = _create_data_array(spectral_data, mode, wavelengths)
-            if mode == "capture":
-                capture = data_array
-                capture_spectral = spectral_data
-                if capture.size == 0:
-                    raise ValueError("No capture data found")
-            elif mode == "black":
-                black = data_array
-                # warning that there is no Dark Reference
-                if black.size == 0:
-                    warnings.warn("No Dark Reference found")
-            elif mode == "white":
-                white = data_array
-                if white.size == 0:
-                    warnings.warn("No White Reference found")
-        except Exception as exception:
-            print(f"An error occurred reading {mode}: {exception}")
+            data_array.name = names[mode]
+        else:
+            break
+
+        if mode == "capture":
+            capture = data_array
+            capture_spectral = spectral_data
+            metadata = metadata
+            wavelengths = wavelengths
+        elif mode == "DARKREF_":
+            black = data_array
+        elif mode == "WHITEREF_":
+            white = data_array
+
+        if len(hdr_file_path) == 0:
+            warnings.warn(f"No {mode} file found")
+
+    if len(capture) == 0:
+        raise ValueError("No capture file found")
     return capture, metadata, wavelengths, black, white, capture_spectral

src/specarray/specarray.py

@@ -14,10 +14,6 @@
 from .io.specim_folder import from_specim_folder
 
 
-def trapz(x, y):
-    return np.trapz(y, x)
-
-
 @dataclass
 class SpecArray:
     """Class for Specim hyperspectral Camera data"""
@@ -34,7 +30,15 @@ class SpecArray:
     @classmethod
     def from_folder(cls, folder: Path):
         """Create a SpecArray from a folder"""
-        return cls(*from_specim_folder(folder))
+        capture, metadata, wavelengths, black, white, capture_spectral = from_specim_folder(folder)
+        return cls(
+            capture=capture,
+            metadata=metadata,
+            wavelengths=wavelengths,
+            black=black,
+            white=white,
+            _capture_spectral=capture_spectral,
+        )
 
     def __len__(self):
         """Return the number of records"""
@@ -62,9 +66,9 @@ def has_white(self) -> bool:
     def spectral_albedo(self) -> xr.DataArray:
         """Calculate and return the spectral albedo. The values are limited to 0 and 1."""
         if self.has_black and self.has_white:
-            spectral_albedo = (self.capture - self.black.mean(dim="sample")) / (
-                self.white.mean(dim="sample") - self.black.mean(dim="sample")
-            )
+            black_mean = self.black.mean(dim="sample")
+            white_mean = self.white.mean(dim="sample")
+            spectral_albedo = (self.capture - black_mean) / (white_mean - black_mean)
             spectral_albedo = xr.where(spectral_albedo < 0.0, 0.0, spectral_albedo)
             spectral_albedo = xr.where(spectral_albedo > 1.0, 1.0, spectral_albedo)
             spectral_albedo.name = "spectral albedo"
@@ -76,20 +80,17 @@ def spectral_albedo(self) -> xr.DataArray:
     def broadband_albedo(self) -> xr.DataArray:
         """Calculate and return the broadband albedo"""
         if self.has_black and self.has_white:
-            spectral_albedo = self.spectral_albedo.chunk({"sample": -1})
-            broadband_albedo = xr.apply_ufunc(
-                trapz,
-                spectral_albedo.coords["wavelength"],
-                spectral_albedo.transpose(
+            broadband_albedo = np.trapz(
+                self.spectral_albedo.transpose(
                     "sample",
                     "point",
                     "wavelength",
                 ),
-                input_core_dims=[["wavelength"], ["wavelength", "sample", "point"]],
-                output_core_dims=[["sample", "point"]],
-                dask="parallelized",
-                output_dtypes=[float],
-            ) / (spectral_albedo.coords["wavelength"].max().values - spectral_albedo.coords["wavelength"].min().values)
+                self.spectral_albedo.coords["wavelength"],
+            ) / (
+                self.spectral_albedo.coords["wavelength"].max().values
+                - self.spectral_albedo.coords["wavelength"].min().values
+            )
             broadband_albedo = xr.DataArray(broadband_albedo, dims=["sample", "point"], name="broadband albedo")
             return broadband_albedo
         else:

tests/test_methods.py

@@ -33,7 +33,6 @@ def test_dims(testdata_specim: SpecArray):
 
 def test_spectral_albedo(testdata_specim: SpecArray):
     spectral_albedo = testdata_specim.spectral_albedo
-
     check.is_instance(spectral_albedo, DataArray)
     check.equal(spectral_albedo.shape, testdata_specim.capture.shape)
     check.equal(spectral_albedo.name, "spectral albedo")
@@ -42,10 +41,12 @@ def test_spectral_albedo(testdata_specim: SpecArray):
     check.less_equal(spectral_albedo.min(), 1.0)
     check.equal(spectral_albedo.dims, ("sample", "wavelength", "point"))
     check.equal(spectral_albedo.dims, testdata_specim.capture.dims)
-    # single_value_min = float(spectral_albedo[0][0].min().compute())
-    # single_value_max = float(spectral_albedo[0][0].max().compute())
-    # check.almost_equal(single_value_min, 0.2482758620689655)
-    # check.almost_equal(single_value_max, 0.7558048525958779)
+    check.almost_equal(float(spectral_albedo.sel(sample=0).max().as_numpy().values), 0.7786328655500226)
+    check.almost_equal(float(spectral_albedo.sel(sample=0).min().as_numpy().values), 0.125)
+    check.almost_equal(
+        float(spectral_albedo.sel(sample=0, point=0).sel(wavelength=1000, method="nearest").as_numpy().values),
+        0.28365385,
+    )
 
 
 def test_broadband_albedo(testdata_specim: SpecArray):