Merge pull request #16 from odhondt/feature/s1-goldstein

Feature/s1 goldstein

docs/index.md

@@ -10,6 +10,8 @@ Some examples can be viewed here:
 
 - [Sentinel-1 custom pipeline](s1-custom-pipeline.ipynb)
 
+- [Sentinel-1 Goldstein phase denoising](s1-goldstein-phase-denoising.ipynb)
+
 - [Sentinel-2 search download, mosaic and crop](discover-and-process-s2.ipynb)  
 
 - [DEM download and mosaic and crop](download-dem.ipynb)  

eo_tools/S1/process.py

@@ -22,6 +22,9 @@
 from osgeo import gdal
 from rasterio.features import geometry_window
 from affine import Affine
+from numpy.fft import fft2, fftshift, ifft2, ifftshift
+from scipy.ndimage import uniform_filter as uflt
+from eo_tools.auxils import block_process
 
 # use child processes
 # USE_CP = False
@@ -870,7 +873,10 @@ def geocode_and_merge_iw(
         multilook (List[int], optional): Multilooking in azimuth and range. Defaults to [1, 4].
         warp_kernel (str, optional): Warping kernel. Defaults to "bicubic".
         clip_to_shape (bool, optional): If set to True, whole bursts intersecting shp will be included. Defaults to True.
-
+    Note:
+        variables starting with the substring 'ifg' are interpreted as
+        interferograms. Their phase will extracted after geocoding. The
+        output file will start with 'phi'.
     """
     if isinstance(pol, str):
         if pol == "full":
@@ -913,8 +919,11 @@ def geocode_and_merge_iw(
                         warnings.warn(
                             "Geocode real-valued phase? If so, the result might not be optimal if the phase is wrapped."
                         )
-                if var == "ifg":
-                    file_out = f"{input_dir}/sar/phi_{postfix}_geo.tif"
+                # if var == "ifg":
+                if var.startswith("ifg"):
+                    file_out = (
+                        f"{input_dir}/sar/{var.replace("ifg", "phi")}_{postfix}_geo.tif"
+                    )
                     sar2geo(
                         file_var,
                         file_lut,
@@ -934,10 +943,11 @@ def geocode_and_merge_iw(
                     )
                 tmp_files.append(file_out)
             if tmp_files:
-                if var != "ifg":
+                # if var != "ifg":
+                if not var.startswith("ifg"):
                     file_out = f"{input_dir}/{var}_{p}.tif"
                 else:
-                    file_out = f"{input_dir}/phi_{p}.tif"
+                    file_out = f"{input_dir}/{var.replace("ifg", "phi")}_{p}.tif"
                 log.info(f"Merge file {Path(file_out).name}")
                 da_to_merge = [riox.open_rasterio(file) for file in tmp_files]
 
@@ -1059,7 +1069,7 @@ def sar2geo(
 
 
 def apply_multilook(file_in: str, file_out: str, multilook: List = [1, 1]) -> None:
-    """Applies multilooking to raster.
+    """Apply multilooking to raster.
 
     Args:
         file_in (str): GeoTiff file of the primary SLC image
@@ -1215,7 +1225,8 @@ def coherence(
     else:
         mlt_az, mlt_rg = multilook
 
-    open_args = dict(lock=False, chunks="auto", cache=True, masked=True)
+    # open_args = dict(lock=False, chunks="auto", cache=True, masked=True)
+    open_args = dict(lock=False, chunks=(1, 1024, 1024), cache=True, masked=True)
 
     warnings.filterwarnings("ignore", category=NotGeoreferencedWarning)
     ds_prm = riox.open_rasterio(file_prm, **open_args)
@@ -1260,6 +1271,7 @@ def coherence(
     nodataval = np.nan
 
     da_coh = xr.DataArray(
+        name="coh",
         data=coh[None],
         dims=("band", "y", "x"),
     )
@@ -1288,14 +1300,61 @@ def coherence(
             ds_prm.rio.transform() * Affine.scale(mlt_rg, mlt_az), inplace=True
         )
         da_ifg.rio.write_nodata(np.nan, inplace=True)
-        da_ifg.rio.to_raster(file_complex_ifg, driver="GTiff")
-        # da_ifg.rio.to_raster(
-        #     file_complex_ifg, driver="GTiff", compress="zstd", num_threads="all_cpus"
-        # )
+        da_ifg.rio.to_raster(
+            file_complex_ifg, driver="GTiff", tiled=True, blockxsize=512, blockysize=512
+        )
 
 
-import os
-from typing import Callable, List, Union
+def goldstein(file_ifg: str, file_out: str, alpha: float = 0.5, overlap: int = 14) -> None:
+    """Apply the Goldstein filter to a complex interferogam to reduce phase noise.
+
+    Args:
+        file_ifg (str): Input file.
+        file_out (str): Output file.
+        alpha (float, optional): Filter parameter. Should be between 0 (no filtering) and 1 (strongest). Defaults to 0.5.
+        overlap (int, optional): Overlap between 64x64 patches. Defaults to 14.
+    Note:
+        The method is described in: 
+        R.M. Goldstein and C.L. Werner, "Radar Interferogram Phase Filtering for Geophysical Applications," Geophysical Research Letters, 25, 4035-4038, 1998
+    """    
+
+    # base filter to be applied on a patch
+    def filter_base(arr, alpha=1):
+        smooth = lambda x: uflt(x, 3)
+        Z = fftshift(fft2(arr))
+        H = smooth(abs(Z)) ** (alpha)
+        arrout = ifft2(ifftshift(H * Z))
+        return arrout
+
+    # base filter to be sequentially applied on a chunk
+    def filter_chunk(chunk, alpha=0.5, overlap=14):
+        # complex phase
+        chunk_ = np.exp(1j * np.angle(chunk))
+        # overlap value found in modified Goldstein paper
+        return block_process(
+            chunk_, (64, 64), (overlap, overlap), filter_base, alpha=alpha
+        )
+
+    # TODO: find a way to automatically tune chunk size
+    open_args = dict(lock=False, chunks=(1, 2048, 2048), masked=True)
+    warnings.filterwarnings("ignore", category=NotGeoreferencedWarning)
+    ds_ifg = riox.open_rasterio(file_ifg, **open_args)
+    ifg = ds_ifg[0].data
+
+    # process multiple chunks in parallel
+    process_args = dict(alpha=alpha, depth=(overlap, overlap), dtype="complex64")
+    ifg_out = da.map_overlap(filter_chunk, ifg, **process_args)
+    da_out = xr.DataArray(
+        name="ifg",
+        data=ifg_out[None],
+        dims=("band", "y", "x"),
+    )
+    da_out.rio.write_transform(ds_ifg.rio.transform(), inplace=True)
+
+    nodataval = np.nan
+    da_out.rio.write_nodata(nodataval, inplace=True)
+    # block size manually set until better solution
+    da_out.rio.to_raster(file_out, tiled=True, blockxsize=512, blockysize=512)
 
 
 def apply_to_patterns_for_pair(

eo_tools/S2.py

@@ -195,7 +195,7 @@ def process_s2_tiles(
                                 )
                             else:
                                 raster = tmp_ds.read()
-                    if proc_bsl > 4:
+                    if proc_bsl >= 4:
                         OFF = dict_pid["offsets"][df_bands.loc[band]["id"]]
                         raster = ((raster + OFF) / QV).clip(0).astype(np.float32)
                     else:

eo_tools/auxils.py

@@ -11,6 +11,7 @@
 import xml.etree.ElementTree as ET
 import pandas as pd
 import geopandas as gpd
+import numpy as np
 
 import logging
 
@@ -293,4 +294,121 @@ def get_burst_geometry(path, target_subswaths, polarization):
                     crs='EPSG:4326'
                 )
         df_all = gpd.GeoDataFrame(pd.concat([df_all, df]), crs='EPSG:4326')
-    return(df_all)
+    return(df_all)
+
+def block_process(img, block_size, overlap_size, fun, *fun_args, **kwargs):
+    """
+    Block processing of a multi-channel 2-D image with an arbitrary function.
+    
+    Args:
+        img (array or tuple): Input image or tuple of arrays with shape (nl, nc, ...),
+            (nl, 1,...) or (1, nc, ...).
+        fun (callable): Function to apply. The first argument must be the img.
+        block_size (tuple of ints): Size of blocks. If an int is provided, it is used for
+            both height and width. If a tuple is provided, it should be (block_height, block_width).
+        overlap_size (tuple of ints, optional): Size of overlaps. If an int is provided, it is used for
+            both height and width. If a tuple is provided, it should be (overlap_height, overlap_width).
+        *fun_args: Additional positional arguments for the function.
+        **kwargs: Additional keyword arguments.
+
+    Returns:
+        array: Processed output image with the same shape as the input image.
+    
+    Raises:
+        ValueError: If overlap is larger than half of the block size or if the output
+            shape is incompatible with the input shape.
+    
+    Notes:
+        The function to be applied has to have arguments of the form (in1, in2, ..., par1, par2, ...)
+        with inputs grouped at the beginning. If not, write a wrapper that follows this order.
+    """
+    # Validate block_size and overlap
+    if not isinstance(block_size, tuple) or not all(isinstance(b, int) and b >= 1 for b in block_size):
+        raise TypeError("block_size must be a tuple of integers, each greater than or equal to 1.")
+
+    if not isinstance(overlap_size, tuple) or not all(isinstance(o, int) and o >= 0 for o in overlap_size):
+        raise TypeError("overlap must be a tuple of non-negative integers.")
+
+    if len(block_size) != 2:
+        raise ValueError("block_size must be of length 2.")
+
+    if len(overlap_size) != 2:
+        raise ValueError("overlap must be of length 2.")
+
+    # Parse block and overlap sizes
+    block_height, block_width = block_size
+    olap_height, olap_width = overlap_size
+
+    # Extract dimensions of the input image
+    if isinstance(img, tuple):
+        ih, iw = img[0].shape[:2]
+    else:
+        ih, iw = img.shape[:2]
+
+    # Check overlap constraints
+    if olap_width > block_width // 2 or olap_height > block_height // 2:
+        raise ValueError("Overlap must be at most half of block size.")
+
+    # Preallocate output image with the same shape as the input image
+    imgout = np.zeros_like(img) if not isinstance(img, tuple) else tuple(np.zeros_like(x) for x in img)
+
+    # nrow = int(np.ceil(float(ih) / block_height))
+    # ncol = int(np.ceil(float(iw) / block_width))
+
+    cnt = 0
+    for i in range(0, ih, block_height):
+        if i == 0:
+            i_top, i_bottom = 0, block_height + olap_height
+            o_top, o_bottom = 0, block_height
+            b_top, b_bottom = 0, block_height
+        elif i + block_height < ih and i + block_height + olap_height > ih:
+            i_top, i_bottom = i - olap_height, ih
+            o_top, o_bottom = i, i + block_height
+            b_top, b_bottom = olap_height, block_height + olap_height
+        elif i + block_height > ih:
+            i_top, i_bottom = i - olap_height, ih
+            o_top, o_bottom = i, ih
+            b_top, b_bottom = olap_height, olap_height + ih - i
+        else:
+            i_top, i_bottom = i - olap_height, i + block_height + olap_height
+            o_top, o_bottom = i, i + block_height
+            b_top, b_bottom = olap_height, block_height + olap_height
+
+        for j in range(0, iw, block_width):
+            if j == 0:
+                i_left, i_right = 0, block_width + olap_width
+                o_left, o_right = 0, block_width
+                b_left, b_right = 0, block_width
+            elif j + block_width < iw and j + block_width + olap_width > iw:
+                i_left, i_right = j - olap_width, iw
+                o_left, o_right = j, j + block_width
+                b_left, b_right = olap_width, block_width + olap_width
+            elif j + block_width > iw:
+                i_left, i_right = j - olap_width, iw
+                o_left, o_right = j, iw
+                b_left, b_right = olap_width, olap_width + iw - j
+            else:
+                i_left, i_right = j - olap_width, j + block_width + olap_width
+                o_left, o_right = j, j + block_width
+                b_left, b_right = olap_width, block_width + olap_width
+
+            cnt += 1
+            # print(f"Processing block # {cnt} of {nrow * ncol}")
+
+            sl = np.s_[i_top:i_bottom, i_left:i_right]
+
+            # Process block with the function
+            if isinstance(img, tuple):
+                blk = tuple(x[sl] for x in img)
+            else:
+                blk = (img[sl],)
+
+            processed_block = fun(*blk, *fun_args, **kwargs)[b_top:b_bottom, b_left:b_right]
+
+            if isinstance(imgout, tuple):
+                for k, output in enumerate(imgout):
+                    imgout[k][o_top:o_bottom, o_left:o_right] = processed_block[k]
+            else:
+                imgout[o_top:o_bottom, o_left:o_right] = processed_block
+
+    return imgout

mkdocs.yml

@@ -8,8 +8,9 @@ nav:
     - Example 2: s1-easy-tops-insar.ipynb
     - Example 3: s1-easy-slc-geocoding.ipynb
     - Example 4: s1-custom-pipeline.ipynb
-    - Example 5: discover-and-process-s2.ipynb
-    - Example 6: download-dem.ipynb
+    - Example 4: s1-goldstein-phase-denoising.ipynb
+    - Example 6: discover-and-process-s2.ipynb
+    - Example 7: download-dem.ipynb
   - API reference: 
     - Sentinel-1 core: s1_core_api.md
     - Sentinel-1 processor: s1_process_api.md