Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

load_data: add mintpy.multilook.method for mean/median #797

Merged
merged 13 commits into from
Jun 22, 2022
Merged

load_data: add mintpy.multilook.method for mean/median #797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
d18a8a6
load_data: multilook x/ystep
yuankailiu Jun 17, 2022
8f077e5
update the template variable names
yuankailiu Jun 20, 2022
b3b1ddb
update tests config files for mintpy.multilook.x/ystep
yunjunz Jun 21, 2022
c0942b7
Update stackDict.py
yunjunz Jun 21, 2022
734ea30
more comments
yunjunz Jun 21, 2022
fc9c4ec
Update stackDict.py
yunjunz Jun 21, 2022
8463cc2
mv mintpy.subset.* in front of mintpy.multilook.*
yunjunz Jun 21, 2022
75f0cb9
update prep_aria for the new mintpy.multilook.*
yunjunz Jun 21, 2022
2333c74
Update prep_aria.py
yunjunz Jun 21, 2022
cfbb558
add "use with caution" note in the template comment
yunjunz Jun 21, 2022
c867337
stackDict: update attribute due to resize
yunjunz Jun 22, 2022
6f56d29
prep_isce: simplify checking before call update_attribute4resize
yunjunz Jun 22, 2022
7587201
Update prep_isce.py
yunjunz Jun 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions mintpy/defaults/smallbaselineApp.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,20 @@ mintpy.load.azAngleFile = auto #[path of azimuth angle file], optional
mintpy.load.shadowMaskFile = auto #[path of shadow mask file], optional but recommended
mintpy.load.waterMaskFile = auto #[path of water mask file], optional but recommended
mintpy.load.bperpFile = auto #[path pattern of 2D perpendicular baseline file], optional
##---------multilook (optional):
## multilook while loading data with nearest interpolation, to reduce dataset size
mintpy.load.ystep = auto #[int >= 1], auto for 1 - no multilooking
mintpy.load.xstep = auto #[int >= 1], auto for 1 - no multilooking
##---------subset (optional):
## if both yx and lalo are specified, use lalo option unless a) no lookup file AND b) dataset is in radar coord
mintpy.subset.yx = auto #[y0:y1,x0:x1 / no], auto for no
mintpy.subset.lalo = auto #[S:N,W:E / no], auto for no
##---------multilook (optional):
## multilook while loading data with the specified method, to reduce dataset size
## method - nearest, mean and median methods are applicable to interferogram/ionosphere/offset stack(s), except for:
## connected components and all geometry datasets, for which nearest is hardwired.
## Use mean / median method with caution! It could smoothen the noise for a better SNR, but it could also smoothen the
## unwrapping errors, breaking the integer 2pi relationship, which is used in the unwrapping error correction.
## If you really want to increase the SNR, consider re-generate your stack of interferograms with more looks instead.
mintpy.multilook.method = auto #[nearest, mean, median], auto for nearest - lines/rows skipping approach
mintpy.multilook.ystep = auto #[int >= 1], auto for 1 - no multilooking
mintpy.multilook.xstep = auto #[int >= 1], auto for 1 - no multilooking


########## 2. modify_network
Expand Down
7 changes: 4 additions & 3 deletions mintpy/defaults/smallbaselineApp_auto.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,13 @@ mintpy.load.processor = isce
mintpy.load.autoPath = no
mintpy.load.updateMode = yes
mintpy.load.compression = no
##---------multilook (optional):
mintpy.load.ystep = 1
mintpy.load.xstep = 1
##-------subset (optional)
mintpy.subset.yx = no
mintpy.subset.lalo = no
##---------multilook (optional):
mintpy.multilook.method = nearest
mintpy.multilook.ystep = 1
mintpy.multilook.xstep = 1


########## modify_network
Expand Down
22 changes: 16 additions & 6 deletions mintpy/load_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,23 +187,31 @@ def read_inps2dict(inps):
if 'processor' in template.keys():
template['mintpy.load.processor'] = template['processor']

# group - load
prefix = 'mintpy.load.'
key_list = [i.split(prefix)[1] for i in template.keys() if i.startswith(prefix)]
for key in key_list:
value = template[prefix+key]
if key in ['processor', 'autoPath', 'updateMode', 'compression']:
iDict[key] = template[prefix+key]
elif key in ['xstep', 'ystep']:
iDict[key] = int(template[prefix+key])
elif value:
iDict[prefix+key] = template[prefix+key]
print('processor : {}'.format(iDict['processor']))

if iDict['compression'] == False:
iDict['compression'] = None

iDict['xstep'] = iDict.get('xstep', 1)
iDict['ystep'] = iDict.get('ystep', 1)
# group - multilook
prefix = 'mintpy.multilook.'
key_list = [i.split(prefix)[1] for i in template.keys() if i.startswith(prefix)]
for key in key_list:
value = template[prefix+key]
if key in ['xstep', 'ystep', 'method']:
iDict[key] = template[prefix+key]

iDict['xstep'] = int(iDict.get('xstep', 1))
iDict['ystep'] = int(iDict.get('ystep', 1))
iDict['method'] = str(iDict.get('method', 'nearest'))

# PROJECT_NAME --> PLATFORM
if not iDict['PROJECT_NAME']:
Expand Down Expand Up @@ -895,13 +903,14 @@ def main(iargs=None):
print('-'*50)
print('updateMode : {}'.format(iDict['updateMode']))
print('compression: {}'.format(iDict['compression']))
print('x/ystep: {}/{}'.format(iDict['xstep'], iDict['ystep']))
print('multilook x/ystep: {}/{}'.format(iDict['xstep'], iDict['ystep']))
print('multilook method : {}'.format(iDict['method']))
kwargs = dict(updateMode=iDict['updateMode'], xstep=iDict['xstep'], ystep=iDict['ystep'])

# read subset info [need the metadata from above]
iDict = read_subset_box(iDict)

# geometry in geo / radar coordinates
# geometry in geo / radar coordinates
geom_dset_name2template_key = {
**GEOM_DSET_NAME2TEMPLATE_KEY,
**IFG_DSET_NAME2TEMPLATE_KEY,
Expand Down Expand Up @@ -957,6 +966,7 @@ def main(iargs=None):
box=iDict['box'],
xstep=iDict['xstep'],
ystep=iDict['ystep'],
mli_method=iDict['method'],
compression=iDict['compression'],
extra_metadata=extraDict,
geom_obj=geom_obj)
Expand Down
108 changes: 74 additions & 34 deletions mintpy/objects/stackDict.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
utils0 as ut,
attribute as attr,
)

from mintpy.multilook import multilook_data

########################################################################################
class ifgramStackDict:
Expand Down Expand Up @@ -97,17 +97,21 @@ def get_dataset_data_type(self, dsName):
dataType = dataTypeDict[metadata.get('DATA_TYPE', 'float32').lower()]
return dataType

def write2hdf5(self, outputFile='ifgramStack.h5', access_mode='w', box=None, xstep=1, ystep=1,
def write2hdf5(self, outputFile='ifgramStack.h5', access_mode='w', box=None, xstep=1, ystep=1, mli_method='nearest',
compression=None, extra_metadata=None, geom_obj=None):
"""Save/write an ifgramStackDict object into an HDF5 file with the structure defined in:

https://mintpy.readthedocs.io/en/latest/api/data_structure/#ifgramstack

Parameters: outputFile : str, Name of the HDF5 file for the InSAR stack
access_mode : str, access mode of output File, e.g. w, r+
box : tuple, subset range in (x0, y0, x1, y1)
extra_metadata : dict, extra metadata to be added into output file
Returns: outputFile
Parameters: outputFile - str, Name of the HDF5 file for the InSAR stack
access_mode - str, access mode of output File, e.g. w, r+
box - tuple, subset range in (x0, y0, x1, y1)
x/ystep - int, multilook number in x/y direction
mli_method - str, multilook method, nearest, mean or median
compression - str, HDF5 dataset compression method, None, lzf or gzip
extra_metadata - dict, extra metadata to be added into output file
geom_obj - geometryDict object, size reference to determine the resizing operation.
Returns: outputFile - str, Name of the HDF5 file for the InSAR stack
"""
print('-'*50)

Expand All @@ -129,18 +133,26 @@ def write2hdf5(self, outputFile='ifgramStack.h5', access_mode='w', box=None, xst
# check if resize is needed for a lower resolution stack, e.g. ionosphere from isce2/topsStack
resize2shape = None
if geom_obj and os.path.basename(outputFile).startswith('ion'):
in_size = self.get_size()[1:]
# compare the original data size between ionosphere and geometry w/o subset/multilook
ion_size = self.get_size()[1:]
geom_size = geom_obj.get_size()
if in_size != geom_size:
if ion_size != geom_size:
msg = 'lower resolution ionosphere file detected'
msg += f' --> resize from {in_size} to {geom_size} via skimage.transform.resize ...'
msg += f' --> resize from {ion_size} to {geom_size} via skimage.transform.resize ...'
print(msg)

# matrix shape for the original geometry size w/o subset/multilook
resize2shape = geom_size
length, width = geom_size
# data size of the output HDF5 file w/ resize/subset/multilook
length, width = self.get_size(
box=box,
xstep=xstep,
ystep=ystep,
geom_obj=geom_obj)[1:]

self.outputFile = outputFile
with h5py.File(self.outputFile, access_mode) as f:
print('create HDF5 file {} with {} mode'.format(self.outputFile, access_mode))
# write HDF5 file
with h5py.File(outputFile, access_mode) as f:
print('create HDF5 file {} with {} mode'.format(outputFile, access_mode))

###############################
# 3D datasets containing unwrapPhase, magnitude, coherence, connectComponent, wrapPhase, etc.
Expand All @@ -151,6 +163,7 @@ def write2hdf5(self, outputFile='ifgramStack.h5', access_mode='w', box=None, xst
if dsName in ['connectComponent']:
dsDataType = np.int16
dsCompression = 'lzf'
mli_method = 'nearest'

print(('create dataset /{d:<{w}} of {t:<25} in size of {s}'
' with compression = {c}').format(d=dsName,
Expand All @@ -172,6 +185,10 @@ def write2hdf5(self, outputFile='ifgramStack.h5', access_mode='w', box=None, xst
if dsFile.endswith('cov.bip'):
print('convert variance to standard deviation.')

# msg
if xstep * ystep > 1:
print(f'apply {xstep} x {ystep} multilooking/downsampling via {mli_method} ...')

prog_bar = ptime.progressBar(maxValue=numIfgram)
for i in range(numIfgram):
# read and/or resize
Expand All @@ -180,6 +197,7 @@ def write2hdf5(self, outputFile='ifgramStack.h5', access_mode='w', box=None, xst
box=box,
xstep=xstep,
ystep=ystep,
mli_method=mli_method,
resize2shape=resize2shape)[0]

# special handling for offset covariance file
Expand Down Expand Up @@ -242,23 +260,35 @@ def write2hdf5(self, outputFile='ifgramStack.h5', access_mode='w', box=None, xst

###############################
# Attributes
self.get_metadata()
# read metadata from original data file w/o resize/subset/multilook
meta = self.get_metadata()
if extra_metadata:
self.metadata.update(extra_metadata)
meta.update(extra_metadata)
print('add extra metadata: {}'.format(extra_metadata))

# update metadata due to resize
# for low resolution ionosphere from isce2/topsStack
if resize2shape:
print('update metadata due to resize')
meta = attr.update_attribute4resize(meta, resize2shape)

# update metadata due to subset
self.metadata = attr.update_attribute4subset(self.metadata, box)
if box:
print('update metadata due to subset')
meta = attr.update_attribute4subset(meta, box)

# update metadata due to multilook
if xstep * ystep > 1:
self.metadata = attr.update_attribute4multilook(self.metadata, ystep, xstep)
print('update metadata due to multilook')
meta = attr.update_attribute4multilook(meta, ystep, xstep)

self.metadata['FILE_TYPE'] = self.name
for key, value in self.metadata.items():
# write metadata to HDF5 file at the root level
meta['FILE_TYPE'] = self.name
for key, value in meta.items():
f.attrs[key] = value

print('Finished writing to {}'.format(self.outputFile))
return self.outputFile
print('Finished writing to {}'.format(outputFile))
return outputFile


########################################################################################
Expand Down Expand Up @@ -292,13 +322,14 @@ def __init__(self, name='ifgram', datasetDict={}, metadata=None):
for key, value in metadata.items():
setattr(self, key, value)

def read(self, family, box=None, xstep=1, ystep=1, resize2shape=None):
def read(self, family, box=None, xstep=1, ystep=1, mli_method='nearest', resize2shape=None):
"""Read data for the given dataset name.

Parameters: self - ifgramDict object
family - str, dataset name
box - tuple of 4 int, in (x0, y0, x1, y1) with respect to the full resolution
x/ystep - int, number of pixels to skip, with respect to the full resolution
mli_method - str, interpolation method, nearest, mean, median
resize2shape - tuple of 2 int, resize the native matrix to the given shape
Set to None for not resizing
Returns: data - 2D np.ndarray
Expand All @@ -307,14 +338,14 @@ def read(self, family, box=None, xstep=1, ystep=1, resize2shape=None):
self.file = self.datasetDict[family]
box2read = None if resize2shape else box

# read input file
# 1. read input file
data, meta = readfile.read(self.file,
datasetName=family,
box=box2read,
xstep=1,
ystep=1)

# resize
# 2. resize
if resize2shape:
# link: https://scikit-image.org/docs/dev/api/skimage.transform.html#skimage.transform.resize
data = resize(data,
Expand All @@ -324,20 +355,29 @@ def read(self, family, box=None, xstep=1, ystep=1, resize2shape=None):
anti_aliasing=True,
preserve_range=True)

# subset by box
# 3. subset by box
if box:
data = data[box[1]:box[3],
box[0]:box[2]]

# multilook - nearest resampling
# 4. multilook
if xstep * ystep > 1:
# output data size
xsize = int(data.shape[1] / xstep)
ysize = int(data.shape[0] / ystep)
# sampling
data = data[int(ystep/2)::ystep,
int(xstep/2)::xstep]
data = data[:ysize, :xsize]
if mli_method == 'nearest':
# multilook - nearest resampling
# output data size
xsize = int(data.shape[1] / xstep)
ysize = int(data.shape[0] / ystep)
# sampling
data = data[int(ystep/2)::ystep,
int(xstep/2)::xstep]
data = data[:ysize, :xsize]

else:
# multilook - mean or median resampling
data = multilook_data(data,
lks_y=ystep,
lks_x=xstep,
method=mli_method)

return data, meta

Expand Down
Loading