Analysis of ICESat-2 crossover track height errors

.github/workflows/push.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [ master ]
   pull_request:
-    branches: [ master ]
+    types: [review_requested, ready_for_review]
 
 jobs:
   build:

.github/workflows/python-app.yml

@@ -2,9 +2,11 @@ name: Test DeepIceDrain
 
 on:
   push:
-    branches: [ master ]
+    branches:
+      - master
   pull_request:
-    branches: [ master ]
+    branches:
+      - "**"
 
 jobs:
   test:

.gitignore

@@ -29,6 +29,7 @@ MANIFEST
 
 # Data files and folders
 **/*.h5
+**/*.tsv
 ATL06.003
 ATL11.001
 ATL11.001z123

README.md

@@ -9,9 +9,9 @@ in Antarctica using remote sensing and machine learning.
 [![Dependabot Status](https://api.dependabot.com/badges/status?host=github&repo=weiji14/deepicedrain)](https://dependabot.com)
 ![License](https://img.shields.io/github/license/weiji14/deepicedrain)
 
-![ICESat-2 ATL11 rate of height change over time in Antarctica 2018-10-14 to 2020-04-04](https://user-images.githubusercontent.com/23487320/84338773-140bb100-abf1-11ea-96eb-5ccfd7b378e4.png)
+![ICESat-2 ATL11 rate of height change over time in Antarctica 2018-10-14 to 2020-05-13](https://user-images.githubusercontent.com/23487320/90118294-2601ff80-ddac-11ea-8b93-7bc9b15f2be0.png)
 
-![DeepIceDrain Pipeline](https://yuml.me/diagram/scruffy;dir:LR/class/[Land-Ice-Elevation|atl06_play.ipynb]->[Convert|atl06_to_atl11.ipynb],[Convert]->[Ice-Sheet-H(t)-Series|atl11_play.ipynb],[Ice-Sheet-H(t)-Series]->[Height-Change-over-Time-(dhdt)|atlxi_dhdt.ipynb])
+![DeepIceDrain Pipeline](https://yuml.me/diagram/scruffy;dir:LR/class/[Land-Ice-Elevation|atl06_play.ipynb]->[Convert|atl06_to_atl11.ipynb],[Convert]->[Ice-Sheet-H(t)-Series|atl11_play.ipynb],[Ice-Sheet-H(t)-Series]->[Height-Change-over-Time-(dhdt)|atlxi_dhdt.ipynb],[Height-Change-over-Time-(dhdt)]->[Subglacial-Lake-Finder|atlxi_lake.ipynb])
 
 # Getting started
 
@@ -57,7 +57,7 @@ To just try out the scripts, download the `environment.yml` file from the reposi
     conda env create --name deepicedrain --file environment.yml
     pip install git+https://github.com/weiji14/deepicedrain.git
 
-### Advanced
+### Intermediate
 
 To help out with development, start by cloning this [repo-url](/../../)
 
@@ -77,11 +77,6 @@ Then install the python libraries listed in the `pyproject.toml`/`poetry.lock` f
 
     poetry install
 
-If you have a [CUDA](https://en.wikipedia.org/wiki/CUDA)-capable GPU,
-you can also install the optional "cuda" packages to accelerate some calculations.
-
-    poetry install --extras cuda
-
 Finally, double-check that the libraries have been installed.
 
     poetry show
@@ -93,6 +88,30 @@ Finally, double-check that the libraries have been installed.
 
     jupyter labextension list  # ensure that extensions are installed
 
+
+### Advanced
+
+This is for those who want full reproducibility of the conda environment,
+and more computing power by using Graphical Processing Units (GPU).
+
+Making an explicit conda-lock file
+(only needed if creating a new conda environment/refreshing an existing one).
+
+    conda env create -f environment.yml
+    conda list --explicit > environment-linux-64.lock
+
+Creating/Installing a virtual environment from a conda lock file.
+See also https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments.
+
+    conda create --name deepicedrain --file environment-linux-64.lock
+    conda install --name deepicedrain --file environment-linux-64.lock
+
+If you have a [CUDA](https://en.wikipedia.org/wiki/CUDA)-capable GPU,
+you can also install the optional "cuda" packages to accelerate some calculations.
+
+    poetry install --extras cuda
+
+
 ## Running jupyter lab
 
     conda activate deepicedrain

X2SYS/ICESAT2/ICESAT2.tag

@@ -0,0 +1,6 @@
+# TAG file for system: ICESAT2
+#
+# Initialized on: Fri Aug 14 12:00:00 2020
+# Initialized by: weiji14
+#
+-Dxyht -Etsv -Nse -Nde -Wd250e -I1/1 -R0/360/-90/90

X2SYS/ICESAT2/ICESAT2_paths.txt

@@ -0,0 +1,3 @@
+# Directories with data files for TAG ICESAT2
+# The current directory is always searched first.
+# Add full paths to search additional directories

X2SYS/ICESAT2/ICESAT2_tracks.d

@@ -0,0 +1 @@
+# ICESAT2

X2SYS/ICESAT2/xyht.fmt

@@ -0,0 +1,14 @@
+#
+# Define file for X2SYS processing of ASCII tsv files
+#
+# This file applies to plain 4-column ASCII files.
+#
+#---------------------------------------------------------------------
+#ASCII		# The input file is ASCII
+#SKIP 1		# The number of header records to skip
+#---------------------------------------------------------------------
+#name	intype	NaN-proxy?	NaN-proxy	scale	offset	oformat
+x	a	N		0		1	0	%g
+y	a	N		0		1	0	%g
+h	a	N		0		1	0	%g
+t	a	N		0		1	0	%g

atl11_play.ipynb

@@ -16,30 +16,29 @@
    "outputs": [],
    "source": [
     "import dataclasses\n",
-    "import os\n",
     "import glob\n",
-    "\n",
-    "import deepicedrain\n",
+    "import os\n",
     "\n",
     "import dask\n",
     "import dask.array\n",
     "import datashader\n",
+    "import geopandas as gpd\n",
     "import holoviews as hv\n",
     "import holoviews.operation\n",
     "import hvplot.dask\n",
     "import hvplot.pandas\n",
     "import hvplot.xarray\n",
-    "\n",
     "import intake\n",
-    "import geopandas as gpd\n",
     "import numpy as np\n",
     "import pandas as pd\n",
     "import pygmt\n",
     "import pyproj\n",
     "import shapely\n",
     "import tqdm\n",
     "import xarray as xr\n",
-    "import zarr"
+    "import zarr\n",
+    "\n",
+    "import deepicedrain"
    ]
   },
   {
@@ -259,28 +258,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Dictionary of Antarctic bounding box locations with EPSG:3031 coordinates\n",
-    "regions: dict = {\n",
-    "    \"kamb\": deepicedrain.Region(\n",
-    "        name=\"Kamb Ice Stream\",\n",
-    "        xmin=-411054.19240523444,\n",
-    "        xmax=-365489.6822096751,\n",
-    "        ymin=-739741.7702261859,\n",
-    "        ymax=-699564.516934089,\n",
-    "    ),\n",
-    "    \"antarctica\": deepicedrain.Region(\n",
-    "        \"Antarctica\", -2700000, 2800000, -2200000, 2300000\n",
-    "    ),\n",
-    "    \"siple_coast\": deepicedrain.Region(\n",
-    "        \"Siple Coast\", -1000000, 250000, -1000000, -100000\n",
-    "    ),\n",
-    "    \"whillans\": deepicedrain.Region(\n",
-    "        \"Whillans Ice Stream\", -350000, -100000, -700000, -450000\n",
-    "    ),\n",
-    "    \"whillans2\": deepicedrain.Region(\n",
-    "        \"Whillans Ice Stream\", -500000, -400000, -600000, -500000\n",
-    "    ),\n",
-    "}\n",
+    "# Antarctic bounding box locations with EPSG:3031 coordinates\n",
+    "regions = gpd.read_file(filename=\"deepicedrain/deepicedrain_regions.geojson\")\n",
+    "regions: gpd.GeoDataFrame = regions.set_index(keys=\"placename\")\n",
+    "\n",
     "# Subset to essential columns\n",
     "essential_columns: list = [\n",
     "    \"x\",\n",
@@ -302,8 +283,8 @@
    "source": [
     "# Do the actual computation to find data points within region of interest\n",
     "placename: str = \"kamb\"  # Select Kamb Ice Stream region\n",
-    "region: deepicedrain.Region = regions[placename]\n",
-    "ds_subset: xr.Dataset = region.subset(ds=ds)\n",
+    "region: deepicedrain.Region = deepicedrain.Region.from_gdf(gdf=regions.loc[placename])\n",
+    "ds_subset: xr.Dataset = region.subset(data=ds)\n",
     "ds_subset = ds_subset.unify_chunks()\n",
     "ds_subset = ds_subset.compute()"
    ]
@@ -648,9 +629,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Select region here, see dictionary of regions at top\n",
+    "# Select region here, see also geodataframe of regions at top\n",
     "placename: str = \"antarctica\"\n",
-    "region: deepicedrain.Region = regions[placename]"
+    "region: deepicedrain.Region = deepicedrain.Region.from_gdf(gdf=regions.loc[placename])"
    ]
   },
   {

atl11_play.py

@@ -20,22 +20,19 @@
 
 # %%
 import dataclasses
-import os
 import glob
-
-import deepicedrain
+import os
 
 import dask
 import dask.array
 import datashader
+import geopandas as gpd
 import holoviews as hv
 import holoviews.operation
 import hvplot.dask
 import hvplot.pandas
 import hvplot.xarray
-
 import intake
-import geopandas as gpd
 import numpy as np
 import pandas as pd
 import pygmt
@@ -45,6 +42,8 @@
 import xarray as xr
 import zarr
 
+import deepicedrain
+
 # %%
 client = dask.distributed.Client(n_workers=72, threads_per_worker=1)
 client
@@ -143,28 +142,10 @@
 # Take a geographical subset and save to a NetCDF/Zarr format for distribution.
 
 # %%
-# Dictionary of Antarctic bounding box locations with EPSG:3031 coordinates
-regions: dict = {
-    "kamb": deepicedrain.Region(
-        name="Kamb Ice Stream",
-        xmin=-411054.19240523444,
-        xmax=-365489.6822096751,
-        ymin=-739741.7702261859,
-        ymax=-699564.516934089,
-    ),
-    "antarctica": deepicedrain.Region(
-        "Antarctica", -2700000, 2800000, -2200000, 2300000
-    ),
-    "siple_coast": deepicedrain.Region(
-        "Siple Coast", -1000000, 250000, -1000000, -100000
-    ),
-    "whillans": deepicedrain.Region(
-        "Whillans Ice Stream", -350000, -100000, -700000, -450000
-    ),
-    "whillans2": deepicedrain.Region(
-        "Whillans Ice Stream", -500000, -400000, -600000, -500000
-    ),
-}
+# Antarctic bounding box locations with EPSG:3031 coordinates
+regions = gpd.read_file(filename="deepicedrain/deepicedrain_regions.geojson")
+regions: gpd.GeoDataFrame = regions.set_index(keys="placename")
+
 # Subset to essential columns
 essential_columns: list = [
     "x",
@@ -180,8 +161,8 @@
 # %%
 # Do the actual computation to find data points within region of interest
 placename: str = "kamb"  # Select Kamb Ice Stream region
-region: deepicedrain.Region = regions[placename]
-ds_subset: xr.Dataset = region.subset(ds=ds)
+region: deepicedrain.Region = deepicedrain.Region.from_gdf(gdf=regions.loc[placename])
+ds_subset: xr.Dataset = region.subset(data=ds)
 ds_subset = ds_subset.unify_chunks()
 ds_subset = ds_subset.compute()
 
@@ -318,9 +299,9 @@
 # from [Smith et al., 2009](https://doi.org/10.3189/002214309789470879).
 
 # %%
-# Select region here, see dictionary of regions at top
+# Select region here, see also geodataframe of regions at top
 placename: str = "antarctica"
-region: deepicedrain.Region = regions[placename]
+region: deepicedrain.Region = deepicedrain.Region.from_gdf(gdf=regions.loc[placename])
 
 # %%
 # Find subglacial lakes (Smith et al., 2009) within region of interest