diff --git a/atl06_play.ipynb b/atl06_play.ipynb new file mode 100644 index 0000000..207abc9 --- /dev/null +++ b/atl06_play.ipynb @@ -0,0 +1,2286 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **ATLAS/ICESat-2 Land Ice Height [ATL06](https://nsidc.org/data/atl06/) Exploratory Data Analysis**\n", + "\n", + "[Yet another](https://xkcd.com/927) take on playing with ICESat-2's Land Ice Height ATL06 data,\n", + "specfically with a focus on analyzing ice elevation changes over Antarctica.\n", + "Specifically, this jupyter notebook will cover:\n", + "\n", + "- Downloading datasets from the web via [intake](https://intake.readthedocs.io)\n", + "- Performing [Exploratory Data Analysis](https://en.wikipedia.org/wiki/Exploratory_data_analysis)\n", + " using the [PyData](https://pydata.org) stack (e.g. [xarray](http://xarray.pydata.org), [dask](https://dask.org))\n", + "- Plotting figures using [Hvplot](https://hvplot.holoviz.org) and [PyGMT](https://www.pygmt.org) (TODO)\n", + "\n", + "This is in contrast with the [icepyx](https://github.com/icesat2py/icepyx) package\n", + "and 'official' 2019/2020 [ICESat-2 Hackweek tutorials](https://github.com/ICESAT-2HackWeek/ICESat2_hackweek_tutorials) (which are also awesome!)\n", + "that tends to use a slightly different approach (e.g. handcoded download scripts, [h5py](http://www.h5py.org) for data reading, etc).\n", + "The core concept here is to run things in a more intuitive and scalable (parallelizable) manner on a continent scale (rather than just a specific region)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "import json\n", + "import logging\n", + "import netrc\n", + "import os\n", + "\n", + "import cartopy\n", + "import dask\n", + "import dask.distributed\n", + "import hvplot.dask\n", + "import hvplot.pandas\n", + "import hvplot.xarray\n", + "import intake\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pyproj\n", + "import requests\n", + "import tqdm\n", + "import xarray as xr\n", + "\n", + "# %matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

Client

\n", + "\n", + "		\n", + "

Cluster

\n", + "
    \n", + "
  • Workers: 10
    • \n", + "
  • Cores: 10
    • \n", + "
  • Memory: 201.22 GB
    • \n", + "
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Configure intake and set number of compute cores for data download\n", + "intake.config.conf[\"download_progress\"] = False # disable automatic tqdm progress bars\n", + "\n", + "logging.basicConfig(level=logging.WARNING)\n", + "\n", + "# Limit compute to 10 cores for download part using intake\n", + "# Can possibly go up to 10 because there are 10 DPs?\n", + "# See https://n5eil02u.ecs.nsidc.org/opendap/hyrax/catalog.xml\n", + "client = dask.distributed.Client(n_workers=10, threads_per_worker=1)\n", + "client" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quick view\n", + "\n", + "Use our [intake catalog](https://intake.readthedocs.io/en/latest/catalog.html) to get some sample ATL06 data\n", + "(while making sure we have our Earthdata credentials set up properly),\n", + "and view it using [xarray](https://xarray.pydata.org) and [hvplot](https://hvplot.pyviz.org)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# open the local catalog file containing ICESat-2 stuff\n", + "catalog = intake.open_catalog(uri=\"catalog.yaml\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "Show/Hide data repr\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Show/Hide attributes\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
xarray.Dataset
    • delta_time: 1683571
    • revision
      ()
      <U2
      '01'
      array('01', dtype='<U2')
    • orbitalsegment
      ()
      <U2
      '11'
      array('11', dtype='<U2')
    • cyclenumber
      ()
      <U2
      '06'
      array('06', dtype='<U2')
    • version
      ()
      <U3
      '003'
      array('003', dtype='<U3')
    • delta_time
      (delta_time)
      datetime64[ns]
      2020-03-06T00:25:18.782277720 .....
      contentType :
      referenceInformation
      description :
      Number of GPS seconds since the ATLAS SDP epoch. The ATLAS Standard Data Products (SDP) epoch offset is defined within /ancillary_data/atlas_sdp_gps_epoch as the number of GPS seconds between the GPS epoch (1980-01-06T00:00:00.000000Z UTC) and the ATLAS SDP epoch. By adding the offset contained within atlas_sdp_gps_epoch to delta time parameters, the time in gps_seconds relative to the GPS epoch can be computed.
      long_name :
      Elapsed GPS seconds
      source :
      section 4.4
      standard_name :
      time
      array(['2020-03-06T00:25:18.782277720', '2020-03-06T00:25:18.785109384',\n",
+       "       '2020-03-06T00:25:18.787941512', ..., '2020-03-07T00:05:23.660334576',\n",
+       "       '2020-03-07T00:05:23.663158032', '2020-03-07T00:05:23.665982160'],\n",
+       "      dtype='datetime64[ns]')
    • latitude
      (delta_time)
      float64
      dask.array<chunksize=(50000,), meta=np.ndarray>
      contentType :
      physicalMeasurement
      description :
      Latitude of segment center, WGS84, North=+,
      long_name :
      Latitude
      source :
      section 3.10
      standard_name :
      latitude
      units :
      degrees_north
      valid_max :
      90.0
      valid_min :
      -90.0
      \n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "
      \n",
+       "\n",
+       "  \n",
+       "    \n",
+       "  \n",
+       "  \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "  \n",
+       "
        Array  Chunk 
       Bytes  13.47 MB   400.00 kB 
       Shape  (1683571,)   (50000,) 
       Count  102 Tasks  43 Chunks 
       Type  float64  numpy.ndarray 
      \n",
+       "      		\n",
+       "\n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  1683571\n",
+       "  1\n",
+       "\n",
+       "
    • longitude
      (delta_time)
      float64
      dask.array<chunksize=(50000,), meta=np.ndarray>
      contentType :
      physicalMeasurement
      description :
      Longitude of segment center, , WGS84, East=+
      long_name :
      Longitude
      source :
      section 3.10
      standard_name :
      longitude
      units :
      degrees_east
      valid_max :
      180.0
      valid_min :
      -180.0
      \n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "
      \n",
+       "\n",
+       "  \n",
+       "    \n",
+       "  \n",
+       "  \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "  \n",
+       "
        Array  Chunk 
       Bytes  13.47 MB   400.00 kB 
       Shape  (1683571,)   (50000,) 
       Count  102 Tasks  43 Chunks 
       Type  float64  numpy.ndarray 
      \n",
+       "      		\n",
+       "\n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  1683571\n",
+       "  1\n",
+       "\n",
+       "
    • datetime
      (delta_time)
      datetime64[ns]
      2020-03-06T00:25:18 ... 2020-03-...
      array(['2020-03-06T00:25:18.000000000', '2020-03-06T00:25:18.000000000',\n",
+       "       '2020-03-06T00:25:18.000000000', ...,\n",
+       "       '2020-03-06T23:59:40.000000000', '2020-03-06T23:59:40.000000000',\n",
+       "       '2020-03-06T23:59:40.000000000'], dtype='datetime64[ns]')
    • referencegroundtrack
      (delta_time)
      <U4
      '1073' '1073' ... '1088' '1088'
      array(['1073', '1073', '1073', ..., '1088', '1088', '1088'], dtype='<U4')
    • atl06_quality_summary
      (delta_time)
      int8
      dask.array<chunksize=(50000,), meta=np.ndarray>
      contentType :
      qualityInformation
      description :
      The ATL06_quality_summary parameter indicates the best-quality subset of all ATL06 data. A zero in this parameter implies that no data-quality tests have found a problem with the segment, a one implies that some potential problem has been found. Users who select only segments with zero values for this flag can be relatively certain of obtaining high-quality data, but will likely miss a significant fraction of usable data, particularly in cloudy, rough, or low-surface-reflectance conditions.
      flag_meanings :
      best_quality potential_problem
      flag_values :
      [0 1]
      long_name :
      ATL06_Quality_Summary
      source :
      section 4.3
      units :
      1
      valid_max :
      1
      valid_min :
      0
      \n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "
      \n",
+       "\n",
+       "  \n",
+       "    \n",
+       "  \n",
+       "  \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "  \n",
+       "
        Array  Chunk 
       Bytes  1.68 MB   50.00 kB 
       Shape  (1683571,)   (50000,) 
       Count  102 Tasks  43 Chunks 
       Type  int8  numpy.ndarray 
      \n",
+       "      		\n",
+       "\n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  1683571\n",
+       "  1\n",
+       "\n",
+       "
    • h_li
      (delta_time)
      float32
      dask.array<chunksize=(50000,), meta=np.ndarray>
      contentType :
      physicalMeasurement
      description :
      Standard land-ice segment height determined by land ice algorithm, corrected for first-photon bias, representing the median- based height of the selected PEs
      long_name :
      Land Ice height
      source :
      section 4.4
      units :
      meters
      \n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "
      \n",
+       "\n",
+       "  \n",
+       "    \n",
+       "  \n",
+       "  \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "  \n",
+       "
        Array  Chunk 
       Bytes  6.73 MB   200.00 kB 
       Shape  (1683571,)   (50000,) 
       Count  102 Tasks  43 Chunks 
       Type  float32  numpy.ndarray 
      \n",
+       "      		\n",
+       "\n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  1683571\n",
+       "  1\n",
+       "\n",
+       "
    • h_li_sigma
      (delta_time)
      float32
      dask.array<chunksize=(50000,), meta=np.ndarray>
      contentType :
      qualityInformation
      description :
      Propagated error due to sampling error and FPB correction from the land ice algorithm
      long_name :
      Expected RMS segment misfit
      source :
      section 4.4
      units :
      meters
      \n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "
      \n",
+       "\n",
+       "  \n",
+       "    \n",
+       "  \n",
+       "  \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "  \n",
+       "
        Array  Chunk 
       Bytes  6.73 MB   200.00 kB 
       Shape  (1683571,)   (50000,) 
       Count  102 Tasks  43 Chunks 
       Type  float32  numpy.ndarray 
      \n",
+       "      		\n",
+       "\n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  1683571\n",
+       "  1\n",
+       "\n",
+       "
    • segment_id
      (delta_time)
      float64
      dask.array<chunksize=(50000,), meta=np.ndarray>
      contentType :
      referenceInformation
      description :
      Segment number, counting from the equator. Equal to the segment_id for the second of the two 20m ATL03 segments included in the 40m ATL06 segment
      long_name :
      Reference Point, m
      source :
      section 3.1.2.1
      units :
      1
      \n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "
      \n",
+       "\n",
+       "  \n",
+       "    \n",
+       "  \n",
+       "  \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "  \n",
+       "
        Array  Chunk 
       Bytes  13.47 MB   400.00 kB 
       Shape  (1683571,)   (50000,) 
       Count  102 Tasks  43 Chunks 
       Type  float64  numpy.ndarray 
      \n",
+       "      		\n",
+       "\n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  1683571\n",
+       "  1\n",
+       "\n",
+       "
    • sigma_geo_h
      (delta_time)
      float32
      dask.array<chunksize=(50000,), meta=np.ndarray>
      contentType :
      qualityInformation
      description :
      Total vertical geolocation error due to PPD and POD, including the effects of horizontal geolocation error on the segment vertical error.
      long_name :
      Vertical Geolocation Error
      source :
      ATBD Section 3.10
      units :
      meters
      \n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "
      \n",
+       "\n",
+       "  \n",
+       "    \n",
+       "  \n",
+       "  \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "    \n",
+       "  \n",
+       "
        Array  Chunk 
       Bytes  6.73 MB   200.00 kB 
       Shape  (1683571,)   (50000,) 
       Count  102 Tasks  43 Chunks 
       Type  float32  numpy.ndarray 
      \n",
+       "      		\n",
+       "\n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  \n",
+       "\n",
+       "  \n",
+       "  1683571\n",
+       "  1\n",
+       "\n",
+       "
  • Description :
    The land_ice_height group contains the primary set of derived ATL06 products. This includes geolocation, height, and standard error and quality measures for each segment. This group is sparse, meaning that parameters are provided only for pairs of segments for which at least one beam has a valid surface-height measurement.
    data_rate :
    Data within this group are sparse. Data values are provided only for those ICESat-2 20m segments where at least one beam has a valid land ice height measurement.
" + ], + "text/plain": [ + "\n", + "Dimensions: (delta_time: 1683571)\n", + "Coordinates:\n", + " revision \n", + " longitude (delta_time) float64 dask.array\n", + " datetime (delta_time) datetime64[ns] 2020-03-06T00:25:18 .....\n", + " referencegroundtrack (delta_time) \n", + " h_li (delta_time) float32 dask.array\n", + " h_li_sigma (delta_time) float32 dask.array\n", + " segment_id (delta_time) float64 dask.array\n", + " sigma_geo_h (delta_time) float32 dask.array\n", + "Attributes:\n", + " Description: The land_ice_height group contains the primary set of deriv...\n", + " data_rate: Data within this group are sparse. Data values are provide..." + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "try:\n", + " netrc.netrc()\n", + "except FileNotFoundError as error_msg:\n", + " print(\n", + " f\"{error_msg}, please follow instructions to create one at \"\n", + " \"https://nsidc.org/support/faq/what-options-are-available-bulk-downloading-data-https-earthdata-login-enabled \"\n", + " 'basically using `echo \"machine urs.earthdata.nasa.gov login password \" >> ~/.netrc`'\n", + " )\n", + " raise\n", + "\n", + "# depends on .netrc file in home folder\n", + "dataset = catalog.icesat2atl06.to_dask().unify_chunks()\n", + "dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# dataset.hvplot.points(\n", + "# x=\"longitude\",\n", + "# y=\"latitude\",\n", + "# c=\"h_li\",\n", + "# cmap=\"Blues\",\n", + "# rasterize=True,\n", + "# hover=True,\n", + "# width=800,\n", + "# height=500,\n", + "# geo=True,\n", + "# coastline=True,\n", + "# crs=cartopy.crs.PlateCarree(),\n", + "# projection=cartopy.crs.Stereographic(central_latitude=-71),\n", + "# )\n", + "catalog.icesat2atl06.hvplot.quickview()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data intake\n", + "\n", + "Pulling in all of the raw ATL06 data (HDF5 format) from the NSIDC servers via an intake catalog file.\n", + "Note that this will involve 100s if not 1000s of GBs of data, so make sure there's enough storage!!" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Download all ICESAT2 ATLAS hdf files from start to end date\n", + "dates1 = pd.date_range(start=\"2018.10.14\", end=\"2018.12.08\") # 1st batch\n", + "dates2 = pd.date_range(start=\"2018.12.10\", end=\"2019.06.26\") # 2nd batch\n", + "dates3 = pd.date_range(start=\"2019.07.26\", end=\"2020.03.06\") # 3rd batch\n", + "dates = dates1.append(other=dates2).append(other=dates3)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Submit download jobs to Client\n", + "futures = []\n", + "for date in dates:\n", + " source = catalog.icesat2atlasdownloader(date=date)\n", + " future = client.submit(\n", + " func=source.discover, key=f\"download-{date}\",\n", + " ) # triggers download of the file(s), or loads from cache\n", + " futures.append(future)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check download progress here, https://stackoverflow.com/a/37901797/6611055\n", + "responses = []\n", + "for f in tqdm.tqdm(\n", + " iterable=dask.distributed.as_completed(futures=futures), total=len(futures)\n", + "):\n", + " responses.append(f.result())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# In case of error, check which downloads are unfinished\n", + "# Manually delete those folders and retry\n", + "unfinished = []\n", + "for foo in futures:\n", + " if foo.status != \"finished\":\n", + " print(foo)\n", + " unfinished.append(foo)\n", + " # foo.retry()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " assert len(unfinished) == 0\n", + "except AssertionError:\n", + " for task in unfinished:\n", + " print(task)\n", + " raise ValueError(\n", + " f\"{len(unfinished)} download tasks are unfinished,\"\n", + " \" please delete those folders and retry again!\"\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exploratory data analysis on local files\n", + "\n", + "Now that we've downloaded a good chunk of data and cached them locally,\n", + "we can have some fun with visualizing the point clouds!" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "lines_to_next_cell": 1 + }, + "outputs": [], + "source": [ + "root_directory = os.path.dirname(\n", + " catalog.icesat2atl06.storage_options[\"simplecache\"][\"cache_storage\"]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "def get_crossing_dates(\n", + " catalog_entry: intake.catalog.local.LocalCatalogEntry,\n", + " root_directory: str,\n", + " referencegroundtrack: str = \"????\",\n", + " datetimestr: str = \"*\",\n", + " cyclenumber: str = \"??\",\n", + " orbitalsegment: str = \"??\",\n", + " version: str = \"003\",\n", + " revision: str = \"01\",\n", + ") -> dict:\n", + " \"\"\"\n", + " Given a 4-digit reference groundtrack (e.g. 1234),\n", + " we output a dictionary where the\n", + " key is the date in \"YYYY.MM.DD\" format when an ICESAT2 crossing was made and the\n", + " value is the filepath to the HDF5 data file.\n", + " \"\"\"\n", + "\n", + " # Get a glob string that looks like \"ATL06_??????????????_XXXX????_002_01.h5\"\n", + " globpath: str = catalog_entry.path_as_pattern\n", + " if datetimestr == \"*\":\n", + " globpath: str = globpath.replace(\"{datetime:%Y%m%d%H%M%S}\", \"??????????????\")\n", + " globpath: str = globpath.format(\n", + " referencegroundtrack=referencegroundtrack,\n", + " cyclenumber=cyclenumber,\n", + " orbitalsegment=orbitalsegment,\n", + " version=version,\n", + " revision=revision,\n", + " )\n", + "\n", + " # Get list of filepaths (dates are contained in the filepath)\n", + " globedpaths: list = glob.glob(os.path.join(root_directory, \"??????????\", globpath))\n", + "\n", + " # Pick out just the dates in \"YYYY.MM.DD\" format from the globedpaths\n", + " # crossingdates = [os.path.basename(os.path.dirname(p=p)) for p in globedpaths]\n", + " crossingdates: dict = {\n", + " os.path.basename(os.path.dirname(p=p)): p for p in sorted(globedpaths)\n", + " }\n", + "\n", + " return crossingdates" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "crossing_dates_dict = {}\n", + "for rgt in range(1, 1388): # ReferenceGroundTrack goes from 0001 to 1387\n", + " referencegroundtrack: str = f\"{rgt}\".zfill(4)\n", + " crossing_dates: dict = dask.delayed(get_crossing_dates)(\n", + " catalog_entry=catalog.icesat2atl06,\n", + " root_directory=root_directory,\n", + " referencegroundtrack=referencegroundtrack,\n", + " )\n", + " crossing_dates_dict[referencegroundtrack] = crossing_dates\n", + "crossing_dates_dict = dask.compute(crossing_dates_dict)[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['2018.10.21', '2019.01.20', '2019.04.21', '2019.10.19', '2020.01.18'])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "crossing_dates_dict[\"0349\"].keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![ICESat-2 Laser Beam Pattern](https://ars.els-cdn.com/content/image/1-s2.0-S0034425719303712-gr1.jpg)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def six_laser_beams(filepaths: list) -> dask.dataframe.DataFrame:\n", + " \"\"\"\n", + " For all 6 lasers along one reference ground track,\n", + " concatenate all points from all crossing dates into one Dask DataFrame\n", + "\n", + " E.g. if there are 5 crossing dates and 6 lasers,\n", + " there would be data from 5 x 6 = 30 files being concatenated together.\n", + " \"\"\"\n", + " lasers: list = [\"gt1l\", \"gt1r\", \"gt2l\", \"gt2r\", \"gt3l\", \"gt3r\"]\n", + "\n", + " objs: list = [\n", + " xr.open_mfdataset(\n", + " paths=filepaths,\n", + " combine=\"by_coords\",\n", + " engine=\"h5netcdf\",\n", + " group=f\"{laser}/land_ice_segments\",\n", + " parallel=True,\n", + " ).assign_coords(coords={\"laser\": laser})\n", + " for laser in lasers\n", + " ]\n", + "\n", + " try:\n", + " da: xr.Dataset = xr.concat(objs=objs, dim=\"laser\")\n", + " df: dask.dataframe.DataFrame = da.unify_chunks().to_dask_dataframe()\n", + " except ValueError:\n", + " # ValueError: cannot reindex or align along dimension 'delta_time'\n", + " # because the index has duplicate values\n", + " df: dask.dataframe.DataFrame = dask.dataframe.concat(\n", + " [obj.unify_chunks().to_dask_dataframe() for obj in objs]\n", + " )\n", + "\n", + " return df" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_dict = {}\n", + "# ReferenceGroundTrack goes from 0001 to 1387\n", + "for referencegroundtrack in list(crossing_dates_dict)[348:349]:\n", + " # print(referencegroundtrack)\n", + " filepaths = list(crossing_dates_dict[referencegroundtrack].values())\n", + " if len(filepaths) > 0:\n", + " dataset_dict[referencegroundtrack] = dask.delayed(obj=six_laser_beams)(\n", + " filepaths=filepaths\n", + " )\n", + " # df = six_laser_beams(filepaths=filepaths)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "df = dataset_dict[\"0349\"].compute() # loads into a dask dataframe (lazy)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Dask DataFrame Structure:
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
delta_timelaserlatitudelongitudeatl06_quality_summaryh_lih_li_sigmasegment_idsigma_geo_h
npartitions=154
0datetime64[ns]objectfloat64float64float64float32float32float64float32
340326...........................
..............................
8445120...........................
8745557...........................
\n", + "
\n", + "
Dask Name: concat-indexed, 21362 tasks
" + ], + "text/plain": [ + "Dask DataFrame Structure:\n", + " delta_time laser latitude longitude atl06_quality_summary h_li h_li_sigma segment_id sigma_geo_h\n", + "npartitions=154 \n", + "0 datetime64[ns] object float64 float64 float64 float32 float32 float64 float32\n", + "340326 ... ... ... ... ... ... ... ... ...\n", + "... ... ... ... ... ... ... ... ... ...\n", + "8445120 ... ... ... ... ... ... ... ... ...\n", + "8745557 ... ... ... ... ... ... ... ... ...\n", + "Dask Name: concat-indexed, 21362 tasks" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# compute every referencegroundtrack, slow... though somewhat parallelized\n", + "# dataset_dict = dask.compute(dataset_dict)[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# big dataframe containing data across all 1387 reference ground tracks!\n", + "# bdf = dask.dataframe.concat(dfs=list(dataset_dict.values()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "lines_to_next_cell": 0 + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "# https://xarray.pydata.org/en/stable/combining.html#concatenate\n", + "# For all 6 lasers one one date ~~along one reference ground track~~,\n", + "# concatenate all points ~~from one dates~~ into one xr.Dataset\n", + "lasers = [\"gt1l\", \"gt1r\", \"gt2l\", \"gt2r\", \"gt3l\", \"gt3r\"]\n", + "da = xr.concat(\n", + " objs=(\n", + " catalog.icesat2atl06(laser=laser, referencegroundtrack=referencegroundtrack)\n", + " .to_dask()\n", + " for laser in lasers\n", + " ),\n", + " dim=pd.Index(data=lasers, name=\"laser\")\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot ATL06 points!" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# Convert dask.DataFrame to pd.DataFrame\n", + "df: pd.DataFrame = df.compute()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Drop points with poor quality\n", + "df = df.dropna(subset=[\"h_li\"]).query(expr=\"atl06_quality_summary == 0\").reset_index()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
indexdelta_timelaserlatitudelongitudeatl06_quality_summaryh_lih_li_sigmasegment_idsigma_geo_h
54285332686112019-01-20 08:07:46.068493512gt2r-73.01619548.0624890.03152.4343260.0285301599374.00.312982
32434019463562018-10-21 12:28:54.317352440gt3l-69.44853146.7692370.02206.0698240.0469081619499.00.480249
1182727096352018-10-21 12:27:17.481998464gt2r-75.54617349.5476220.03388.3486330.0105081585012.00.307785
86847452378372019-04-21 03:47:11.232018304gt3r-74.74813949.0803370.03388.1450200.0273481589537.00.304449
129735380802322020-01-18 14:47:39.217810632gt2l-70.28478946.9081610.02500.0930180.0300891614817.00.359534
\n", + "
" + ], + "text/plain": [ + " index delta_time laser latitude longitude \\\n", + "542853 3268611 2019-01-20 08:07:46.068493512 gt2r -73.016195 48.062489 \n", + "324340 1946356 2018-10-21 12:28:54.317352440 gt3l -69.448531 46.769237 \n", + "118272 709635 2018-10-21 12:27:17.481998464 gt2r -75.546173 49.547622 \n", + "868474 5237837 2019-04-21 03:47:11.232018304 gt3r -74.748139 49.080337 \n", + "1297353 8080232 2020-01-18 14:47:39.217810632 gt2l -70.284789 46.908161 \n", + "\n", + " atl06_quality_summary h_li h_li_sigma segment_id \\\n", + "542853 0.0 3152.434326 0.028530 1599374.0 \n", + "324340 0.0 2206.069824 0.046908 1619499.0 \n", + "118272 0.0 3388.348633 0.010508 1585012.0 \n", + "868474 0.0 3388.145020 0.027348 1589537.0 \n", + "1297353 0.0 2500.093018 0.030089 1614817.0 \n", + "\n", + " sigma_geo_h \n", + "542853 0.312982 \n", + "324340 0.480249 \n", + "118272 0.307785 \n", + "868474 0.304449 \n", + "1297353 0.359534 " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get a small random sample of our data\n", + "dfs = df.sample(n=1_000, random_state=42)\n", + "dfs.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dfs.hvplot.scatter(\n", + " x=\"longitude\",\n", + " y=\"latitude\",\n", + " by=\"laser\",\n", + " hover_cols=[\"delta_time\", \"segment_id\"],\n", + " # datashade=True, dynspread=True,\n", + " # width=800, height=500, colorbar=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Transform from EPSG:4326 (lat/lon) to EPSG:3031 (Antarctic Polar Stereographic)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "transformer = pyproj.Transformer.from_crs(\n", + " crs_from=pyproj.CRS.from_epsg(4326),\n", + " crs_to=pyproj.CRS.from_epsg(3031),\n", + " always_xy=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "dfs[\"x\"], dfs[\"y\"] = transformer.transform(\n", + " xx=dfs.longitude.values, yy=dfs.latitude.values\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
indexdelta_timelaserlatitudelongitudeatl06_quality_summaryh_lih_li_sigmasegment_idsigma_geo_hxy
54285332686112019-01-20 08:07:46.068493512gt2r-73.01619548.0624890.03152.4343260.0285301599374.00.3129821.382429e+061.242017e+06
32434019463562018-10-21 12:28:54.317352440gt3l-69.44853146.7692370.02206.0698240.0469081619499.00.4802491.643908e+061.545394e+06
1182727096352018-10-21 12:27:17.481998464gt2r-75.54617349.5476220.03388.3486330.0105081585012.00.3077851.201144e+061.024148e+06
86847452378372019-04-21 03:47:11.232018304gt3r-74.74813949.0803370.03388.1450200.0273481589537.00.3044491.259340e+061.091631e+06
129735380802322020-01-18 14:47:39.217810632gt2l-70.28478946.9081610.02500.0930180.0300891614817.00.3595341.579289e+061.477451e+06
\n", + "
" + ], + "text/plain": [ + " index delta_time laser latitude longitude \\\n", + "542853 3268611 2019-01-20 08:07:46.068493512 gt2r -73.016195 48.062489 \n", + "324340 1946356 2018-10-21 12:28:54.317352440 gt3l -69.448531 46.769237 \n", + "118272 709635 2018-10-21 12:27:17.481998464 gt2r -75.546173 49.547622 \n", + "868474 5237837 2019-04-21 03:47:11.232018304 gt3r -74.748139 49.080337 \n", + "1297353 8080232 2020-01-18 14:47:39.217810632 gt2l -70.284789 46.908161 \n", + "\n", + " atl06_quality_summary h_li h_li_sigma segment_id \\\n", + "542853 0.0 3152.434326 0.028530 1599374.0 \n", + "324340 0.0 2206.069824 0.046908 1619499.0 \n", + "118272 0.0 3388.348633 0.010508 1585012.0 \n", + "868474 0.0 3388.145020 0.027348 1589537.0 \n", + "1297353 0.0 2500.093018 0.030089 1614817.0 \n", + "\n", + " sigma_geo_h x y \n", + "542853 0.312982 1.382429e+06 1.242017e+06 \n", + "324340 0.480249 1.643908e+06 1.545394e+06 \n", + "118272 0.307785 1.201144e+06 1.024148e+06 \n", + "868474 0.304449 1.259340e+06 1.091631e+06 \n", + "1297353 0.359534 1.579289e+06 1.477451e+06 " + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dfs.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dfs.hvplot.scatter(\n", + " x=\"x\",\n", + " y=\"y\",\n", + " by=\"laser\",\n", + " hover_cols=[\"delta_time\", \"segment_id\", \"h_li\"],\n", + " # datashade=True, dynspread=True,\n", + " # width=800, height=500, colorbar=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Plot cross section view\n", + "dfs.hvplot.scatter(x=\"x\", y=\"h_li\", by=\"laser\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "dfs.to_pickle(path=\"icesat2_sample.pkl\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Experimental Work-in-Progress stuff below" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Old way of making a DEM grid surface from points" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import scipy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# https://github.com/ICESAT-2HackWeek/gridding/blob/master/notebook/utils.py#L23\n", + "def make_grid(xmin, xmax, ymin, ymax, dx, dy):\n", + " \"\"\"Construct output grid-coordinates.\"\"\"\n", + "\n", + " # Setup grid dimensions\n", + " Nn = int((np.abs(ymax - ymin)) / dy) + 1\n", + " Ne = int((np.abs(xmax - xmin)) / dx) + 1\n", + "\n", + " # Initiate x/y vectors for grid\n", + " x_i = np.linspace(xmin, xmax, num=Ne)\n", + " y_i = np.linspace(ymin, ymax, num=Nn)\n", + "\n", + " return np.meshgrid(x_i, y_i)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xi, yi = make_grid(\n", + " xmin=dfs.x.min(), xmax=dfs.x.max(), ymin=dfs.y.max(), ymax=dfs.y.min(), dx=10, dy=10\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ar = scipy.interpolate.griddata(points=(dfs.x, dfs.y), values=dfs.h_li, xi=(xi, yi))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.imshow(ar, extent=(dfs.x.min(), dfs.x.max(), dfs.y.min(), dfs.y.max()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import plotly.express as px" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "px.scatter_3d(data_frame=dfs, x=\"longitude\", y=\"latitude\", z=\"h_li\", color=\"laser\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Play using XrViz\n", + "\n", + "Install the PyViz JupyterLab extension first using the [extension manager](https://jupyterlab.readthedocs.io/en/stable/user/extensions.html#using-the-extension-manager) or via the command below:\n", + "\n", + "```bash\n", + "jupyter labextension install @pyviz/jupyterlab_pyviz@v0.8.0 --no-build\n", + "jupyter labextension list # check to see that extension is installed\n", + "jupyter lab build --debug # build extension ??? with debug messages printed\n", + "```\n", + "\n", + "Note: Had to add `network-timeout 600000` to `.yarnrc` file to resolve university network issues." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xrviz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xrviz.example()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# https://xrviz.readthedocs.io/en/latest/set_initial_parameters.html\n", + "initial_params = {\n", + " # Select variable to plot\n", + " \"Variables\": \"h_li\",\n", + " # Set coordinates\n", + " \"Set Coords\": [\"longitude\", \"latitude\"],\n", + " # Axes\n", + " \"x\": \"longitude\",\n", + " \"y\": \"latitude\",\n", + " # \"sigma\": \"animate\",\n", + " # Projection\n", + " # \"is_geo\": True,\n", + " # \"basemap\": True,\n", + " # \"crs\": \"PlateCarree\"\n", + "}\n", + "dashboard = xrviz.dashboard.Dashboard(data=dataset) # , initial_params=initial_params)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dashboard.panel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dashboard.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OpenAltimetry" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\"minx=-154.56678505984297&miny=-88.82881451427136&maxx=-125.17872921546498&maxy=-81.34051361301398&date=2019-05-02&trackId=516\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Paste the OpenAltimetry selection parameters here\n", + "OA_REFERENCE_URL = \"minx=-177.64275595145213&miny=-88.12014866942751&maxx=-128.25920892322736&maxy=-85.52394234080862&date=2019-05-02&trackId=515\"\n", + "# We populate a list with the photon data using the OpenAltimetry API, no HDF!\n", + "OA_URL = (\n", + " \"https://openaltimetry.org/data/icesat2/getPhotonData?client=jupyter&\"\n", + " + OA_REFERENCE_URL\n", + ")\n", + "OA_PHOTONS = [\"Noise\", \"Low\", \"Medium\", \"High\"]\n", + "# OA_PLOTTED_BEAMS = [1,2,3,4,5,6] you can select up to 6 beams for each ground track.\n", + "# Some beams may not be usable due cloud covering or QC issues.\n", + "OA_BEAMS = [3, 4]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "minx, miny, maxx, maxy = [-156, -88, -127, -84]\n", + "date = \"2019-05-02\" # UTC date?\n", + "track = 515 #\n", + "beam = 1 # 1 to 6\n", + "params = {\n", + " \"client\": \"jupyter\",\n", + " \"minx\": minx,\n", + " \"miny\": miny,\n", + " \"maxx\": maxx,\n", + " \"maxy\": maxy,\n", + " \"date\": date,\n", + " \"trackId\": str(track),\n", + " \"beam\": str(beam),\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = requests.get(\n", + " url=\"https://openaltimetry.org/data/icesat2/getPhotonData\", params=params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# OpenAltimetry Data cleansing\n", + "df = pd.io.json.json_normalize(data=r.json()[\"series\"], meta=\"name\", record_path=\"data\")\n", + "df.name = df.name.str.split().str.get(0) # Get e.g. just \"Low\" instead of \"Low [12345]\"\n", + "df.query(\n", + " expr=\"name in ('Low', 'Medium', 'High')\", inplace=True\n", + ") # filter out Noise and Buffer points\n", + "\n", + "df.rename(columns={0: \"latitude\", 1: \"elevation\", 2: \"longitude\"}, inplace=True)\n", + "df = df.reindex(\n", + " columns=[\"longitude\", \"latitude\", \"elevation\", \"name\"]\n", + ") # reorder columns\n", + "df.reset_index(inplace=True)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.hvplot.scatter(x=\"latitude\", y=\"elevation\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "formats": "ipynb,py:hydrogen" + }, + "kernelspec": { + "display_name": "deepicedrain", + "language": "python", + "name": "deepicedrain" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/atl06_play.py b/atl06_play.py new file mode 100644 index 0000000..14562e2 --- /dev/null +++ b/atl06_play.py @@ -0,0 +1,531 @@ +# --- +# jupyter: +# jupytext: +# formats: ipynb,py:hydrogen +# text_representation: +# extension: .py +# format_name: hydrogen +# format_version: '1.3' +# jupytext_version: 1.4.2 +# kernelspec: +# display_name: deepicedrain +# language: python +# name: deepicedrain +# --- + +# %% [markdown] +# # **ATLAS/ICESat-2 Land Ice Height [ATL06](https://nsidc.org/data/atl06/) Exploratory Data Analysis** +# +# [Yet another](https://xkcd.com/927) take on playing with ICESat-2's Land Ice Height ATL06 data, +# specfically with a focus on analyzing ice elevation changes over Antarctica. +# Specifically, this jupyter notebook will cover: +# +# - Downloading datasets from the web via [intake](https://intake.readthedocs.io) +# - Performing [Exploratory Data Analysis](https://en.wikipedia.org/wiki/Exploratory_data_analysis) +# using the [PyData](https://pydata.org) stack (e.g. [xarray](http://xarray.pydata.org), [dask](https://dask.org)) +# - Plotting figures using [Hvplot](https://hvplot.holoviz.org) and [PyGMT](https://www.pygmt.org) (TODO) +# +# This is in contrast with the [icepyx](https://github.com/icesat2py/icepyx) package +# and 'official' 2019/2020 [ICESat-2 Hackweek tutorials](https://github.com/ICESAT-2HackWeek/ICESat2_hackweek_tutorials) (which are also awesome!) +# that tends to use a slightly different approach (e.g. handcoded download scripts, [h5py](http://www.h5py.org) for data reading, etc). +# The core concept here is to run things in a more intuitive and scalable (parallelizable) manner on a continent scale (rather than just a specific region). + +# %% +import glob +import json +import logging +import netrc +import os + +import cartopy +import dask +import dask.distributed +import hvplot.dask +import hvplot.pandas +import hvplot.xarray +import intake +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pyproj +import requests +import tqdm +import xarray as xr + +# %matplotlib inline + +# %% +# Configure intake and set number of compute cores for data download +intake.config.conf["download_progress"] = False # disable automatic tqdm progress bars + +logging.basicConfig(level=logging.WARNING) + +# Limit compute to 10 cores for download part using intake +# Can possibly go up to 10 because there are 10 DPs? +# See https://n5eil02u.ecs.nsidc.org/opendap/hyrax/catalog.xml +client = dask.distributed.Client(n_workers=10, threads_per_worker=1) +client + +# %% [markdown] +# ## Quick view +# +# Use our [intake catalog](https://intake.readthedocs.io/en/latest/catalog.html) to get some sample ATL06 data +# (while making sure we have our Earthdata credentials set up properly), +# and view it using [xarray](https://xarray.pydata.org) and [hvplot](https://hvplot.pyviz.org). + +# %% +# open the local catalog file containing ICESat-2 stuff +catalog = intake.open_catalog(uri="catalog.yaml") + +# %% +try: + netrc.netrc() +except FileNotFoundError as error_msg: + print( + f"{error_msg}, please follow instructions to create one at " + "https://nsidc.org/support/faq/what-options-are-available-bulk-downloading-data-https-earthdata-login-enabled " + 'basically using `echo "machine urs.earthdata.nasa.gov login password " >> ~/.netrc`' + ) + raise + +# depends on .netrc file in home folder +dataset = catalog.icesat2atl06.to_dask().unify_chunks() +dataset + +# %% +# dataset.hvplot.points( +# x="longitude", +# y="latitude", +# c="h_li", +# cmap="Blues", +# rasterize=True, +# hover=True, +# width=800, +# height=500, +# geo=True, +# coastline=True, +# crs=cartopy.crs.PlateCarree(), +# projection=cartopy.crs.Stereographic(central_latitude=-71), +# ) +catalog.icesat2atl06.hvplot.quickview() + +# %% [markdown] +# ## Data intake +# +# Pulling in all of the raw ATL06 data (HDF5 format) from the NSIDC servers via an intake catalog file. +# Note that this will involve 100s if not 1000s of GBs of data, so make sure there's enough storage!! + +# %% +# Download all ICESAT2 ATLAS hdf files from start to end date +dates1 = pd.date_range(start="2018.10.14", end="2018.12.08") # 1st batch +dates2 = pd.date_range(start="2018.12.10", end="2019.06.26") # 2nd batch +dates3 = pd.date_range(start="2019.07.26", end="2020.03.06") # 3rd batch +dates = dates1.append(other=dates2).append(other=dates3) + +# %% +# Submit download jobs to Client +futures = [] +for date in dates: + source = catalog.icesat2atlasdownloader(date=date) + future = client.submit( + func=source.discover, key=f"download-{date}", + ) # triggers download of the file(s), or loads from cache + futures.append(future) + +# %% +# Check download progress here, https://stackoverflow.com/a/37901797/6611055 +responses = [] +for f in tqdm.tqdm( + iterable=dask.distributed.as_completed(futures=futures), total=len(futures) +): + responses.append(f.result()) + +# %% +# In case of error, check which downloads are unfinished +# Manually delete those folders and retry +unfinished = [] +for foo in futures: + if foo.status != "finished": + print(foo) + unfinished.append(foo) + # foo.retry() + +# %% +try: + assert len(unfinished) == 0 +except AssertionError: + for task in unfinished: + print(task) + raise ValueError( + f"{len(unfinished)} download tasks are unfinished," + " please delete those folders and retry again!" + ) + +# %% + +# %% [markdown] +# ## Exploratory data analysis on local files +# +# Now that we've downloaded a good chunk of data and cached them locally, +# we can have some fun with visualizing the point clouds! + +# %% +root_directory = os.path.dirname( + catalog.icesat2atl06.storage_options["simplecache"]["cache_storage"] +) + +# %% +def get_crossing_dates( + catalog_entry: intake.catalog.local.LocalCatalogEntry, + root_directory: str, + referencegroundtrack: str = "????", + datetimestr: str = "*", + cyclenumber: str = "??", + orbitalsegment: str = "??", + version: str = "003", + revision: str = "01", +) -> dict: + """ + Given a 4-digit reference groundtrack (e.g. 1234), + we output a dictionary where the + key is the date in "YYYY.MM.DD" format when an ICESAT2 crossing was made and the + value is the filepath to the HDF5 data file. + """ + + # Get a glob string that looks like "ATL06_??????????????_XXXX????_002_01.h5" + globpath: str = catalog_entry.path_as_pattern + if datetimestr == "*": + globpath: str = globpath.replace("{datetime:%Y%m%d%H%M%S}", "??????????????") + globpath: str = globpath.format( + referencegroundtrack=referencegroundtrack, + cyclenumber=cyclenumber, + orbitalsegment=orbitalsegment, + version=version, + revision=revision, + ) + + # Get list of filepaths (dates are contained in the filepath) + globedpaths: list = glob.glob(os.path.join(root_directory, "??????????", globpath)) + + # Pick out just the dates in "YYYY.MM.DD" format from the globedpaths + # crossingdates = [os.path.basename(os.path.dirname(p=p)) for p in globedpaths] + crossingdates: dict = { + os.path.basename(os.path.dirname(p=p)): p for p in sorted(globedpaths) + } + + return crossingdates + + +# %% +crossing_dates_dict = {} +for rgt in range(1, 1388): # ReferenceGroundTrack goes from 0001 to 1387 + referencegroundtrack: str = f"{rgt}".zfill(4) + crossing_dates: dict = dask.delayed(get_crossing_dates)( + catalog_entry=catalog.icesat2atl06, + root_directory=root_directory, + referencegroundtrack=referencegroundtrack, + ) + crossing_dates_dict[referencegroundtrack] = crossing_dates +crossing_dates_dict = dask.compute(crossing_dates_dict)[0] + +# %% +crossing_dates_dict["0349"].keys() + + +# %% [markdown] +# ![ICESat-2 Laser Beam Pattern](https://ars.els-cdn.com/content/image/1-s2.0-S0034425719303712-gr1.jpg) + +# %% +def six_laser_beams(filepaths: list) -> dask.dataframe.DataFrame: + """ + For all 6 lasers along one reference ground track, + concatenate all points from all crossing dates into one Dask DataFrame + + E.g. if there are 5 crossing dates and 6 lasers, + there would be data from 5 x 6 = 30 files being concatenated together. + """ + lasers: list = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] + + objs: list = [ + xr.open_mfdataset( + paths=filepaths, + combine="by_coords", + engine="h5netcdf", + group=f"{laser}/land_ice_segments", + parallel=True, + ).assign_coords(coords={"laser": laser}) + for laser in lasers + ] + + try: + da: xr.Dataset = xr.concat(objs=objs, dim="laser") + df: dask.dataframe.DataFrame = da.unify_chunks().to_dask_dataframe() + except ValueError: + # ValueError: cannot reindex or align along dimension 'delta_time' + # because the index has duplicate values + df: dask.dataframe.DataFrame = dask.dataframe.concat( + [obj.unify_chunks().to_dask_dataframe() for obj in objs] + ) + + return df + + +# %% +dataset_dict = {} +# ReferenceGroundTrack goes from 0001 to 1387 +for referencegroundtrack in list(crossing_dates_dict)[348:349]: + # print(referencegroundtrack) + filepaths = list(crossing_dates_dict[referencegroundtrack].values()) + if len(filepaths) > 0: + dataset_dict[referencegroundtrack] = dask.delayed(obj=six_laser_beams)( + filepaths=filepaths + ) + # df = six_laser_beams(filepaths=filepaths) + +# %% +df = dataset_dict["0349"].compute() # loads into a dask dataframe (lazy) + +# %% +df + +# %% + +# %% +# compute every referencegroundtrack, slow... though somewhat parallelized +# dataset_dict = dask.compute(dataset_dict)[0] + +# %% +# big dataframe containing data across all 1387 reference ground tracks! +# bdf = dask.dataframe.concat(dfs=list(dataset_dict.values())) + +# %% +# %% [raw] +# # https://xarray.pydata.org/en/stable/combining.html#concatenate +# # For all 6 lasers one one date ~~along one reference ground track~~, +# # concatenate all points ~~from one dates~~ into one xr.Dataset +# lasers = ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] +# da = xr.concat( +# objs=( +# catalog.icesat2atl06(laser=laser, referencegroundtrack=referencegroundtrack) +# .to_dask() +# for laser in lasers +# ), +# dim=pd.Index(data=lasers, name="laser") +# ) + +# %% + +# %% [markdown] +# ## Plot ATL06 points! + +# %% +# Convert dask.DataFrame to pd.DataFrame +df: pd.DataFrame = df.compute() + +# %% +# Drop points with poor quality +df = df.dropna(subset=["h_li"]).query(expr="atl06_quality_summary == 0").reset_index() + +# %% +# Get a small random sample of our data +dfs = df.sample(n=1_000, random_state=42) +dfs.head() + +# %% +dfs.hvplot.scatter( + x="longitude", + y="latitude", + by="laser", + hover_cols=["delta_time", "segment_id"], + # datashade=True, dynspread=True, + # width=800, height=500, colorbar=True +) + +# %% [markdown] +# ### Transform from EPSG:4326 (lat/lon) to EPSG:3031 (Antarctic Polar Stereographic) + +# %% +transformer = pyproj.Transformer.from_crs( + crs_from=pyproj.CRS.from_epsg(4326), + crs_to=pyproj.CRS.from_epsg(3031), + always_xy=True, +) + +# %% +dfs["x"], dfs["y"] = transformer.transform( + xx=dfs.longitude.values, yy=dfs.latitude.values +) + +# %% +dfs.head() + +# %% +dfs.hvplot.scatter( + x="x", + y="y", + by="laser", + hover_cols=["delta_time", "segment_id", "h_li"], + # datashade=True, dynspread=True, + # width=800, height=500, colorbar=True +) + +# %% +# Plot cross section view +dfs.hvplot.scatter(x="x", y="h_li", by="laser") + +# %% +dfs.to_pickle(path="icesat2_sample.pkl") + +# %% + +# %% [markdown] +# ## Experimental Work-in-Progress stuff below + +# %% [markdown] +# ### Old way of making a DEM grid surface from points + +# %% +import scipy + + +# %% +# https://github.com/ICESAT-2HackWeek/gridding/blob/master/notebook/utils.py#L23 +def make_grid(xmin, xmax, ymin, ymax, dx, dy): + """Construct output grid-coordinates.""" + + # Setup grid dimensions + Nn = int((np.abs(ymax - ymin)) / dy) + 1 + Ne = int((np.abs(xmax - xmin)) / dx) + 1 + + # Initiate x/y vectors for grid + x_i = np.linspace(xmin, xmax, num=Ne) + y_i = np.linspace(ymin, ymax, num=Nn) + + return np.meshgrid(x_i, y_i) + + +# %% +xi, yi = make_grid( + xmin=dfs.x.min(), xmax=dfs.x.max(), ymin=dfs.y.max(), ymax=dfs.y.min(), dx=10, dy=10 +) + +# %% +ar = scipy.interpolate.griddata(points=(dfs.x, dfs.y), values=dfs.h_li, xi=(xi, yi)) + +# %% +plt.imshow(ar, extent=(dfs.x.min(), dfs.x.max(), dfs.y.min(), dfs.y.max())) + +# %% + +# %% +import plotly.express as px + +# %% +px.scatter_3d(data_frame=dfs, x="longitude", y="latitude", z="h_li", color="laser") + +# %% + +# %% [markdown] +# ### Play using XrViz +# +# Install the PyViz JupyterLab extension first using the [extension manager](https://jupyterlab.readthedocs.io/en/stable/user/extensions.html#using-the-extension-manager) or via the command below: +# +# ```bash +# jupyter labextension install @pyviz/jupyterlab_pyviz@v0.8.0 --no-build +# jupyter labextension list # check to see that extension is installed +# jupyter lab build --debug # build extension ??? with debug messages printed +# ``` +# +# Note: Had to add `network-timeout 600000` to `.yarnrc` file to resolve university network issues. + +# %% +import xrviz + +# %% +xrviz.example() + +# %% +# https://xrviz.readthedocs.io/en/latest/set_initial_parameters.html +initial_params = { + # Select variable to plot + "Variables": "h_li", + # Set coordinates + "Set Coords": ["longitude", "latitude"], + # Axes + "x": "longitude", + "y": "latitude", + # "sigma": "animate", + # Projection + # "is_geo": True, + # "basemap": True, + # "crs": "PlateCarree" +} +dashboard = xrviz.dashboard.Dashboard(data=dataset) # , initial_params=initial_params) + +# %% +dashboard.panel + +# %% +dashboard.show() + +# %% + +# %% [markdown] +# ## OpenAltimetry + +# %% +"minx=-154.56678505984297&miny=-88.82881451427136&maxx=-125.17872921546498&maxy=-81.34051361301398&date=2019-05-02&trackId=516" + +# %% +# Paste the OpenAltimetry selection parameters here +OA_REFERENCE_URL = "minx=-177.64275595145213&miny=-88.12014866942751&maxx=-128.25920892322736&maxy=-85.52394234080862&date=2019-05-02&trackId=515" +# We populate a list with the photon data using the OpenAltimetry API, no HDF! +OA_URL = ( + "https://openaltimetry.org/data/icesat2/getPhotonData?client=jupyter&" + + OA_REFERENCE_URL +) +OA_PHOTONS = ["Noise", "Low", "Medium", "High"] +# OA_PLOTTED_BEAMS = [1,2,3,4,5,6] you can select up to 6 beams for each ground track. +# Some beams may not be usable due cloud covering or QC issues. +OA_BEAMS = [3, 4] + +# %% +minx, miny, maxx, maxy = [-156, -88, -127, -84] +date = "2019-05-02" # UTC date? +track = 515 # +beam = 1 # 1 to 6 +params = { + "client": "jupyter", + "minx": minx, + "miny": miny, + "maxx": maxx, + "maxy": maxy, + "date": date, + "trackId": str(track), + "beam": str(beam), +} + +# %% +r = requests.get( + url="https://openaltimetry.org/data/icesat2/getPhotonData", params=params +) + +# %% +# OpenAltimetry Data cleansing +df = pd.io.json.json_normalize(data=r.json()["series"], meta="name", record_path="data") +df.name = df.name.str.split().str.get(0) # Get e.g. just "Low" instead of "Low [12345]" +df.query( + expr="name in ('Low', 'Medium', 'High')", inplace=True +) # filter out Noise and Buffer points + +df.rename(columns={0: "latitude", 1: "elevation", 2: "longitude"}, inplace=True) +df = df.reindex( + columns=["longitude", "latitude", "elevation", "name"] +) # reorder columns +df.reset_index(inplace=True) +df + +# %% +df.hvplot.scatter(x="latitude", y="elevation") + +# %% diff --git a/catalog.yaml b/catalog.yaml index 07f87ec..4c7efa9 100644 --- a/catalog.yaml +++ b/catalog.yaml @@ -1,27 +1,75 @@ metadata: version: 1 sources: + icesat2atlasdownloader: + args: + urlpath: + - simplecache::https://n5eil01u.ecs.nsidc.org/ATLAS/ATL{{atlproduct}}.00{{version}}/{{date.strftime("%Y.%m.%d")}}/ATL{{atlproduct}}_*_*10_00{{version}}_0{{revision}}.h5 + - simplecache::https://n5eil01u.ecs.nsidc.org/ATLAS/ATL{{atlproduct}}.00{{version}}/{{date.strftime("%Y.%m.%d")}}/ATL{{atlproduct}}_*_*11_00{{version}}_0{{revision}}.h5 + - simplecache::https://n5eil01u.ecs.nsidc.org/ATLAS/ATL{{atlproduct}}.00{{version}}/{{date.strftime("%Y.%m.%d")}}/ATL{{atlproduct}}_*_*12_00{{version}}_0{{revision}}.h5 + xarray_kwargs: + combine: nested + concat_dim: "" + engine: h5netcdf + storage_options: + simplecache: + cache_storage: ATL{{atlproduct}}.00{{version}}/{{date.strftime("%Y.%m.%d")}} + same_names: True + parameters: + atlproduct: + description: ICESat-2/ATLAS product number + type: str + default: "06" + allowed: ["02", "03", "04", "06", "07", "08", "09", "10", "12", "13"] + date: + description: Year, month, and day of data acquisition + type: datetime + default: 2019.06.26 + min: 2018.10.14 + max: 2020.03.06 # note gap from 2019.06.27 to 2019.07.25 (inclusive) + orbitalsegment: + description: Orbital Segment + type: list + default: [10, 11, 12] + version: + description: Version number + type: int + default: 3 + allowed: [1, 2, 3] + revision: + description: Revision number + type: int + default: 1 + driver: intake_xarray.netcdf.NetCDFSource icesat2atl06: args: chunks: delta_time: 50000 path_as_pattern: ATL06_{datetime:%Y%m%d%H%M%S}_{referencegroundtrack:4}{cyclenumber:2}{orbitalsegment:2}_{version:3}_{revision:2}.h5 - # urlpath: https://n5eil02u.ecs.nsidc.org/opendap/hyrax/ATLAS/ATL06.002/{{date.strftime("%Y.%m.%d")}}/ATL06_*_*{{orbitalsegment}}_002_01.h5 - urlpath: https://n5eil01u.ecs.nsidc.org/ATLAS/ATL06.002/{{date.strftime("%Y.%m.%d")}}/ATL06_*_*{{orbitalsegment}}_002_01.h5 + # urlpath: https://n5eil02u.ecs.nsidc.org/opendap/hyrax/ATLAS/ATL06.003/{{date.strftime("%Y.%m.%d")}}/ATL06_*_*{{orbitalsegment}}_003_01.h5 + urlpath: simplecache::https://n5eil01u.ecs.nsidc.org/ATLAS/ATL06.00{{version}}/{{date.strftime("%Y.%m.%d")}}/ATL06_*_{{referencegroundtrack}}*{{orbitalsegment}}_00{{version}}_01.h5 xarray_kwargs: - combine: nested - concat_dim: referencegroundtrack # from 0000 to 1387 + combine: by_coords engine: h5netcdf group: /{{laser}}/land_ice_segments + mask_and_scale: true parallel: true + storage_options: + simplecache: + cache_storage: ATL06.00{{version}}/{{date.strftime("%Y.%m.%d")}} + same_names: True # https://intake.readthedocs.io/en/latest/catalog.html#parameter-definition parameters: date: description: Year, month, and day of data acquisition type: datetime - default: 2019.06.26 + default: 2020.03.06 min: 2018.10.14 - max: 2019.11.15 # note gap from 2019.06.27 to 2019.07.25 (inclusive) + max: 2020.03.06 # note missing 2018.12.09, and gap from 2019.06.27 to 2019.07.25 (inclusive) + referencegroundtrack: + description: ICESat-2 Reference Ground Track number + type: str + default: "" # Default: "" (all), min: "0000", max: "1387" orbitalsegment: description: Orbital Segment type: str @@ -32,18 +80,24 @@ sources: type: str default: gt2l allowed: ["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"] + version: + description: Version number + type: int + default: 3 + allowed: [1, 2, 3] description: '' driver: intake_xarray.netcdf.NetCDFSource - cache: - - argkey: urlpath - regex: 'n5eil01u.ecs.nsidc.org' - type: file metadata: plots: quickview: kind: points x: longitude y: latitude - datashade: True + c: h_li + cmap: Blues + rasterize: True + hover: True width: 800 height: 500 + geo: True + coastline: True