Merge pull request #174 from martindurant/references

Add reference HDF recipe type

.github/workflows/main.yaml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
     steps:
       - uses: actions/checkout@v2
       - name: Setup Python

.readthedocs.yml

@@ -5,6 +5,6 @@ sphinx:
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.7
+  version: 3.8
   install:
     - requirements: docs/requirements.txt

ci/py3.8.yml

@@ -14,8 +14,11 @@ dependencies:
   - h5netcdf
   - h5py
   - hdf5
+  - intake
+  - intake-xarray
   - lxml
   - netcdf4
+  - numcodecs
   - numpy
   - pandas
   - pip
@@ -37,3 +40,4 @@ dependencies:
   - pip
   - pip:
     - pytest-timeout
+    - fsspec-reference-maker

ci/py3.9.yml

@@ -14,8 +14,11 @@ dependencies:
   - h5netcdf
   - h5py
   - hdf5
+  - intake
+  - intake-xarray
   - lxml    # Optional dep of pydap
   - netcdf4
+  - numcodecs
   - numpy
   - pandas
   - pip
@@ -36,3 +39,4 @@ dependencies:
   - zarr>=2.6.0
   - pip:
     - pytest-timeout
+    - fsspec-reference-maker

docs/conf.py

@@ -25,7 +25,7 @@
 
 # we always have to manually run the notebooks because they are slow / expensive
 jupyter_execute_notebooks = "auto"
-execution_excludepatterns = ["tutorials/*"]
+execution_excludepatterns = ["tutorials/xarray_zarr/*", "tutorials/hdf_reference/*"]
 
 # -- Options for HTML output -------------------------------------------------
 

docs/recipe_user_guide/recipes.md

@@ -44,11 +44,14 @@ The target Zarr dataset will conform to the
 
 The best way to really understand how recipes work is to go through the relevant
 tutorials for this recipe class. These are, in order of increasing complexity
-- {doc}`../tutorials/netcdf_zarr_sequential`
-- {doc}`../tutorials/multi_variable_recipe`
-- {doc}`../tutorials/terraclimate`
 
-Below we give a very basic overview of how recipes are used.
+- {doc}`../tutorials/xarray_zarr/netcdf_zarr_sequential`
+- {doc}`../tutorials/xarray_zarr/cmip6-recipe`
+- {doc}`../tutorials/xarray_zarr/multi_variable_recipe`
+- {doc}`../tutorials/xarray_zarr/terraclimate`
+- {doc}`../tutorials/xarray_zarr/opendap_subset_recipe`
+
+Below we give a very basic overview of how this recipe is used.
 
 First you must define a {doc}`file pattern <file_patterns>`.
 Once you have a {class}`file pattern <pangeo_forge_recipes.patterns.FilePattern>` object,
@@ -82,6 +85,26 @@ move on to {doc}`execution`.
 The API documentation below explains all of the possible options for `XarrayZarrRecipe`.
 Many of these options are explored further in the {doc}`../tutorials/index`.
 
+#### API Documentation
+
 ```{eval-rst}
 .. autoclass:: pangeo_forge_recipes.recipes.XarrayZarrRecipe
 ```
+
+### HDF Reference Recipe
+
+Like the `XarrayZarrRecipe`, this recipe allows us to more efficiently access data from a bunch of NetCDF / HDF files.
+However, this recipe does not actually copy the original source data.
+Instead, it generates metadata files which reference and index the original data, allowing it to be accessed more quickly and easily.
+For more background, see [this blog post](https://medium.com/pangeo/fake-it-until-you-make-it-reading-goes-netcdf4-data-on-aws-s3-as-zarr-for-rapid-data-access-61e33f8fe685).
+
+There is currently one tutorial for this recipe:
+
+- - {doc}`../tutorials/hdf_reference/reference_cmip6`
+
+#### API Documentation
+
+
+```{eval-rst}
+.. autoclass:: pangeo_forge_recipes.recipes.HDFReferenceRecipe
+```

docs/requirements.txt

@@ -1,10 +1,7 @@
-sphinx
-pangeo-sphinx-book-theme>=0.1.1
-myst-parser
-myst-nb>=0.12.3
-sphinx-copybutton
+sphinx==3.5.4
+pangeo-sphinx-book-theme==0.1.1
+myst-parser==0.13.6
+myst-nb==0.12.3
+sphinx-copybutton==0.3.1
 sphinx-autodoc-typehints
-fsspec
-prefect
-rechunker>=0.4.2
 -e .