Merge pull request #90 from ioos/netcdf-example-usage

Fix the netCDF example usage notebook

docs/requirements.txt

@@ -1,7 +1,9 @@
 bokeh
+erddapy
 ipykernel
-matplotlib
+matplotlib>=3
 nbsphinx
+pocean-core>=3
 pynco
 sphinx
 sphinx-autodoc-typehints

docs/source/development.rst

@@ -23,8 +23,7 @@ Build docs::
     conda activate ioosqc38_docs
     conda install -y --file requirements.txt \
         --file tests/requirements.txt \
-        --file docs/requirements.txt \
-        --file docs/source/examples/requirements.txt
+        --file docs/requirements.txt
     cd docs
     make html
 

docs/source/examples.rst

@@ -8,8 +8,7 @@ You can download the source for these notebooks `here <https://github.com/ioos/i
 
 * :doc:`Read water level data from CSV into Pandas, run QARTOD tests, and plot results in Bokeh </examples/QartodTestExample_WaterLevel>`
 
-..
-    * :doc:`Run QARTOD on a netCDF file, and store the results </examples/Qartod_netCDF_example>`
+* :doc:`Run QARTOD on a netCDF file, and store the results </examples/QartodTestExample_netCDF>`
 
 * :doc:`Run Attenuated Signal test on Salinity data to detect biofouling in an instrument </examples/QartodTestExample_SalinityAttenuation>`
 

docs/source/examples/usage/Qartod_netCDF_example.ipynb → docs/source/examples/QartodTestExample_netCDF.ipynb

@@ -6,7 +6,7 @@
    "source": [
     "# QARTOD - NetCDF Examples\n",
     "\n",
-    "This notebook provides examples of running QARTOD on a netCDF file. For background, see [NcQcConfig Usage](https://ioos.github.io/ioos_qc/usage.html#ncqcconfig) in the docs.\n",
+    "This notebook provides examples of running QARTOD on a netCDF file. For background, see [XarrayStream](https://ioos.github.io/ioos_qc/usage.html#xarraystream) and [CFNetCDFStore](https://ioos.github.io/ioos_qc/usage.html#cfnetcdfstore) in the docs.\n",
     "\n",
     "There are multiple ways that you can integrate `ioos_qc` into your netcdf-based workflow. \n",
     "\n",
@@ -54,25 +54,27 @@
    "source": [
     "def plot_ncresults(ncdata, var_name, results, title, test_name):\n",
     "    \"\"\"Helper method to plot QC results using Bokeh.\"\"\"\n",
-    "    time = np.array(ncdata.variables[\"time\"])\n",
-    "    obs = np.array(ncdata.variables[var_name])\n",
-    "    qc_test = results[var_name][\"qartod\"][test_name]\n",
-    "\n",
-    "    qc_pass = np.ma.masked_where(qc_test != 1, obs)\n",
-    "    num_pass = (qc_test == 1).sum()\n",
-    "    qc_suspect = np.ma.masked_where(qc_test != 3, obs)\n",
-    "    num_suspect = (qc_test == 3).sum()\n",
-    "    qc_fail = np.ma.masked_where(qc_test != 4, obs)\n",
-    "    num_fail = (qc_test == 4).sum()\n",
-    "    qc_notrun = np.ma.masked_where(qc_test != 2, obs)\n",
+    "    \n",
+    "    qc_test = next(r for r in results if r.stream_id == var_name and r.test == test_name)\n",
+    "    time = np.array(qc_test.tinp)\n",
+    "    obs = np.array(qc_test.data)\n",
+    "    results = qc_test.results\n",
+    "\n",
+    "    qc_pass = np.ma.masked_where(results != 1, obs)\n",
+    "    num_pass = (results == 1).sum()\n",
+    "    qc_suspect = np.ma.masked_where(results != 3, obs)\n",
+    "    num_suspect = (results == 3).sum()\n",
+    "    qc_fail = np.ma.masked_where(results != 4, obs)\n",
+    "    num_fail = (results == 4).sum()\n",
+    "    qc_notrun = np.ma.masked_where(results != 2, obs)\n",
     "\n",
     "    p1 = figure(\n",
     "        x_axis_type=\"datetime\",\n",
     "        title=f\"{test_name} : {title} : p/s/f= {num_pass}/{num_suspect}/{num_fail}\")\n",
     "    p1.grid.grid_line_alpha = 0.3\n",
     "    p1.xaxis.axis_label = \"Time\"\n",
     "    p1.yaxis.axis_label = \"Observation Value\"\n",
-    "\n",
+    "    \n",
     "    p1.line(time, obs, legend_label=\"obs\", color=\"#A6CEE3\")\n",
     "    p1.circle(\n",
     "        time, qc_notrun, size=2, legend_label=\"qc not run\", color=\"gray\", alpha=0.2\n",
@@ -130,7 +132,6 @@
     "url = \"https://github.com/ioos/ioos_qc/raw/master/docs/source/examples\"\n",
     "fname = f\"{url}/pco2_netcdf_example.nc#mode=bytes\"\n",
     "\n",
-    "\n",
     "pco2 = open_from_https(fname)\n",
     "pco2"
    ]
@@ -244,11 +245,28 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from ioos_qc.qartod import aggregate\n",
     "from ioos_qc.streams import XarrayStream\n",
+    "from ioos_qc.results import collect_results, CollectedResult\n",
     "\n",
     "qc = XarrayStream(pco2, lon=\"lon\", lat=\"lat\")\n",
     "\n",
-    "results = qc.run(c)"
+    "# Store as a list to run QC now\n",
+    "runner = list(qc.run(c))\n",
+    "\n",
+    "results = collect_results(runner, how='list')\n",
+    "\n",
+    "agg = CollectedResult(\n",
+    "    stream_id='',\n",
+    "    package='qartod',\n",
+    "    test='qc_rollup',\n",
+    "    function=aggregate,\n",
+    "    results=aggregate(results),\n",
+    "    tinp=qc.time(),\n",
+    "    data=data\n",
+    ")\n",
+    "results.append(agg)\n",
+    "results"
    ]
   },
   {
@@ -294,7 +312,7 @@
    "outputs": [],
    "source": [
     "# To see overall results, use the aggregate test\n",
-    "plot_ncresults(pco2, \"pco2_seawater\", results, \"pCO2 seawater\", \"aggregate\")"
+    "plot_ncresults(pco2, \"\", results, \"pCO2 seawater\", \"qc_rollup\")"
    ]
   },
   {
@@ -306,26 +324,14 @@
     "# Store results manually\n",
     "# This is just a simple example and stores the aggregate test flag as a variable.\n",
     "# You can expand upon this, or use the ioos_qc library to store the results for you (see subsequent examples)\n",
+    "# and use xarray's to_xxx methods to serialize results to whichever format you prefer\n",
     "\n",
-    "# Create output file\n",
-    "outfile_a = os.path.join(tempfile.gettempdir(), \"out_a.nc\")\n",
-    "shutil.copy(filename, outfile_a)\n",
+    "agg_da = xr.DataArray(agg.results, {}, ('time',))\n",
+    "output_xds = pco2.assign(\n",
+    "    qartod_aggregate=agg_da\n",
+    ")\n",
     "\n",
-    "# Store results\n",
-    "with Dataset(outfile_a, \"r+\") as nc_file:\n",
-    "    qc_agg = nc_file.createVariable(\"qartod_aggregate\", \"u1\", (\"time\",), fill_value=2)\n",
-    "    qc_agg[:] = results[\"pco2_seawater\"][\"qartod\"][\"aggregate\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Print results\n",
-    "out_a = xr.open_dataset(outfile_a)\n",
-    "print(out_a[\"qartod_aggregate\"])"
+    "output_xds.qartod_aggregate"
    ]
   },
   {
@@ -343,24 +349,35 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# We already have results from the previous run, but re-create them here for completeness\n",
-    "qc = NcQcConfig(config, lon=\"lon\", lat=\"lat\")\n",
-    "results = qc.run(filename)\n",
-    "results"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Create output file\n",
-    "outfile_b = os.path.join(tempfile.gettempdir(), \"out_b.nc\")\n",
-    "shutil.copy(filename, outfile_b)\n",
+    "# Using the CFNetCDFStore Store we can serialize our results back to a CF compliant netCDF file easily\n",
+    "from ioos_qc.stores import CFNetCDFStore\n",
+    "from pocean.dsg import OrthogonalMultidimensionalTimeseries\n",
     "\n",
-    "# Use the library to store the results to the netcdf file\n",
-    "qc.save_to_netcdf(outfile_b, results)"
+    "# We use the `results` from Option A so we don't repeat ourselves.\n",
+    "store = CFNetCDFStore(runner)\n",
+    "\n",
+    "outfile_b = os.path.join(tempfile.gettempdir(), \"out_b.nc\")\n",
+    "if os.path.exists(outfile_b):\n",
+    "    os.remove(outfile_b)\n",
+    "\n",
+    "qc_all = store.save(\n",
+    "    # The netCDF file to export OR append to\n",
+    "    outfile_b,\n",
+    "    # The DSG class to save the results as\n",
+    "    OrthogonalMultidimensionalTimeseries,\n",
+    "    # The QC config that was run\n",
+    "    c,\n",
+    "    # Should we write the data or just metadata? Defaults to false\n",
+    "    write_data=True,\n",
+    "    # Compute a total aggregate?\n",
+    "    compute_aggregate=True,\n",
+    "    # Any kwargs to pass to the DSG class\n",
+    "    dsg_kwargs=dict(\n",
+    "        reduce_dims=True,  # Remove dimensions of size 1\n",
+    "        unlimited=False,  # Don't make the record dimension unlimited\n",
+    "        unique_dims=True,  # Support loading into xarray\n",
+    "    ),\n",
+    ")"
    ]
   },
   {
@@ -371,7 +388,7 @@
    "source": [
     "# Explore results: qc test variables are named [variable_name]_qartod_[test_name]\n",
     "out_b = xr.open_dataset(outfile_b)\n",
-    "print(out_b)"
+    "out_b"
    ]
   },
   {
@@ -392,7 +409,7 @@
    "outputs": [],
    "source": [
     "# Aggregate/rollup flag\n",
-    "out_b[\"pco2_seawater_qartod_aggregate\"]"
+    "out_b[\"qartod_qc_rollup\"]"
    ]
   },
   {
@@ -413,15 +430,85 @@
    "outputs": [],
    "source": [
     "# Create a copy of the output from B\n",
-    "outfile_c = os.path.join(tempfile.gettempdir(), \"out_c.nc\")\n",
-    "shutil.copy(outfile_b, outfile_c)\n",
+    "infile_c = os.path.join(tempfile.gettempdir(), \"in_c.nc\")\n",
+    "shutil.copy(outfile_b, infile_c)\n",
     "\n",
-    "# Load this file into the NcQcConfig object\n",
-    "qc = NcQcConfig(outfile_c, lon=\"lon\", lat=\"lat\")\n",
+    "# Load this file into the Config object\n",
+    "input_c = xr.open_dataset(infile_c)\n",
+    "qc_config_c = Config(input_c)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The QC functions that will be run are extracted from the netCDF attributes\n",
+    "qc_config_c.calls"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# We can use that Config just like any other config object.\n",
+    "# Here we will re-run the netCDF file with the config extracted from the same netCDF file\n",
+    "\n",
+    "# Setup input stream from file\n",
+    "xrs_c = XarrayStream(input_c, lon=\"lon\", lat=\"lat\")\n",
+    "\n",
+    "# Setup config run\n",
+    "runner_c = list(xrs_c.run(qc_config_c))\n",
+    "\n",
+    "# Collect QC run results\n",
+    "results = collect_results(runner_c, how='list')\n",
+    "\n",
+    "# Compute Aggregate\n",
+    "agg = CollectedResult(\n",
+    "    stream_id='',\n",
+    "    package='qartod',\n",
+    "    test='qc_rollup',\n",
+    "    function=aggregate,\n",
+    "    results=aggregate(results),\n",
+    "    tinp=xrs_c.time(),\n",
+    "    data=data\n",
+    ")\n",
+    "results.append(agg)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "store = CFNetCDFStore(runner)\n",
     "\n",
-    "# Run tests and store results\n",
-    "results_c = qc.run(outfile_c)\n",
-    "qc.save_to_netcdf(outfile_c, results_c)"
+    "outfile_c = os.path.join(tempfile.gettempdir(), \"out_c.nc\")\n",
+    "if os.path.exists(outfile_c):\n",
+    "    os.remove(outfile_c)\n",
+    "\n",
+    "qc_all = store.save(\n",
+    "    # The netCDF file to export OR append to\n",
+    "    outfile_c,\n",
+    "    # The DSG class to save the results as\n",
+    "    OrthogonalMultidimensionalTimeseries,\n",
+    "    # The QC config that was run\n",
+    "    c,\n",
+    "    # Should we write the data or just metadata? Defaults to false\n",
+    "    write_data=True,\n",
+    "    # Compute a total aggregate?\n",
+    "    compute_aggregate=True,\n",
+    "    # Any kwargs to pass to the DSG class\n",
+    "    dsg_kwargs=dict(\n",
+    "        reduce_dims=True,  # Remove dimensions of size 1\n",
+    "        unlimited=False,  # Don't make the record dimension unlimited\n",
+    "        unique_dims=True,  # Support loading into xarray\n",
+    "    ),\n",
+    ")"
    ]
   },
   {
@@ -430,9 +517,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Explore results\n",
+    "# Explore results: qc test variables are named [variable_name]_qartod_[test_name]\n",
     "out_c = xr.open_dataset(outfile_c)\n",
-    "print(out_c)"
+    "out_c"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Aggregate/rollup flag\n",
+    "out_c[\"qartod_qc_rollup\"]"
    ]
   }
  ],
@@ -453,6 +550,9 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.10.5"
+  },
+  "nbsphinx": {
+   "orphan": true
   }
  },
  "nbformat": 4,

ioos_qc/config.py

@@ -131,7 +131,18 @@ def __repr__(self):
             ret += f' ending={self.window.ending}'
         if self.context.region is not None:
             ret += ' region=True'
-        ret += f' {self.module}.{self.method}({self.args}, {self.kwargs})>'
+
+        ret += f' function={self.module}.{self.method}('
+
+        if self.args:
+            ret += ', '.join([ x for x in self.args if x ])
+
+        if self.kwargs:
+            ret += ', '.join([
+                f'{k}={v}' for k, v in self.kwargs.items()
+            ])
+
+        ret += ')>'
         return ret
 
     def run(self, **passedkwargs):