LTERsite-EDI-diagnostics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import sys\n",
    "sys.path.append('/Users/gmaurer/GitHub/')\n",
    "import pyEDIutils.changes as edi\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "siteabbrev = 'JRN'\n",
    "scopestr = 'knb-lter-' + siteabbrev.lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Archive a request for earlier data\n",
    "\n",
    "Do this if needed to update the request archives for the current report ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#curl -i -X GET \"https://pasta.lternet.edu/package/changes/eml?fromDate=2017-02-01T12:00:00&toDate=2020-01-28&scope=knb-lter-jrn\"\n",
    "# This is from the \"List recent changes\" call  https://pastaplus-core.readthedocs.io/en/latest/doc_tree/pasta_api/data_package_manager_api.html#list-recent-changes\n",
    "# Only do this if it isn't archived yet!\n",
    "edi.archive_requested_changes('2023-06-02', '2023-07-15')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get edi activity since a given date\n",
    "# Note that for some sites this may return more than the API\n",
    "# can handle in one go and you'll have to go in stages\n",
    "lter_edi = edi.request_changes('2012-01-01', todt=None, scope=scopestr,\n",
    "    dedup = True, parsedt=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lter_edi.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add method count columns\n",
    "df = edi.get_counts(lter_edi)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count deleted packages\n",
    "deleted_pkgs = df.loc[df.action=='deleteDataPackage','pkgid']\n",
    "print(deleted_pkgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create daily data file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Sum to daily methods\n",
    "daily = edi.counts_to_daily(df)\n",
    "daily.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot activity since 2013\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "# Figure config\n",
    "#font = {'family' : 'normal',\n",
    "#        'weight' : 'normal',\n",
    "#        'size'   : 14}\n",
    "\n",
    "#matplotlib.rc('font', **font)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Get a datestamp for saving figures\n",
    "import datetime\n",
    "now = datetime.datetime.now().strftime(\"%Y%m%d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# All time plot:\n",
    "fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(7, 7), sharex=True)\n",
    "\n",
    "ax[0].plot(daily.index, daily.n_update, label='Update pkg')\n",
    "ax[0].plot(daily.index, daily.n_create, label='Create pkg')\n",
    "ax[0].set_ylabel(siteabbrev + ' events')\n",
    "\n",
    "ax[1].plot(daily.index, daily.n_update.cumsum(), label='Update pkg')\n",
    "ax[1].plot(daily.index, daily.n_create.cumsum(), label='Create pkg')\n",
    "ax[1].plot(daily.index, daily.n_tot.cumsum(), label='Total')\n",
    "\n",
    "plt.ylabel(siteabbrev + ' Cumulative')\n",
    "fig.autofmt_xdate()\n",
    "plt.suptitle(siteabbrev + ' activity on EDI (all time)')\n",
    "plt.legend()\n",
    "\n",
    "import os\n",
    "#fig.savefig(os.path.expanduser(\"~/data/GD_gmaurer@nmsu/IM/figures/JRN_EDI_all_{0}.png\".format(now)),\n",
    "#            dpi=150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total number of packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "daily.n_tot.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot activity since proposal review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "startdt = \"2022-05-30\"\n",
    "daily_subs = edi.counts_to_daily(df[df.date > startdt])\n",
    "daily_subs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Recent plot:\n",
    "fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(7, 7), sharex=True)\n",
    "\n",
    "ax[0].plot(daily_subs.index, daily_subs.n_update, label='Update pkg')\n",
    "ax[0].plot(daily_subs.index, daily_subs.n_create, label='Create pkg')\n",
    "ax[0].set_ylabel(siteabbrev + ' events')\n",
    "\n",
    "ax[1].plot(daily_subs.index, daily_subs.n_update.cumsum(), label='Update pkg')\n",
    "ax[1].plot(daily_subs.index, daily_subs.n_create.cumsum(), label='Create pkg')\n",
    "ax[1].plot(daily_subs.index, daily_subs.n_tot.cumsum(), label='$Delta$ Total')\n",
    "#ax[1].axhline(y=63, xmin=0, xmax=1, ls=':', lw=1.5, color='k',\n",
    "#             label='Proposal goal')\n",
    "\n",
    "plt.ylabel(siteabbrev + ' Cumulative')\n",
    "fig.autofmt_xdate()\n",
    "plt.suptitle(siteabbrev + ' activity on EDI since renewal')\n",
    "plt.legend()\n",
    "\n",
    "#fig.savefig(os.path.expanduser(\n",
    "#    \"~/data/GD_gmaurer@nmsu/IM/figures/JRN_EDI_recNoMet_{0}.png\".format(now)),\n",
    "#            dpi=150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unique packages updated\n",
    "uniqueUp = df.loc[df.action=='updateDataPackage',:].pkgid.unique()\n",
    "# Since proposal\n",
    "recent = df.loc[df.date > startdt,]\n",
    "uniqueUp_subs = recent.loc[recent.action=='updateDataPackage',:].pkgid.unique()\n",
    "#rEDI.loc[rEDI.action=='updateDataPackage',].pkgid.unique()\n",
    "uniqueUp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pkgtbl = pd.DataFrame({'Total':[daily.n_create.sum(), daily.n_update.sum(),len(uniqueUp)],\n",
    "                       'Since renewal':[daily_subs.n_create.sum(), daily_subs.n_update.sum(),len(uniqueUp_subs)]})\n",
    "pkgtbl.index = ['New packages created', 'Package updates', 'Unique package updates']\n",
    "\n",
    "print(pkgtbl)\n",
    "\n",
    "# Some comparison totals as of late 2024:\n",
    "VCR: 344\n",
    "KBS: 83\n",
    "BNZ: 669\n",
    "ARC: 651\n",
    "SEV: 231\n",
    "LUQ: 204\n",
    "HFR: 451\n",
    "HBR: 310\n",
    "MCM: 332\n",
    "PAL: 94\n",
    "CAP: 265\n",
    "PIE: 599\n",
    "JRN: 393\n",
    "MCR: 112\n",
    "CCE: 101\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}