dataset_priorities.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "81c82e87-6d29-4909-aae5-51dfec815b5d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sys\n",
    "import os\n",
    "sys.path.append('/home/greg/GitHub/')\n",
    "import pyEDIutils.search as edi\n",
    "import datetime\n",
    "import psycopg2\n",
    "from sqlalchemy import create_engine\n",
    "now = datetime.datetime.now().strftime(\"%Y%m%d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "2bff3c90-a3eb-43bb-a1dd-f293c80bab75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://pasta.lternet.edu/package/search/eml?defType=edismax&q=%2A&fq=scope%3Aknb-lter-jrn&fl=packageid%2Ctitle%2Cpubdate%2Cauthor%2Cbegindate%2Cenddate%2Cdoi&sort=packageid%2Casc&rows=500\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packageid</th>\n",
       "      <th>title</th>\n",
       "      <th>pubdate</th>\n",
       "      <th>authors</th>\n",
       "      <th>begindate</th>\n",
       "      <th>enddate</th>\n",
       "      <th>doi</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>knb-lter-jrn.100.3</td>\n",
       "      <td>High resolution shrub cover raster maps of the...</td>\n",
       "      <td>2020</td>\n",
       "      <td>Ji, Wenjie;Hanan, Niall P.</td>\n",
       "      <td>2011-01-01</td>\n",
       "      <td>2011-12-31</td>\n",
       "      <td>doi:10.6073/pasta/313fec8669bc7b4d8debf7393dd2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>knb-lter-jrn.101.1</td>\n",
       "      <td>Thresholds and alternative states in neotropic...</td>\n",
       "      <td>2021</td>\n",
       "      <td>Peinetti, H. Raúl;Bestelmeyer, Brandon;Chirino...</td>\n",
       "      <td>2005-09-01</td>\n",
       "      <td>2018-03-31</td>\n",
       "      <td>doi:10.6073/pasta/e1978467bd09fc4f6e0cd8509b06...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>knb-lter-jrn.102.1</td>\n",
       "      <td>Carbon exchange responses of rehydrated and in...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Hoellrich, Mikaela;Pietrasiak, Nicole</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>2022-12-31</td>\n",
       "      <td>doi:10.6073/pasta/c6ffd88dc80df1ed1ec32ccdc477...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>knb-lter-jrn.200021001.5</td>\n",
       "      <td>Data to explore circular manureshed management...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Spiegal, Sheri;Vendramini, João M.B.;Bittman, ...</td>\n",
       "      <td>2010-01-01</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>doi:10.6073/pasta/6025924b89e313f25bddf1912c8a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>knb-lter-jrn.200022001.3</td>\n",
       "      <td>Criollo and Crossbred Steer Comparison: Weight...</td>\n",
       "      <td>2022</td>\n",
       "      <td>McIntosh, Matt M;Cibils, Andres F;Estell, Rick...</td>\n",
       "      <td>2015-12-01</td>\n",
       "      <td>2017-01-31</td>\n",
       "      <td>doi:10.6073/pasta/66ffe320906785bb708c55a61787...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  packageid  \\\n",
       "0        knb-lter-jrn.100.3   \n",
       "1        knb-lter-jrn.101.1   \n",
       "2        knb-lter-jrn.102.1   \n",
       "3  knb-lter-jrn.200021001.5   \n",
       "4  knb-lter-jrn.200022001.3   \n",
       "\n",
       "                                               title pubdate  \\\n",
       "0  High resolution shrub cover raster maps of the...    2020   \n",
       "1  Thresholds and alternative states in neotropic...    2021   \n",
       "2  Carbon exchange responses of rehydrated and in...    2023   \n",
       "3  Data to explore circular manureshed management...    2023   \n",
       "4  Criollo and Crossbred Steer Comparison: Weight...    2022   \n",
       "\n",
       "                                             authors   begindate     enddate  \\\n",
       "0                         Ji, Wenjie;Hanan, Niall P.  2011-01-01  2011-12-31   \n",
       "1  Peinetti, H. Raúl;Bestelmeyer, Brandon;Chirino...  2005-09-01  2018-03-31   \n",
       "2              Hoellrich, Mikaela;Pietrasiak, Nicole  2020-01-01  2022-12-31   \n",
       "3  Spiegal, Sheri;Vendramini, João M.B.;Bittman, ...  2010-01-01  2019-12-31   \n",
       "4  McIntosh, Matt M;Cibils, Andres F;Estell, Rick...  2015-12-01  2017-01-31   \n",
       "\n",
       "                                                 doi  \n",
       "0  doi:10.6073/pasta/313fec8669bc7b4d8debf7393dd2...  \n",
       "1  doi:10.6073/pasta/e1978467bd09fc4f6e0cd8509b06...  \n",
       "2  doi:10.6073/pasta/c6ffd88dc80df1ed1ec32ccdc477...  \n",
       "3  doi:10.6073/pasta/6025924b89e313f25bddf1912c8a...  \n",
       "4  doi:10.6073/pasta/66ffe320906785bb708c55a61787...  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df = edi.search_pasta(fields=['packageid','title','pubdate','author',\n",
    "                                   'begindate','enddate','doi'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "282174a5-d805-4444-b343-a35fb8c7d044",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DataSetID</th>\n",
       "      <th>Revision</th>\n",
       "      <th>Title</th>\n",
       "      <th>PubDate</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>ShortName</th>\n",
       "      <th>UpdateFrequency</th>\n",
       "      <th>MaintenanceDescription</th>\n",
       "      <th>AbstractType</th>\n",
       "      <th>BoilerplateSetting</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>210011005</td>\n",
       "      <td>105</td>\n",
       "      <td>Annual ground-based photographs taken at 15 ne...</td>\n",
       "      <td>2019-10-09</td>\n",
       "      <td>abstract.210011005.md</td>\n",
       "      <td>None</td>\n",
       "      <td>annually</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>file</td>\n",
       "      <td>jrn-default</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>210086004</td>\n",
       "      <td>105</td>\n",
       "      <td>Leaf litter cover data on 1m x 1m plots from t...</td>\n",
       "      <td>2022-04-11</td>\n",
       "      <td>abstract.210086004.md</td>\n",
       "      <td>None</td>\n",
       "      <td>asNeeded</td>\n",
       "      <td>ongoing - New data are added every 5 years</td>\n",
       "      <td>file</td>\n",
       "      <td>jrn-default</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>210013001</td>\n",
       "      <td>62</td>\n",
       "      <td>Raw neutron counts from a soil water content h...</td>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>abstract.210013001.md</td>\n",
       "      <td>None</td>\n",
       "      <td>asNeeded</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>file</td>\n",
       "      <td>jrn-default</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>210013003</td>\n",
       "      <td>16</td>\n",
       "      <td>Soil volumetric water content calculated from ...</td>\n",
       "      <td>2019-10-29</td>\n",
       "      <td>abstract.210013003.md</td>\n",
       "      <td>None</td>\n",
       "      <td>notPlanned</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>file</td>\n",
       "      <td>jrn-default</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>210011002</td>\n",
       "      <td>109</td>\n",
       "      <td>Seasonal non-destructive vegetation measuremen...</td>\n",
       "      <td>2023-08-23</td>\n",
       "      <td>abstract.210011002.md</td>\n",
       "      <td>None</td>\n",
       "      <td>annually</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>file</td>\n",
       "      <td>jrn-default</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   DataSetID  Revision                                              Title  \\\n",
       "0  210011005       105  Annual ground-based photographs taken at 15 ne...   \n",
       "1  210086004       105  Leaf litter cover data on 1m x 1m plots from t...   \n",
       "2  210013001        62  Raw neutron counts from a soil water content h...   \n",
       "3  210013003        16  Soil volumetric water content calculated from ...   \n",
       "4  210011002       109  Seasonal non-destructive vegetation measuremen...   \n",
       "\n",
       "      PubDate               Abstract ShortName UpdateFrequency  \\\n",
       "0  2019-10-09  abstract.210011005.md      None        annually   \n",
       "1  2022-04-11  abstract.210086004.md      None        asNeeded   \n",
       "2  2019-09-30  abstract.210013001.md      None        asNeeded   \n",
       "3  2019-10-29  abstract.210013003.md      None      notPlanned   \n",
       "4  2023-08-23  abstract.210011002.md      None        annually   \n",
       "\n",
       "                       MaintenanceDescription AbstractType BoilerplateSetting  \n",
       "0                                     ongoing         file        jrn-default  \n",
       "1  ongoing - New data are added every 5 years         file        jrn-default  \n",
       "2                                     ongoing         file        jrn-default  \n",
       "3                                     ongoing         file        jrn-default  \n",
       "4                                     ongoing         file        jrn-default  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sqlalchemy as sq\n",
    "import yaml\n",
    "\n",
    "# Get the metabase credentials\n",
    "with open(\"/home/greg/admin/jrn_metabase_keys.yaml\", \"r\") as stream:\n",
    "    try:\n",
    "        mbcred = yaml.safe_load(stream)\n",
    "    except yaml.YAMLError as exc:\n",
    "        print(exc)\n",
    "\n",
    "# Database engine\n",
    "engine = sq.create_engine(\"postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}\".format(mbcred['user'],\n",
    "                                                                             mbcred['password'],\n",
    "                                                                             mbcred['host'],\n",
    "                                                                             mbcred['port'],\n",
    "                                                                             mbcred['mbname']))\n",
    "\n",
    "# connect to the database using the newly created Engine instance\n",
    "conn = engine.connect()\n",
    "\n",
    "# run SQL query\n",
    "mbdf = pd.read_sql_query(sql='SELECT * FROM lter_metabase.\"DataSet\"', con=conn)\n",
    "mbdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "8f2f5b46-67a0-4caf-886c-e97c62e3e1f6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['DataSetID'] = df.packageid.str.split('.', n=2, expand=True).iloc[:,1].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "bc7cafe4-6afb-49f5-87e1-894ee3ccfffc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.merge(mbdf[['DataSetID','MaintenanceDescription']], on='DataSetID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c9f83fc8-4ad5-4e07-8715-adda924383e3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      False\n",
       "1      False\n",
       "2      False\n",
       "3      False\n",
       "4      False\n",
       "       ...  \n",
       "380     True\n",
       "381    False\n",
       "382    False\n",
       "383    False\n",
       "384    False\n",
       "Name: MaintenanceDescription, Length: 385, dtype: bool"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.isnull(df.MaintenanceDescription)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "06daa01e-1fcf-424b-b1d5-6e961d6b07a3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Find John's met packages and assign \"ongoing\"\n",
    "test = (df['packageid'].str.contains('210437') | df['packageid'].str.contains('210548'))\n",
    "df.loc[test, 'MaintenanceDescription'] = 'ongoing'\n",
    "# Find \"None\" Maintenance descriptions and change to ''\n",
    "test = pd.isnull(df['MaintenanceDescription'])\n",
    "df.loc[test, 'MaintenanceDescription'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "65e839e1-364d-48ae-adf2-3c16ec187950",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# Create a priority column with empty values\n",
    "df['priority'] = ''\n",
    "# Find high priority datasets\n",
    "test = np.logical_and(df['pubdate'].astype(int) < 2022, df['MaintenanceDescription'].str.contains('ongoing'))\n",
    "df.loc[test, 'priority'] = 'high'\n",
    "# Find low priority datasets\n",
    "test = np.logical_or(df['pubdate'].astype(int) > 2021, df['MaintenanceDescription'].str.contains('complete'))\n",
    "df.loc[test, 'priority'] = 'low'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "8508a034-36d2-4559-80c1-ca48de5d6bd1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packageid</th>\n",
       "      <th>title</th>\n",
       "      <th>pubdate</th>\n",
       "      <th>authors</th>\n",
       "      <th>begindate</th>\n",
       "      <th>enddate</th>\n",
       "      <th>doi</th>\n",
       "      <th>DataSetID</th>\n",
       "      <th>MaintenanceDescription</th>\n",
       "      <th>priority</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>knb-lter-jrn.100.3</td>\n",
       "      <td>High resolution shrub cover raster maps of the...</td>\n",
       "      <td>2020</td>\n",
       "      <td>Ji, Wenjie;Hanan, Niall P.</td>\n",
       "      <td>2011-01-01</td>\n",
       "      <td>2011-12-31</td>\n",
       "      <td>doi:10.6073/pasta/313fec8669bc7b4d8debf7393dd2...</td>\n",
       "      <td>100</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>knb-lter-jrn.101.1</td>\n",
       "      <td>Thresholds and alternative states in neotropic...</td>\n",
       "      <td>2021</td>\n",
       "      <td>Peinetti, H. Raúl;Bestelmeyer, Brandon;Chirino...</td>\n",
       "      <td>2005-09-01</td>\n",
       "      <td>2018-03-31</td>\n",
       "      <td>doi:10.6073/pasta/e1978467bd09fc4f6e0cd8509b06...</td>\n",
       "      <td>101</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>knb-lter-jrn.102.1</td>\n",
       "      <td>Carbon exchange responses of rehydrated and in...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Hoellrich, Mikaela;Pietrasiak, Nicole</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>2022-12-31</td>\n",
       "      <td>doi:10.6073/pasta/c6ffd88dc80df1ed1ec32ccdc477...</td>\n",
       "      <td>102</td>\n",
       "      <td>complete</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>knb-lter-jrn.200021001.5</td>\n",
       "      <td>Data to explore circular manureshed management...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Spiegal, Sheri;Vendramini, João M.B.;Bittman, ...</td>\n",
       "      <td>2010-01-01</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>doi:10.6073/pasta/6025924b89e313f25bddf1912c8a...</td>\n",
       "      <td>200021001</td>\n",
       "      <td>complete</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>knb-lter-jrn.200022001.3</td>\n",
       "      <td>Criollo and Crossbred Steer Comparison: Weight...</td>\n",
       "      <td>2022</td>\n",
       "      <td>McIntosh, Matt M;Cibils, Andres F;Estell, Rick...</td>\n",
       "      <td>2015-12-01</td>\n",
       "      <td>2017-01-31</td>\n",
       "      <td>doi:10.6073/pasta/66ffe320906785bb708c55a61787...</td>\n",
       "      <td>200022001</td>\n",
       "      <td>complete</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  packageid  \\\n",
       "0        knb-lter-jrn.100.3   \n",
       "1        knb-lter-jrn.101.1   \n",
       "2        knb-lter-jrn.102.1   \n",
       "3  knb-lter-jrn.200021001.5   \n",
       "4  knb-lter-jrn.200022001.3   \n",
       "\n",
       "                                               title pubdate  \\\n",
       "0  High resolution shrub cover raster maps of the...    2020   \n",
       "1  Thresholds and alternative states in neotropic...    2021   \n",
       "2  Carbon exchange responses of rehydrated and in...    2023   \n",
       "3  Data to explore circular manureshed management...    2023   \n",
       "4  Criollo and Crossbred Steer Comparison: Weight...    2022   \n",
       "\n",
       "                                             authors   begindate     enddate  \\\n",
       "0                         Ji, Wenjie;Hanan, Niall P.  2011-01-01  2011-12-31   \n",
       "1  Peinetti, H. Raúl;Bestelmeyer, Brandon;Chirino...  2005-09-01  2018-03-31   \n",
       "2              Hoellrich, Mikaela;Pietrasiak, Nicole  2020-01-01  2022-12-31   \n",
       "3  Spiegal, Sheri;Vendramini, João M.B.;Bittman, ...  2010-01-01  2019-12-31   \n",
       "4  McIntosh, Matt M;Cibils, Andres F;Estell, Rick...  2015-12-01  2017-01-31   \n",
       "\n",
       "                                                 doi  DataSetID  \\\n",
       "0  doi:10.6073/pasta/313fec8669bc7b4d8debf7393dd2...        100   \n",
       "1  doi:10.6073/pasta/e1978467bd09fc4f6e0cd8509b06...        101   \n",
       "2  doi:10.6073/pasta/c6ffd88dc80df1ed1ec32ccdc477...        102   \n",
       "3  doi:10.6073/pasta/6025924b89e313f25bddf1912c8a...  200021001   \n",
       "4  doi:10.6073/pasta/66ffe320906785bb708c55a61787...  200022001   \n",
       "\n",
       "  MaintenanceDescription priority  \n",
       "0                ongoing     high  \n",
       "1                ongoing     high  \n",
       "2               complete      low  \n",
       "3               complete      low  \n",
       "4               complete      low  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packageid</th>\n",
       "      <th>title</th>\n",
       "      <th>pubdate</th>\n",
       "      <th>authors</th>\n",
       "      <th>begindate</th>\n",
       "      <th>enddate</th>\n",
       "      <th>doi</th>\n",
       "      <th>DataSetID</th>\n",
       "      <th>MaintenanceDescription</th>\n",
       "      <th>priority</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>knb-lter-jrn.210520001.1</td>\n",
       "      <td>Vascular Plant Species of the Jornada Basin, 1...</td>\n",
       "      <td>2018</td>\n",
       "      <td>Anderson, John;Van Zee, Justin</td>\n",
       "      <td>1982-01-01</td>\n",
       "      <td>2018-09-30</td>\n",
       "      <td>doi:10.6073/pasta/6981502ed9fcf69a6bef71cc133f...</td>\n",
       "      <td>210520001</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>knb-lter-jrn.210011005.105</td>\n",
       "      <td>Annual ground-based photographs taken at 15 ne...</td>\n",
       "      <td>2019</td>\n",
       "      <td>Anderson, John</td>\n",
       "      <td>2003-10-14</td>\n",
       "      <td>2018-09-18</td>\n",
       "      <td>doi:10.6073/pasta/69ca7ac7254a2b236d3f44b21fa8...</td>\n",
       "      <td>210011005</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>knb-lter-jrn.210012001.139</td>\n",
       "      <td>Raw neutron counts from a soil water content h...</td>\n",
       "      <td>2019</td>\n",
       "      <td>Wierenga, Peter;Virginia, Ross;Duniway, Michae...</td>\n",
       "      <td>1985-11-22</td>\n",
       "      <td>2019-04-30</td>\n",
       "      <td>doi:10.6073/pasta/963587e508b2b01b1bb7f8941f54...</td>\n",
       "      <td>210012001</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>knb-lter-jrn.210012002.127</td>\n",
       "      <td>Soil volumetric water content calculated from ...</td>\n",
       "      <td>2019</td>\n",
       "      <td>Wierenga, Peter;Virginia, Ross;Duniway, Michae...</td>\n",
       "      <td>1982-04-30</td>\n",
       "      <td>2019-04-30</td>\n",
       "      <td>doi:10.6073/pasta/2e37c18c14eea1d1e4fccb35ed94...</td>\n",
       "      <td>210012002</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>knb-lter-jrn.210013001.62</td>\n",
       "      <td>Raw neutron counts from a soil water content h...</td>\n",
       "      <td>2019</td>\n",
       "      <td>Virginia, Ross;Duniway, Mike;Vivoni, Enrique</td>\n",
       "      <td>1989-07-05</td>\n",
       "      <td>2019-05-03</td>\n",
       "      <td>doi:10.6073/pasta/0f4474b774dab3d097fad39efcf5...</td>\n",
       "      <td>210013001</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>knb-lter-jrn.210548101.28</td>\n",
       "      <td>Jornada Basin LTER Cross-scale Interactions St...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Anderson, John</td>\n",
       "      <td>2017-08-11</td>\n",
       "      <td>2023-09-21</td>\n",
       "      <td>doi:10.6073/pasta/5fa75fe767589be1baaaa57e85de...</td>\n",
       "      <td>210548101</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>knb-lter-jrn.210548102.27</td>\n",
       "      <td>Jornada Basin LTER Cross-scale Interactions St...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Anderson, John</td>\n",
       "      <td>2017-08-11</td>\n",
       "      <td>2023-09-21</td>\n",
       "      <td>doi:10.6073/pasta/e1236353b6a15d53aa8864788236...</td>\n",
       "      <td>210548102</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>380</th>\n",
       "      <td>knb-lter-jrn.210548103.28</td>\n",
       "      <td>Jornada Basin LTER Cross-scale Interactions St...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Anderson, John</td>\n",
       "      <td>2017-08-11</td>\n",
       "      <td>2023-09-21</td>\n",
       "      <td>doi:10.6073/pasta/029c896133cb4571259ffe077822...</td>\n",
       "      <td>210548103</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>knb-lter-jrn.210586001.3</td>\n",
       "      <td>Soil and foliar carbon and nitrogen content an...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Currier, Courtney;Reichmann, Lara;Sala, Osvaldo E</td>\n",
       "      <td>2011-06-01</td>\n",
       "      <td>2020-09-30</td>\n",
       "      <td>doi:10.6073/pasta/8929d93801db656ae1ad0f3a12a1...</td>\n",
       "      <td>210586001</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>knb-lter-jrn.210586002.3</td>\n",
       "      <td>Dataset and analyses for publication entitled:...</td>\n",
       "      <td>2023</td>\n",
       "      <td>Currier, Courtney;Reichmann, Lara;Sala, Osvaldo E</td>\n",
       "      <td>1876-01-01</td>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>doi:10.6073/pasta/f2c47d19074337832ee4e720e2a0...</td>\n",
       "      <td>210586002</td>\n",
       "      <td>complete</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>385 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      packageid  \\\n",
       "271    knb-lter-jrn.210520001.1   \n",
       "28   knb-lter-jrn.210011005.105   \n",
       "29   knb-lter-jrn.210012001.139   \n",
       "30   knb-lter-jrn.210012002.127   \n",
       "31    knb-lter-jrn.210013001.62   \n",
       "..                          ...   \n",
       "378   knb-lter-jrn.210548101.28   \n",
       "379   knb-lter-jrn.210548102.27   \n",
       "380   knb-lter-jrn.210548103.28   \n",
       "383    knb-lter-jrn.210586001.3   \n",
       "384    knb-lter-jrn.210586002.3   \n",
       "\n",
       "                                                 title pubdate  \\\n",
       "271  Vascular Plant Species of the Jornada Basin, 1...    2018   \n",
       "28   Annual ground-based photographs taken at 15 ne...    2019   \n",
       "29   Raw neutron counts from a soil water content h...    2019   \n",
       "30   Soil volumetric water content calculated from ...    2019   \n",
       "31   Raw neutron counts from a soil water content h...    2019   \n",
       "..                                                 ...     ...   \n",
       "378  Jornada Basin LTER Cross-scale Interactions St...    2023   \n",
       "379  Jornada Basin LTER Cross-scale Interactions St...    2023   \n",
       "380  Jornada Basin LTER Cross-scale Interactions St...    2023   \n",
       "383  Soil and foliar carbon and nitrogen content an...    2023   \n",
       "384  Dataset and analyses for publication entitled:...    2023   \n",
       "\n",
       "                                               authors   begindate  \\\n",
       "271                     Anderson, John;Van Zee, Justin  1982-01-01   \n",
       "28                                      Anderson, John  2003-10-14   \n",
       "29   Wierenga, Peter;Virginia, Ross;Duniway, Michae...  1985-11-22   \n",
       "30   Wierenga, Peter;Virginia, Ross;Duniway, Michae...  1982-04-30   \n",
       "31        Virginia, Ross;Duniway, Mike;Vivoni, Enrique  1989-07-05   \n",
       "..                                                 ...         ...   \n",
       "378                                     Anderson, John  2017-08-11   \n",
       "379                                     Anderson, John  2017-08-11   \n",
       "380                                     Anderson, John  2017-08-11   \n",
       "383  Currier, Courtney;Reichmann, Lara;Sala, Osvaldo E  2011-06-01   \n",
       "384  Currier, Courtney;Reichmann, Lara;Sala, Osvaldo E  1876-01-01   \n",
       "\n",
       "        enddate                                                doi  DataSetID  \\\n",
       "271  2018-09-30  doi:10.6073/pasta/6981502ed9fcf69a6bef71cc133f...  210520001   \n",
       "28   2018-09-18  doi:10.6073/pasta/69ca7ac7254a2b236d3f44b21fa8...  210011005   \n",
       "29   2019-04-30  doi:10.6073/pasta/963587e508b2b01b1bb7f8941f54...  210012001   \n",
       "30   2019-04-30  doi:10.6073/pasta/2e37c18c14eea1d1e4fccb35ed94...  210012002   \n",
       "31   2019-05-03  doi:10.6073/pasta/0f4474b774dab3d097fad39efcf5...  210013001   \n",
       "..          ...                                                ...        ...   \n",
       "378  2023-09-21  doi:10.6073/pasta/5fa75fe767589be1baaaa57e85de...  210548101   \n",
       "379  2023-09-21  doi:10.6073/pasta/e1236353b6a15d53aa8864788236...  210548102   \n",
       "380  2023-09-21  doi:10.6073/pasta/029c896133cb4571259ffe077822...  210548103   \n",
       "383  2020-09-30  doi:10.6073/pasta/8929d93801db656ae1ad0f3a12a1...  210586001   \n",
       "384  2021-12-31  doi:10.6073/pasta/f2c47d19074337832ee4e720e2a0...  210586002   \n",
       "\n",
       "    MaintenanceDescription priority  \n",
       "271                ongoing     high  \n",
       "28                 ongoing     high  \n",
       "29                 ongoing     high  \n",
       "30                 ongoing     high  \n",
       "31                 ongoing     high  \n",
       "..                     ...      ...  \n",
       "378                ongoing      low  \n",
       "379                ongoing      low  \n",
       "380                ongoing      low  \n",
       "383                ongoing      low  \n",
       "384               complete      low  \n",
       "\n",
       "[385 rows x 10 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values(by=['priority','pubdate'], ascending=[True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "caa62538-ed83-4d42-ba15-97a5165dd9cd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.sort_values(by=['priority','pubdate'], ascending=[True, True]).to_csv('tables_out/jrn_edi_catalog_bypubdate_20231103.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "standard",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}