datastore_lazy_loading.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a7f37c2c-705c-49be-8992-99ab915d9b73",
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import copy\n",
    "import astropy.units as u\n",
    "from gammapy.utils.fits import HDULocation\n",
    "from gammapy.data import DataStore, GTI\n",
    "from gammapy.data.data_store import REQUIRED_IRFS, ALL_HDUS, ALL_IRFS\n",
    "from ctao_data_model import IRFGroup, IRFGroups, VODFObservation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55a801d6-6e70-41f1-982b-15d0817b5283",
   "metadata": {},
   "source": [
    "Here we create a function to extract  from a `HDUIndexTable` a GADF observation as a `VODFObservation`. \n",
    "The `IRFGroup` is created from the list of IRFs locations to use the lazy loading mechanism.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eaf85aeb-04cb-4711-89a0-e329a3f366b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_obs(hdu_table, obs_id, required_irf=\"full-enclosure\", require_events=True):\n",
    "    \"\"\"Access a given `~gammapy.data.Observation` from a HDU index table.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    hdu_table : `~gammapy.data.`\n",
    "        Input index table\n",
    "    obs_id : int\n",
    "        Observation ID.\n",
    "    required_irf : list of str or str, optional\n",
    "        The list can include the following options:\n",
    "\n",
    "        * `\"events\"` : Events\n",
    "        * `\"gti\"` :  Good time intervals\n",
    "        * `\"aeff\"` : Effective area\n",
    "        * `\"bkg\"` : Background\n",
    "        * `\"edisp\"` : Energy dispersion\n",
    "        * `\"psf\"` : Point Spread Function\n",
    "        * `\"rad_max\"` : Maximal radius\n",
    "\n",
    "        Alternatively single string can be used as shortcut:\n",
    "\n",
    "        * `\"full-enclosure\"` : includes `[\"events\", \"gti\", \"aeff\", \"edisp\", \"psf\", \"bkg\"]`\n",
    "        * `\"point-like\"` : includes `[\"events\", \"gti\", \"aeff\", \"edisp\"]`\n",
    "\n",
    "        Default is `\"full-enclosure\"`.\n",
    "    require_events : bool, optional\n",
    "        Require events and gti table or not. Default is True.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    VODFobservation : `~cato_data_model.VODFObservation`\n",
    "        Observation container.\n",
    "\n",
    "    \"\"\"\n",
    "    if obs_id not in hdu_table[\"OBS_ID\"]:\n",
    "        raise ValueError(f\"OBS_ID = {obs_id} not in HDU index table.\")\n",
    "\n",
    "    kwargs = {\"obs_id\": int(obs_id)}\n",
    "\n",
    "    # check for the \"short forms\"\n",
    "    if isinstance(required_irf, str):\n",
    "        required_irf = REQUIRED_IRFS[required_irf]\n",
    "\n",
    "    if not set(required_irf).issubset(ALL_IRFS):\n",
    "        difference = set(required_irf).difference(ALL_IRFS)\n",
    "        raise ValueError(\n",
    "            f\"{difference} is not a valid hdu key. Choose from: {ALL_IRFS}\"\n",
    "        )\n",
    "\n",
    "    if require_events:\n",
    "        required_hdus = {\"events\", \"gti\"}.union(required_irf)\n",
    "    else:\n",
    "        required_hdus = required_irf\n",
    "\n",
    "    missing_hdus = []\n",
    "    kwargs_irf = {}\n",
    "    for hdu in ALL_HDUS:\n",
    "        hdu_location = hdu_table.hdu_location(\n",
    "            obs_id=obs_id,\n",
    "            hdu_type=hdu,\n",
    "            warn_missing=False,\n",
    "        )\n",
    "        if hdu_location is not None:\n",
    "            if hdu in ALL_IRFS:\n",
    "                kwargs_irf[hdu] = hdu_location\n",
    "            else:\n",
    "                kwargs[hdu] = hdu_location\n",
    "        elif hdu in required_hdus:\n",
    "            missing_hdus.append(hdu)\n",
    "\n",
    "\n",
    "    \n",
    "    if len(missing_hdus) > 0:\n",
    "        raise MissingRequiredHDU(\n",
    "            f\"Required HDUs {missing_hdus} not found in observation {obs_id}\"\n",
    "        )\n",
    "\n",
    "    # TODO: right now, gammapy doesn't support using the pointing table of GADF\n",
    "    # so we always pass the events location here to be read into a FixedPointingInfo\n",
    "    if \"events\" in kwargs:\n",
    "        pointing_location = copy(kwargs[\"events\"])\n",
    "        pointing_location.hdu_class = \"pointing\"\n",
    "        kwargs[\"pointing\"] = pointing_location\n",
    "\n",
    "    kwargs[\"irf_groups\"] = IRFGroups([IRFGroup(**kwargs_irf)])\n",
    "    \n",
    "    return VODFObservation(**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7578ae01-63ee-4234-ad57-6bf26f6f9775",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "da4d5185-9bf6-4300-8faf-79f54af3d6b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "datastore = DataStore.from_dir(\"$GAMMAPY_DATA/hess-dl3-dr1/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "76cdf4b4-ebc3-4004-837e-69bd3f748de3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'gti': <gammapy.utils.fits.HDULocation object at 0x1509a13f0>, 'aeff': <gammapy.utils.fits.HDULocation object at 0x15092d930>, 'edisp': <gammapy.utils.fits.HDULocation object at 0x15092ee00>, 'psf': <gammapy.utils.fits.HDULocation object at 0x15092ed40>, 'bkg': <gammapy.utils.fits.HDULocation object at 0x15092dc90>}\n"
     ]
    }
   ],
   "source": [
    "grp = IRFGroup.from_hdu_index_table(datastore.hdu_table, 23523)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "84ff6858-7d6e-47be-96fe-8ebb1ecfb4db",
   "metadata": {},
   "outputs": [],
   "source": [
    "time_range1 = GTI.from_time_intervals([grp.gti.time_start + [0, 5]*u.min])\n",
    "grp.gti = time_range1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7c4e3462-d536-48aa-82c5-1ec89e99f67f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>GTI info:\n",
       "- Number of GTIs: 1\n",
       "- Duration: 299.9999999999989 s\n",
       "- Start: 155513290.18399984 s MET\n",
       "- Start: 2004-12-04T22:08:10.184 (time standard: TT)\n",
       "- Stop: 155513590.18399984 s MET\n",
       "- Stop: 2004-12-04T22:13:10.184 (time standard: TT)\n",
       "</pre>"
      ],
      "text/plain": [
       "<gammapy.data.gti.GTI at 0x150fb60e0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grp.gti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8d373f3-1b9a-45f6-99f1-1ba3ec2adde6",
   "metadata": {},
   "outputs": [],
   "source": [
    "isinstance(obs.edisp, HDULocation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eabe6f18-5924-4e18-8d80-7bbc8ec5bbf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "grp.gti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d07e2bf-cf91-44e0-9493-6d8b63f2f36f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}