feast/Feast.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fb922a2c-4968-430f-84bc-a2eb3ca80a75",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install protobuf feast[aws,redis] pyarrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "89a7805d-4a63-4a64-865a-61cc072f63ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# configure aws cli via terminal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2edadd1b-ef57-497d-b360-279520ff60a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# don't forget to port-forward redis service in terminal: kubectl port-forward svc/kf-feast-redis-master 6379 -n feast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25eba9bc-c256-43e7-b3df-88950683461c",
   "metadata": {},
   "outputs": [],
   "source": [
    "1) \n",
    "# let's start with local registry and redis in .yaml file\n",
    "# and local paths in main.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "917a5cf3-d192-453c-805e-0f4641502acf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HouseId</th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>MedHouseVal</th>\n",
       "      <th>EventTimestamp</th>\n",
       "      <th>Created</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2.4792</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3.454704</td>\n",
       "      <td>1.134146</td>\n",
       "      <td>2251.0</td>\n",
       "      <td>3.921603</td>\n",
       "      <td>2.000</td>\n",
       "      <td>2021-12-11 18:40:03</td>\n",
       "      <td>2022-04-12 12:04:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3.4630</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.363636</td>\n",
       "      <td>1.166297</td>\n",
       "      <td>1307.0</td>\n",
       "      <td>2.898004</td>\n",
       "      <td>2.017</td>\n",
       "      <td>2021-12-11 18:57:30</td>\n",
       "      <td>2022-04-12 12:04:13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   HouseId  MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  \\\n",
       "0        1  2.4792      24.0  3.454704   1.134146      2251.0  3.921603   \n",
       "1        2  3.4630       8.0  6.363636   1.166297      1307.0  2.898004   \n",
       "\n",
       "   MedHouseVal      EventTimestamp             Created  \n",
       "0        2.000 2021-12-11 18:40:03 2022-04-12 12:04:13  \n",
       "1        2.017 2021-12-11 18:57:30 2022-04-12 12:04:13  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.read_parquet('data/house_dataset_main.parquet').head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "91ccf276-9304-4cb8-961b-17030f7419dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HouseId</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>EventTimestamp</th>\n",
       "      <th>Created</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>34.18</td>\n",
       "      <td>-118.38</td>\n",
       "      <td>2021-12-11 18:40:03</td>\n",
       "      <td>2022-02-13 22:43:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>39.08</td>\n",
       "      <td>-121.04</td>\n",
       "      <td>2021-12-11 18:57:30</td>\n",
       "      <td>2022-02-13 22:43:53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   HouseId  Latitude  Longitude      EventTimestamp             Created\n",
       "0        1     34.18    -118.38 2021-12-11 18:40:03 2022-02-13 22:43:53\n",
       "1        2     39.08    -121.04 2021-12-11 18:57:30 2022-02-13 22:43:53"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_parquet('data/house_dataset_lat_lon.parquet').head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e2c4f9d7-154c-4c89-83fd-7330979c3374",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/mlops_workshops/feast/feature_store\n"
     ]
    }
   ],
   "source": [
    "cd feature_store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "164a6e54-4b8b-4df6-aab6-493591feca89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[94mNo changes to registry\n",
      "Deploying infrastructure for \u001b[1m\u001b[32mhouse_lat_lon_view\u001b[0m\n",
      "Deploying infrastructure for \u001b[1m\u001b[32mhouse_main_view\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!feast apply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "870a0572-d562-466d-aaa0-2e3129a2b4c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from feast import FeatureStore\n",
    "\n",
    "store = FeatureStore(repo_path=\".\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adb726e6-ac8d-4d4b-81c5-8b30e7981b32",
   "metadata": {},
   "source": [
    "## Offline queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "120df41c-ab1d-4884-aa79-93beb32774ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import feast\n",
    "import pandas as pd\n",
    "from feast.infra.offline_stores.file_source import SavedDatasetFileStorage\n",
    "\n",
    "entity_df = pd.DataFrame.from_dict({\"HouseId\": [i for i in range(1, 1000)]})\n",
    "entity_df['event_timestamp'] = pd.to_datetime('now', utc=True)\n",
    "\n",
    "retrieval_job = store.get_historical_features(\n",
    "    entity_df=entity_df,\n",
    "    features=store.get_feature_service('house_service')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3c3b7c77-8e29-4491-958c-01c3c31b5a87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['house_main_view:MedInc',\n",
       " 'house_main_view:HouseAge',\n",
       " 'house_main_view:AveRooms',\n",
       " 'house_main_view:AveBedrms',\n",
       " 'house_main_view:Population',\n",
       " 'house_main_view:AveOccup',\n",
       " 'house_main_view:MedHouseVal',\n",
       " 'house_lat_lon_view:Latitude',\n",
       " 'house_lat_lon_view:Longitude']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retrieval_job.metadata.features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "47c08cf7-0f64-48e5-9f25-adf21525fc5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/feast/feature_store.py:952: RuntimeWarning: Saving dataset is an experimental feature. This API is unstable and it could and most probably will be changed in the future. We do not guarantee that future changes will maintain backward compatibility.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "999\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HouseId</th>\n",
       "      <th>event_timestamp</th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>MedHouseVal</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2022-06-23 19:34:23.483386+00:00</td>\n",
       "      <td>2.4792</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3.454704</td>\n",
       "      <td>1.134146</td>\n",
       "      <td>2251.0</td>\n",
       "      <td>3.921603</td>\n",
       "      <td>2.000</td>\n",
       "      <td>34.18</td>\n",
       "      <td>-118.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2022-06-23 19:34:23.483386+00:00</td>\n",
       "      <td>3.4630</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.363636</td>\n",
       "      <td>1.166297</td>\n",
       "      <td>1307.0</td>\n",
       "      <td>2.898004</td>\n",
       "      <td>2.017</td>\n",
       "      <td>39.08</td>\n",
       "      <td>-121.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2022-06-23 19:34:23.483386+00:00</td>\n",
       "      <td>3.7500</td>\n",
       "      <td>16.0</td>\n",
       "      <td>5.768719</td>\n",
       "      <td>1.023295</td>\n",
       "      <td>1478.0</td>\n",
       "      <td>2.459235</td>\n",
       "      <td>1.473</td>\n",
       "      <td>38.68</td>\n",
       "      <td>-121.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2022-06-23 19:34:23.483386+00:00</td>\n",
       "      <td>2.8542</td>\n",
       "      <td>34.0</td>\n",
       "      <td>3.858779</td>\n",
       "      <td>1.045802</td>\n",
       "      <td>1164.0</td>\n",
       "      <td>4.442748</td>\n",
       "      <td>1.469</td>\n",
       "      <td>34.04</td>\n",
       "      <td>-118.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2022-06-23 19:34:23.483386+00:00</td>\n",
       "      <td>1.3375</td>\n",
       "      <td>18.0</td>\n",
       "      <td>4.567625</td>\n",
       "      <td>1.087327</td>\n",
       "      <td>2707.0</td>\n",
       "      <td>2.882854</td>\n",
       "      <td>0.596</td>\n",
       "      <td>39.13</td>\n",
       "      <td>-121.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   HouseId                  event_timestamp  MedInc  HouseAge  AveRooms  \\\n",
       "0        1 2022-06-23 19:34:23.483386+00:00  2.4792      24.0  3.454704   \n",
       "1        2 2022-06-23 19:34:23.483386+00:00  3.4630       8.0  6.363636   \n",
       "2        3 2022-06-23 19:34:23.483386+00:00  3.7500      16.0  5.768719   \n",
       "3        4 2022-06-23 19:34:23.483386+00:00  2.8542      34.0  3.858779   \n",
       "4        5 2022-06-23 19:34:23.483386+00:00  1.3375      18.0  4.567625   \n",
       "\n",
       "   AveBedrms  Population  AveOccup  MedHouseVal  Latitude  Longitude  \n",
       "0   1.134146      2251.0  3.921603        2.000     34.18    -118.38  \n",
       "1   1.166297      1307.0  2.898004        2.017     39.08    -121.04  \n",
       "2   1.023295      1478.0  2.459235        1.473     38.68    -121.28  \n",
       "3   1.045802      1164.0  4.442748        1.469     34.04    -118.19  \n",
       "4   1.087327      2707.0  2.882854        0.596     39.13    -121.54  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = store.create_saved_dataset(\n",
    "    from_=retrieval_job,\n",
    "    name='merged_dataset_v1',\n",
    "    # storage=SavedDatasetFileStorage('s3://dvoitekh-kubeflow/feast/data/merged_dataset_v1.parquet')\n",
    "    storage=SavedDatasetFileStorage('merged_dataset_v1.parquet')\n",
    ")\n",
    "\n",
    "training_df = retrieval_job.to_df()\n",
    "print(len(training_df))\n",
    "training_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "04cee75c-83a8-4017-8f8a-10f56fa3f25a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views to \u001b[1m\u001b[32m2022-02-01 00:00:00+00:00\u001b[0m into the \u001b[1m\u001b[32mredis\u001b[0m online store.\n",
      "\n",
      "Since the ttl is 0 for feature view \u001b[1m\u001b[32mhouse_lat_lon_view\u001b[0m, the start date will be set to 1 year before the current time.\n",
      "\u001b[1m\u001b[32mhouse_lat_lon_view\u001b[0m from \u001b[1m\u001b[32m2021-06-24 19:34:57+00:00\u001b[0m to \u001b[1m\u001b[32m2022-02-01 00:00:00+00:00\u001b[0m:\n",
      "100%|████████████████████████████████████████████████████████| 5821/5821 [00:00<00:00, 15857.22it/s]\n",
      "Since the ttl is 0 for feature view \u001b[1m\u001b[32mhouse_main_view\u001b[0m, the start date will be set to 1 year before the current time.\n",
      "\u001b[1m\u001b[32mhouse_main_view\u001b[0m from \u001b[1m\u001b[32m2021-06-24 19:34:57+00:00\u001b[0m to \u001b[1m\u001b[32m2022-02-01 00:00:00+00:00\u001b[0m:\n",
      "100%|████████████████████████████████████████████████████████| 5821/5821 [00:00<00:00, 10446.02it/s]\n"
     ]
    }
   ],
   "source": [
    "!feast materialize-incremental 2022-02-01T00:00:00"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "180b0499-b6a1-43df-808b-a8df3b132de4",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Online Queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3216c95f-8095-462e-9f2c-59038e091e69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HouseId</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>MedHouseVal</th>\n",
       "      <th>MedInc</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>24.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.4792</td>\n",
       "      <td>34.18</td>\n",
       "      <td>-118.379997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20640</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   HouseId  HouseAge  MedHouseVal  MedInc  Latitude   Longitude\n",
       "0        1      24.0          2.0  2.4792     34.18 -118.379997\n",
       "1    20640       NaN          NaN     NaN       NaN         NaN"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "online_df = store.get_online_features(\n",
    "    features=[\n",
    "        \"house_main_view:MedInc\",\n",
    "        \"house_main_view:HouseAge\",\n",
    "        \"house_main_view:MedHouseVal\",\n",
    "        \"house_lat_lon_view:Latitude\",\n",
    "        \"house_lat_lon_view:Longitude\",\n",
    "    ],\n",
    "    entity_rows=[\n",
    "        {\"HouseId\": 1},\n",
    "        {\"HouseId\": 20640},\n",
    "    ]\n",
    ")\n",
    "online_df.to_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9ffbf63b-dc00-42b8-861d-f8b6e6fb0e17",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views to \u001b[1m\u001b[32m2022-06-11 00:00:00+00:00\u001b[0m into the \u001b[1m\u001b[32mredis\u001b[0m online store.\n",
      "\n",
      "\u001b[1m\u001b[32mhouse_lat_lon_view\u001b[0m from \u001b[1m\u001b[32m2022-02-01 00:00:00+00:00\u001b[0m to \u001b[1m\u001b[32m2022-06-11 00:00:00+00:00\u001b[0m:\n",
      "100%|██████████████████████████████████████████████████████| 14819/14819 [00:01<00:00, 13844.75it/s]\n",
      "\u001b[1m\u001b[32mhouse_main_view\u001b[0m from \u001b[1m\u001b[32m2022-02-01 00:00:00+00:00\u001b[0m to \u001b[1m\u001b[32m2022-06-11 00:00:00+00:00\u001b[0m:\n",
      "100%|███████████████████████████████████████████████████████| 14819/14819 [00:01<00:00, 9714.25it/s]\n"
     ]
    }
   ],
   "source": [
    "!feast materialize-incremental 2022-06-11T00:00:00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "89f59f12-74fc-413f-94da-e7df74c30270",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HouseId</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>MedHouseVal</th>\n",
       "      <th>MedInc</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>24.0</td>\n",
       "      <td>2.000</td>\n",
       "      <td>2.4792</td>\n",
       "      <td>34.180000</td>\n",
       "      <td>-118.379997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20640</td>\n",
       "      <td>13.0</td>\n",
       "      <td>3.333</td>\n",
       "      <td>3.3438</td>\n",
       "      <td>33.689999</td>\n",
       "      <td>-117.800003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   HouseId  HouseAge  MedHouseVal  MedInc   Latitude   Longitude\n",
       "0        1      24.0        2.000  2.4792  34.180000 -118.379997\n",
       "1    20640      13.0        3.333  3.3438  33.689999 -117.800003"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "online_df = store.get_online_features(\n",
    "    features=[\n",
    "        \"house_main_view:MedInc\",\n",
    "        \"house_main_view:HouseAge\",\n",
    "        \"house_main_view:MedHouseVal\",\n",
    "        \"house_lat_lon_view:Latitude\",\n",
    "        \"house_lat_lon_view:Longitude\",\n",
    "    ],\n",
    "    entity_rows=[\n",
    "        {\"HouseId\": 1},\n",
    "        {\"HouseId\": 20640},\n",
    "    ]\n",
    ")\n",
    "online_df.to_df()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dd8884a-0ac8-48dd-b45f-fb1c88069de5",
   "metadata": {},
   "source": [
    "## Push Source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "784859fd-a1a2-438e-a8eb-8482b57b7d14",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n",
      "/opt/conda/lib/python3.8/site-packages/feast/feature_store.py:1219: RuntimeWarning: Push source is an experimental feature. This API is unstable and it could and might change in the future. We do not guarantee that future changes will maintain backward compatibility.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "\n",
    "store.push('main_push_source', pd.DataFrame([{\n",
    "    'HouseId': 1,\n",
    "    'MedInc': 1.0,\n",
    "    'HouseAge': 1.0,\n",
    "    'AveRooms': 1.0,\n",
    "    'AveBedrms': 1.0,\n",
    "    'Population': 1.0,\n",
    "    'AveOccup': 1.0,\n",
    "    'MedHouseVal': 1.0,\n",
    "    'Latitude': 1.0,\n",
    "    'Longitude': 1.0,\n",
    "    'EventTimestamp': datetime.datetime.now(),\n",
    "    'Created': datetime.datetime.now(),\n",
    "}]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9c43a651-1242-479e-a82f-9a98bf8ac7ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HouseId</th>\n",
       "      <th>MedInc</th>\n",
       "      <th>MedHouseVal</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Latitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-118.379997</td>\n",
       "      <td>34.180000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20640</td>\n",
       "      <td>3.3438</td>\n",
       "      <td>3.333</td>\n",
       "      <td>13.0</td>\n",
       "      <td>-117.800003</td>\n",
       "      <td>33.689999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   HouseId  MedInc  MedHouseVal  HouseAge   Longitude   Latitude\n",
       "0        1  1.0000        1.000       1.0 -118.379997  34.180000\n",
       "1    20640  3.3438        3.333      13.0 -117.800003  33.689999"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "online_df = store.get_online_features(\n",
    "    features=[\n",
    "        \"house_main_view:MedInc\",\n",
    "        \"house_main_view:HouseAge\",\n",
    "        \"house_main_view:MedHouseVal\",\n",
    "        \"house_lat_lon_view:Latitude\",\n",
    "        \"house_lat_lon_view:Longitude\",\n",
    "    ],\n",
    "    entity_rows=[\n",
    "        {\"HouseId\": 1},\n",
    "        {\"HouseId\": 20640},\n",
    "    ]\n",
    ")\n",
    "online_df.to_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ff1aa1c6-4831-4b59-b685-3a7b819c83ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n",
      "/opt/conda/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py:1722: DeprecationWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.\n",
      "  if dataset.partitions is not None:\n",
      "/opt/conda/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py:1797: DeprecationWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.\n",
      "  if len(dataset.pieces) > 1:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HouseId</th>\n",
       "      <th>event_timestamp</th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>MedHouseVal</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2022-06-14 20:40:51.937687+00:00</td>\n",
       "      <td>2.4792</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3.454704</td>\n",
       "      <td>1.134146</td>\n",
       "      <td>2251.0</td>\n",
       "      <td>3.921603</td>\n",
       "      <td>2.0</td>\n",
       "      <td>34.18</td>\n",
       "      <td>-118.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   HouseId                  event_timestamp  MedInc  HouseAge  AveRooms  \\\n",
       "0        1 2022-06-14 20:40:51.937687+00:00  2.4792      24.0  3.454704   \n",
       "\n",
       "   AveBedrms  Population  AveOccup  MedHouseVal  Latitude  Longitude  \n",
       "0   1.134146      2251.0  3.921603          2.0     34.18    -118.38  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entity_df = pd.DataFrame.from_dict({\"HouseId\": [1]})\n",
    "entity_df['event_timestamp'] = pd.to_datetime('now', utc=True)\n",
    "\n",
    "store.get_historical_features(\n",
    "    entity_df=entity_df,\n",
    "    features=store.get_feature_service('house_service')\n",
    ").to_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85f38200-708c-4f4d-b39b-4daadfa4fe71",
   "metadata": {},
   "outputs": [],
   "source": [
    "# now we can add actual aws credentials file (with s3 access)\n",
    "# replace links in main.py with s3 (not pachyderm)\n",
    "# replace repo in .yaml with s3 (not pachyderm)\n",
    "# repeat everything in this notebook\n",
    "# show gitops flow in the repo"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}