vectorize_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import os\n",
    "from PIL import Image\n",
    "from torchvision import models, transforms\n",
    "from src import helpers"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In this notebook we will interactively create vector representation of images in our dog breed dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's get a pretrained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = # initialize a pretrained model, e.g. resnet50, from torchvision.models with default weights https://pytorch.org/vision/stable/models.html\n",
    "model.eval()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to iterate through dataset and create vector representations of every image. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = # define a relative path to data folder\n",
    "dataset_dir = f\"{data_dir}/dogs_dataset\"\n",
    "\n",
    "vectors = []\n",
    "for subdir, dirs, files in os.walk(dataset_dir):\n",
    "    for file in files:\n",
    "        img_path = os.path.join(subdir, file)\n",
    "        # use helper function to create vector from image using img_path and model - look into src/helpers.py\n",
    "        try:\n",
    "            vectors.append({\"img_path\": img_path, \"vector\": vector})\n",
    "        except Exception as e:\n",
    "            print(f\"Skipping {img_path}, {e}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We stored the vectors with the corresponding image paths to the list - look at its shape and structure >>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As the last step, we want to save the list into a file (e.g. parquet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_records(vectors)\n",
    "df.to_parquet(f\"{data_dir}/vectors.parquet\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check vectors.parquet in /data/ folder."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "streamlit-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a00bf2f4ae519fa774afe132cf63de2c1e3bc15ee70f41375f14ccc5832ccfa3"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}