CLIP-zeroshot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import CLIPTokenizerFast, CLIPModel\n",
    "import torch\n",
    "from torchvision.datasets import ImageFolder\n",
    "from torchvision.transforms import Compose, ToTensor, Lambda, Resize, Normalize\n",
    "from PIL import Image, ImageDraw\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "from sklearn.metrics import accuracy_score,f1_score\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Zero-shot image classification with CLIP\n",
    "\n",
    "This is rather prompting than training. \n",
    "### Parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIRECTROY = 'data'\n",
    "MODEL_PATH = 'models'\n",
    "BATCH_SIZE = 32\n",
    "IMG_SIZE = 224\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(f'{DIRECTROY}/reduced_train.csv') \n",
    "df_test = pd.read_csv(f'{DIRECTROY}/reduced_test.csv') \n",
    "num_classes = len(df_train['newid'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class2label = {c:l for c, l in zip(df_train['newid'], df_train['label'])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_class2label = dict(sorted(class2label.items()))\n",
    "labels = []\n",
    "for k, v in sorted_class2label.items():\n",
    "    labels.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(649, 649)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(class2label), num_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "649"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_train['label'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_public = df_test[df_test['Usage'] == 'Public']\n",
    "df_test_private = df_test[df_test['Usage'] == 'Private']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, df, transforms, directory):\n",
    "        self.tokenizer =  CLIPTokenizerFast.from_pretrained(\"openai/clip-vit-base-patch16\")\n",
    "        self.df = df\n",
    "        self.transforms = transforms\n",
    "        self.directory = directory\n",
    "        self.labels = torch.Tensor(df['newid'].values).long()\n",
    "        self.imgs = torch.cat([ self.transforms(self.resize_img(Image.open(f'{DIRECTROY}/{self.directory}/{x}')).convert('RGB')).half().reshape(1,3,IMG_SIZE,IMG_SIZE) for x in tqdm(df['name'].values)])\n",
    "        self.tokenized = self.tokenizer(df['label'].tolist(), padding=True, truncation=True, return_tensors=\"pt\")\n",
    "        self.input_ids = self.tokenized['input_ids']\n",
    "        self.attention_mask = self.tokenized['attention_mask']\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img = self.imgs[idx]\n",
    "        label = self.labels[idx]\n",
    "        input_ids = self.input_ids[idx]\n",
    "        attention_mask = self.attention_mask[idx]\n",
    "        return img, label, input_ids, attention_mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CLIPModel.from_pretrained(\"openai/clip-vit-base-patch16\")\n",
    "model = model.to(device)\n",
    "tokenizer = CLIPTokenizerFast.from_pretrained(\"openai/clip-vit-base-patch16\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Module.parameters of CLIPModel(\n",
       "  (text_model): CLIPTextTransformer(\n",
       "    (embeddings): CLIPTextEmbeddings(\n",
       "      (token_embedding): Embedding(49408, 512)\n",
       "      (position_embedding): Embedding(77, 512)\n",
       "    )\n",
       "    (encoder): CLIPEncoder(\n",
       "      (layers): ModuleList(\n",
       "        (0-11): 12 x CLIPEncoderLayer(\n",
       "          (self_attn): CLIPAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (layer_norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): CLIPMLP(\n",
       "            (activation_fn): QuickGELUActivation()\n",
       "            (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "            (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          )\n",
       "          (layer_norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (vision_model): CLIPVisionTransformer(\n",
       "    (embeddings): CLIPVisionEmbeddings(\n",
       "      (patch_embedding): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)\n",
       "      (position_embedding): Embedding(197, 768)\n",
       "    )\n",
       "    (pre_layrnorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "    (encoder): CLIPEncoder(\n",
       "      (layers): ModuleList(\n",
       "        (0-11): 12 x CLIPEncoderLayer(\n",
       "          (self_attn): CLIPAttention(\n",
       "            (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (layer_norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): CLIPMLP(\n",
       "            (activation_fn): QuickGELUActivation()\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (layer_norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (post_layernorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (visual_projection): Linear(in_features=768, out_features=512, bias=False)\n",
       "  (text_projection): Linear(in_features=512, out_features=512, bias=False)\n",
       ")>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prompting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = tokenizer([f'a photo of {x}' for x in labels], return_tensors=\"pt\", padding=True, truncation=True)\n",
    "prompts_input_ids = prompts['input_ids'].to(device)\n",
    "prompts_attention_mask = prompts['attention_mask'].to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = torch.load(f'{DIRECTROY}/test_public_dataset/test_public_reduced_dataset_0.pth')\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:29<00:00,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.36541625534715366\n",
      "F1 Score Weighted: 0.33904288265469684\n",
      "F1 Score Macro: 0.3289402167768423\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "true_labels = []\n",
    "pred_labels = []\n",
    "test_loss = 0\n",
    "len_dataset = 0\n",
    "with torch.no_grad():\n",
    "        len_test = 0\n",
    "        model.eval()\n",
    "        for inputs, labels, input_ids, attention_mask  in tqdm(test_dataloader):\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "            input_ids = input_ids.to(device)\n",
    "            attention_mask = attention_mask.to(device)\n",
    "            \n",
    "            logits = model(pixel_values=inputs, input_ids=prompts_input_ids, attention_mask=prompts_attention_mask)\n",
    "            \n",
    "            outputs = torch.argmax(logits.logits_per_image, 1).flatten().cpu().numpy()\n",
    "            labels = labels.flatten().cpu().numpy()\n",
    "            \n",
    "            true_labels.extend(labels)\n",
    "            pred_labels.extend(outputs)\n",
    "        len_test += len(test_dataset)\n",
    "        \n",
    "        \n",
    "        print(f'Accuracy: {accuracy_score(true_labels, pred_labels)}')\n",
    "        print(f'F1 Score Weighted: {f1_score(true_labels, pred_labels, average=\"weighted\")}')\n",
    "        print(f'F1 Score Macro: {f1_score(true_labels, pred_labels, average=\"macro\")}')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Zero-shot can only get 34.5% in accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}