DISTIL_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vYmMCnfg1PN8"
   },
   "source": [
    "# Preface\n",
    "\n",
    "The locations requiring configuration for your experiment are commented in capital text."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kgYWNPhf801A"
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZYsutkIJrGvK"
   },
   "source": [
    "**Experiment-Specific Imports**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "lfQKdd0DrKsa"
   },
   "outputs": [],
   "source": [
    "from distil.utils.models.resnet import ResNet18                                 # IMPORT YOUR MODEL HERE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Maz6VJxS787x"
   },
   "source": [
    "## Main Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "V9-8qRo8KD3a"
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import copy\n",
    "from torch.utils.data import Dataset, DataLoader, Subset, ConcatDataset\n",
    "import torch.nn.functional as F\n",
    "from torch import nn\n",
    "from torchvision import transforms\n",
    "from torchvision import datasets\n",
    "from PIL import Image\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import math\n",
    "import random\n",
    "import os\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "import torch.utils.data as data\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from numpy.linalg import cond\n",
    "from numpy.linalg import inv\n",
    "from numpy.linalg import norm\n",
    "from scipy import sparse as sp\n",
    "from scipy.linalg import lstsq\n",
    "from scipy.linalg import solve\n",
    "from scipy.optimize import nnls\n",
    "\n",
    "from distil.active_learning_strategies.badge import BADGE\n",
    "from distil.active_learning_strategies.glister import GLISTER\n",
    "from distil.active_learning_strategies.margin_sampling import MarginSampling\n",
    "from distil.active_learning_strategies.entropy_sampling import EntropySampling\n",
    "from distil.active_learning_strategies.random_sampling import RandomSampling\n",
    "from distil.active_learning_strategies.gradmatch_active import GradMatchActive\n",
    "from distil.active_learning_strategies.fass import FASS\n",
    "from distil.active_learning_strategies.adversarial_bim import AdversarialBIM\n",
    "from distil.active_learning_strategies.adversarial_deepfool import AdversarialDeepFool\n",
    "from distil.active_learning_strategies.core_set import CoreSet\n",
    "from distil.active_learning_strategies.least_confidence_sampling import LeastConfidenceSampling\n",
    "from distil.active_learning_strategies.margin_sampling import MarginSampling\n",
    "from distil.active_learning_strategies.bayesian_active_learning_disagreement_dropout import BALDDropout\n",
    "from distil.utils.train_helper import data_train\n",
    "from distil.utils.utils import LabeledToUnlabeledDataset\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ecvumggc6UhF"
   },
   "source": [
    "## Checkpointing and Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "ohuLHm5E58bj",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Checkpoint:\n",
    "\n",
    "    def __init__(self, acc_list=None, indices=None, state_dict=None, experiment_name=None, path=None):\n",
    "\n",
    "        # If a path is supplied, load a checkpoint from there.\n",
    "        if path is not None:\n",
    "\n",
    "            if experiment_name is not None:\n",
    "                self.load_checkpoint(path, experiment_name)\n",
    "            else:\n",
    "                raise ValueError(\"Checkpoint contains None value for experiment_name\")\n",
    "\n",
    "            return\n",
    "\n",
    "        if acc_list is None:\n",
    "            raise ValueError(\"Checkpoint contains None value for acc_list\")\n",
    "\n",
    "        if indices is None:\n",
    "            raise ValueError(\"Checkpoint contains None value for indices\")\n",
    "\n",
    "        if state_dict is None:\n",
    "            raise ValueError(\"Checkpoint contains None value for state_dict\")\n",
    "\n",
    "        if experiment_name is None:\n",
    "            raise ValueError(\"Checkpoint contains None value for experiment_name\")\n",
    "\n",
    "        self.acc_list = acc_list\n",
    "        self.indices = indices\n",
    "        self.state_dict = state_dict\n",
    "        self.experiment_name = experiment_name\n",
    "\n",
    "    def __eq__(self, other):\n",
    "\n",
    "        # Check if the accuracy lists are equal\n",
    "        acc_lists_equal = self.acc_list == other.acc_list\n",
    "\n",
    "        # Check if the indices are equal\n",
    "        indices_equal = self.indices == other.indices\n",
    "\n",
    "        # Check if the experiment names are equal\n",
    "        experiment_names_equal = self.experiment_name == other.experiment_name\n",
    "\n",
    "        return acc_lists_equal and indices_equal and experiment_names_equal\n",
    "\n",
    "    def save_checkpoint(self, path):\n",
    "\n",
    "        # Get current time to use in file timestamp\n",
    "        timestamp = time.time_ns()\n",
    "\n",
    "        # Create the path supplied\n",
    "        os.makedirs(path, exist_ok=True)\n",
    "\n",
    "        # Name saved files using timestamp to add recency information\n",
    "        save_path = os.path.join(path, F\"c{timestamp}1\")\n",
    "        copy_save_path = os.path.join(path, F\"c{timestamp}2\")\n",
    "\n",
    "        # Write this checkpoint to the first save location\n",
    "        with open(save_path, 'wb') as save_file:\n",
    "            pickle.dump(self, save_file)\n",
    "\n",
    "        # Write this checkpoint to the second save location\n",
    "        with open(copy_save_path, 'wb') as copy_save_file:\n",
    "            pickle.dump(self, copy_save_file)\n",
    "\n",
    "    def load_checkpoint(self, path, experiment_name):\n",
    "\n",
    "        # Obtain a list of all files present at the path\n",
    "        timestamp_save_no = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]\n",
    "\n",
    "        # If there are no such files, set values to None and return\n",
    "        if len(timestamp_save_no) == 0:\n",
    "            self.acc_list = None\n",
    "            self.indices = None\n",
    "            self.state_dict = None\n",
    "            return\n",
    "\n",
    "        # Sort the list of strings to get the most recent\n",
    "        timestamp_save_no.sort(reverse=True)\n",
    "\n",
    "        # Read in two files at a time, checking if they are equal to one another. \n",
    "        # If they are equal, then it means that the save operation finished correctly.\n",
    "        # If they are not, then it means that the save operation failed (could not be \n",
    "        # done atomically). Repeat this action until no possible pair can exist.\n",
    "        while len(timestamp_save_no) > 1:\n",
    "\n",
    "            # Pop a most recent checkpoint copy\n",
    "            first_file = timestamp_save_no.pop(0)\n",
    "\n",
    "            # Keep popping until two copies with equal timestamps are present\n",
    "            while True:\n",
    "                \n",
    "                second_file = timestamp_save_no.pop(0)\n",
    "                \n",
    "                # Timestamps match if the removal of the \"1\" or \"2\" results in equal numbers\n",
    "                if (second_file[:-1]) == (first_file[:-1]):\n",
    "                    break\n",
    "                else:\n",
    "                    first_file = second_file\n",
    "\n",
    "                    # If there are no more checkpoints to examine, set to None and return\n",
    "                    if len(timestamp_save_no) == 0:\n",
    "                        self.acc_list = None\n",
    "                        self.indices = None\n",
    "                        self.state_dict = None\n",
    "                        return\n",
    "\n",
    "            # Form the paths to the files\n",
    "            load_path = os.path.join(path, first_file)\n",
    "            copy_load_path = os.path.join(path, second_file)\n",
    "\n",
    "            # Load the two checkpoints\n",
    "            with open(load_path, 'rb') as load_file:\n",
    "                checkpoint = pickle.load(load_file)\n",
    "\n",
    "            with open(copy_load_path, 'rb') as copy_load_file:\n",
    "                checkpoint_copy = pickle.load(copy_load_file)\n",
    "\n",
    "            # Do not check this experiment if it is not the one we need to restore\n",
    "            if checkpoint.experiment_name != experiment_name:\n",
    "                continue\n",
    "\n",
    "            # Check if they are equal\n",
    "            if checkpoint == checkpoint_copy:\n",
    "\n",
    "                # This checkpoint will suffice. Populate this checkpoint's fields \n",
    "                # with the selected checkpoint's fields.\n",
    "                self.acc_list = checkpoint.acc_list\n",
    "                self.indices = checkpoint.indices\n",
    "                self.state_dict = checkpoint.state_dict\n",
    "                return\n",
    "\n",
    "        # Instantiate None values in acc_list, indices, and model\n",
    "        self.acc_list = None\n",
    "        self.indices = None\n",
    "        self.state_dict = None\n",
    "\n",
    "    def get_saved_values(self):\n",
    "\n",
    "        return (self.acc_list, self.indices, self.state_dict)\n",
    "\n",
    "def delete_checkpoints(checkpoint_directory, experiment_name):\n",
    "\n",
    "    # Iteratively go through each checkpoint, deleting those whose experiment name matches.\n",
    "    timestamp_save_no = [f for f in os.listdir(checkpoint_directory) if os.path.isfile(os.path.join(checkpoint_directory, f))]\n",
    "\n",
    "    for file in timestamp_save_no:\n",
    "\n",
    "        delete_file = False\n",
    "\n",
    "        # Get file location\n",
    "        file_path = os.path.join(checkpoint_directory, file)\n",
    "\n",
    "        if not os.path.exists(file_path):\n",
    "            continue\n",
    "\n",
    "        # Unpickle the checkpoint and see if its experiment name matches\n",
    "        with open(file_path, \"rb\") as load_file:\n",
    "\n",
    "            checkpoint_copy = pickle.load(load_file)\n",
    "            if checkpoint_copy.experiment_name == experiment_name:\n",
    "                delete_file = True\n",
    "\n",
    "        # Delete this file only if the experiment name matched\n",
    "        if delete_file:\n",
    "            os.remove(file_path)\n",
    "\n",
    "#Logs\n",
    "def write_logs(logs, save_directory, rd):\n",
    "  file_path = save_directory + 'run_'+'.txt'\n",
    "  with open(file_path, 'a') as f:\n",
    "    f.write('---------------------\\n')\n",
    "    f.write('Round '+str(rd)+'\\n')\n",
    "    f.write('---------------------\\n')\n",
    "    for key, val in logs.items():\n",
    "      if key == 'Training':\n",
    "        f.write(str(key)+ '\\n')\n",
    "        for epoch in val:\n",
    "          f.write(str(epoch)+'\\n')       \n",
    "      else:\n",
    "        f.write(str(key) + ' - '+ str(val) +'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Bd-8vetN6cP-"
   },
   "source": [
    "## AL Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "jWdgKV2M6PFu"
   },
   "outputs": [],
   "source": [
    "def train_one(full_train_dataset, initial_train_indices, test_dataset, net, n_rounds, budget, args, nclasses, strategy, save_directory, checkpoint_directory, experiment_name):\n",
    "\n",
    "    # Split the full training dataset into an initial training dataset and an unlabeled dataset\n",
    "    train_dataset = Subset(full_train_dataset, initial_train_indices)\n",
    "    initial_unlabeled_indices = list(set(range(len(full_train_dataset))) - set(initial_train_indices))\n",
    "    unlabeled_dataset = Subset(full_train_dataset, initial_unlabeled_indices)\n",
    "\n",
    "    # Set up the AL strategy\n",
    "    if strategy == \"random\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = RandomSampling(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"entropy\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = EntropySampling(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"margin\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = MarginSampling(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"least_confidence\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = LeastConfidenceSampling(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"badge\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = BADGE(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"coreset\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = CoreSet(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"fass\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = FASS(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"glister\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'lr': args['lr'], 'device':args['device']}\n",
    "        strategy = GLISTER(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args, typeOf='rand', lam=0.1)\n",
    "    elif strategy == \"adversarial_bim\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = AdversarialBIM(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"adversarial_deepfool\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = AdversarialDeepFool(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "    elif strategy == \"bald\":\n",
    "        strategy_args = {'batch_size' : args['batch_size'], 'device':args['device']}\n",
    "        strategy = BALDDropout(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset), net, nclasses, strategy_args)\n",
    "\n",
    "    # Define acc initially\n",
    "    acc = np.zeros(n_rounds+1)\n",
    "\n",
    "    initial_unlabeled_size = len(unlabeled_dataset)\n",
    "\n",
    "    initial_round = 1\n",
    "\n",
    "    # Define an index map\n",
    "    index_map = np.array([x for x in range(initial_unlabeled_size)])\n",
    "\n",
    "    # Attempt to load a checkpoint. If one exists, then the experiment crashed.\n",
    "    training_checkpoint = Checkpoint(experiment_name=experiment_name, path=checkpoint_directory)\n",
    "    rec_acc, rec_indices, rec_state_dict = training_checkpoint.get_saved_values()\n",
    "\n",
    "    # Check if there are values to recover\n",
    "    if rec_acc is not None:\n",
    "\n",
    "        # Restore the accuracy list\n",
    "        for i in range(len(rec_acc)):\n",
    "            acc[i] = rec_acc[i]\n",
    "\n",
    "        # Restore the indices list and shift those unlabeled points to the labeled set.\n",
    "        index_map = np.delete(index_map, rec_indices)\n",
    "\n",
    "        # Record initial size of the training dataset\n",
    "        intial_seed_size = len(train_dataset)\n",
    "\n",
    "        restored_unlabeled_points = Subset(unlabeled_dataset, rec_indices)\n",
    "        train_dataset = ConcatDataset([train_dataset, restored_unlabeled_points])\n",
    "\n",
    "        remaining_unlabeled_indices = list(set(range(len(unlabeled_dataset))) - set(rec_indices))\n",
    "        unlabeled_dataset = Subset(unlabeled_dataset, remaining_unlabeled_indices)\n",
    "\n",
    "        # Restore the model\n",
    "        net.load_state_dict(rec_state_dict) \n",
    "\n",
    "        # Fix the initial round\n",
    "        initial_round = (len(train_dataset) - initial_seed_size) // budget + 1\n",
    "\n",
    "        # Ensure loaded model is moved to GPU\n",
    "        if torch.cuda.is_available():\n",
    "            net = net.cuda()     \n",
    "\n",
    "        strategy.update_model(net)\n",
    "        strategy.update_data(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset)) \n",
    "\n",
    "        dt = data_train(train_dataset, net, args)\n",
    "\n",
    "    else:\n",
    "\n",
    "        if torch.cuda.is_available():\n",
    "            net = net.cuda()\n",
    "\n",
    "        dt = data_train(train_dataset, net, args)\n",
    "\n",
    "        acc[0] = dt.get_acc_on_set(test_dataset)\n",
    "        print('Initial Testing accuracy:', round(acc[0]*100, 2), flush=True)\n",
    "\n",
    "        logs = {}\n",
    "        logs['Training Points'] = len(train_dataset)\n",
    "        logs['Test Accuracy'] =  str(round(acc[0]*100, 2))\n",
    "        write_logs(logs, save_directory, 0)\n",
    "          \n",
    "        #Updating the trained model in strategy class\n",
    "        strategy.update_model(net)\n",
    "\n",
    "    # Record the training transform and test transform for disabling purposes\n",
    "    train_transform = full_train_dataset.transform\n",
    "    test_transform = test_dataset.transform\n",
    "\n",
    "    ##User Controlled Loop\n",
    "    for rd in range(initial_round, n_rounds+1):\n",
    "        print('-------------------------------------------------')\n",
    "        print('Round', rd) \n",
    "        print('-------------------------------------------------')\n",
    "\n",
    "        sel_time = time.time()\n",
    "        full_train_dataset.transform = test_transform # Disable any augmentation while selecting points\n",
    "        idx = strategy.select(budget)            \n",
    "        full_train_dataset.transform = train_transform # Re-enable any augmentation done during training\n",
    "        sel_time = time.time() - sel_time\n",
    "        print(\"Selection Time:\", sel_time)\n",
    "\n",
    "        selected_unlabeled_points = Subset(unlabeled_dataset, idx)\n",
    "        train_dataset = ConcatDataset([train_dataset, selected_unlabeled_points])\n",
    "\n",
    "        remaining_unlabeled_indices = list(set(range(len(unlabeled_dataset))) - set(idx))\n",
    "        unlabeled_dataset = Subset(unlabeled_dataset, remaining_unlabeled_indices)\n",
    "\n",
    "        # Update the index map\n",
    "        index_map = np.delete(index_map, idx, axis = 0)\n",
    "\n",
    "        print('Number of training points -', len(train_dataset))\n",
    "\n",
    "        # Start training\n",
    "        strategy.update_data(train_dataset, LabeledToUnlabeledDataset(unlabeled_dataset))\n",
    "        dt.update_data(train_dataset)\n",
    "        t1 = time.time()\n",
    "        clf, train_logs = dt.train(None)\n",
    "        t2 = time.time()\n",
    "        acc[rd] = dt.get_acc_on_set(test_dataset)\n",
    "        logs = {}\n",
    "        logs['Training Points'] = len(train_dataset)\n",
    "        logs['Test Accuracy'] =  str(round(acc[rd]*100, 2))\n",
    "        logs['Selection Time'] = str(sel_time)\n",
    "        logs['Trainining Time'] = str(t2 - t1) \n",
    "        logs['Training'] = train_logs\n",
    "        write_logs(logs, save_directory, rd)\n",
    "        strategy.update_model(clf)\n",
    "        print('Testing accuracy:', round(acc[rd]*100, 2), flush=True)\n",
    "\n",
    "        # Create a checkpoint\n",
    "        used_indices = np.array([x for x in range(initial_unlabeled_size)])\n",
    "        used_indices = np.delete(used_indices, index_map).tolist()\n",
    "\n",
    "        round_checkpoint = Checkpoint(acc.tolist(), used_indices, clf.state_dict(), experiment_name=experiment_name)\n",
    "        round_checkpoint.save_checkpoint(checkpoint_directory)\n",
    "\n",
    "    print('Training Completed')\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-rFh9y0M3ZVH"
   },
   "source": [
    "# MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "E-e_sDnGsC_N"
   },
   "source": [
    "## Parameter Definitions\n",
    "\n",
    "Parameters related to the specific experiment are placed here. You should examine each and modify them as needed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "0cHXLa_YsIQG"
   },
   "outputs": [],
   "source": [
    "data_set_name = \"MNIST\" # DSET NAME HERE\n",
    "dataset_root_path = '../downloaded_data/'\n",
    "net = ResNet18() # MODEL HERE\n",
    "\n",
    "# MODIFY AS NECESSARY\n",
    "logs_directory = 'distil_checkpoints/logs/'\n",
    "checkpoint_directory = 'distil_checkpoints/check/'\n",
    "# model_directory = \"/home/atharv/distil_checkpoints/model/\"\n",
    "\n",
    "experiment_name = \"MNIST DISTIL IMPLEMENTATION 4\"\n",
    "\n",
    "# Only train a new model if one does not exist.\n",
    "\n",
    "model_directory = 'distil_checkpoints/model/MNIST4'\n",
    "\n",
    "initial_seed_size = 50 # INIT SEED SIZE HERE\n",
    "training_size_cap = 1250 # TRAIN SIZE CAP HERE\n",
    "\n",
    "budget = 50 # BUDGET HERE\n",
    "\n",
    "# CHANGE ARGS AS NECESSARY\n",
    "args = {'n_epoch':300, 'lr':float(0.01), 'batch_size':32, 'max_accuracy':float(0.99), 'islogs':True, 'isreset':False, 'isverbose':True, 'device':'cuda', 'optimizer':'adam'} \n",
    "\n",
    "# Train on approximately the full dataset given the budget contraints\n",
    "n_rounds = (training_size_cap - initial_seed_size) // budget"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O0WfH3eq3nv_",
    "tags": []
   },
   "source": [
    "## Initial Loading and Training\n",
    "\n",
    "You may choose to train a new initial model or to continue to load a specific model. If this notebook is being executed in Colab, you should consider whether or not you need the gdown line."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = time.monotonic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "K1522SUk3nwF",
    "outputId": "c680275b-c646-4141-8cc5-9eb68b8bf277"
   },
   "outputs": [],
   "source": [
    "# # Mount drive containing possible saved model and define file path.\n",
    "# colab_model_storage_mount = \"/content/gdrive\"\n",
    "# drive.mount(colab_model_storage_mount)\n",
    "\n",
    "# Retrieve the model from a download link and save it to the drive\n",
    "# os.makedirs(logs_directory, exist_ok = True)\n",
    "# os.makedirs(checkpoint_directory, exist_ok = True)\n",
    "# os.makedirs(model_directory, exist_ok = True)\n",
    "# model_directory = F\"{model_directory}/{data_set_name}\"\n",
    "#!/content/gdown.pl/gdown.pl \"INSERT SHARABLE LINK HERE\" \"INSERT DOWNLOAD LOCATION HERE (ideally, same as model_directory)\" # MAY NOT NEED THIS LINE IF NOT CLONING MODEL FROM COLAB\n",
    "\n",
    "# Load the dataset\n",
    "if data_set_name == \"MNIST\":\n",
    "\n",
    "    image_dim=28\n",
    "    train_transform = transforms.Compose([transforms.RandomCrop(image_dim, padding=4), transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)), transforms.Lambda(lambda x: x.repeat(3, 1, 1))])\n",
    "    test_transform = transforms.Compose([transforms.Resize((image_dim, image_dim)), transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)), transforms.Lambda(lambda x: x.repeat(3, 1, 1))])\n",
    "\n",
    "    full_train_dataset = datasets.MNIST(dataset_root_path, download=True, train=True, transform=train_transform, target_transform=torch.tensor)\n",
    "    test_dataset = datasets.MNIST(dataset_root_path, download=True, train=False, transform=test_transform, target_transform=torch.tensor)\n",
    "\n",
    "    nclasses = 10 # NUM CLASSES HERE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training..\n",
      "50 training accuracy: 0.74\n",
      "Epoch: 88 Training accuracy: 1.0\n",
      "Training for 24 rounds with budget 50 on unlabeled set size 1250\n"
     ]
    }
   ],
   "source": [
    "args['nclasses'] = nclasses\n",
    "\n",
    "dim = full_train_dataset[0][0].shape\n",
    "\n",
    "# Seed the random number generator for reproducibility and create the initial seed set\n",
    "np.random.seed(42)\n",
    "initial_train_indices = np.random.choice(len(full_train_dataset), replace=False, size=initial_seed_size)\n",
    "\n",
    "# COMMENT OUT ONE OR THE OTHER IF YOU WANT TO TRAIN A NEW INITIAL MODEL\n",
    "load_model = False\n",
    "# load_model = True\n",
    "\n",
    "if load_model:\n",
    "    net.load_state_dict(torch.load(model_directory))\n",
    "    initial_model = net\n",
    "else:\n",
    "    dt = data_train(Subset(full_train_dataset, initial_train_indices), net, args)\n",
    "    initial_model, _ = dt.train(None)\n",
    "    torch.save(initial_model.state_dict(), model_directory)\n",
    "\n",
    "print(\"Training for\", n_rounds, \"rounds with budget\", budget, \"on unlabeled set size\", training_size_cap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nclasses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B9N-4eTMPrZZ"
   },
   "source": [
    "## Random Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "i4eKSOaiPruO",
    "outputId": "b5bb0f59-85f2-4c31-c648-0a4a7bd93482"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Testing accuracy: 48.01\n",
      "-------------------------------------------------\n",
      "Round 1\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0007522106170654297\n",
      "Number of training points - 100\n",
      "Training..\n",
      "50 training accuracy: 0.9\n",
      "100 training accuracy: 0.93\n",
      "Epoch: 101 Training accuracy: 1.0\n",
      "Testing accuracy: 84.42\n",
      "-------------------------------------------------\n",
      "Round 2\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0007653236389160156\n",
      "Number of training points - 150\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 1.0\n",
      "Testing accuracy: 90.42\n",
      "-------------------------------------------------\n",
      "Round 3\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.00101470947265625\n",
      "Number of training points - 200\n",
      "Training..\n",
      "Epoch: 28 Training accuracy: 0.995\n",
      "Testing accuracy: 91.2\n",
      "-------------------------------------------------\n",
      "Round 4\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009143352508544922\n",
      "Number of training points - 250\n",
      "Training..\n",
      "Epoch: 13 Training accuracy: 0.992\n",
      "Testing accuracy: 92.63\n",
      "-------------------------------------------------\n",
      "Round 5\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009365081787109375\n",
      "Number of training points - 300\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.993\n",
      "Testing accuracy: 90.75\n",
      "-------------------------------------------------\n",
      "Round 6\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009469985961914062\n",
      "Number of training points - 350\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.991\n",
      "Testing accuracy: 92.96\n",
      "-------------------------------------------------\n",
      "Round 7\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009093284606933594\n",
      "Number of training points - 400\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.99\n",
      "Testing accuracy: 92.12\n",
      "-------------------------------------------------\n",
      "Round 8\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009171962738037109\n",
      "Number of training points - 450\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.996\n",
      "Testing accuracy: 94.59\n",
      "-------------------------------------------------\n",
      "Round 9\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009989738464355469\n",
      "Number of training points - 500\n",
      "Training..\n",
      "Epoch: 7 Training accuracy: 0.99\n",
      "Testing accuracy: 94.55\n",
      "-------------------------------------------------\n",
      "Round 10\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009975433349609375\n",
      "Number of training points - 550\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.991\n",
      "Testing accuracy: 95.66\n",
      "-------------------------------------------------\n",
      "Round 11\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0008723735809326172\n",
      "Number of training points - 600\n",
      "Training..\n",
      "Epoch: 13 Training accuracy: 0.99\n",
      "Testing accuracy: 95.71\n",
      "-------------------------------------------------\n",
      "Round 12\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0008847713470458984\n",
      "Number of training points - 650\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.997\n",
      "Testing accuracy: 95.74\n",
      "-------------------------------------------------\n",
      "Round 13\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0008809566497802734\n",
      "Number of training points - 700\n",
      "Training..\n",
      "Epoch: 3 Training accuracy: 0.991\n",
      "Testing accuracy: 94.7\n",
      "-------------------------------------------------\n",
      "Round 14\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0008547306060791016\n",
      "Number of training points - 750\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.993\n",
      "Testing accuracy: 97.3\n",
      "-------------------------------------------------\n",
      "Round 15\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0007596015930175781\n",
      "Number of training points - 800\n",
      "Training..\n",
      "Epoch: 4 Training accuracy: 0.99\n",
      "Testing accuracy: 96.5\n",
      "-------------------------------------------------\n",
      "Round 16\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009577274322509766\n",
      "Number of training points - 850\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.994\n",
      "Testing accuracy: 96.69\n",
      "-------------------------------------------------\n",
      "Round 17\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009372234344482422\n",
      "Number of training points - 900\n",
      "Training..\n",
      "Epoch: 4 Training accuracy: 0.99\n",
      "Testing accuracy: 97.21\n",
      "-------------------------------------------------\n",
      "Round 18\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0007121562957763672\n",
      "Number of training points - 950\n",
      "Training..\n",
      "Epoch: 4 Training accuracy: 0.992\n",
      "Testing accuracy: 96.35\n",
      "-------------------------------------------------\n",
      "Round 19\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0008018016815185547\n",
      "Number of training points - 1000\n",
      "Training..\n",
      "Epoch: 5 Training accuracy: 0.99\n",
      "Testing accuracy: 96.73\n",
      "-------------------------------------------------\n",
      "Round 20\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009818077087402344\n",
      "Number of training points - 1050\n",
      "Training..\n",
      "Epoch: 3 Training accuracy: 0.99\n",
      "Testing accuracy: 96.86\n",
      "-------------------------------------------------\n",
      "Round 21\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0008256435394287109\n",
      "Number of training points - 1100\n",
      "Training..\n",
      "Epoch: 5 Training accuracy: 0.991\n",
      "Testing accuracy: 96.94\n",
      "-------------------------------------------------\n",
      "Round 22\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009927749633789062\n",
      "Number of training points - 1150\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.994\n",
      "Testing accuracy: 96.99\n",
      "-------------------------------------------------\n",
      "Round 23\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0009374618530273438\n",
      "Number of training points - 1200\n",
      "Training..\n",
      "Epoch: 4 Training accuracy: 0.994\n",
      "Testing accuracy: 97.72\n",
      "-------------------------------------------------\n",
      "Round 24\n",
      "-------------------------------------------------\n",
      "Selection Time: 0.0007061958312988281\n",
      "Number of training points - 1250\n",
      "Training..\n",
      "Epoch: 2 Training accuracy: 0.99\n",
      "Testing accuracy: 96.53\n",
      "Training Completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4801, 0.8442, 0.9042, 0.912 , 0.9263, 0.9075, 0.9296, 0.9212,\n",
       "       0.9459, 0.9455, 0.9566, 0.9571, 0.9574, 0.947 , 0.973 , 0.965 ,\n",
       "       0.9669, 0.9721, 0.9635, 0.9673, 0.9686, 0.9694, 0.9699, 0.9772,\n",
       "       0.9653])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strategy = \"random\"\n",
    "strat_logs = logs_directory+F'{data_set_name}/{strategy}/'\n",
    "os.makedirs(strat_logs, exist_ok = True)\n",
    "train_one(full_train_dataset, initial_train_indices, test_dataset, copy.deepcopy(initial_model), n_rounds, budget, args, nclasses, strategy, strat_logs, checkpoint_directory, F\"{experiment_name}_{strategy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bg1XH87hPsCe"
   },
   "source": [
    "## Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mRAKMe2RPsTp",
    "outputId": "9bdcc56b-b907-4b84-fe25-3a43cb4673c7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Testing accuracy: 48.01\n",
      "-------------------------------------------------\n",
      "Round 1\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.68111228942871\n",
      "Number of training points - 100\n",
      "Training..\n",
      "50 training accuracy: 0.91\n",
      "Epoch: 58 Training accuracy: 1.0\n",
      "Testing accuracy: 76.53\n",
      "-------------------------------------------------\n",
      "Round 2\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.767117261886597\n",
      "Number of training points - 150\n",
      "Training..\n",
      "Epoch: 27 Training accuracy: 0.993\n",
      "Testing accuracy: 85.14\n",
      "-------------------------------------------------\n",
      "Round 3\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.32067370414734\n",
      "Number of training points - 200\n",
      "Training..\n",
      "Epoch: 26 Training accuracy: 0.99\n",
      "Testing accuracy: 90.64\n",
      "-------------------------------------------------\n",
      "Round 4\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.91009521484375\n",
      "Number of training points - 250\n",
      "Training..\n",
      "Epoch: 27 Training accuracy: 0.992\n",
      "Testing accuracy: 90.12\n",
      "-------------------------------------------------\n",
      "Round 5\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.7394015789032\n",
      "Number of training points - 300\n",
      "Training..\n",
      "Epoch: 12 Training accuracy: 0.99\n",
      "Testing accuracy: 95.14\n",
      "-------------------------------------------------\n",
      "Round 6\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.830536603927612\n",
      "Number of training points - 350\n",
      "Training..\n",
      "Epoch: 33 Training accuracy: 0.991\n",
      "Testing accuracy: 94.6\n",
      "-------------------------------------------------\n",
      "Round 7\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.61891746520996\n",
      "Number of training points - 400\n",
      "Training..\n",
      "Epoch: 7 Training accuracy: 0.993\n",
      "Testing accuracy: 96.37\n",
      "-------------------------------------------------\n",
      "Round 8\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.6986346244812\n",
      "Number of training points - 450\n",
      "Training..\n",
      "Epoch: 21 Training accuracy: 0.991\n",
      "Testing accuracy: 97.48\n",
      "-------------------------------------------------\n",
      "Round 9\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.801111221313477\n",
      "Number of training points - 500\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.99\n",
      "Testing accuracy: 97.37\n",
      "-------------------------------------------------\n",
      "Round 10\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.757136583328247\n",
      "Number of training points - 550\n",
      "Training..\n",
      "Epoch: 28 Training accuracy: 0.993\n",
      "Testing accuracy: 98.1\n",
      "-------------------------------------------------\n",
      "Round 11\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.80772042274475\n",
      "Number of training points - 600\n",
      "Training..\n",
      "Epoch: 28 Training accuracy: 0.993\n",
      "Testing accuracy: 98.1\n",
      "-------------------------------------------------\n",
      "Round 12\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.859212398529053\n",
      "Number of training points - 650\n",
      "Training..\n",
      "Epoch: 25 Training accuracy: 0.994\n",
      "Testing accuracy: 98.06\n",
      "-------------------------------------------------\n",
      "Round 13\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.9264395236969\n",
      "Number of training points - 700\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.99\n",
      "Testing accuracy: 97.69\n",
      "-------------------------------------------------\n",
      "Round 14\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.918351411819458\n",
      "Number of training points - 750\n",
      "Training..\n",
      "Epoch: 19 Training accuracy: 0.995\n",
      "Testing accuracy: 98.7\n",
      "-------------------------------------------------\n",
      "Round 15\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.960086584091187\n",
      "Number of training points - 800\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.993\n",
      "Testing accuracy: 98.53\n",
      "-------------------------------------------------\n",
      "Round 16\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.721630811691284\n",
      "Number of training points - 850\n",
      "Training..\n",
      "Epoch: 17 Training accuracy: 0.991\n",
      "Testing accuracy: 98.47\n",
      "-------------------------------------------------\n",
      "Round 17\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.484690189361572\n",
      "Number of training points - 900\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.992\n",
      "Testing accuracy: 98.83\n",
      "-------------------------------------------------\n",
      "Round 18\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.849460124969482\n",
      "Number of training points - 950\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.991\n",
      "Testing accuracy: 99.02\n",
      "-------------------------------------------------\n",
      "Round 19\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.791348934173584\n",
      "Number of training points - 1000\n",
      "Training..\n",
      "Epoch: 12 Training accuracy: 0.991\n",
      "Testing accuracy: 98.96\n",
      "-------------------------------------------------\n",
      "Round 20\n",
      "-------------------------------------------------\n",
      "Selection Time: 18.654710054397583\n",
      "Number of training points - 1050\n",
      "Training..\n",
      "Epoch: 24 Training accuracy: 0.995\n",
      "Testing accuracy: 99.24\n",
      "-------------------------------------------------\n",
      "Round 21\n",
      "-------------------------------------------------\n",
      "Selection Time: 18.574126958847046\n",
      "Number of training points - 1100\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.992\n",
      "Testing accuracy: 99.06\n",
      "-------------------------------------------------\n",
      "Round 22\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.588383674621582\n",
      "Number of training points - 1150\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.995\n",
      "Testing accuracy: 99.08\n",
      "-------------------------------------------------\n",
      "Round 23\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.110356092453003\n",
      "Number of training points - 1200\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.994\n",
      "Testing accuracy: 98.91\n",
      "-------------------------------------------------\n",
      "Round 24\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.754735708236694\n",
      "Number of training points - 1250\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.99\n",
      "Testing accuracy: 99.22\n",
      "Training Completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4801, 0.7653, 0.8514, 0.9064, 0.9012, 0.9514, 0.946 , 0.9637,\n",
       "       0.9748, 0.9737, 0.981 , 0.981 , 0.9806, 0.9769, 0.987 , 0.9853,\n",
       "       0.9847, 0.9883, 0.9902, 0.9896, 0.9924, 0.9906, 0.9908, 0.9891,\n",
       "       0.9922])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strategy = \"entropy\"\n",
    "strat_logs = logs_directory+F'{data_set_name}/{strategy}/'\n",
    "os.makedirs(strat_logs, exist_ok = True)\n",
    "train_one(full_train_dataset, initial_train_indices, test_dataset, copy.deepcopy(initial_model), n_rounds, budget, args, nclasses, strategy, strat_logs, checkpoint_directory, F\"{experiment_name}_{strategy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NkMYHHwyP5Bd",
    "tags": []
   },
   "source": [
    "## GLISTER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 432
    },
    "id": "D0KstkZWP5fT",
    "outputId": "84121e03-d684-43ca-8d2a-4af051cf37d6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Testing accuracy: 48.01\n",
      "-------------------------------------------------\n",
      "Round 1\n",
      "-------------------------------------------------\n",
      "Selection Time: 77.3955557346344\n",
      "Number of training points - 100\n",
      "Training..\n",
      "50 training accuracy: 0.78\n",
      "Epoch: 98 Training accuracy: 1.0\n",
      "Testing accuracy: 71.1\n",
      "-------------------------------------------------\n",
      "Round 2\n",
      "-------------------------------------------------\n",
      "Selection Time: 72.95324802398682\n",
      "Number of training points - 150\n",
      "Training..\n",
      "Epoch: 28 Training accuracy: 0.993\n",
      "Testing accuracy: 86.09\n",
      "-------------------------------------------------\n",
      "Round 3\n",
      "-------------------------------------------------\n",
      "Selection Time: 72.91101479530334\n",
      "Number of training points - 200\n",
      "Training..\n",
      "Epoch: 27 Training accuracy: 0.995\n",
      "Testing accuracy: 89.2\n",
      "-------------------------------------------------\n",
      "Round 4\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.02025389671326\n",
      "Number of training points - 250\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.992\n",
      "Testing accuracy: 91.41\n",
      "-------------------------------------------------\n",
      "Round 5\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.53708934783936\n",
      "Number of training points - 300\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.997\n",
      "Testing accuracy: 93.35\n",
      "-------------------------------------------------\n",
      "Round 6\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.65018558502197\n",
      "Number of training points - 350\n",
      "Training..\n",
      "Epoch: 21 Training accuracy: 0.991\n",
      "Testing accuracy: 90.75\n",
      "-------------------------------------------------\n",
      "Round 7\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.7317247390747\n",
      "Number of training points - 400\n",
      "Training..\n",
      "Epoch: 17 Training accuracy: 0.993\n",
      "Testing accuracy: 93.87\n",
      "-------------------------------------------------\n",
      "Round 8\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.86955833435059\n",
      "Number of training points - 450\n",
      "Training..\n",
      "Epoch: 25 Training accuracy: 0.991\n",
      "Testing accuracy: 96.07\n",
      "-------------------------------------------------\n",
      "Round 9\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.74392294883728\n",
      "Number of training points - 500\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.992\n",
      "Testing accuracy: 96.72\n",
      "-------------------------------------------------\n",
      "Round 10\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.88196229934692\n",
      "Number of training points - 550\n",
      "Training..\n",
      "Epoch: 18 Training accuracy: 0.991\n",
      "Testing accuracy: 97.05\n",
      "-------------------------------------------------\n",
      "Round 11\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.72958183288574\n",
      "Number of training points - 600\n",
      "Training..\n",
      "Epoch: 13 Training accuracy: 0.993\n",
      "Testing accuracy: 97.07\n",
      "-------------------------------------------------\n",
      "Round 12\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.85044264793396\n",
      "Number of training points - 650\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.991\n",
      "Testing accuracy: 97.48\n",
      "-------------------------------------------------\n",
      "Round 13\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.86135244369507\n",
      "Number of training points - 700\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.993\n",
      "Testing accuracy: 98.41\n",
      "-------------------------------------------------\n",
      "Round 14\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.86749172210693\n",
      "Number of training points - 750\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.996\n",
      "Testing accuracy: 98.53\n",
      "-------------------------------------------------\n",
      "Round 15\n",
      "-------------------------------------------------\n",
      "Selection Time: 74.04784870147705\n",
      "Number of training points - 800\n",
      "Training..\n",
      "Epoch: 12 Training accuracy: 0.993\n",
      "Testing accuracy: 98.65\n",
      "-------------------------------------------------\n",
      "Round 16\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.90284585952759\n",
      "Number of training points - 850\n",
      "Training..\n",
      "Epoch: 19 Training accuracy: 0.993\n",
      "Testing accuracy: 98.52\n",
      "-------------------------------------------------\n",
      "Round 17\n",
      "-------------------------------------------------\n",
      "Selection Time: 74.12188243865967\n",
      "Number of training points - 900\n",
      "Training..\n",
      "Epoch: 16 Training accuracy: 0.99\n",
      "Testing accuracy: 98.86\n",
      "-------------------------------------------------\n",
      "Round 18\n",
      "-------------------------------------------------\n",
      "Selection Time: 74.1294264793396\n",
      "Number of training points - 950\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.991\n",
      "Testing accuracy: 98.39\n",
      "-------------------------------------------------\n",
      "Round 19\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.84829545021057\n",
      "Number of training points - 1000\n",
      "Training..\n",
      "Epoch: 20 Training accuracy: 0.99\n",
      "Testing accuracy: 98.83\n",
      "-------------------------------------------------\n",
      "Round 20\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.95661163330078\n",
      "Number of training points - 1050\n",
      "Training..\n",
      "Epoch: 4 Training accuracy: 0.99\n",
      "Testing accuracy: 98.31\n",
      "-------------------------------------------------\n",
      "Round 21\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.63000440597534\n",
      "Number of training points - 1100\n",
      "Training..\n",
      "Epoch: 5 Training accuracy: 0.993\n",
      "Testing accuracy: 99.02\n",
      "-------------------------------------------------\n",
      "Round 22\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.52579689025879\n",
      "Number of training points - 1150\n",
      "Training..\n",
      "Epoch: 5 Training accuracy: 0.994\n",
      "Testing accuracy: 98.86\n",
      "-------------------------------------------------\n",
      "Round 23\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.27287220954895\n",
      "Number of training points - 1200\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.994\n",
      "Testing accuracy: 99.03\n",
      "-------------------------------------------------\n",
      "Round 24\n",
      "-------------------------------------------------\n",
      "Selection Time: 73.46849155426025\n",
      "Number of training points - 1250\n",
      "Training..\n",
      "Epoch: 22 Training accuracy: 0.995\n",
      "Testing accuracy: 99.06\n",
      "Training Completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4801, 0.711 , 0.8609, 0.892 , 0.9141, 0.9335, 0.9075, 0.9387,\n",
       "       0.9607, 0.9672, 0.9705, 0.9707, 0.9748, 0.9841, 0.9853, 0.9865,\n",
       "       0.9852, 0.9886, 0.9839, 0.9883, 0.9831, 0.9902, 0.9886, 0.9903,\n",
       "       0.9906])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strategy = \"glister\"\n",
    "strat_logs = logs_directory+F'{data_set_name}/{strategy}/'\n",
    "os.makedirs(strat_logs, exist_ok = True)\n",
    "train_one(full_train_dataset, initial_train_indices, test_dataset, copy.deepcopy(initial_model), n_rounds, budget, args, nclasses, strategy, strat_logs, checkpoint_directory, F\"{experiment_name}_{strategy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AQmHTnnOP9KU"
   },
   "source": [
    "## FASS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tJSuDWowP9fD",
    "outputId": "ee837817-970b-4e58-c971-b372888b8feb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Testing accuracy: 48.01\n",
      "-------------------------------------------------\n",
      "Round 1\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.80303406715393\n",
      "Number of training points - 100\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50 training accuracy: 0.92\n",
      "Epoch: 94 Training accuracy: 0.99\n",
      "Testing accuracy: 72.41\n",
      "-------------------------------------------------\n",
      "Round 2\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.40721035003662\n",
      "Number of training points - 150\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Training accuracy: 1.0\n",
      "Testing accuracy: 88.47\n",
      "-------------------------------------------------\n",
      "Round 3\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.941866397857666\n",
      "Number of training points - 200\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48 Training accuracy: 0.99\n",
      "Testing accuracy: 92.07\n",
      "-------------------------------------------------\n",
      "Round 4\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.908734798431396\n",
      "Number of training points - 250\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Training accuracy: 0.996\n",
      "Testing accuracy: 94.7\n",
      "-------------------------------------------------\n",
      "Round 5\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.232004642486572\n",
      "Number of training points - 300\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training accuracy: 0.99\n",
      "Testing accuracy: 95.74\n",
      "-------------------------------------------------\n",
      "Round 6\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.877830028533936\n",
      "Number of training points - 350\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 Training accuracy: 0.991\n",
      "Testing accuracy: 95.88\n",
      "-------------------------------------------------\n",
      "Round 7\n",
      "-------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selection Time: 22.92329716682434\n",
      "Number of training points - 400\n",
      "Training..\n",
      "Epoch: 34 Training accuracy: 0.99\n",
      "Testing accuracy: 97.87\n",
      "-------------------------------------------------\n",
      "Round 8\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.426288843154907\n",
      "Number of training points - 450\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Training accuracy: 0.991\n",
      "Testing accuracy: 98.06\n",
      "-------------------------------------------------\n",
      "Round 9\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.77938151359558\n",
      "Number of training points - 500\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 Training accuracy: 0.992\n",
      "Testing accuracy: 98.24\n",
      "-------------------------------------------------\n",
      "Round 10\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.502869129180908\n",
      "Number of training points - 550\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 Training accuracy: 0.996\n",
      "Testing accuracy: 98.54\n",
      "-------------------------------------------------\n",
      "Round 11\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.085646152496338\n",
      "Number of training points - 600\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training accuracy: 0.993\n",
      "Testing accuracy: 98.08\n",
      "-------------------------------------------------\n",
      "Round 12\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.88173484802246\n",
      "Number of training points - 650\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training accuracy: 0.995\n",
      "Testing accuracy: 98.23\n",
      "-------------------------------------------------\n",
      "Round 13\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.232613563537598\n",
      "Number of training points - 700\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Training accuracy: 0.994\n",
      "Testing accuracy: 98.69\n",
      "-------------------------------------------------\n",
      "Round 14\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.85296082496643\n",
      "Number of training points - 750\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 Training accuracy: 0.997\n",
      "Testing accuracy: 99.05\n",
      "-------------------------------------------------\n",
      "Round 15\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.13641905784607\n",
      "Number of training points - 800\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Training accuracy: 0.991\n",
      "Testing accuracy: 98.95\n",
      "-------------------------------------------------\n",
      "Round 16\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.530940532684326\n",
      "Number of training points - 850\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 Training accuracy: 0.996\n",
      "Testing accuracy: 98.5\n",
      "-------------------------------------------------\n",
      "Round 17\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.868223667144775\n",
      "Number of training points - 900\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 Training accuracy: 0.991\n",
      "Testing accuracy: 99.2\n",
      "-------------------------------------------------\n",
      "Round 18\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.79943871498108\n",
      "Number of training points - 950\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 Training accuracy: 0.993\n",
      "Testing accuracy: 98.95\n",
      "-------------------------------------------------\n",
      "Round 19\n",
      "-------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selection Time: 18.921785354614258\n",
      "Number of training points - 1000\n",
      "Training..\n",
      "Epoch: 15 Training accuracy: 0.99\n",
      "Testing accuracy: 99.09\n",
      "-------------------------------------------------\n",
      "Round 20\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.811479330062866\n",
      "Number of training points - 1050\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 Training accuracy: 0.993\n",
      "Testing accuracy: 99.03\n",
      "-------------------------------------------------\n",
      "Round 21\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.66963505744934\n",
      "Number of training points - 1100\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 Training accuracy: 0.994\n",
      "Testing accuracy: 99.1\n",
      "-------------------------------------------------\n",
      "Round 22\n",
      "-------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selection Time: 20.73637366294861\n",
      "Number of training points - 1150\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.993\n",
      "Testing accuracy: 99.25\n",
      "-------------------------------------------------\n",
      "Round 23\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.999318838119507\n",
      "Number of training points - 1200\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 Training accuracy: 0.99\n",
      "Testing accuracy: 99.33\n",
      "-------------------------------------------------\n",
      "Round 24\n",
      "-------------------------------------------------\n",
      "Selection Time: 18.458323001861572\n",
      "Number of training points - 1250\n",
      "Training..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[||||||||||||||||||||]100% [Iteration 50 of 50]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Training accuracy: 0.993\n",
      "Testing accuracy: 99.21\n",
      "Training Completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4801, 0.7241, 0.8847, 0.9207, 0.947 , 0.9574, 0.9588, 0.9787,\n",
       "       0.9806, 0.9824, 0.9854, 0.9808, 0.9823, 0.9869, 0.9905, 0.9895,\n",
       "       0.985 , 0.992 , 0.9895, 0.9909, 0.9903, 0.991 , 0.9925, 0.9933,\n",
       "       0.9921])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strategy = \"fass\"\n",
    "strat_logs = logs_directory+F'{data_set_name}/{strategy}/'\n",
    "os.makedirs(strat_logs, exist_ok = True)\n",
    "train_one(full_train_dataset, initial_train_indices, test_dataset, copy.deepcopy(initial_model), n_rounds, budget, args, 10, strategy, strat_logs, checkpoint_directory, F\"{experiment_name}_{strategy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6ZSiRahu3nwK"
   },
   "source": [
    "## BADGE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "b5c8AckN3nwK",
    "outputId": "63fb1411-4ede-4074-c8f3-43d7c0b98636"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Testing accuracy: 48.01\n",
      "-------------------------------------------------\n",
      "Round 1\n",
      "-------------------------------------------------\n",
      "Selection Time: 36.72698473930359\n",
      "Number of training points - 100\n",
      "Training..\n",
      "50 training accuracy: 0.91\n",
      "Epoch: 63 Training accuracy: 0.99\n",
      "Testing accuracy: 79.03\n",
      "-------------------------------------------------\n",
      "Round 2\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.39088034629822\n",
      "Number of training points - 150\n",
      "Training..\n",
      "Epoch: 19 Training accuracy: 1.0\n",
      "Testing accuracy: 87.02\n",
      "-------------------------------------------------\n",
      "Round 3\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.40498423576355\n",
      "Number of training points - 200\n",
      "Training..\n",
      "Epoch: 32 Training accuracy: 0.99\n",
      "Testing accuracy: 92.33\n",
      "-------------------------------------------------\n",
      "Round 4\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.49461555480957\n",
      "Number of training points - 250\n",
      "Training..\n",
      "Epoch: 17 Training accuracy: 0.992\n",
      "Testing accuracy: 95.51\n",
      "-------------------------------------------------\n",
      "Round 5\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.312052726745605\n",
      "Number of training points - 300\n",
      "Training..\n",
      "Epoch: 15 Training accuracy: 0.997\n",
      "Testing accuracy: 96.25\n",
      "-------------------------------------------------\n",
      "Round 6\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.383562088012695\n",
      "Number of training points - 350\n",
      "Training..\n",
      "Epoch: 22 Training accuracy: 0.994\n",
      "Testing accuracy: 97.0\n",
      "-------------------------------------------------\n",
      "Round 7\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.00301790237427\n",
      "Number of training points - 400\n",
      "Training..\n",
      "Epoch: 13 Training accuracy: 0.99\n",
      "Testing accuracy: 96.9\n",
      "-------------------------------------------------\n",
      "Round 8\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.06913876533508\n",
      "Number of training points - 450\n",
      "Training..\n",
      "Epoch: 40 Training accuracy: 0.993\n",
      "Testing accuracy: 97.97\n",
      "-------------------------------------------------\n",
      "Round 9\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.09768724441528\n",
      "Number of training points - 500\n",
      "Training..\n",
      "Epoch: 19 Training accuracy: 0.992\n",
      "Testing accuracy: 97.73\n",
      "-------------------------------------------------\n",
      "Round 10\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.006465911865234\n",
      "Number of training points - 550\n",
      "Training..\n",
      "Epoch: 21 Training accuracy: 0.993\n",
      "Testing accuracy: 98.43\n",
      "-------------------------------------------------\n",
      "Round 11\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.00904941558838\n",
      "Number of training points - 600\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.995\n",
      "Testing accuracy: 98.59\n",
      "-------------------------------------------------\n",
      "Round 12\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.22048997879028\n",
      "Number of training points - 650\n",
      "Training..\n",
      "Epoch: 21 Training accuracy: 0.991\n",
      "Testing accuracy: 98.48\n",
      "-------------------------------------------------\n",
      "Round 13\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.3409059047699\n",
      "Number of training points - 700\n",
      "Training..\n",
      "Epoch: 12 Training accuracy: 0.991\n",
      "Testing accuracy: 98.51\n",
      "-------------------------------------------------\n",
      "Round 14\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.27739429473877\n",
      "Number of training points - 750\n",
      "Training..\n",
      "Epoch: 12 Training accuracy: 0.992\n",
      "Testing accuracy: 98.63\n",
      "-------------------------------------------------\n",
      "Round 15\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.110997676849365\n",
      "Number of training points - 800\n",
      "Training..\n",
      "Epoch: 17 Training accuracy: 0.993\n",
      "Testing accuracy: 98.86\n",
      "-------------------------------------------------\n",
      "Round 16\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.062365770339966\n",
      "Number of training points - 850\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.995\n",
      "Testing accuracy: 98.99\n",
      "-------------------------------------------------\n",
      "Round 17\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.116422176361084\n",
      "Number of training points - 900\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.993\n",
      "Testing accuracy: 99.05\n",
      "-------------------------------------------------\n",
      "Round 18\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.041098833084106\n",
      "Number of training points - 950\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.995\n",
      "Testing accuracy: 99.05\n",
      "-------------------------------------------------\n",
      "Round 19\n",
      "-------------------------------------------------\n",
      "Selection Time: 32.17426133155823\n",
      "Number of training points - 1000\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.992\n",
      "Testing accuracy: 99.22\n",
      "-------------------------------------------------\n",
      "Round 20\n",
      "-------------------------------------------------\n",
      "Selection Time: 31.939390182495117\n",
      "Number of training points - 1050\n",
      "Training..\n",
      "Epoch: 7 Training accuracy: 0.993\n",
      "Testing accuracy: 99.11\n",
      "-------------------------------------------------\n",
      "Round 21\n",
      "-------------------------------------------------\n",
      "Selection Time: 31.903162956237793\n",
      "Number of training points - 1100\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.995\n",
      "Testing accuracy: 99.36\n",
      "-------------------------------------------------\n",
      "Round 22\n",
      "-------------------------------------------------\n",
      "Selection Time: 31.97156047821045\n",
      "Number of training points - 1150\n",
      "Training..\n",
      "Epoch: 4 Training accuracy: 0.993\n",
      "Testing accuracy: 99.38\n",
      "-------------------------------------------------\n",
      "Round 23\n",
      "-------------------------------------------------\n",
      "Selection Time: 31.886812686920166\n",
      "Number of training points - 1200\n",
      "Training..\n",
      "Epoch: 16 Training accuracy: 0.991\n",
      "Testing accuracy: 99.21\n",
      "-------------------------------------------------\n",
      "Round 24\n",
      "-------------------------------------------------\n",
      "Selection Time: 31.90269684791565\n",
      "Number of training points - 1250\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.994\n",
      "Testing accuracy: 99.35\n",
      "Training Completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4801, 0.7903, 0.8702, 0.9233, 0.9551, 0.9625, 0.97  , 0.969 ,\n",
       "       0.9797, 0.9773, 0.9843, 0.9859, 0.9848, 0.9851, 0.9863, 0.9886,\n",
       "       0.9899, 0.9905, 0.9905, 0.9922, 0.9911, 0.9936, 0.9938, 0.9921,\n",
       "       0.9935])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strategy = \"badge\"\n",
    "strat_logs = logs_directory+F'{data_set_name}/{strategy}/'\n",
    "os.makedirs(strat_logs, exist_ok = True)\n",
    "train_one(full_train_dataset, initial_train_indices, test_dataset, copy.deepcopy(initial_model), n_rounds, budget, args, nclasses, strategy, strat_logs, checkpoint_directory, F\"{experiment_name}_{strategy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RQeyff_gWp2E"
   },
   "source": [
    "## CoreSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "T4osHfoHWp2F"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Testing accuracy: 48.01\n",
      "-------------------------------------------------\n",
      "Round 1\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.748229265213013\n",
      "Number of training points - 100\n",
      "Training..\n",
      "50 training accuracy: 0.9\n",
      "Epoch: 60 Training accuracy: 0.99\n",
      "Testing accuracy: 85.09\n",
      "-------------------------------------------------\n",
      "Round 2\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.747932434082031\n",
      "Number of training points - 150\n",
      "Training..\n",
      "Epoch: 24 Training accuracy: 0.993\n",
      "Testing accuracy: 88.68\n",
      "-------------------------------------------------\n",
      "Round 3\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.782348155975342\n",
      "Number of training points - 200\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.99\n",
      "Testing accuracy: 82.51\n",
      "-------------------------------------------------\n",
      "Round 4\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.821179866790771\n",
      "Number of training points - 250\n",
      "Training..\n",
      "Epoch: 34 Training accuracy: 0.996\n",
      "Testing accuracy: 91.11\n",
      "-------------------------------------------------\n",
      "Round 5\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.750189542770386\n",
      "Number of training points - 300\n",
      "Training..\n",
      "Epoch: 22 Training accuracy: 0.99\n",
      "Testing accuracy: 95.44\n",
      "-------------------------------------------------\n",
      "Round 6\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.785975933074951\n",
      "Number of training points - 350\n",
      "Training..\n",
      "Epoch: 20 Training accuracy: 0.991\n",
      "Testing accuracy: 95.49\n",
      "-------------------------------------------------\n",
      "Round 7\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.756095886230469\n",
      "Number of training points - 400\n",
      "Training..\n",
      "Epoch: 23 Training accuracy: 0.993\n",
      "Testing accuracy: 95.59\n",
      "-------------------------------------------------\n",
      "Round 8\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.820149183273315\n",
      "Number of training points - 450\n",
      "Training..\n",
      "Epoch: 15 Training accuracy: 0.996\n",
      "Testing accuracy: 96.89\n",
      "-------------------------------------------------\n",
      "Round 9\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.806877851486206\n",
      "Number of training points - 500\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.992\n",
      "Testing accuracy: 96.97\n",
      "-------------------------------------------------\n",
      "Round 10\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.790761947631836\n",
      "Number of training points - 550\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.993\n",
      "Testing accuracy: 97.66\n",
      "-------------------------------------------------\n",
      "Round 11\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.769989728927612\n",
      "Number of training points - 600\n",
      "Training..\n",
      "Epoch: 19 Training accuracy: 0.99\n",
      "Testing accuracy: 97.48\n",
      "-------------------------------------------------\n",
      "Round 12\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.823616981506348\n",
      "Number of training points - 650\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.991\n",
      "Testing accuracy: 97.33\n",
      "-------------------------------------------------\n",
      "Round 13\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.823919534683228\n",
      "Number of training points - 700\n",
      "Training..\n",
      "Epoch: 20 Training accuracy: 0.993\n",
      "Testing accuracy: 98.25\n",
      "-------------------------------------------------\n",
      "Round 14\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.837539196014404\n",
      "Number of training points - 750\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.993\n",
      "Testing accuracy: 98.22\n",
      "-------------------------------------------------\n",
      "Round 15\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.822772979736328\n",
      "Number of training points - 800\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.991\n",
      "Testing accuracy: 98.15\n",
      "-------------------------------------------------\n",
      "Round 16\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.799297094345093\n",
      "Number of training points - 850\n",
      "Training..\n",
      "Epoch: 7 Training accuracy: 0.991\n",
      "Testing accuracy: 98.59\n",
      "-------------------------------------------------\n",
      "Round 17\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.83381986618042\n",
      "Number of training points - 900\n",
      "Training..\n",
      "Epoch: 18 Training accuracy: 0.992\n",
      "Testing accuracy: 98.65\n",
      "-------------------------------------------------\n",
      "Round 18\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.802804231643677\n",
      "Number of training points - 950\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.993\n",
      "Testing accuracy: 98.31\n",
      "-------------------------------------------------\n",
      "Round 19\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.8774573802948\n",
      "Number of training points - 1000\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.993\n",
      "Testing accuracy: 98.56\n",
      "-------------------------------------------------\n",
      "Round 20\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.831302881240845\n",
      "Number of training points - 1050\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.992\n",
      "Testing accuracy: 97.98\n",
      "-------------------------------------------------\n",
      "Round 21\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.811014652252197\n",
      "Number of training points - 1100\n",
      "Training..\n",
      "Epoch: 5 Training accuracy: 0.993\n",
      "Testing accuracy: 98.52\n",
      "-------------------------------------------------\n",
      "Round 22\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.846909046173096\n",
      "Number of training points - 1150\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.993\n",
      "Testing accuracy: 98.56\n",
      "-------------------------------------------------\n",
      "Round 23\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.806569814682007\n",
      "Number of training points - 1200\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.99\n",
      "Testing accuracy: 97.61\n",
      "-------------------------------------------------\n",
      "Round 24\n",
      "-------------------------------------------------\n",
      "Selection Time: 13.921243667602539\n",
      "Number of training points - 1250\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.991\n",
      "Testing accuracy: 98.45\n",
      "Training Completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4801, 0.8509, 0.8868, 0.8251, 0.9111, 0.9544, 0.9549, 0.9559,\n",
       "       0.9689, 0.9697, 0.9766, 0.9748, 0.9733, 0.9825, 0.9822, 0.9815,\n",
       "       0.9859, 0.9865, 0.9831, 0.9856, 0.9798, 0.9852, 0.9856, 0.9761,\n",
       "       0.9845])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strategy = \"coreset\"\n",
    "strat_logs = logs_directory+F'{data_set_name}/{strategy}/'\n",
    "os.makedirs(strat_logs, exist_ok = True)\n",
    "train_one(full_train_dataset, initial_train_indices, test_dataset, copy.deepcopy(initial_model), n_rounds, budget, args, nclasses, strategy, strat_logs, checkpoint_directory, F\"{experiment_name}_{strategy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TwlszjoBWqMW"
   },
   "source": [
    "## Least Confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "myi_zJ2UWqMW"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Testing accuracy: 48.01\n",
      "-------------------------------------------------\n",
      "Round 1\n",
      "-------------------------------------------------\n",
      "Selection Time: 23.25375533103943\n",
      "Number of training points - 100\n",
      "Training..\n",
      "50 training accuracy: 0.9\n",
      "Epoch: 72 Training accuracy: 0.99\n",
      "Testing accuracy: 57.55\n",
      "-------------------------------------------------\n",
      "Round 2\n",
      "-------------------------------------------------\n",
      "Selection Time: 23.06514883041382\n",
      "Number of training points - 150\n",
      "Training..\n",
      "Epoch: 35 Training accuracy: 0.993\n",
      "Testing accuracy: 89.98\n",
      "-------------------------------------------------\n",
      "Round 3\n",
      "-------------------------------------------------\n",
      "Selection Time: 23.014482498168945\n",
      "Number of training points - 200\n",
      "Training..\n",
      "Epoch: 42 Training accuracy: 0.99\n",
      "Testing accuracy: 86.61\n",
      "-------------------------------------------------\n",
      "Round 4\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.961421251296997\n",
      "Number of training points - 250\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.996\n",
      "Testing accuracy: 90.84\n",
      "-------------------------------------------------\n",
      "Round 5\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.91942596435547\n",
      "Number of training points - 300\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.997\n",
      "Testing accuracy: 96.31\n",
      "-------------------------------------------------\n",
      "Round 6\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.826937198638916\n",
      "Number of training points - 350\n",
      "Training..\n",
      "Epoch: 26 Training accuracy: 0.991\n",
      "Testing accuracy: 95.0\n",
      "-------------------------------------------------\n",
      "Round 7\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.942938804626465\n",
      "Number of training points - 400\n",
      "Training..\n",
      "Epoch: 21 Training accuracy: 0.993\n",
      "Testing accuracy: 97.71\n",
      "-------------------------------------------------\n",
      "Round 8\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.974841117858887\n",
      "Number of training points - 450\n",
      "Training..\n",
      "Epoch: 28 Training accuracy: 0.993\n",
      "Testing accuracy: 98.01\n",
      "-------------------------------------------------\n",
      "Round 9\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.092551708221436\n",
      "Number of training points - 500\n",
      "Training..\n",
      "Epoch: 32 Training accuracy: 0.996\n",
      "Testing accuracy: 97.84\n",
      "-------------------------------------------------\n",
      "Round 10\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.17208218574524\n",
      "Number of training points - 550\n",
      "Training..\n",
      "Epoch: 31 Training accuracy: 0.995\n",
      "Testing accuracy: 98.32\n",
      "-------------------------------------------------\n",
      "Round 11\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.727576732635498\n",
      "Number of training points - 600\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.992\n",
      "Testing accuracy: 98.74\n",
      "-------------------------------------------------\n",
      "Round 12\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.588507413864136\n",
      "Number of training points - 650\n",
      "Training..\n",
      "Epoch: 13 Training accuracy: 0.994\n",
      "Testing accuracy: 98.81\n",
      "-------------------------------------------------\n",
      "Round 13\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.095683097839355\n",
      "Number of training points - 700\n",
      "Training..\n",
      "Epoch: 24 Training accuracy: 0.991\n",
      "Testing accuracy: 99.0\n",
      "-------------------------------------------------\n",
      "Round 14\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.67910122871399\n",
      "Number of training points - 750\n",
      "Training..\n",
      "Epoch: 11 Training accuracy: 0.992\n",
      "Testing accuracy: 98.68\n",
      "-------------------------------------------------\n",
      "Round 15\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.916128873825073\n",
      "Number of training points - 800\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.995\n",
      "Testing accuracy: 99.06\n",
      "-------------------------------------------------\n",
      "Round 16\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.728395700454712\n",
      "Number of training points - 850\n",
      "Training..\n",
      "Epoch: 16 Training accuracy: 0.995\n",
      "Testing accuracy: 99.12\n",
      "-------------------------------------------------\n",
      "Round 17\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.789413452148438\n",
      "Number of training points - 900\n",
      "Training..\n",
      "Epoch: 18 Training accuracy: 0.99\n",
      "Testing accuracy: 99.05\n",
      "-------------------------------------------------\n",
      "Round 18\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.135721445083618\n",
      "Number of training points - 950\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.997\n",
      "Testing accuracy: 99.21\n",
      "-------------------------------------------------\n",
      "Round 19\n",
      "-------------------------------------------------\n",
      "Selection Time: 18.80766010284424\n",
      "Number of training points - 1000\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.994\n",
      "Testing accuracy: 99.01\n",
      "-------------------------------------------------\n",
      "Round 20\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.501537322998047\n",
      "Number of training points - 1050\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.995\n",
      "Testing accuracy: 99.04\n",
      "-------------------------------------------------\n",
      "Round 21\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.135730266571045\n",
      "Number of training points - 1100\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.995\n",
      "Testing accuracy: 99.27\n",
      "-------------------------------------------------\n",
      "Round 22\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.026052713394165\n",
      "Number of training points - 1150\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.991\n",
      "Testing accuracy: 99.16\n",
      "-------------------------------------------------\n",
      "Round 23\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.28543210029602\n",
      "Number of training points - 1200\n",
      "Training..\n",
      "Epoch: 7 Training accuracy: 0.99\n",
      "Testing accuracy: 99.17\n",
      "-------------------------------------------------\n",
      "Round 24\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.052167654037476\n",
      "Number of training points - 1250\n",
      "Training..\n",
      "Epoch: 12 Training accuracy: 0.99\n",
      "Testing accuracy: 99.38\n",
      "Training Completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4801, 0.5755, 0.8998, 0.8661, 0.9084, 0.9631, 0.95  , 0.9771,\n",
       "       0.9801, 0.9784, 0.9832, 0.9874, 0.9881, 0.99  , 0.9868, 0.9906,\n",
       "       0.9912, 0.9905, 0.9921, 0.9901, 0.9904, 0.9927, 0.9916, 0.9917,\n",
       "       0.9938])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strategy = \"least_confidence\"\n",
    "strat_logs = logs_directory+F'{data_set_name}/{strategy}/'\n",
    "os.makedirs(strat_logs, exist_ok = True)\n",
    "train_one(full_train_dataset, initial_train_indices, test_dataset, copy.deepcopy(initial_model), n_rounds, budget, args, nclasses, strategy, strat_logs, checkpoint_directory, F\"{experiment_name}_{strategy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rv-NjQBFWqWW"
   },
   "source": [
    "## Margin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "IE0NVJW5WqWW",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Testing accuracy: 48.01\n",
      "-------------------------------------------------\n",
      "Round 1\n",
      "-------------------------------------------------\n",
      "Selection Time: 23.16196632385254\n",
      "Number of training points - 100\n",
      "Training..\n",
      "50 training accuracy: 0.94\n",
      "Epoch: 51 Training accuracy: 0.99\n",
      "Testing accuracy: 87.72\n",
      "-------------------------------------------------\n",
      "Round 2\n",
      "-------------------------------------------------\n",
      "Selection Time: 23.13338041305542\n",
      "Number of training points - 150\n",
      "Training..\n",
      "Epoch: 30 Training accuracy: 1.0\n",
      "Testing accuracy: 88.45\n",
      "-------------------------------------------------\n",
      "Round 3\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.73502540588379\n",
      "Number of training points - 200\n",
      "Training..\n",
      "Epoch: 40 Training accuracy: 0.99\n",
      "Testing accuracy: 94.0\n",
      "-------------------------------------------------\n",
      "Round 4\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.577221870422363\n",
      "Number of training points - 250\n",
      "Training..\n",
      "Epoch: 19 Training accuracy: 0.992\n",
      "Testing accuracy: 93.53\n",
      "-------------------------------------------------\n",
      "Round 5\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.499353885650635\n",
      "Number of training points - 300\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.997\n",
      "Testing accuracy: 95.06\n",
      "-------------------------------------------------\n",
      "Round 6\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.501869916915894\n",
      "Number of training points - 350\n",
      "Training..\n",
      "Epoch: 30 Training accuracy: 0.991\n",
      "Testing accuracy: 96.1\n",
      "-------------------------------------------------\n",
      "Round 7\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.74063229560852\n",
      "Number of training points - 400\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.993\n",
      "Testing accuracy: 97.24\n",
      "-------------------------------------------------\n",
      "Round 8\n",
      "-------------------------------------------------\n",
      "Selection Time: 22.534908294677734\n",
      "Number of training points - 450\n",
      "Training..\n",
      "Epoch: 22 Training accuracy: 0.993\n",
      "Testing accuracy: 97.88\n",
      "-------------------------------------------------\n",
      "Round 9\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.890463829040527\n",
      "Number of training points - 500\n",
      "Training..\n",
      "Epoch: 25 Training accuracy: 0.994\n",
      "Testing accuracy: 97.88\n",
      "-------------------------------------------------\n",
      "Round 10\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.87356996536255\n",
      "Number of training points - 550\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.993\n",
      "Testing accuracy: 98.01\n",
      "-------------------------------------------------\n",
      "Round 11\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.278749227523804\n",
      "Number of training points - 600\n",
      "Training..\n",
      "Epoch: 19 Training accuracy: 0.992\n",
      "Testing accuracy: 98.49\n",
      "-------------------------------------------------\n",
      "Round 12\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.357516765594482\n",
      "Number of training points - 650\n",
      "Training..\n",
      "Epoch: 18 Training accuracy: 0.992\n",
      "Testing accuracy: 98.71\n",
      "-------------------------------------------------\n",
      "Round 13\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.07812190055847\n",
      "Number of training points - 700\n",
      "Training..\n",
      "Epoch: 14 Training accuracy: 0.99\n",
      "Testing accuracy: 98.11\n",
      "-------------------------------------------------\n",
      "Round 14\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.75791358947754\n",
      "Number of training points - 750\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.992\n",
      "Testing accuracy: 98.73\n",
      "-------------------------------------------------\n",
      "Round 15\n",
      "-------------------------------------------------\n",
      "Selection Time: 21.027082204818726\n",
      "Number of training points - 800\n",
      "Training..\n",
      "Epoch: 22 Training accuracy: 0.993\n",
      "Testing accuracy: 98.85\n",
      "-------------------------------------------------\n",
      "Round 16\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.159159421920776\n",
      "Number of training points - 850\n",
      "Training..\n",
      "Epoch: 6 Training accuracy: 0.992\n",
      "Testing accuracy: 99.05\n",
      "-------------------------------------------------\n",
      "Round 17\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.56709313392639\n",
      "Number of training points - 900\n",
      "Training..\n",
      "Epoch: 15 Training accuracy: 0.996\n",
      "Testing accuracy: 98.85\n",
      "-------------------------------------------------\n",
      "Round 18\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.563642024993896\n",
      "Number of training points - 950\n",
      "Training..\n",
      "Epoch: 15 Training accuracy: 0.992\n",
      "Testing accuracy: 98.68\n",
      "-------------------------------------------------\n",
      "Round 19\n",
      "-------------------------------------------------\n",
      "Selection Time: 20.225004196166992\n",
      "Number of training points - 1000\n",
      "Training..\n",
      "Epoch: 10 Training accuracy: 0.996\n",
      "Testing accuracy: 99.08\n",
      "-------------------------------------------------\n",
      "Round 20\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.160227060317993\n",
      "Number of training points - 1050\n",
      "Training..\n",
      "Epoch: 12 Training accuracy: 0.993\n",
      "Testing accuracy: 98.92\n",
      "-------------------------------------------------\n",
      "Round 21\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.698264598846436\n",
      "Number of training points - 1100\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.992\n",
      "Testing accuracy: 98.83\n",
      "-------------------------------------------------\n",
      "Round 22\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.758565425872803\n",
      "Number of training points - 1150\n",
      "Training..\n",
      "Epoch: 8 Training accuracy: 0.992\n",
      "Testing accuracy: 99.05\n",
      "-------------------------------------------------\n",
      "Round 23\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.627365350723267\n",
      "Number of training points - 1200\n",
      "Training..\n",
      "Epoch: 9 Training accuracy: 0.99\n",
      "Testing accuracy: 99.11\n",
      "-------------------------------------------------\n",
      "Round 24\n",
      "-------------------------------------------------\n",
      "Selection Time: 19.571457624435425\n",
      "Number of training points - 1250\n",
      "Training..\n",
      "Epoch: 23 Training accuracy: 0.993\n",
      "Testing accuracy: 99.27\n",
      "Training Completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4801, 0.8772, 0.8845, 0.94  , 0.9353, 0.9506, 0.961 , 0.9724,\n",
       "       0.9788, 0.9788, 0.9801, 0.9849, 0.9871, 0.9811, 0.9873, 0.9885,\n",
       "       0.9905, 0.9885, 0.9868, 0.9908, 0.9892, 0.9883, 0.9905, 0.9911,\n",
       "       0.9927])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strategy = \"margin\"\n",
    "strat_logs = logs_directory+F'{data_set_name}/{strategy}/'\n",
    "os.makedirs(strat_logs, exist_ok = True)\n",
    "train_one(full_train_dataset, initial_train_indices, test_dataset, copy.deepcopy(initial_model), n_rounds, budget, args, nclasses, strategy, strat_logs, checkpoint_directory, F\"{experiment_name}_{strategy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_time = time.monotonic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0116025433813856"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(end_time - start_time)/3600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "DISTIL BloodMNIST Demo Atharv.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}