diff --git a/Leaderboard.ipynb b/Leaderboard.ipynb index 3bcbad8..9dc20b8 100644 --- a/Leaderboard.ipynb +++ b/Leaderboard.ipynb @@ -5360,8 +5360,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/bigboi/opt/anaconda3/lib/python3.9/site-packages/pandas/core/indexing.py:719: FutureWarning: Slicing a positional slice with .loc is not supported, and will raise TypeError in a future version. Use .loc with labels or .iloc with positions instead.\n", - " indexer = self._get_setitem_indexer(key)\n" + "/tmp/ipykernel_3014651/2713444043.py:6: FutureWarning: Slicing a positional slice with .loc is not supported, and will raise TypeError in a future version. Use .loc with labels or .iloc with positions instead.\n", + " triM_all_df.loc[1:, ['mean (%)', '95% CI']] = [\n" ] }, { @@ -5829,6 +5829,326 @@ "transductive_cnaps_imagenet_df" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Results from Hu et al. (2022)\n", + "\n", + "Shell Xu Hu, Da Li, Jan Stühmer, Minyoung Kim and Timothy Hospedales. Pushing the Limits of Simple Pipelines for Few-Shot Learning: External Data and Fine-Tuning Make a Difference. CVPR 2022." + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "ref = (\"Hu et al. (2022)\",\n", + " \"Shell Xu Hu, Da Li, Jan Stühmer, Minyoung Kim and Timothy Hospedales.; \"\n", + " \"[_Pushing the Limits of Simple Pipelines for Few-Shot Learning: External Data and Fine-Tuning Make a Difference._]\"\n", + " \"(https://arxiv.org/abs/2204.07305); \"\n", + " \"CVPR 2022.\")\n", + "references.append(ref)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### P>M>F pipeline with DINO ViT-Small (`PMF_DINOSmall`)\n", + "\n", + "* 128x128 input size\n", + "* ViT-Small backbone pre-trained with DINO self-supervised loss on ILSVRC\n", + "* Backbone is fine-tuned for each episode on its support set with domain-specifically validated learning rate" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
mean (%)95% CI# episodes
ILSVRC (valid)NaNNaN600
ILSVRC (test)73.520.8600
Omniglot92.170.57600
Aircraft89.490.52600
Birds91.040.37600
Textures85.730.62600
QuickDraw79.430.67600
Fungi74.990.94600
VGG Flower95.30.44600
Traffic signs89.850.76600
MSCOCO59.691.02600
\n", + "
" + ], + "text/plain": [ + " mean (%) 95% CI # episodes\n", + "ILSVRC (valid) NaN NaN 600\n", + "ILSVRC (test) 73.52 0.8 600\n", + "Omniglot 92.17 0.57 600\n", + "Aircraft 89.49 0.52 600\n", + "Birds 91.04 0.37 600\n", + "Textures 85.73 0.62 600\n", + "QuickDraw 79.43 0.67 600\n", + "Fungi 74.99 0.94 600\n", + "VGG Flower 95.3 0.44 600\n", + "Traffic signs 89.85 0.76 600\n", + "MSCOCO 59.69 1.02 600" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pmf_dinosmall_all_df = pd.DataFrame(\n", + " columns=['mean (%)', '95% CI', '# episodes'],\n", + " index=datasets\n", + ")\n", + "pmf_dinosmall_all_df['# episodes'] = 600\n", + "pmf_dinosmall_all_df.loc[datasets[1:], ['mean (%)', '95% CI']] = [\n", + " [73.52, 0.80],\n", + " [92.17, 0.57],\n", + " [89.49, 0.52],\n", + " [91.04, 0.37],\n", + " [85.73, 0.62],\n", + " [79.43, 0.67],\n", + " [74.99, 0.94],\n", + " [95.30, 0.44],\n", + " [89.85, 0.76],\n", + " [59.69, 1.02],\n", + "]\n", + "pmf_dinosmall_all_df" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
mean (%)95% CI# episodes
ILSVRC (valid)NaNNaN600
ILSVRC (test)75.510.72600
Omniglot82.811.1600
Aircraft78.381.09600
Birds85.180.77600
Textures86.950.6600
QuickDraw74.470.83600
Fungi55.161.09600
VGG Flower94.660.48600
Traffic signs90.040.81600
MSCOCO62.60.96600
\n", + "
" + ], + "text/plain": [ + " mean (%) 95% CI # episodes\n", + "ILSVRC (valid) NaN NaN 600\n", + "ILSVRC (test) 75.51 0.72 600\n", + "Omniglot 82.81 1.1 600\n", + "Aircraft 78.38 1.09 600\n", + "Birds 85.18 0.77 600\n", + "Textures 86.95 0.6 600\n", + "QuickDraw 74.47 0.83 600\n", + "Fungi 55.16 1.09 600\n", + "VGG Flower 94.66 0.48 600\n", + "Traffic signs 90.04 0.81 600\n", + "MSCOCO 62.6 0.96 600" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pmf_dinosmall_imagenet_df = pd.DataFrame(\n", + " columns=['mean (%)', '95% CI', '# episodes'],\n", + " index=datasets\n", + ")\n", + "pmf_dinosmall_imagenet_df['# episodes'] = 600\n", + "pmf_dinosmall_imagenet_df.loc[datasets[1:], ['mean (%)', '95% CI']] = [\n", + " [75.51, 0.72],\n", + " [82.81, 1.10],\n", + " [78.38, 1.09],\n", + " [85.18, 0.77],\n", + " [86.95, 0.60],\n", + " [74.47, 0.83],\n", + " [55.16, 1.09],\n", + " [94.66, 0.48],\n", + " [90.04, 0.81],\n", + " [62.60, 0.96],\n", + "]\n", + "pmf_dinosmall_imagenet_df" + ] + }, { "cell_type": "markdown", "metadata": { @@ -5876,7 +6196,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 57, "metadata": { "id": "bU1CMkCk3nP6" }, @@ -5898,12 +6218,13 @@ " 'TransductiveCNAPS': transductive_cnaps_imagenet_df,\n", " 'TSA_resnet18': tsa_resnet18_imagenet_df,\n", " 'TSA_resnet34': tsa_resnet34_imagenet_df,\n", + " 'PMF-DINOSmall': pmf_dinosmall_imagenet_df,\n", "}" ] }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 58, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -5939,10 +6260,10 @@ " MatchingNet\n", " ProtoNet\n", " ...\n", - " SimpleCNAPS\n", - " TransductiveCNAPS\n", + " TransductiveCNAPS\n", " TSA_resnet18\n", " TSA_resnet34\n", + " PMF-DINOSmall\n", " \n", " \n", " \n", @@ -6008,15 +6329,15 @@ " 50.5\n", " ...\n", " 600\n", - " 54.1\n", - " 1.1\n", - " 600\n", " 59.5\n", " 1.1\n", " 600\n", " 63.73\n", " 0.99\n", " 600\n", + " 75.51\n", + " 0.72\n", + " 600\n", " \n", " \n", " Omniglot\n", @@ -6032,15 +6353,15 @@ " 59.98\n", " ...\n", " 600\n", - " 62.9\n", - " 1.3\n", - " 600\n", " 78.2\n", " 1.2\n", " 600\n", " 82.58\n", " 1.11\n", " 600\n", + " 82.81\n", + " 1.1\n", + " 600\n", " \n", " \n", " Aircraft\n", @@ -6056,15 +6377,15 @@ " 53.1\n", " ...\n", " 600\n", - " 48.4\n", - " 0.9\n", - " 600\n", " 72.2\n", " 1.0\n", " 600\n", " 80.13\n", " 1.01\n", " 600\n", + " 78.38\n", + " 1.09\n", + " 600\n", " \n", " \n", " Birds\n", @@ -6080,15 +6401,15 @@ " 68.79\n", " ...\n", " 600\n", - " 67.3\n", - " 0.9\n", - " 600\n", " 74.9\n", " 0.9\n", " 600\n", " 83.39\n", " 0.8\n", " 600\n", + " 85.18\n", + " 0.77\n", + " 600\n", " \n", " \n", " Textures\n", @@ -6104,15 +6425,15 @@ " 66.56\n", " ...\n", " 600\n", - " 72.5\n", - " 0.7\n", - " 600\n", " 77.3\n", " 0.7\n", " 600\n", " 79.61\n", " 0.68\n", " 600\n", + " 86.95\n", + " 0.6\n", + " 600\n", " \n", " \n", " QuickDraw\n", @@ -6128,15 +6449,15 @@ " 48.96\n", " ...\n", " 600\n", - " 58.0\n", - " 1.0\n", - " 600\n", " 67.6\n", " 0.9\n", " 600\n", " 71.03\n", " 0.84\n", " 600\n", + " 74.47\n", + " 0.83\n", + " 600\n", " \n", " \n", " Fungi\n", @@ -6152,15 +6473,15 @@ " 39.71\n", " ...\n", " 600\n", - " 37.7\n", - " 1.1\n", - " 600\n", " 44.7\n", " 1.0\n", " 600\n", " 51.38\n", " 1.17\n", " 600\n", + " 55.16\n", + " 1.09\n", + " 600\n", " \n", " \n", " VGG Flower\n", @@ -6176,15 +6497,15 @@ " 85.27\n", " ...\n", " 600\n", - " 82.8\n", - " 0.8\n", - " 600\n", " 90.9\n", " 0.6\n", " 600\n", " 94.05\n", " 0.45\n", " 600\n", + " 94.66\n", + " 0.48\n", + " 600\n", " \n", " \n", " Traffic signs\n", @@ -6200,15 +6521,15 @@ " 47.12\n", " ...\n", " 600\n", - " 61.8\n", - " 1.1\n", - " 600\n", " 82.5\n", " 0.8\n", " 600\n", " 81.71\n", " 0.95\n", " 600\n", + " 90.04\n", + " 0.81\n", + " 600\n", " \n", " \n", " MSCOCO\n", @@ -6224,19 +6545,19 @@ " 41.0\n", " ...\n", " 600\n", - " 45.8\n", - " 1.0\n", - " 600\n", " 59.0\n", " 1.0\n", " 600\n", " 61.67\n", " 0.95\n", " 600\n", + " 62.6\n", + " 0.96\n", + " 600\n", " \n", " \n", "\n", - "

11 rows × 45 columns

\n", + "

11 rows × 48 columns

\n", "" ], "text/plain": [ @@ -6254,52 +6575,52 @@ "Traffic signs 44.59 1.19 600 66.79 1.31 600 \n", "MSCOCO 30.38 0.99 600 34.86 0.97 600 \n", "\n", - " MatchingNet ProtoNet ... SimpleCNAPS \\\n", - " mean (%) 95% CI # episodes mean (%) ... # episodes \n", - "ILSVRC (valid) NaN NaN 600 NaN ... 600 \n", - "ILSVRC (test) 45.0 1.1 600 50.5 ... 600 \n", - "Omniglot 52.27 1.28 600 59.98 ... 600 \n", - "Aircraft 48.97 0.93 600 53.1 ... 600 \n", - "Birds 62.21 0.95 600 68.79 ... 600 \n", - "Textures 64.15 0.85 600 66.56 ... 600 \n", - "QuickDraw 42.87 1.09 600 48.96 ... 600 \n", - "Fungi 33.97 1.0 600 39.71 ... 600 \n", - "VGG Flower 80.13 0.71 600 85.27 ... 600 \n", - "Traffic signs 47.8 1.14 600 47.12 ... 600 \n", - "MSCOCO 34.99 1.0 600 41.0 ... 600 \n", + " MatchingNet ProtoNet ... TransductiveCNAPS \\\n", + " mean (%) 95% CI # episodes mean (%) ... # episodes \n", + "ILSVRC (valid) NaN NaN 600 NaN ... 600 \n", + "ILSVRC (test) 45.0 1.1 600 50.5 ... 600 \n", + "Omniglot 52.27 1.28 600 59.98 ... 600 \n", + "Aircraft 48.97 0.93 600 53.1 ... 600 \n", + "Birds 62.21 0.95 600 68.79 ... 600 \n", + "Textures 64.15 0.85 600 66.56 ... 600 \n", + "QuickDraw 42.87 1.09 600 48.96 ... 600 \n", + "Fungi 33.97 1.0 600 39.71 ... 600 \n", + "VGG Flower 80.13 0.71 600 85.27 ... 600 \n", + "Traffic signs 47.8 1.14 600 47.12 ... 600 \n", + "MSCOCO 34.99 1.0 600 41.0 ... 600 \n", "\n", - " TransductiveCNAPS TSA_resnet18 \\\n", - " mean (%) 95% CI # episodes mean (%) 95% CI \n", - "ILSVRC (valid) NaN NaN 600 NaN NaN \n", - "ILSVRC (test) 54.1 1.1 600 59.5 1.1 \n", - "Omniglot 62.9 1.3 600 78.2 1.2 \n", - "Aircraft 48.4 0.9 600 72.2 1.0 \n", - "Birds 67.3 0.9 600 74.9 0.9 \n", - "Textures 72.5 0.7 600 77.3 0.7 \n", - "QuickDraw 58.0 1.0 600 67.6 0.9 \n", - "Fungi 37.7 1.1 600 44.7 1.0 \n", - "VGG Flower 82.8 0.8 600 90.9 0.6 \n", - "Traffic signs 61.8 1.1 600 82.5 0.8 \n", - "MSCOCO 45.8 1.0 600 59.0 1.0 \n", + " TSA_resnet18 TSA_resnet34 \\\n", + " mean (%) 95% CI # episodes mean (%) 95% CI # episodes \n", + "ILSVRC (valid) NaN NaN 600 NaN NaN 600 \n", + "ILSVRC (test) 59.5 1.1 600 63.73 0.99 600 \n", + "Omniglot 78.2 1.2 600 82.58 1.11 600 \n", + "Aircraft 72.2 1.0 600 80.13 1.01 600 \n", + "Birds 74.9 0.9 600 83.39 0.8 600 \n", + "Textures 77.3 0.7 600 79.61 0.68 600 \n", + "QuickDraw 67.6 0.9 600 71.03 0.84 600 \n", + "Fungi 44.7 1.0 600 51.38 1.17 600 \n", + "VGG Flower 90.9 0.6 600 94.05 0.45 600 \n", + "Traffic signs 82.5 0.8 600 81.71 0.95 600 \n", + "MSCOCO 59.0 1.0 600 61.67 0.95 600 \n", "\n", - " TSA_resnet34 \n", - " # episodes mean (%) 95% CI # episodes \n", - "ILSVRC (valid) 600 NaN NaN 600 \n", - "ILSVRC (test) 600 63.73 0.99 600 \n", - "Omniglot 600 82.58 1.11 600 \n", - "Aircraft 600 80.13 1.01 600 \n", - "Birds 600 83.39 0.8 600 \n", - "Textures 600 79.61 0.68 600 \n", - "QuickDraw 600 71.03 0.84 600 \n", - "Fungi 600 51.38 1.17 600 \n", - "VGG Flower 600 94.05 0.45 600 \n", - "Traffic signs 600 81.71 0.95 600 \n", - "MSCOCO 600 61.67 0.95 600 \n", + " PMF-DINOSmall \n", + " mean (%) 95% CI # episodes \n", + "ILSVRC (valid) NaN NaN 600 \n", + "ILSVRC (test) 75.51 0.72 600 \n", + "Omniglot 82.81 1.1 600 \n", + "Aircraft 78.38 1.09 600 \n", + "Birds 85.18 0.77 600 \n", + "Textures 86.95 0.6 600 \n", + "QuickDraw 74.47 0.83 600 \n", + "Fungi 55.16 1.09 600 \n", + "VGG Flower 94.66 0.48 600 \n", + "Traffic signs 90.04 0.81 600 \n", + "MSCOCO 62.6 0.96 600 \n", "\n", - "[11 rows x 45 columns]" + "[11 rows x 48 columns]" ] }, - "execution_count": 55, + "execution_count": 58, "metadata": {}, "output_type": "execute_result" } @@ -6314,7 +6635,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 59, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -6350,10 +6671,10 @@ " MatchingNet\n", " ProtoNet\n", " ...\n", - " FLUTE\n", - " URL\n", + " URL\n", " TSA\n", " TriM\n", + " PMF-DINOSmall\n", " \n", " \n", " \n", @@ -6419,15 +6740,15 @@ " 44.5\n", " ...\n", " 600\n", - " 57.51\n", - " 1.08\n", - " 600\n", " 57.35\n", " 1.05\n", " 600\n", " 58.6\n", " 1.0\n", " 600\n", + " 73.52\n", + " 0.8\n", + " 600\n", " \n", " \n", " Omniglot\n", @@ -6443,15 +6764,15 @@ " 79.56\n", " ...\n", " 600\n", - " 94.51\n", - " 0.41\n", - " 600\n", " 94.96\n", " 0.38\n", " 600\n", " 92.0\n", " 0.6\n", " 600\n", + " 92.17\n", + " 0.57\n", + " 600\n", " \n", " \n", " Aircraft\n", @@ -6467,15 +6788,15 @@ " 71.14\n", " ...\n", " 600\n", - " 88.59\n", - " 0.46\n", - " 600\n", " 89.33\n", " 0.44\n", " 600\n", " 82.8\n", " 0.7\n", " 600\n", + " 89.49\n", + " 0.52\n", + " 600\n", " \n", " \n", " Birds\n", @@ -6491,15 +6812,15 @@ " 67.01\n", " ...\n", " 600\n", - " 80.54\n", - " 0.69\n", - " 600\n", " 81.42\n", " 0.74\n", " 600\n", " 75.3\n", " 0.8\n", " 600\n", + " 91.04\n", + " 0.37\n", + " 600\n", " \n", " \n", " Textures\n", @@ -6515,15 +6836,15 @@ " 65.18\n", " ...\n", " 600\n", - " 76.17\n", - " 0.67\n", - " 600\n", " 76.74\n", " 0.72\n", " 600\n", " 71.2\n", " 0.8\n", " 600\n", + " 85.73\n", + " 0.62\n", + " 600\n", " \n", " \n", " QuickDraw\n", @@ -6539,15 +6860,15 @@ " 64.88\n", " ...\n", " 600\n", - " 81.94\n", - " 0.56\n", - " 600\n", " 82.01\n", " 0.57\n", " 600\n", " 77.3\n", " 0.7\n", " 600\n", + " 79.43\n", + " 0.67\n", + " 600\n", " \n", " \n", " Fungi\n", @@ -6563,15 +6884,15 @@ " 40.26\n", " ...\n", " 600\n", - " 68.75\n", - " 0.95\n", - " 600\n", " 67.4\n", " 0.99\n", " 600\n", " 48.5\n", " 1.0\n", " 600\n", + " 74.99\n", + " 0.94\n", + " 600\n", " \n", " \n", " VGG Flower\n", @@ -6587,15 +6908,15 @@ " 86.85\n", " ...\n", " 600\n", - " 92.11\n", - " 0.48\n", - " 600\n", " 92.18\n", " 0.52\n", " 600\n", " 90.5\n", " 0.5\n", " 600\n", + " 95.3\n", + " 0.44\n", + " 600\n", " \n", " \n", " Traffic signs\n", @@ -6611,15 +6932,15 @@ " 46.48\n", " ...\n", " 600\n", - " 63.34\n", - " 1.19\n", - " 600\n", " 83.55\n", " 0.9\n", " 600\n", " 63.0\n", " 1.0\n", " 600\n", + " 89.85\n", + " 0.76\n", + " 600\n", " \n", " \n", " MSCOCO\n", @@ -6635,19 +6956,19 @@ " 39.87\n", " ...\n", " 600\n", - " 54.03\n", - " 0.96\n", - " 600\n", " 55.75\n", " 1.06\n", " 600\n", " 52.8\n", " 1.1\n", " 600\n", + " 59.69\n", + " 1.02\n", + " 600\n", " \n", " \n", "\n", - "

11 rows × 54 columns

\n", + "

11 rows × 57 columns

\n", "" ], "text/plain": [ @@ -6665,7 +6986,7 @@ "Traffic signs 40.11 1.1 600 66.74 1.23 600 \n", "MSCOCO 29.55 0.96 600 35.17 1.08 600 \n", "\n", - " MatchingNet ProtoNet ... FLUTE \\\n", + " MatchingNet ProtoNet ... URL \\\n", " mean (%) 95% CI # episodes mean (%) ... # episodes \n", "ILSVRC (valid) NaN NaN 600 NaN ... 600 \n", "ILSVRC (test) 36.08 1.0 600 44.5 ... 600 \n", @@ -6679,38 +7000,38 @@ "Traffic signs 55.57 1.08 600 46.48 ... 600 \n", "MSCOCO 28.79 0.96 600 39.87 ... 600 \n", "\n", - " URL TSA TriM \\\n", - " mean (%) 95% CI # episodes mean (%) 95% CI # episodes mean (%) \n", - "ILSVRC (valid) NaN NaN 600 NaN NaN 600 NaN \n", - "ILSVRC (test) 57.51 1.08 600 57.35 1.05 600 58.6 \n", - "Omniglot 94.51 0.41 600 94.96 0.38 600 92.0 \n", - "Aircraft 88.59 0.46 600 89.33 0.44 600 82.8 \n", - "Birds 80.54 0.69 600 81.42 0.74 600 75.3 \n", - "Textures 76.17 0.67 600 76.74 0.72 600 71.2 \n", - "QuickDraw 81.94 0.56 600 82.01 0.57 600 77.3 \n", - "Fungi 68.75 0.95 600 67.4 0.99 600 48.5 \n", - "VGG Flower 92.11 0.48 600 92.18 0.52 600 90.5 \n", - "Traffic signs 63.34 1.19 600 83.55 0.9 600 63.0 \n", - "MSCOCO 54.03 0.96 600 55.75 1.06 600 52.8 \n", + " TSA TriM \\\n", + " mean (%) 95% CI # episodes mean (%) 95% CI # episodes \n", + "ILSVRC (valid) NaN NaN 600 NaN NaN 600 \n", + "ILSVRC (test) 57.35 1.05 600 58.6 1.0 600 \n", + "Omniglot 94.96 0.38 600 92.0 0.6 600 \n", + "Aircraft 89.33 0.44 600 82.8 0.7 600 \n", + "Birds 81.42 0.74 600 75.3 0.8 600 \n", + "Textures 76.74 0.72 600 71.2 0.8 600 \n", + "QuickDraw 82.01 0.57 600 77.3 0.7 600 \n", + "Fungi 67.4 0.99 600 48.5 1.0 600 \n", + "VGG Flower 92.18 0.52 600 90.5 0.5 600 \n", + "Traffic signs 83.55 0.9 600 63.0 1.0 600 \n", + "MSCOCO 55.75 1.06 600 52.8 1.1 600 \n", "\n", - " \n", - " 95% CI # episodes \n", - "ILSVRC (valid) NaN 600 \n", - "ILSVRC (test) 1.0 600 \n", - "Omniglot 0.6 600 \n", - "Aircraft 0.7 600 \n", - "Birds 0.8 600 \n", - "Textures 0.8 600 \n", - "QuickDraw 0.7 600 \n", - "Fungi 1.0 600 \n", - "VGG Flower 0.5 600 \n", - "Traffic signs 1.0 600 \n", - "MSCOCO 1.1 600 \n", + " PMF-DINOSmall \n", + " mean (%) 95% CI # episodes \n", + "ILSVRC (valid) NaN NaN 600 \n", + "ILSVRC (test) 73.52 0.8 600 \n", + "Omniglot 92.17 0.57 600 \n", + "Aircraft 89.49 0.52 600 \n", + "Birds 91.04 0.37 600 \n", + "Textures 85.73 0.62 600 \n", + "QuickDraw 79.43 0.67 600 \n", + "Fungi 74.99 0.94 600 \n", + "VGG Flower 95.3 0.44 600 \n", + "Traffic signs 89.85 0.76 600 \n", + "MSCOCO 59.69 1.02 600 \n", "\n", - "[11 rows x 54 columns]" + "[11 rows x 57 columns]" ] }, - "execution_count": 56, + "execution_count": 59, "metadata": {}, "output_type": "execute_result" } @@ -6735,6 +7056,7 @@ " 'URL': url_all_df,\n", " 'TSA': tsa_all_df,\n", " 'TriM': triM_all_df,\n", + " 'PMF-DINOSmall': pmf_dinosmall_all_df,\n", "}\n", "all_df = pd.concat(\n", " all_dfs.values(),\n", @@ -6745,7 +7067,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 60, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -6876,6 +7198,10 @@ " TransductiveCNAPS\n", " [Bateni et al. (2022b), Bateni et al. (2022a)]\n", " \n", + " \n", + " PMF-DINOSmall\n", + " [Hu et al. (2022)]\n", + " \n", " \n", "\n", "" @@ -6905,10 +7231,11 @@ "TSA_resnet34 Li et al. (2021b)\n", "TriM Liu et al. (2021b)\n", "SimpleCNAPS [Bateni et al. (2022b), Bateni et al. (2020)]\n", - "TransductiveCNAPS [Bateni et al. (2022b), Bateni et al. (2022a)]" + "TransductiveCNAPS [Bateni et al. (2022b), Bateni et al. (2022a)]\n", + "PMF-DINOSmall [Hu et al. (2022)]" ] }, - "execution_count": 57, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } @@ -6942,6 +7269,7 @@ " 'TriM': 'Liu et al. (2021b)',\n", " 'SimpleCNAPS': ['Bateni et al. (2022b)', 'Bateni et al. (2020)'],\n", " 'TransductiveCNAPS': ['Bateni et al. (2022b)', 'Bateni et al. (2022a)'],\n", + " 'PMF-DINOSmall': ['Hu et al. (2022)'],\n", " })\n", "models_df" ] @@ -6957,7 +7285,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 61, "metadata": { "id": "E6sHKzIvPiYD" }, @@ -6979,7 +7307,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 62, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -7014,9 +7342,9 @@ " Finetune\n", " MatchingNet\n", " ...\n", - " TransductiveCNAPS\n", - " TSA_resnet18\n", + " TSA_resnet18\n", " TSA_resnet34\n", + " PMF-DINOSmall\n", " \n", " \n", " \n", @@ -7081,16 +7409,16 @@ " 600\n", " 1.1\n", " ...\n", - " 54.1\n", - " 13.747136\n", - " 600\n", - " 1.1\n", " 59.5\n", " 13.747136\n", " 600\n", " 0.99\n", " 63.73\n", " 12.372423\n", + " 600\n", + " 0.72\n", + " 75.51\n", + " 8.998126\n", " \n", " \n", " Omniglot\n", @@ -7105,16 +7433,16 @@ " 600\n", " 1.28\n", " ...\n", - " 62.9\n", - " 16.246616\n", - " 600\n", - " 1.2\n", " 78.2\n", " 14.996876\n", " 600\n", " 1.11\n", " 82.58\n", " 13.87211\n", + " 600\n", + " 1.1\n", + " 82.81\n", + " 13.747136\n", " \n", " \n", " Aircraft\n", @@ -7129,16 +7457,16 @@ " 600\n", " 0.93\n", " ...\n", - " 48.4\n", - " 11.247657\n", - " 600\n", - " 1.0\n", " 72.2\n", " 12.497397\n", " 600\n", " 1.01\n", " 80.13\n", " 12.622371\n", + " 600\n", + " 1.09\n", + " 78.38\n", + " 13.622162\n", " \n", " \n", " Birds\n", @@ -7153,16 +7481,16 @@ " 600\n", " 0.95\n", " ...\n", - " 67.3\n", - " 11.247657\n", - " 600\n", - " 0.9\n", " 74.9\n", " 11.247657\n", " 600\n", " 0.8\n", " 83.39\n", " 9.997917\n", + " 600\n", + " 0.77\n", + " 85.18\n", + " 9.622995\n", " \n", " \n", " Textures\n", @@ -7177,16 +7505,16 @@ " 600\n", " 0.85\n", " ...\n", - " 72.5\n", - " 8.748178\n", - " 600\n", - " 0.7\n", " 77.3\n", " 8.748178\n", " 600\n", " 0.68\n", " 79.61\n", " 8.49823\n", + " 600\n", + " 0.6\n", + " 86.95\n", + " 7.498438\n", " \n", " \n", " QuickDraw\n", @@ -7201,16 +7529,16 @@ " 600\n", " 1.09\n", " ...\n", - " 58.0\n", - " 12.497397\n", - " 600\n", - " 0.9\n", " 67.6\n", " 11.247657\n", " 600\n", " 0.84\n", " 71.03\n", " 10.497813\n", + " 600\n", + " 0.83\n", + " 74.47\n", + " 10.372839\n", " \n", " \n", " Fungi\n", @@ -7225,16 +7553,16 @@ " 600\n", " 1.0\n", " ...\n", - " 37.7\n", - " 13.747136\n", - " 600\n", - " 1.0\n", " 44.7\n", " 12.497397\n", " 600\n", " 1.17\n", " 51.38\n", " 14.621954\n", + " 600\n", + " 1.09\n", + " 55.16\n", + " 13.622162\n", " \n", " \n", " VGG Flower\n", @@ -7249,16 +7577,16 @@ " 600\n", " 0.71\n", " ...\n", - " 82.8\n", - " 9.997917\n", - " 600\n", - " 0.6\n", " 90.9\n", " 7.498438\n", " 600\n", " 0.45\n", " 94.05\n", " 5.623828\n", + " 600\n", + " 0.48\n", + " 94.66\n", + " 5.99875\n", " \n", " \n", " Traffic signs\n", @@ -7273,16 +7601,16 @@ " 600\n", " 1.14\n", " ...\n", - " 61.8\n", - " 13.747136\n", - " 600\n", - " 0.8\n", " 82.5\n", " 9.997917\n", " 600\n", " 0.95\n", " 81.71\n", " 11.872527\n", + " 600\n", + " 0.81\n", + " 90.04\n", + " 10.122891\n", " \n", " \n", " MSCOCO\n", @@ -7297,20 +7625,20 @@ " 600\n", " 1.0\n", " ...\n", - " 45.8\n", - " 12.497397\n", - " 600\n", - " 1.0\n", " 59.0\n", " 12.497397\n", " 600\n", " 0.95\n", " 61.67\n", " 11.872527\n", + " 600\n", + " 0.96\n", + " 62.6\n", + " 11.997501\n", " \n", " \n", "\n", - "

11 rows × 60 columns

\n", + "

11 rows × 64 columns

\n", "" ], "text/plain": [ @@ -7328,52 +7656,52 @@ "Traffic signs 600 1.19 44.59 14.871902 600 1.31 \n", "MSCOCO 600 0.99 30.38 12.372423 600 0.97 \n", "\n", - " MatchingNet ... TransductiveCNAPS \\\n", - " mean (%) stddev # episodes 95% CI ... mean (%) \n", - "ILSVRC (valid) NaN NaN 600 NaN ... NaN \n", - "ILSVRC (test) 45.78 13.747136 600 1.1 ... 54.1 \n", - "Omniglot 60.85 19.745887 600 1.28 ... 62.9 \n", - "Aircraft 68.69 15.74672 600 0.93 ... 48.4 \n", - "Birds 57.31 15.74672 600 0.95 ... 67.3 \n", - "Textures 69.05 11.247657 600 0.85 ... 72.5 \n", - "QuickDraw 42.6 14.621954 600 1.09 ... 58.0 \n", - "Fungi 38.2 12.747345 600 1.0 ... 37.7 \n", - "VGG Flower 85.51 8.49823 600 0.71 ... 82.8 \n", - "Traffic signs 66.79 16.37159 600 1.14 ... 61.8 \n", - "MSCOCO 34.86 12.122475 600 1.0 ... 45.8 \n", + " MatchingNet ... TSA_resnet18 \\\n", + " mean (%) stddev # episodes 95% CI ... mean (%) \n", + "ILSVRC (valid) NaN NaN 600 NaN ... NaN \n", + "ILSVRC (test) 45.78 13.747136 600 1.1 ... 59.5 \n", + "Omniglot 60.85 19.745887 600 1.28 ... 78.2 \n", + "Aircraft 68.69 15.74672 600 0.93 ... 72.2 \n", + "Birds 57.31 15.74672 600 0.95 ... 74.9 \n", + "Textures 69.05 11.247657 600 0.85 ... 77.3 \n", + "QuickDraw 42.6 14.621954 600 1.09 ... 67.6 \n", + "Fungi 38.2 12.747345 600 1.0 ... 44.7 \n", + "VGG Flower 85.51 8.49823 600 0.71 ... 90.9 \n", + "Traffic signs 66.79 16.37159 600 1.14 ... 82.5 \n", + "MSCOCO 34.86 12.122475 600 1.0 ... 59.0 \n", "\n", - " TSA_resnet18 \\\n", + " TSA_resnet34 \\\n", " stddev # episodes 95% CI mean (%) stddev \n", "ILSVRC (valid) NaN 600 NaN NaN NaN \n", - "ILSVRC (test) 13.747136 600 1.1 59.5 13.747136 \n", - "Omniglot 16.246616 600 1.2 78.2 14.996876 \n", - "Aircraft 11.247657 600 1.0 72.2 12.497397 \n", - "Birds 11.247657 600 0.9 74.9 11.247657 \n", - "Textures 8.748178 600 0.7 77.3 8.748178 \n", - "QuickDraw 12.497397 600 0.9 67.6 11.247657 \n", - "Fungi 13.747136 600 1.0 44.7 12.497397 \n", - "VGG Flower 9.997917 600 0.6 90.9 7.498438 \n", - "Traffic signs 13.747136 600 0.8 82.5 9.997917 \n", - "MSCOCO 12.497397 600 1.0 59.0 12.497397 \n", + "ILSVRC (test) 13.747136 600 0.99 63.73 12.372423 \n", + "Omniglot 14.996876 600 1.11 82.58 13.87211 \n", + "Aircraft 12.497397 600 1.01 80.13 12.622371 \n", + "Birds 11.247657 600 0.8 83.39 9.997917 \n", + "Textures 8.748178 600 0.68 79.61 8.49823 \n", + "QuickDraw 11.247657 600 0.84 71.03 10.497813 \n", + "Fungi 12.497397 600 1.17 51.38 14.621954 \n", + "VGG Flower 7.498438 600 0.45 94.05 5.623828 \n", + "Traffic signs 9.997917 600 0.95 81.71 11.872527 \n", + "MSCOCO 12.497397 600 0.95 61.67 11.872527 \n", "\n", - " TSA_resnet34 \n", - " # episodes 95% CI mean (%) stddev \n", - "ILSVRC (valid) 600 NaN NaN NaN \n", - "ILSVRC (test) 600 0.99 63.73 12.372423 \n", - "Omniglot 600 1.11 82.58 13.87211 \n", - "Aircraft 600 1.01 80.13 12.622371 \n", - "Birds 600 0.8 83.39 9.997917 \n", - "Textures 600 0.68 79.61 8.49823 \n", - "QuickDraw 600 0.84 71.03 10.497813 \n", - "Fungi 600 1.17 51.38 14.621954 \n", - "VGG Flower 600 0.45 94.05 5.623828 \n", - "Traffic signs 600 0.95 81.71 11.872527 \n", - "MSCOCO 600 0.95 61.67 11.872527 \n", + " PMF-DINOSmall \n", + " # episodes 95% CI mean (%) stddev \n", + "ILSVRC (valid) 600 NaN NaN NaN \n", + "ILSVRC (test) 600 0.72 75.51 8.998126 \n", + "Omniglot 600 1.1 82.81 13.747136 \n", + "Aircraft 600 1.09 78.38 13.622162 \n", + "Birds 600 0.77 85.18 9.622995 \n", + "Textures 600 0.6 86.95 7.498438 \n", + "QuickDraw 600 0.83 74.47 10.372839 \n", + "Fungi 600 1.09 55.16 13.622162 \n", + "VGG Flower 600 0.48 94.66 5.99875 \n", + "Traffic signs 600 0.81 90.04 10.122891 \n", + "MSCOCO 600 0.96 62.6 11.997501 \n", "\n", - "[11 rows x 60 columns]" + "[11 rows x 64 columns]" ] }, - "execution_count": 59, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } @@ -7385,7 +7713,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 63, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -7420,9 +7748,9 @@ " Finetune\n", " MatchingNet\n", " ...\n", - " URL\n", - " TSA\n", + " TSA\n", " TriM\n", + " PMF-DINOSmall\n", " \n", " \n", " \n", @@ -7487,16 +7815,16 @@ " 600\n", " 1.0\n", " ...\n", - " 57.51\n", - " 13.497188\n", - " 600\n", - " 1.05\n", " 57.35\n", " 13.122266\n", " 600\n", " 1.0\n", " 58.6\n", " 12.497397\n", + " 600\n", + " 0.8\n", + " 73.52\n", + " 9.997917\n", " \n", " \n", " Omniglot\n", @@ -7511,16 +7839,16 @@ " 600\n", " 1.01\n", " ...\n", - " 94.51\n", - " 5.123933\n", - " 600\n", - " 0.38\n", " 94.96\n", " 4.749011\n", " 600\n", " 0.6\n", " 92.0\n", " 7.498438\n", + " 600\n", + " 0.57\n", + " 92.17\n", + " 7.123516\n", " \n", " \n", " Aircraft\n", @@ -7535,16 +7863,16 @@ " 600\n", " 0.96\n", " ...\n", - " 88.59\n", - " 5.748802\n", - " 600\n", - " 0.44\n", " 89.33\n", " 5.498855\n", " 600\n", " 0.7\n", " 82.8\n", " 8.748178\n", + " 600\n", + " 0.52\n", + " 89.49\n", + " 6.498646\n", " \n", " \n", " Birds\n", @@ -7559,16 +7887,16 @@ " 600\n", " 1.0\n", " ...\n", - " 80.54\n", - " 8.623204\n", - " 600\n", - " 0.74\n", " 81.42\n", " 9.248074\n", " 600\n", " 0.8\n", " 75.3\n", " 9.997917\n", + " 600\n", + " 0.37\n", + " 91.04\n", + " 4.624037\n", " \n", " \n", " Textures\n", @@ -7583,16 +7911,16 @@ " 600\n", " 0.74\n", " ...\n", - " 76.17\n", - " 8.373256\n", - " 600\n", - " 0.72\n", " 76.74\n", " 8.998126\n", " 600\n", " 0.8\n", " 71.2\n", " 9.997917\n", + " 600\n", + " 0.62\n", + " 85.73\n", + " 7.748386\n", " \n", " \n", " QuickDraw\n", @@ -7607,16 +7935,16 @@ " 600\n", " 1.03\n", " ...\n", - " 81.94\n", - " 6.998542\n", - " 600\n", - " 0.57\n", " 82.01\n", " 7.123516\n", " 600\n", " 0.7\n", " 77.3\n", " 8.748178\n", + " 600\n", + " 0.67\n", + " 79.43\n", + " 8.373256\n", " \n", " \n", " Fungi\n", @@ -7631,16 +7959,16 @@ " 600\n", " 1.04\n", " ...\n", - " 68.75\n", - " 11.872527\n", - " 600\n", - " 0.99\n", " 67.4\n", " 12.372423\n", " 600\n", " 1.0\n", " 48.5\n", " 12.497397\n", + " 600\n", + " 0.94\n", + " 74.99\n", + " 11.747553\n", " \n", " \n", " VGG Flower\n", @@ -7655,16 +7983,16 @@ " 600\n", " 0.72\n", " ...\n", - " 92.11\n", - " 5.99875\n", - " 600\n", - " 0.52\n", " 92.18\n", " 6.498646\n", " 600\n", " 0.5\n", " 90.5\n", " 6.248698\n", + " 600\n", + " 0.44\n", + " 95.3\n", + " 5.498855\n", " \n", " \n", " Traffic signs\n", @@ -7679,16 +8007,16 @@ " 600\n", " 1.08\n", " ...\n", - " 63.34\n", - " 14.871902\n", - " 600\n", - " 0.9\n", " 83.55\n", " 11.247657\n", " 600\n", " 1.0\n", " 63.0\n", " 12.497397\n", + " 600\n", + " 0.76\n", + " 89.85\n", + " 9.498021\n", " \n", " \n", " MSCOCO\n", @@ -7703,20 +8031,20 @@ " 600\n", " 0.96\n", " ...\n", - " 54.03\n", - " 11.997501\n", - " 600\n", - " 1.06\n", " 55.75\n", " 13.24724\n", " 600\n", " 1.1\n", " 52.8\n", " 13.747136\n", + " 600\n", + " 1.02\n", + " 59.69\n", + " 12.747345\n", " \n", " \n", "\n", - "

11 rows × 72 columns

\n", + "

11 rows × 76 columns

\n", "" ], "text/plain": [ @@ -7734,52 +8062,52 @@ "Traffic signs 600 1.1 40.11 13.747136 600 1.23 \n", "MSCOCO 600 0.96 29.55 11.997501 600 1.08 \n", "\n", - " MatchingNet ... URL \\\n", + " MatchingNet ... TSA \\\n", " mean (%) stddev # episodes 95% CI ... mean (%) \n", "ILSVRC (valid) NaN NaN 600 NaN ... NaN \n", - "ILSVRC (test) 43.08 13.497188 600 1.0 ... 57.51 \n", - "Omniglot 71.11 17.121433 600 1.01 ... 94.51 \n", - "Aircraft 72.03 13.372214 600 0.96 ... 88.59 \n", - "Birds 59.82 14.372006 600 1.0 ... 80.54 \n", - "Textures 69.14 10.622787 600 0.74 ... 76.17 \n", - "QuickDraw 47.05 14.49698 600 1.03 ... 81.94 \n", - "Fungi 38.16 12.997293 600 1.04 ... 68.75 \n", - "VGG Flower 85.28 8.623204 600 0.72 ... 92.11 \n", - "Traffic signs 66.74 15.371798 600 1.08 ... 63.34 \n", - "MSCOCO 35.17 13.497188 600 0.96 ... 54.03 \n", + "ILSVRC (test) 43.08 13.497188 600 1.0 ... 57.35 \n", + "Omniglot 71.11 17.121433 600 1.01 ... 94.96 \n", + "Aircraft 72.03 13.372214 600 0.96 ... 89.33 \n", + "Birds 59.82 14.372006 600 1.0 ... 81.42 \n", + "Textures 69.14 10.622787 600 0.74 ... 76.74 \n", + "QuickDraw 47.05 14.49698 600 1.03 ... 82.01 \n", + "Fungi 38.16 12.997293 600 1.04 ... 67.4 \n", + "VGG Flower 85.28 8.623204 600 0.72 ... 92.18 \n", + "Traffic signs 66.74 15.371798 600 1.08 ... 83.55 \n", + "MSCOCO 35.17 13.497188 600 0.96 ... 55.75 \n", "\n", - " TSA TriM \\\n", - " stddev # episodes 95% CI mean (%) stddev # episodes \n", - "ILSVRC (valid) NaN 600 NaN NaN NaN 600 \n", - "ILSVRC (test) 13.497188 600 1.05 57.35 13.122266 600 \n", - "Omniglot 5.123933 600 0.38 94.96 4.749011 600 \n", - "Aircraft 5.748802 600 0.44 89.33 5.498855 600 \n", - "Birds 8.623204 600 0.74 81.42 9.248074 600 \n", - "Textures 8.373256 600 0.72 76.74 8.998126 600 \n", - "QuickDraw 6.998542 600 0.57 82.01 7.123516 600 \n", - "Fungi 11.872527 600 0.99 67.4 12.372423 600 \n", - "VGG Flower 5.99875 600 0.52 92.18 6.498646 600 \n", - "Traffic signs 14.871902 600 0.9 83.55 11.247657 600 \n", - "MSCOCO 11.997501 600 1.06 55.75 13.24724 600 \n", + " TriM PMF-DINOSmall \\\n", + " stddev # episodes 95% CI mean (%) stddev # episodes \n", + "ILSVRC (valid) NaN 600 NaN NaN NaN 600 \n", + "ILSVRC (test) 13.122266 600 1.0 58.6 12.497397 600 \n", + "Omniglot 4.749011 600 0.6 92.0 7.498438 600 \n", + "Aircraft 5.498855 600 0.7 82.8 8.748178 600 \n", + "Birds 9.248074 600 0.8 75.3 9.997917 600 \n", + "Textures 8.998126 600 0.8 71.2 9.997917 600 \n", + "QuickDraw 7.123516 600 0.7 77.3 8.748178 600 \n", + "Fungi 12.372423 600 1.0 48.5 12.497397 600 \n", + "VGG Flower 6.498646 600 0.5 90.5 6.248698 600 \n", + "Traffic signs 11.247657 600 1.0 63.0 12.497397 600 \n", + "MSCOCO 13.24724 600 1.1 52.8 13.747136 600 \n", "\n", " \n", " 95% CI mean (%) stddev \n", "ILSVRC (valid) NaN NaN NaN \n", - "ILSVRC (test) 1.0 58.6 12.497397 \n", - "Omniglot 0.6 92.0 7.498438 \n", - "Aircraft 0.7 82.8 8.748178 \n", - "Birds 0.8 75.3 9.997917 \n", - "Textures 0.8 71.2 9.997917 \n", - "QuickDraw 0.7 77.3 8.748178 \n", - "Fungi 1.0 48.5 12.497397 \n", - "VGG Flower 0.5 90.5 6.248698 \n", - "Traffic signs 1.0 63.0 12.497397 \n", - "MSCOCO 1.1 52.8 13.747136 \n", + "ILSVRC (test) 0.8 73.52 9.997917 \n", + "Omniglot 0.57 92.17 7.123516 \n", + "Aircraft 0.52 89.49 6.498646 \n", + "Birds 0.37 91.04 4.624037 \n", + "Textures 0.62 85.73 7.748386 \n", + "QuickDraw 0.67 79.43 8.373256 \n", + "Fungi 0.94 74.99 11.747553 \n", + "VGG Flower 0.44 95.3 5.498855 \n", + "Traffic signs 0.76 89.85 9.498021 \n", + "MSCOCO 1.02 59.69 12.747345 \n", "\n", - "[11 rows x 72 columns]" + "[11 rows x 76 columns]" ] }, - "execution_count": 60, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -7800,7 +8128,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 64, "metadata": { "id": "u0k0DqdXt07o" }, @@ -7816,7 +8144,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 65, "metadata": { "id": "0vn3fGqHnzJe" }, @@ -7859,7 +8187,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 66, "metadata": { "id": "0Ra3dxcIwYE4" }, @@ -7877,7 +8205,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 67, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -7970,23 +8298,23 @@ " 1.11\n", " 52.8\n", " 13.87211\n", - " 7.5\n", + " 8.5\n", " 600\n", " 1.05\n", " 51.92\n", " 13.122266\n", - " 7.5\n", + " 8.5\n", " ...\n", " 600\n", " 1.05\n", " 49.53\n", " 13.122266\n", - " 9.5\n", + " 10.5\n", " 600\n", " 1.01\n", " 41.03\n", " 12.622371\n", - " 14.0\n", + " 15.0\n", " \n", " \n", " Omniglot\n", @@ -7994,23 +8322,23 @@ " 1.51\n", " 61.87\n", " 18.871069\n", - " 7.5\n", + " 8.5\n", " 600\n", " 1.21\n", " 67.57\n", " 15.12185\n", - " 4.5\n", + " 5.5\n", " ...\n", " 600\n", " 1.33\n", " 63.37\n", " 16.621538\n", - " 7.5\n", + " 8.5\n", " 600\n", " 1.15\n", " 37.07\n", " 14.372006\n", - " 15.0\n", + " 16.0\n", " \n", " \n", " Aircraft\n", @@ -8018,23 +8346,23 @@ " 1.1\n", " 63.43\n", " 13.747136\n", - " 5.0\n", + " 6.0\n", " 600\n", " 0.9\n", " 54.12\n", " 11.247657\n", - " 9.5\n", + " 10.5\n", " ...\n", " 600\n", " 0.99\n", " 55.95\n", " 12.372423\n", - " 7.5\n", + " 8.5\n", " 600\n", " 0.89\n", " 46.81\n", " 11.122683\n", - " 14.0\n", + " 15.0\n", " \n", " \n", " Birds\n", @@ -8042,23 +8370,23 @@ " 1.05\n", " 69.75\n", " 13.122266\n", - " 5.5\n", + " 6.5\n", " 600\n", " 0.9\n", " 70.69\n", " 11.247657\n", - " 5.5\n", + " 6.5\n", " ...\n", " 600\n", " 0.96\n", " 68.66\n", " 11.997501\n", - " 7.5\n", + " 8.5\n", " 600\n", " 1.0\n", " 50.13\n", " 12.497397\n", - " 14.5\n", + " 15.5\n", " \n", " \n", " Textures\n", @@ -8066,23 +8394,23 @@ " 0.88\n", " 70.78\n", " 10.997709\n", - " 6.0\n", + " 7.0\n", " 600\n", " 0.76\n", " 68.34\n", " 9.498021\n", - " 8.5\n", + " 9.5\n", " ...\n", " 600\n", " 0.83\n", " 66.49\n", " 10.372839\n", - " 12.0\n", + " 13.0\n", " 600\n", " 0.75\n", " 66.36\n", " 9.373047\n", - " 12.0\n", + " 13.0\n", " \n", " \n", " QuickDraw\n", @@ -8090,23 +8418,23 @@ " 1.16\n", " 59.17\n", " 14.49698\n", - " 4.5\n", + " 5.5\n", " 600\n", " 1.04\n", " 50.33\n", " 12.997293\n", - " 8.5\n", + " 9.5\n", " ...\n", " 600\n", " 1.0\n", " 51.52\n", " 12.497397\n", - " 8.5\n", + " 9.5\n", " 600\n", " 1.08\n", " 32.06\n", " 13.497188\n", - " 15.0\n", + " 16.0\n", " \n", " \n", " Fungi\n", @@ -8114,23 +8442,23 @@ " 1.17\n", " 41.49\n", " 14.621954\n", - " 5.5\n", + " 6.5\n", " 600\n", " 1.12\n", " 41.38\n", " 13.997084\n", - " 5.5\n", + " 6.5\n", " ...\n", " 600\n", " 1.14\n", " 39.96\n", " 14.247032\n", - " 5.5\n", + " 6.5\n", " 600\n", " 1.02\n", " 36.16\n", " 12.747345\n", - " 12.0\n", + " 13.0\n", " \n", " \n", " VGG Flower\n", @@ -8138,23 +8466,23 @@ " 0.77\n", " 85.96\n", " 9.622995\n", - " 8.0\n", + " 9.0\n", " 600\n", " 0.59\n", " 87.34\n", " 7.373464\n", - " 5.0\n", + " 6.0\n", " ...\n", " 600\n", " 0.69\n", " 87.15\n", " 8.623204\n", - " 5.0\n", + " 6.0\n", " 600\n", " 0.68\n", " 83.1\n", " 8.49823\n", - " 11.0\n", + " 12.0\n", " \n", " \n", " Traffic signs\n", @@ -8162,23 +8490,23 @@ " 1.29\n", " 60.78\n", " 16.121642\n", - " 7.0\n", + " 8.0\n", " 600\n", " 1.04\n", " 51.8\n", " 12.997293\n", - " 9.5\n", + " 10.5\n", " ...\n", " 600\n", " 1.09\n", " 48.83\n", " 13.622162\n", - " 11.5\n", + " 12.5\n", " 600\n", " 1.19\n", " 44.59\n", " 14.871902\n", - " 14.0\n", + " 15.0\n", " \n", " \n", " MSCOCO\n", @@ -8186,90 +8514,90 @@ " 1.14\n", " 48.11\n", " 14.247032\n", - " 4.5\n", + " 5.5\n", " 600\n", " 0.99\n", " 48.03\n", " 12.372423\n", - " 4.5\n", + " 5.5\n", " ...\n", " 600\n", " 1.12\n", " 43.74\n", " 13.997084\n", - " 8.0\n", + " 9.0\n", " 600\n", " 0.99\n", " 30.38\n", " 12.372423\n", - " 14.5\n", + " 15.5\n", " \n", " \n", "\n", - "

11 rows × 75 columns

\n", + "

11 rows × 80 columns

\n", "" ], "text/plain": [ " ALFA+fo-Proto-MAML BOHB \\\n", " # episodes 95% CI mean (%) stddev rank # episodes \n", "ILSVRC (valid) 600 NaN NaN NaN NaN 600 \n", - "ILSVRC (test) 600 1.11 52.8 13.87211 7.5 600 \n", - "Omniglot 600 1.51 61.87 18.871069 7.5 600 \n", - "Aircraft 600 1.1 63.43 13.747136 5.0 600 \n", - "Birds 600 1.05 69.75 13.122266 5.5 600 \n", - "Textures 600 0.88 70.78 10.997709 6.0 600 \n", - "QuickDraw 600 1.16 59.17 14.49698 4.5 600 \n", - "Fungi 600 1.17 41.49 14.621954 5.5 600 \n", - "VGG Flower 600 0.77 85.96 9.622995 8.0 600 \n", - "Traffic signs 600 1.29 60.78 16.121642 7.0 600 \n", - "MSCOCO 600 1.14 48.11 14.247032 4.5 600 \n", + "ILSVRC (test) 600 1.11 52.8 13.87211 8.5 600 \n", + "Omniglot 600 1.51 61.87 18.871069 8.5 600 \n", + "Aircraft 600 1.1 63.43 13.747136 6.0 600 \n", + "Birds 600 1.05 69.75 13.122266 6.5 600 \n", + "Textures 600 0.88 70.78 10.997709 7.0 600 \n", + "QuickDraw 600 1.16 59.17 14.49698 5.5 600 \n", + "Fungi 600 1.17 41.49 14.621954 6.5 600 \n", + "VGG Flower 600 0.77 85.96 9.622995 9.0 600 \n", + "Traffic signs 600 1.29 60.78 16.121642 8.0 600 \n", + "MSCOCO 600 1.14 48.11 14.247032 5.5 600 \n", "\n", - " ... fo-Proto-MAML \\\n", - " 95% CI mean (%) stddev rank ... # episodes 95% CI \n", - "ILSVRC (valid) NaN NaN NaN NaN ... 600 NaN \n", - "ILSVRC (test) 1.05 51.92 13.122266 7.5 ... 600 1.05 \n", - "Omniglot 1.21 67.57 15.12185 4.5 ... 600 1.33 \n", - "Aircraft 0.9 54.12 11.247657 9.5 ... 600 0.99 \n", - "Birds 0.9 70.69 11.247657 5.5 ... 600 0.96 \n", - "Textures 0.76 68.34 9.498021 8.5 ... 600 0.83 \n", - "QuickDraw 1.04 50.33 12.997293 8.5 ... 600 1.0 \n", - "Fungi 1.12 41.38 13.997084 5.5 ... 600 1.14 \n", - "VGG Flower 0.59 87.34 7.373464 5.0 ... 600 0.69 \n", - "Traffic signs 1.04 51.8 12.997293 9.5 ... 600 1.09 \n", - "MSCOCO 0.99 48.03 12.372423 4.5 ... 600 1.12 \n", + " ... fo-Proto-MAML \\\n", + " 95% CI mean (%) stddev rank ... # episodes 95% CI \n", + "ILSVRC (valid) NaN NaN NaN NaN ... 600 NaN \n", + "ILSVRC (test) 1.05 51.92 13.122266 8.5 ... 600 1.05 \n", + "Omniglot 1.21 67.57 15.12185 5.5 ... 600 1.33 \n", + "Aircraft 0.9 54.12 11.247657 10.5 ... 600 0.99 \n", + "Birds 0.9 70.69 11.247657 6.5 ... 600 0.96 \n", + "Textures 0.76 68.34 9.498021 9.5 ... 600 0.83 \n", + "QuickDraw 1.04 50.33 12.997293 9.5 ... 600 1.0 \n", + "Fungi 1.12 41.38 13.997084 6.5 ... 600 1.14 \n", + "VGG Flower 0.59 87.34 7.373464 6.0 ... 600 0.69 \n", + "Traffic signs 1.04 51.8 12.997293 10.5 ... 600 1.09 \n", + "MSCOCO 0.99 48.03 12.372423 5.5 ... 600 1.12 \n", "\n", " k-NN \\\n", " mean (%) stddev rank # episodes 95% CI mean (%) \n", "ILSVRC (valid) NaN NaN NaN 600 NaN NaN \n", - "ILSVRC (test) 49.53 13.122266 9.5 600 1.01 41.03 \n", - "Omniglot 63.37 16.621538 7.5 600 1.15 37.07 \n", - "Aircraft 55.95 12.372423 7.5 600 0.89 46.81 \n", - "Birds 68.66 11.997501 7.5 600 1.0 50.13 \n", - "Textures 66.49 10.372839 12.0 600 0.75 66.36 \n", - "QuickDraw 51.52 12.497397 8.5 600 1.08 32.06 \n", - "Fungi 39.96 14.247032 5.5 600 1.02 36.16 \n", - "VGG Flower 87.15 8.623204 5.0 600 0.68 83.1 \n", - "Traffic signs 48.83 13.622162 11.5 600 1.19 44.59 \n", - "MSCOCO 43.74 13.997084 8.0 600 0.99 30.38 \n", + "ILSVRC (test) 49.53 13.122266 10.5 600 1.01 41.03 \n", + "Omniglot 63.37 16.621538 8.5 600 1.15 37.07 \n", + "Aircraft 55.95 12.372423 8.5 600 0.89 46.81 \n", + "Birds 68.66 11.997501 8.5 600 1.0 50.13 \n", + "Textures 66.49 10.372839 13.0 600 0.75 66.36 \n", + "QuickDraw 51.52 12.497397 9.5 600 1.08 32.06 \n", + "Fungi 39.96 14.247032 6.5 600 1.02 36.16 \n", + "VGG Flower 87.15 8.623204 6.0 600 0.68 83.1 \n", + "Traffic signs 48.83 13.622162 12.5 600 1.19 44.59 \n", + "MSCOCO 43.74 13.997084 9.0 600 0.99 30.38 \n", "\n", " \n", " stddev rank \n", "ILSVRC (valid) NaN NaN \n", - "ILSVRC (test) 12.622371 14.0 \n", - "Omniglot 14.372006 15.0 \n", - "Aircraft 11.122683 14.0 \n", - "Birds 12.497397 14.5 \n", - "Textures 9.373047 12.0 \n", - "QuickDraw 13.497188 15.0 \n", - "Fungi 12.747345 12.0 \n", - "VGG Flower 8.49823 11.0 \n", - "Traffic signs 14.871902 14.0 \n", - "MSCOCO 12.372423 14.5 \n", + "ILSVRC (test) 12.622371 15.0 \n", + "Omniglot 14.372006 16.0 \n", + "Aircraft 11.122683 15.0 \n", + "Birds 12.497397 15.5 \n", + "Textures 9.373047 13.0 \n", + "QuickDraw 13.497188 16.0 \n", + "Fungi 12.747345 13.0 \n", + "VGG Flower 8.49823 12.0 \n", + "Traffic signs 14.871902 15.0 \n", + "MSCOCO 12.372423 15.5 \n", "\n", - "[11 rows x 75 columns]" + "[11 rows x 80 columns]" ] }, - "execution_count": 64, + "execution_count": 67, "metadata": {}, "output_type": "execute_result" } @@ -8281,7 +8609,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 68, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -8374,23 +8702,23 @@ " 1.1\n", " 50.8\n", " 13.747136\n", - " 10.5\n", + " 11.5\n", " 600\n", " 1.1\n", " 51.8\n", " 13.747136\n", - " 10.5\n", + " 11.5\n", " ...\n", " 600\n", " 1.05\n", " 46.52\n", " 13.122266\n", - " 12.0\n", + " 13.0\n", " 600\n", " 0.94\n", " 38.55\n", " 11.747553\n", - " 15.5\n", + " 16.5\n", " \n", " \n", " Omniglot\n", @@ -8398,7 +8726,7 @@ " 0.5\n", " 91.7\n", " 6.248698\n", - " 8.0\n", + " 8.5\n", " 600\n", " 0.5\n", " 93.2\n", @@ -8409,12 +8737,12 @@ " 0.97\n", " 82.69\n", " 12.122475\n", - " 13.5\n", + " 14.5\n", " 600\n", " 1.08\n", " 74.6\n", " 13.497188\n", - " 17.0\n", + " 18.0\n", " \n", " \n", " Aircraft\n", @@ -8422,23 +8750,23 @@ " 0.6\n", " 83.7\n", " 7.498438\n", - " 7.5\n", + " 8.5\n", " 600\n", " 0.5\n", " 87.2\n", " 6.248698\n", - " 3.0\n", + " 4.0\n", " ...\n", " 600\n", " 0.76\n", " 75.23\n", " 9.498021\n", - " 13.0\n", + " 14.0\n", " 600\n", " 0.82\n", " 64.98\n", " 10.247865\n", - " 18.0\n", + " 19.0\n", " \n", " \n", " Birds\n", @@ -8446,23 +8774,23 @@ " 0.9\n", " 73.6\n", " 11.247657\n", - " 10.0\n", + " 11.0\n", " 600\n", " 0.8\n", " 79.2\n", " 9.997917\n", - " 3.5\n", + " 4.5\n", " ...\n", " 600\n", " 1.02\n", " 69.88\n", " 12.747345\n", - " 11.5\n", + " 12.5\n", " 600\n", " 0.92\n", " 66.35\n", " 11.497605\n", - " 13.5\n", + " 14.5\n", " \n", " \n", " Textures\n", @@ -8470,23 +8798,23 @@ " 0.7\n", " 59.5\n", " 8.748178\n", - " 17.0\n", + " 18.0\n", " 600\n", " 0.8\n", " 68.8\n", " 9.997917\n", - " 10.5\n", + " 11.5\n", " ...\n", " 600\n", " 0.81\n", " 68.25\n", " 10.122891\n", - " 10.5\n", + " 11.5\n", " 600\n", " 0.79\n", " 63.58\n", " 9.872943\n", - " 14.5\n", + " 15.5\n", " \n", " \n", " QuickDraw\n", @@ -8494,23 +8822,23 @@ " 0.8\n", " 74.7\n", " 9.997917\n", - " 11.0\n", + " 12.0\n", " 600\n", " 0.7\n", " 79.5\n", " 8.748178\n", - " 5.0\n", + " 5.5\n", " ...\n", " 600\n", " 0.94\n", " 66.84\n", " 11.747553\n", - " 12.0\n", + " 13.0\n", " 600\n", " 1.05\n", " 44.88\n", " 13.122266\n", - " 18.0\n", + " 19.0\n", " \n", " \n", " Fungi\n", @@ -8518,23 +8846,23 @@ " 1.1\n", " 50.2\n", " 13.747136\n", - " 7.5\n", + " 8.5\n", " 600\n", " 1.1\n", " 58.1\n", " 13.747136\n", - " 5.0\n", + " 6.0\n", " ...\n", " 600\n", " 1.17\n", " 41.99\n", " 14.621954\n", - " 12.0\n", + " 13.0\n", " 600\n", " 1.06\n", " 37.12\n", " 13.24724\n", - " 14.5\n", + " 15.5\n", " \n", " \n", " VGG Flower\n", @@ -8542,23 +8870,23 @@ " 0.5\n", " 88.9\n", " 6.248698\n", - " 10.0\n", + " 11.0\n", " 600\n", " 0.6\n", " 91.6\n", " 7.498438\n", - " 3.0\n", + " 4.0\n", " ...\n", " 600\n", " 0.67\n", " 88.72\n", " 8.373256\n", - " 10.0\n", + " 11.0\n", " 600\n", " 0.61\n", " 83.47\n", " 7.623412\n", - " 14.5\n", + " 15.5\n", " \n", " \n", " Traffic signs\n", @@ -8566,23 +8894,23 @@ " 1.1\n", " 56.5\n", " 13.747136\n", - " 8.5\n", + " 9.5\n", " 600\n", " 1.1\n", " 58.4\n", " 13.747136\n", - " 6.0\n", + " 7.0\n", " ...\n", " 600\n", " 1.08\n", " 52.42\n", " 13.497188\n", - " 11.5\n", + " 12.5\n", " 600\n", " 1.1\n", " 40.11\n", " 13.747136\n", - " 17.0\n", + " 18.0\n", " \n", " \n", " MSCOCO\n", @@ -8590,76 +8918,76 @@ " 1.0\n", " 39.4\n", " 12.497397\n", - " 12.5\n", + " 13.5\n", " 600\n", " 1.0\n", " 50.0\n", " 12.497397\n", - " 7.0\n", + " 8.0\n", " ...\n", " 600\n", " 1.13\n", " 41.74\n", " 14.122058\n", - " 10.0\n", + " 11.0\n", " 600\n", " 0.96\n", " 29.55\n", " 11.997501\n", - " 16.0\n", + " 17.0\n", " \n", " \n", "\n", - "

11 rows × 90 columns

\n", + "

11 rows × 95 columns

\n", "" ], "text/plain": [ " CNAPs FLUTE \\\n", " # episodes 95% CI mean (%) stddev rank # episodes 95% CI \n", "ILSVRC (valid) 600 NaN NaN NaN NaN 600 NaN \n", - "ILSVRC (test) 600 1.1 50.8 13.747136 10.5 600 1.1 \n", - "Omniglot 600 0.5 91.7 6.248698 8.0 600 0.5 \n", - "Aircraft 600 0.6 83.7 7.498438 7.5 600 0.5 \n", - "Birds 600 0.9 73.6 11.247657 10.0 600 0.8 \n", - "Textures 600 0.7 59.5 8.748178 17.0 600 0.8 \n", - "QuickDraw 600 0.8 74.7 9.997917 11.0 600 0.7 \n", - "Fungi 600 1.1 50.2 13.747136 7.5 600 1.1 \n", - "VGG Flower 600 0.5 88.9 6.248698 10.0 600 0.6 \n", - "Traffic signs 600 1.1 56.5 13.747136 8.5 600 1.1 \n", - "MSCOCO 600 1.0 39.4 12.497397 12.5 600 1.0 \n", + "ILSVRC (test) 600 1.1 50.8 13.747136 11.5 600 1.1 \n", + "Omniglot 600 0.5 91.7 6.248698 8.5 600 0.5 \n", + "Aircraft 600 0.6 83.7 7.498438 8.5 600 0.5 \n", + "Birds 600 0.9 73.6 11.247657 11.0 600 0.8 \n", + "Textures 600 0.7 59.5 8.748178 18.0 600 0.8 \n", + "QuickDraw 600 0.8 74.7 9.997917 12.0 600 0.7 \n", + "Fungi 600 1.1 50.2 13.747136 8.5 600 1.1 \n", + "VGG Flower 600 0.5 88.9 6.248698 11.0 600 0.6 \n", + "Traffic signs 600 1.1 56.5 13.747136 9.5 600 1.1 \n", + "MSCOCO 600 1.0 39.4 12.497397 13.5 600 1.0 \n", "\n", " ... fo-Proto-MAML \\\n", " mean (%) stddev rank ... # episodes 95% CI mean (%) \n", "ILSVRC (valid) NaN NaN NaN ... 600 NaN NaN \n", - "ILSVRC (test) 51.8 13.747136 10.5 ... 600 1.05 46.52 \n", + "ILSVRC (test) 51.8 13.747136 11.5 ... 600 1.05 46.52 \n", "Omniglot 93.2 6.248698 5.5 ... 600 0.97 82.69 \n", - "Aircraft 87.2 6.248698 3.0 ... 600 0.76 75.23 \n", - "Birds 79.2 9.997917 3.5 ... 600 1.02 69.88 \n", - "Textures 68.8 9.997917 10.5 ... 600 0.81 68.25 \n", - "QuickDraw 79.5 8.748178 5.0 ... 600 0.94 66.84 \n", - "Fungi 58.1 13.747136 5.0 ... 600 1.17 41.99 \n", - "VGG Flower 91.6 7.498438 3.0 ... 600 0.67 88.72 \n", - "Traffic signs 58.4 13.747136 6.0 ... 600 1.08 52.42 \n", - "MSCOCO 50.0 12.497397 7.0 ... 600 1.13 41.74 \n", + "Aircraft 87.2 6.248698 4.0 ... 600 0.76 75.23 \n", + "Birds 79.2 9.997917 4.5 ... 600 1.02 69.88 \n", + "Textures 68.8 9.997917 11.5 ... 600 0.81 68.25 \n", + "QuickDraw 79.5 8.748178 5.5 ... 600 0.94 66.84 \n", + "Fungi 58.1 13.747136 6.0 ... 600 1.17 41.99 \n", + "VGG Flower 91.6 7.498438 4.0 ... 600 0.67 88.72 \n", + "Traffic signs 58.4 13.747136 7.0 ... 600 1.08 52.42 \n", + "MSCOCO 50.0 12.497397 8.0 ... 600 1.13 41.74 \n", "\n", " k-NN \n", " stddev rank # episodes 95% CI mean (%) stddev rank \n", "ILSVRC (valid) NaN NaN 600 NaN NaN NaN NaN \n", - "ILSVRC (test) 13.122266 12.0 600 0.94 38.55 11.747553 15.5 \n", - "Omniglot 12.122475 13.5 600 1.08 74.6 13.497188 17.0 \n", - "Aircraft 9.498021 13.0 600 0.82 64.98 10.247865 18.0 \n", - "Birds 12.747345 11.5 600 0.92 66.35 11.497605 13.5 \n", - "Textures 10.122891 10.5 600 0.79 63.58 9.872943 14.5 \n", - "QuickDraw 11.747553 12.0 600 1.05 44.88 13.122266 18.0 \n", - "Fungi 14.621954 12.0 600 1.06 37.12 13.24724 14.5 \n", - "VGG Flower 8.373256 10.0 600 0.61 83.47 7.623412 14.5 \n", - "Traffic signs 13.497188 11.5 600 1.1 40.11 13.747136 17.0 \n", - "MSCOCO 14.122058 10.0 600 0.96 29.55 11.997501 16.0 \n", + "ILSVRC (test) 13.122266 13.0 600 0.94 38.55 11.747553 16.5 \n", + "Omniglot 12.122475 14.5 600 1.08 74.6 13.497188 18.0 \n", + "Aircraft 9.498021 14.0 600 0.82 64.98 10.247865 19.0 \n", + "Birds 12.747345 12.5 600 0.92 66.35 11.497605 14.5 \n", + "Textures 10.122891 11.5 600 0.79 63.58 9.872943 15.5 \n", + "QuickDraw 11.747553 13.0 600 1.05 44.88 13.122266 19.0 \n", + "Fungi 14.621954 13.0 600 1.06 37.12 13.24724 15.5 \n", + "VGG Flower 8.373256 11.0 600 0.61 83.47 7.623412 15.5 \n", + "Traffic signs 13.497188 12.5 600 1.1 40.11 13.747136 18.0 \n", + "MSCOCO 14.122058 11.0 600 0.96 29.55 11.997501 17.0 \n", "\n", - "[11 rows x 90 columns]" + "[11 rows x 95 columns]" ] }, - "execution_count": 65, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -8671,7 +8999,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 69, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -8683,25 +9011,26 @@ { "data": { "text/plain": [ - "ALFA+fo-Proto-MAML 6.10\n", - "BOHB 6.85\n", - "CTX 1.75\n", - "Finetune 9.45\n", - "MatchingNet 12.55\n", - "ProtoNet 9.75\n", - "ProtoNet (large) 6.25\n", - "RelationNet 14.55\n", - "SimpleCNAPS 7.75\n", - "TSA_resnet18 2.80\n", - "TSA_resnet34 1.50\n", - "TransductiveCNAPS 7.60\n", - "fo-MAML 11.25\n", - "fo-Proto-MAML 8.25\n", - "k-NN 13.60\n", + "ALFA+fo-Proto-MAML 7.10\n", + "BOHB 7.85\n", + "CTX 2.50\n", + "Finetune 10.45\n", + "MatchingNet 13.55\n", + "PMF-DINOSmall 1.50\n", + "ProtoNet 10.75\n", + "ProtoNet (large) 7.25\n", + "RelationNet 15.55\n", + "SimpleCNAPS 8.75\n", + "TSA_resnet18 3.80\n", + "TSA_resnet34 2.25\n", + "TransductiveCNAPS 8.60\n", + "fo-MAML 12.25\n", + "fo-Proto-MAML 9.25\n", + "k-NN 14.60\n", "dtype: float64" ] }, - "execution_count": 66, + "execution_count": 69, "metadata": {}, "output_type": "execute_result" } @@ -8712,7 +9041,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 70, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -8724,28 +9053,29 @@ { "data": { "text/plain": [ - "CNAPs 10.25\n", - "FLUTE 5.90\n", - "Finetune 13.10\n", - "MatchingNet 15.40\n", - "ProtoNet 13.50\n", - "RelationNet 16.80\n", - "SUR 7.65\n", - "SUR-pnf 8.20\n", - "SimpleCNAPS 7.45\n", - "TSA 1.65\n", - "TransductiveCNAPS 6.05\n", - "TriM 6.60\n", - "URL 2.15\n", - "URT 6.05\n", - "URT-pf 7.55\n", - "fo-MAML 15.25\n", - "fo-Proto-MAML 11.60\n", - "k-NN 15.85\n", + "CNAPs 11.20\n", + "FLUTE 6.75\n", + "Finetune 14.10\n", + "MatchingNet 16.40\n", + "PMF-DINOSmall 2.25\n", + "ProtoNet 14.50\n", + "RelationNet 17.80\n", + "SUR 8.45\n", + "SUR-pnf 9.20\n", + "SimpleCNAPS 8.40\n", + "TSA 2.40\n", + "TransductiveCNAPS 6.95\n", + "TriM 7.55\n", + "URL 2.95\n", + "URT 6.85\n", + "URT-pf 8.55\n", + "fo-MAML 16.25\n", + "fo-Proto-MAML 12.60\n", + "k-NN 16.85\n", "dtype: float64" ] }, - "execution_count": 67, + "execution_count": 70, "metadata": {}, "output_type": "execute_result" } @@ -8768,7 +9098,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 71, "metadata": { "id": "u0S8Uno9yem3" }, @@ -8789,7 +9119,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 72, "metadata": { "id": "_uFqczyza6XZ" }, @@ -8819,7 +9149,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 73, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -8833,246 +9163,260 @@ "data": { "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
 Avg rankILSVRC (test)OmniglotAircraftBirdsTexturesQuickDrawFungiVGG FlowerTraffic signsMSCOCOAvg rankILSVRC (test)OmniglotAircraftBirdsTexturesQuickDrawFungiVGG FlowerTraffic signsMSCOCO
k-NN13.641.03±1.01 (14)37.07±1.15 (15)46.81±0.89 (14)50.13±1.0 (14.5)66.36±0.75 (12)32.06±1.08 (15)36.16±1.02 (12)83.1±0.68 (11)44.59±1.19 (14)30.38±0.99 (14.5)
Finetune9.4545.78±1.1 (12)60.85±1.58 (10.5)68.69±1.26 (4)57.31±1.26 (13)69.05±0.9 (8.5)42.6±1.17 (12.5)38.2±1.02 (10)85.51±0.68 (8)66.79±1.31 (4)34.86±0.97 (12)
MatchingNet12.5545.0±1.1 (12)52.27±1.28 (13)48.97±0.93 (12)62.21±0.95 (11.5)64.15±0.85 (14)42.87±1.09 (12.5)33.97±1.0 (13)80.13±0.71 (14)47.8±1.14 (11.5)34.99±1.0 (12)
ProtoNet9.7550.5±1.08 (9.5)59.98±1.35 (10.5)53.1±1.0 (9.5)68.79±1.01 (7.5)66.56±0.83 (12)48.96±1.08 (10)39.71±1.11 (8)85.27±0.77 (8)47.12±1.1 (13)41.0±1.1 (9.5)
fo-MAML11.2545.51±1.11 (12)55.55±1.54 (12)56.24±1.11 (7.5)63.61±1.06 (11.5)68.04±0.81 (8.5)43.96±1.29 (12.5)32.1±1.1 (14)81.74±0.83 (13)50.93±1.51 (9.5)35.3±1.23 (12)
RelationNet14.5534.69±1.01 (15)45.35±1.36 (14)40.73±0.83 (15)49.51±1.05 (14.5)52.97±0.69 (15)43.3±1.08 (12.5)30.55±1.04 (15)68.76±0.83 (15)33.67±1.05 (15)29.15±1.01 (14.5)
fo-Proto-MAML8.2549.53±1.05 (9.5)63.37±1.33 (7.5)55.95±0.99 (7.5)68.66±0.96 (7.5)66.49±0.83 (12)51.52±1.0 (8.5)39.96±1.14 (5.5)87.15±0.69 (5)48.83±1.09 (11.5)43.74±1.12 (8)
ALFA+fo-Proto-MAML6.152.8±1.11 (7.5)61.87±1.51 (7.5)63.43±1.1 (5)69.75±1.05 (5.5)70.78±0.88 (6)59.17±1.16 (4.5)41.49±1.17 (5.5)85.96±0.77 (8)60.78±1.29 (7)48.11±1.14 (4.5)
ProtoNet (large)6.2553.69±1.07 (5)68.5±1.27 (4.5)58.04±0.96 (6)74.07±0.92 (3.5)68.76±0.77 (8.5)53.3±1.06 (7)40.73±1.15 (5.5)86.96±0.73 (5)58.11±1.05 (8)41.7±1.08 (9.5)
CTX1.7562.76±0.99 (1.5)82.21±1.0 (1.5)79.49±0.89 (1.5)80.63±0.88 (2)75.57±0.64 (3)72.68±0.82 (1)51.58±1.11 (1.5)95.34±0.37 (1)82.65±0.76 (2)59.9±1.02 (2.5)
BOHB6.8551.92±1.05 (7.5)67.57±1.21 (4.5)54.12±0.9 (9.5)70.69±0.9 (5.5)68.34±0.76 (8.5)50.33±1.04 (8.5)41.38±1.12 (5.5)87.34±0.59 (5)51.8±1.04 (9.5)48.03±0.99 (4.5)
SimpleCNAPS7.7554.8±1.2 (5)62.0±1.3 (7.5)49.2±0.9 (12)66.5±1.0 (9.5)71.6±0.7 (4.5)56.6±1.0 (6)37.5±1.2 (10)82.1±0.9 (11)63.1±1.1 (5.5)45.8±1.0 (6.5)
TransductiveCNAPS7.654.1±1.1 (5)62.9±1.3 (7.5)48.4±0.9 (12)67.3±0.9 (9.5)72.5±0.7 (4.5)58.0±1.0 (4.5)37.7±1.1 (10)82.8±0.8 (11)61.8±1.1 (5.5)45.8±1.0 (6.5)
TSA_resnet182.859.5±1.1 (3)78.2±1.2 (3)72.2±1.0 (3)74.9±0.9 (3.5)77.3±0.7 (2)67.6±0.9 (3)44.7±1.0 (3)90.9±0.6 (3)82.5±0.8 (2)59.0±1.0 (2.5)
TSA_resnet341.563.73±0.99 (1.5)82.58±1.11 (1.5)80.13±1.01 (1.5)83.39±0.8 (1)79.61±0.68 (1)71.03±0.84 (2)51.38±1.17 (1.5)94.05±0.45 (2)81.71±0.95 (2)61.67±0.95 (1)k-NN14.641.03±1.01 (15)37.07±1.15 (16)46.81±0.89 (15)50.13±1.0 (15.5)66.36±0.75 (13)32.06±1.08 (16)36.16±1.02 (13)83.1±0.68 (12)44.59±1.19 (15)30.38±0.99 (15.5)
Finetune10.4545.78±1.1 (13)60.85±1.58 (11.5)68.69±1.26 (5)57.31±1.26 (14)69.05±0.9 (9.5)42.6±1.17 (13.5)38.2±1.02 (11)85.51±0.68 (9)66.79±1.31 (5)34.86±0.97 (13)
MatchingNet13.5545.0±1.1 (13)52.27±1.28 (14)48.97±0.93 (13)62.21±0.95 (12.5)64.15±0.85 (15)42.87±1.09 (13.5)33.97±1.0 (14)80.13±0.71 (15)47.8±1.14 (12.5)34.99±1.0 (13)
ProtoNet10.7550.5±1.08 (10.5)59.98±1.35 (11.5)53.1±1.0 (10.5)68.79±1.01 (8.5)66.56±0.83 (13)48.96±1.08 (11)39.71±1.11 (9)85.27±0.77 (9)47.12±1.1 (14)41.0±1.1 (10.5)
fo-MAML12.2545.51±1.11 (13)55.55±1.54 (13)56.24±1.11 (8.5)63.61±1.06 (12.5)68.04±0.81 (9.5)43.96±1.29 (13.5)32.1±1.1 (15)81.74±0.83 (14)50.93±1.51 (10.5)35.3±1.23 (13)
RelationNet15.5534.69±1.01 (16)45.35±1.36 (15)40.73±0.83 (16)49.51±1.05 (15.5)52.97±0.69 (16)43.3±1.08 (13.5)30.55±1.04 (16)68.76±0.83 (16)33.67±1.05 (16)29.15±1.01 (15.5)
fo-Proto-MAML9.2549.53±1.05 (10.5)63.37±1.33 (8.5)55.95±0.99 (8.5)68.66±0.96 (8.5)66.49±0.83 (13)51.52±1.0 (9.5)39.96±1.14 (6.5)87.15±0.69 (6)48.83±1.09 (12.5)43.74±1.12 (9)
ALFA+fo-Proto-MAML7.152.8±1.11 (8.5)61.87±1.51 (8.5)63.43±1.1 (6)69.75±1.05 (6.5)70.78±0.88 (7)59.17±1.16 (5.5)41.49±1.17 (6.5)85.96±0.77 (9)60.78±1.29 (8)48.11±1.14 (5.5)
ProtoNet (large)7.2553.69±1.07 (6)68.5±1.27 (5.5)58.04±0.96 (7)74.07±0.92 (4.5)68.76±0.77 (9.5)53.3±1.06 (8)40.73±1.15 (6.5)86.96±0.73 (6)58.11±1.05 (9)41.7±1.08 (10.5)
CTX2.562.76±0.99 (2.5)82.21±1.0 (2)79.49±0.89 (1.5)80.63±0.88 (3)75.57±0.64 (4)72.68±0.82 (2)51.58±1.11 (2.5)95.34±0.37 (1)82.65±0.76 (3)59.9±1.02 (3.5)
BOHB7.8551.92±1.05 (8.5)67.57±1.21 (5.5)54.12±0.9 (10.5)70.69±0.9 (6.5)68.34±0.76 (9.5)50.33±1.04 (9.5)41.38±1.12 (6.5)87.34±0.59 (6)51.8±1.04 (10.5)48.03±0.99 (5.5)
SimpleCNAPS8.7554.8±1.2 (6)62.0±1.3 (8.5)49.2±0.9 (13)66.5±1.0 (10.5)71.6±0.7 (5.5)56.6±1.0 (7)37.5±1.2 (11)82.1±0.9 (12)63.1±1.1 (6.5)45.8±1.0 (7.5)
TransductiveCNAPS8.654.1±1.1 (6)62.9±1.3 (8.5)48.4±0.9 (13)67.3±0.9 (10.5)72.5±0.7 (5.5)58.0±1.0 (5.5)37.7±1.1 (11)82.8±0.8 (12)61.8±1.1 (6.5)45.8±1.0 (7.5)
TSA_resnet183.859.5±1.1 (4)78.2±1.2 (4)72.2±1.0 (4)74.9±0.9 (4.5)77.3±0.7 (3)67.6±0.9 (4)44.7±1.0 (4)90.9±0.6 (4)82.5±0.8 (3)59.0±1.0 (3.5)
TSA_resnet342.2563.73±0.99 (2.5)82.58±1.11 (2)80.13±1.01 (1.5)83.39±0.8 (2)79.61±0.68 (2)71.03±0.84 (3)51.38±1.17 (2.5)94.05±0.45 (2.5)81.71±0.95 (3)61.67±0.95 (1.5)
PMF-DINOSmall1.575.51±0.72 (1)82.81±1.1 (2)78.38±1.09 (3)85.18±0.77 (1)86.95±0.6 (1)74.47±0.83 (1)55.16±1.09 (1)94.66±0.48 (2.5)90.04±0.81 (1)62.6±0.96 (1.5)
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 70, + "execution_count": 73, "metadata": {}, "output_type": "execute_result" } @@ -9084,7 +9428,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 74, "metadata": { "id": "dMjhdJkiimQx" }, @@ -9095,7 +9439,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 75, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -9109,288 +9453,302 @@ "data": { "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
 Avg rankILSVRC (test)OmniglotAircraftBirdsTexturesQuickDrawFungiVGG FlowerTraffic signsMSCOCOAvg rankILSVRC (test)OmniglotAircraftBirdsTexturesQuickDrawFungiVGG FlowerTraffic signsMSCOCO
k-NN15.8538.55±0.94 (15.5)74.6±1.08 (17)64.98±0.82 (18)66.35±0.92 (13.5)63.58±0.79 (14.5)44.88±1.05 (18)37.12±1.06 (14.5)83.47±0.61 (14.5)40.11±1.1 (17)29.55±0.96 (16)
Finetune13.143.08±1.08 (13.5)71.11±1.37 (18)72.03±1.07 (14.5)59.82±1.15 (16)69.14±0.85 (8.5)47.05±1.16 (17)38.16±1.04 (14.5)85.28±0.69 (13)66.74±1.23 (2)35.17±1.08 (14)
MatchingNet15.436.08±1.0 (17)78.25±1.01 (15.5)69.17±0.96 (16.5)56.4±1.0 (17)61.8±0.74 (16)60.81±1.03 (14.5)33.7±1.04 (17)81.9±0.72 (16)55.57±1.08 (8.5)28.79±0.96 (16)
ProtoNet13.544.5±1.05 (13.5)79.56±1.12 (15.5)71.14±0.86 (14.5)67.01±1.02 (13.5)65.18±0.84 (12.5)64.88±0.89 (13)40.26±1.13 (13)86.85±0.71 (12)46.48±1.0 (15)39.87±1.06 (12.5)
fo-MAML15.2537.83±1.01 (15.5)83.92±0.95 (13.5)76.41±0.69 (12)62.43±1.08 (15)64.16±0.83 (14.5)59.73±1.1 (16)33.54±1.11 (17)79.94±0.84 (17)42.91±1.31 (16)29.37±1.08 (16)
RelationNet16.830.89±0.93 (18)86.57±0.79 (12)69.71±0.83 (16.5)54.14±0.99 (18)56.56±0.73 (18)61.75±0.97 (14.5)32.56±1.08 (17)76.08±0.76 (18)37.48±0.93 (18)27.41±0.89 (18)
fo-Proto-MAML11.646.52±1.05 (12)82.69±0.97 (13.5)75.23±0.76 (13)69.88±1.02 (11.5)68.25±0.81 (10.5)66.84±0.94 (12)41.99±1.17 (12)88.72±0.67 (10)52.42±1.08 (11.5)41.74±1.13 (10)
CNAPs10.2550.8±1.1 (10.5)91.7±0.5 (8)83.7±0.6 (7.5)73.6±0.9 (10)59.5±0.7 (17)74.7±0.8 (11)50.2±1.1 (7.5)88.9±0.5 (10)56.5±1.1 (8.5)39.4±1.0 (12.5)
SUR7.6556.1±1.1 (7)93.1±0.5 (5.5)84.6±0.7 (5.5)70.6±1.0 (11.5)71.0±0.8 (6.5)81.3±0.6 (4)64.2±1.1 (3.5)82.8±0.8 (14.5)53.4±1.0 (11.5)50.1±1.0 (7)
SUR-pnf8.256.0±1.1 (7)90.0±0.6 (10.5)79.7±0.8 (10.5)75.9±0.9 (7.5)72.5±0.7 (4.5)76.7±0.7 (8.5)49.8±1.1 (7.5)90.0±0.6 (7.5)52.2±0.8 (11.5)50.2±1.1 (7)
SimpleCNAPS7.4556.5±1.1 (7)91.9±0.6 (8)83.8±0.6 (7.5)76.1±0.9 (7.5)70.0±0.8 (8.5)78.3±0.7 (6.5)49.1±1.2 (7.5)91.3±0.6 (6)59.2±1.0 (6)42.4±1.1 (10)
TransductiveCNAPS6.0557.9±1.1 (2.5)94.3±0.4 (3.5)84.7±0.5 (5.5)78.8±0.7 (3.5)66.2±0.8 (12.5)77.9±0.6 (6.5)48.9±1.2 (7.5)92.3±0.4 (3)59.7±1.1 (6)42.5±1.1 (10)
URT6.0555.7±1.0 (7)94.4±0.4 (3.5)85.8±0.6 (4)76.3±0.8 (7.5)71.8±0.7 (4.5)82.5±0.6 (2)63.5±1.0 (3.5)88.2±0.6 (10)51.1±1.1 (14)52.2±1.1 (4.5)
URT-pf7.5555.5±1.1 (7)90.2±0.6 (10.5)79.8±0.7 (10.5)77.5±0.8 (5)73.5±0.7 (3)75.8±0.7 (10)48.1±0.9 (10.5)91.9±0.5 (3)52.0±1.4 (11.5)52.1±1.0 (4.5)
FLUTE5.951.8±1.1 (10.5)93.2±0.5 (5.5)87.2±0.5 (3)79.2±0.8 (3.5)68.8±0.8 (10.5)79.5±0.7 (5)58.1±1.1 (5)91.6±0.6 (3)58.4±1.1 (6)50.0±1.0 (7)
URL2.1557.51±1.08 (2.5)94.51±0.41 (1.5)88.59±0.46 (2)80.54±0.69 (1.5)76.17±0.67 (1.5)81.94±0.56 (2)68.75±0.95 (1.5)92.11±0.48 (3)63.34±1.19 (3.5)54.03±0.96 (2.5)
TSA1.6557.35±1.05 (2.5)94.96±0.38 (1.5)89.33±0.44 (1)81.42±0.74 (1.5)76.74±0.72 (1.5)82.01±0.57 (2)67.4±0.99 (1.5)92.18±0.52 (3)83.55±0.9 (1)55.75±1.06 (1)
TriM6.658.6±1.0 (2.5)92.0±0.6 (8)82.8±0.7 (9)75.3±0.8 (7.5)71.2±0.8 (6.5)77.3±0.7 (8.5)48.5±1.0 (10.5)90.5±0.5 (7.5)63.0±1.0 (3.5)52.8±1.1 (2.5)k-NN16.8538.55±0.94 (16.5)74.6±1.08 (18)64.98±0.82 (19)66.35±0.92 (14.5)63.58±0.79 (15.5)44.88±1.05 (19)37.12±1.06 (15.5)83.47±0.61 (15.5)40.11±1.1 (18)29.55±0.96 (17)
Finetune14.143.08±1.08 (14.5)71.11±1.37 (19)72.03±1.07 (15.5)59.82±1.15 (17)69.14±0.85 (9.5)47.05±1.16 (18)38.16±1.04 (15.5)85.28±0.69 (14)66.74±1.23 (3)35.17±1.08 (15)
MatchingNet16.436.08±1.0 (18)78.25±1.01 (16.5)69.17±0.96 (17.5)56.4±1.0 (18)61.8±0.74 (17)60.81±1.03 (15.5)33.7±1.04 (18)81.9±0.72 (17)55.57±1.08 (9.5)28.79±0.96 (17)
ProtoNet14.544.5±1.05 (14.5)79.56±1.12 (16.5)71.14±0.86 (15.5)67.01±1.02 (14.5)65.18±0.84 (13.5)64.88±0.89 (14)40.26±1.13 (14)86.85±0.71 (13)46.48±1.0 (16)39.87±1.06 (13.5)
fo-MAML16.2537.83±1.01 (16.5)83.92±0.95 (14.5)76.41±0.69 (13)62.43±1.08 (16)64.16±0.83 (15.5)59.73±1.1 (17)33.54±1.11 (18)79.94±0.84 (18)42.91±1.31 (17)29.37±1.08 (17)
RelationNet17.830.89±0.93 (19)86.57±0.79 (13)69.71±0.83 (17.5)54.14±0.99 (19)56.56±0.73 (19)61.75±0.97 (15.5)32.56±1.08 (18)76.08±0.76 (19)37.48±0.93 (19)27.41±0.89 (19)
fo-Proto-MAML12.646.52±1.05 (13)82.69±0.97 (14.5)75.23±0.76 (14)69.88±1.02 (12.5)68.25±0.81 (11.5)66.84±0.94 (13)41.99±1.17 (13)88.72±0.67 (11)52.42±1.08 (12.5)41.74±1.13 (11)
CNAPs11.250.8±1.1 (11.5)91.7±0.5 (8.5)83.7±0.6 (8.5)73.6±0.9 (11)59.5±0.7 (18)74.7±0.8 (12)50.2±1.1 (8.5)88.9±0.5 (11)56.5±1.1 (9.5)39.4±1.0 (13.5)
SUR8.4556.1±1.1 (8)93.1±0.5 (5.5)84.6±0.7 (6.5)70.6±1.0 (12.5)71.0±0.8 (7.5)81.3±0.6 (4)64.2±1.1 (4.5)82.8±0.8 (15.5)53.4±1.0 (12.5)50.1±1.0 (8)
SUR-pnf9.256.0±1.1 (8)90.0±0.6 (11.5)79.7±0.8 (11.5)75.9±0.9 (8.5)72.5±0.7 (5.5)76.7±0.7 (9.5)49.8±1.1 (8.5)90.0±0.6 (8.5)52.2±0.8 (12.5)50.2±1.1 (8)
SimpleCNAPS8.456.5±1.1 (8)91.9±0.6 (8.5)83.8±0.6 (8.5)76.1±0.9 (8.5)70.0±0.8 (9.5)78.3±0.7 (7.5)49.1±1.2 (8.5)91.3±0.6 (7)59.2±1.0 (7)42.4±1.1 (11)
TransductiveCNAPS6.9557.9±1.1 (3.5)94.3±0.4 (3.5)84.7±0.5 (6.5)78.8±0.7 (4.5)66.2±0.8 (13.5)77.9±0.6 (7.5)48.9±1.2 (8.5)92.3±0.4 (4)59.7±1.1 (7)42.5±1.1 (11)
URT6.8555.7±1.0 (8)94.4±0.4 (3.5)85.8±0.6 (5)76.3±0.8 (8.5)71.8±0.7 (5.5)82.5±0.6 (2)63.5±1.0 (4.5)88.2±0.6 (11)51.1±1.1 (15)52.2±1.1 (5.5)
URT-pf8.5555.5±1.1 (8)90.2±0.6 (11.5)79.8±0.7 (11.5)77.5±0.8 (6)73.5±0.7 (4)75.8±0.7 (11)48.1±0.9 (11.5)91.9±0.5 (4)52.0±1.4 (12.5)52.1±1.0 (5.5)
FLUTE6.7551.8±1.1 (11.5)93.2±0.5 (5.5)87.2±0.5 (4)79.2±0.8 (4.5)68.8±0.8 (11.5)79.5±0.7 (5.5)58.1±1.1 (6)91.6±0.6 (4)58.4±1.1 (7)50.0±1.0 (8)
URL2.9557.51±1.08 (3.5)94.51±0.41 (1.5)88.59±0.46 (3)80.54±0.69 (2.5)76.17±0.67 (2.5)81.94±0.56 (2)68.75±0.95 (2.5)92.11±0.48 (4)63.34±1.19 (4.5)54.03±0.96 (3.5)
TSA2.457.35±1.05 (3.5)94.96±0.38 (1.5)89.33±0.44 (1.5)81.42±0.74 (2.5)76.74±0.72 (2.5)82.01±0.57 (2)67.4±0.99 (2.5)92.18±0.52 (4)83.55±0.9 (2)55.75±1.06 (2)
TriM7.5558.6±1.0 (3.5)92.0±0.6 (8.5)82.8±0.7 (10)75.3±0.8 (8.5)71.2±0.8 (7.5)77.3±0.7 (9.5)48.5±1.0 (11.5)90.5±0.5 (8.5)63.0±1.0 (4.5)52.8±1.1 (3.5)
PMF-DINOSmall2.2573.52±0.8 (1)92.17±0.57 (8.5)89.49±0.52 (1.5)91.04±0.37 (1)85.73±0.62 (1)79.43±0.67 (5.5)74.99±0.94 (1)95.3±0.44 (1)89.85±0.76 (1)59.69±1.02 (1)
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 72, + "execution_count": 75, "metadata": {}, "output_type": "execute_result" } @@ -9402,7 +9760,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 76, "metadata": { "id": "ASEngzGKifwU" }, @@ -9423,7 +9781,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 77, "metadata": { "id": "cB_SuLA4GNQ7" }, @@ -9444,7 +9802,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 78, "metadata": { "id": "2xe8elpLLFkH" }, @@ -9495,7 +9853,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 79, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -9510,21 +9868,22 @@ "text": [ "Method |Avg rank |ILSVRC (test) |Omniglot |Aircraft |Birds |Textures |QuickDraw |Fungi |VGG Flower |Traffic signs |MSCOCO \n", "---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------\n", - "k-NN [[1]] |13.6 |41.03±1.01 (14) |37.07±1.15 (15) |46.81±0.89 (14) |50.13±1.00 (14.5) |66.36±0.75 (12) |32.06±1.08 (15) |36.16±1.02 (12) |83.10±0.68 (11) |44.59±1.19 (14) |30.38±0.99 (14.5) \n", - "Finetune [[1]] |9.45 |45.78±1.10 (12) |60.85±1.58 (10.5) |68.69±1.26 (4) |57.31±1.26 (13) |69.05±0.90 (8.5) |42.60±1.17 (12.5) |38.20±1.02 (10) |85.51±0.68 (8) |66.79±1.31 (4) |34.86±0.97 (12) \n", - "MatchingNet [[1]] |12.55 |45.00±1.10 (12) |52.27±1.28 (13) |48.97±0.93 (12) |62.21±0.95 (11.5) |64.15±0.85 (14) |42.87±1.09 (12.5) |33.97±1.00 (13) |80.13±0.71 (14) |47.80±1.14 (11.5) |34.99±1.00 (12) \n", - "ProtoNet [[1]] |9.75 |50.50±1.08 (9.5) |59.98±1.35 (10.5) |53.10±1.00 (9.5) |68.79±1.01 (7.5) |66.56±0.83 (12) |48.96±1.08 (10) |39.71±1.11 (8) |85.27±0.77 (8) |47.12±1.10 (13) |41.00±1.10 (9.5) \n", - "fo-MAML [[1]] |11.25 |45.51±1.11 (12) |55.55±1.54 (12) |56.24±1.11 (7.5) |63.61±1.06 (11.5) |68.04±0.81 (8.5) |43.96±1.29 (12.5) |32.10±1.10 (14) |81.74±0.83 (13) |50.93±1.51 (9.5) |35.30±1.23 (12) \n", - "RelationNet [[1]] |14.55 |34.69±1.01 (15) |45.35±1.36 (14) |40.73±0.83 (15) |49.51±1.05 (14.5) |52.97±0.69 (15) |43.30±1.08 (12.5) |30.55±1.04 (15) |68.76±0.83 (15) |33.67±1.05 (15) |29.15±1.01 (14.5) \n", - "fo-Proto-MAML [[1]] |8.25 |49.53±1.05 (9.5) |63.37±1.33 (7.5) |55.95±0.99 (7.5) |68.66±0.96 (7.5) |66.49±0.83 (12) |51.52±1.00 (8.5) |39.96±1.14 (5.5) |87.15±0.69 (5) |48.83±1.09 (11.5) |43.74±1.12 (8) \n", - "ALFA+fo-Proto-MAML [[3]] |6.1 |52.80±1.11 (7.5) |61.87±1.51 (7.5) |63.43±1.10 (5) |69.75±1.05 (5.5) |70.78±0.88 (6) |59.17±1.16 (4.5) |41.49±1.17 (5.5) |85.96±0.77 (8) |60.78±1.29 (7) |48.11±1.14 (4.5) \n", - "ProtoNet (large) [[4]] |6.25 |53.69±1.07 (5) |68.50±1.27 (4.5) |58.04±0.96 (6) |74.07±0.92 (3.5) |68.76±0.77 (8.5) |53.30±1.06 (7) |40.73±1.15 (5.5) |86.96±0.73 (5) |58.11±1.05 (8) |41.70±1.08 (9.5) \n", - "CTX [[4]] |1.75 |**62.76**±0.99 (1.5) |**82.21**±1.00 (1.5) |**79.49**±0.89 (1.5) |80.63±0.88 (2) |75.57±0.64 (3) |**72.68**±0.82 (1) |**51.58**±1.11 (1.5) |**95.34**±0.37 (1) |**82.65**±0.76 (2) |59.90±1.02 (2.5) \n", - "BOHB [[5]] |6.85 |51.92±1.05 (7.5) |67.57±1.21 (4.5) |54.12±0.90 (9.5) |70.69±0.90 (5.5) |68.34±0.76 (8.5) |50.33±1.04 (8.5) |41.38±1.12 (5.5) |87.34±0.59 (5) |51.80±1.04 (9.5) |48.03±0.99 (4.5) \n", - "SimpleCNAPS [[14],[7]] |7.75 |54.80±1.20 (5) |62.00±1.30 (7.5) |49.20±0.90 (12) |66.50±1.00 (9.5) |71.60±0.70 (4.5) |56.60±1.00 (6) |37.50±1.20 (10) |82.10±0.90 (11) |63.10±1.10 (5.5) |45.80±1.00 (6.5) \n", - "TransductiveCNAPS [[14],[8]]|7.6 |54.10±1.10 (5) |62.90±1.30 (7.5) |48.40±0.90 (12) |67.30±0.90 (9.5) |72.50±0.70 (4.5) |58.00±1.00 (4.5) |37.70±1.10 (10) |82.80±0.80 (11) |61.80±1.10 (5.5) |45.80±1.00 (6.5) \n", - "TSA_resnet18 [[12]] |2.8 |59.50±1.10 (3) |78.20±1.20 (3) |72.20±1.00 (3) |74.90±0.90 (3.5) |77.30±0.70 (2) |67.60±0.90 (3) |44.70±1.00 (3) |90.90±0.60 (3) |**82.50**±0.80 (2) |59.00±1.00 (2.5) \n", - "TSA_resnet34 [[12]] |**1.5** |**63.73**±0.99 (1.5) |**82.58**±1.11 (1.5) |**80.13**±1.01 (1.5) |**83.39**±0.80 (1) |**79.61**±0.68 (1) |71.03±0.84 (2) |**51.38**±1.17 (1.5) |94.05±0.45 (2) |**81.71**±0.95 (2) |**61.67**±0.95 (1) \n" + "k-NN [[1]] |14.6 |41.03±1.01 (15) |37.07±1.15 (16) |46.81±0.89 (15) |50.13±1.00 (15.5) |66.36±0.75 (13) |32.06±1.08 (16) |36.16±1.02 (13) |83.10±0.68 (12) |44.59±1.19 (15) |30.38±0.99 (15.5) \n", + "Finetune [[1]] |10.45 |45.78±1.10 (13) |60.85±1.58 (11.5) |68.69±1.26 (5) |57.31±1.26 (14) |69.05±0.90 (9.5) |42.60±1.17 (13.5) |38.20±1.02 (11) |85.51±0.68 (9) |66.79±1.31 (5) |34.86±0.97 (13) \n", + "MatchingNet [[1]] |13.55 |45.00±1.10 (13) |52.27±1.28 (14) |48.97±0.93 (13) |62.21±0.95 (12.5) |64.15±0.85 (15) |42.87±1.09 (13.5) |33.97±1.00 (14) |80.13±0.71 (15) |47.80±1.14 (12.5) |34.99±1.00 (13) \n", + "ProtoNet [[1]] |10.75 |50.50±1.08 (10.5) |59.98±1.35 (11.5) |53.10±1.00 (10.5) |68.79±1.01 (8.5) |66.56±0.83 (13) |48.96±1.08 (11) |39.71±1.11 (9) |85.27±0.77 (9) |47.12±1.10 (14) |41.00±1.10 (10.5) \n", + "fo-MAML [[1]] |12.25 |45.51±1.11 (13) |55.55±1.54 (13) |56.24±1.11 (8.5) |63.61±1.06 (12.5) |68.04±0.81 (9.5) |43.96±1.29 (13.5) |32.10±1.10 (15) |81.74±0.83 (14) |50.93±1.51 (10.5) |35.30±1.23 (13) \n", + "RelationNet [[1]] |15.55 |34.69±1.01 (16) |45.35±1.36 (15) |40.73±0.83 (16) |49.51±1.05 (15.5) |52.97±0.69 (16) |43.30±1.08 (13.5) |30.55±1.04 (16) |68.76±0.83 (16) |33.67±1.05 (16) |29.15±1.01 (15.5) \n", + "fo-Proto-MAML [[1]] |9.25 |49.53±1.05 (10.5) |63.37±1.33 (8.5) |55.95±0.99 (8.5) |68.66±0.96 (8.5) |66.49±0.83 (13) |51.52±1.00 (9.5) |39.96±1.14 (6.5) |87.15±0.69 (6) |48.83±1.09 (12.5) |43.74±1.12 (9) \n", + "ALFA+fo-Proto-MAML [[3]] |7.1 |52.80±1.11 (8.5) |61.87±1.51 (8.5) |63.43±1.10 (6) |69.75±1.05 (6.5) |70.78±0.88 (7) |59.17±1.16 (5.5) |41.49±1.17 (6.5) |85.96±0.77 (9) |60.78±1.29 (8) |48.11±1.14 (5.5) \n", + "ProtoNet (large) [[4]] |7.25 |53.69±1.07 (6) |68.50±1.27 (5.5) |58.04±0.96 (7) |74.07±0.92 (4.5) |68.76±0.77 (9.5) |53.30±1.06 (8) |40.73±1.15 (6.5) |86.96±0.73 (6) |58.11±1.05 (9) |41.70±1.08 (10.5) \n", + "CTX [[4]] |2.5 |62.76±0.99 (2.5) |**82.21**±1.00 (2) |**79.49**±0.89 (1.5) |80.63±0.88 (3) |75.57±0.64 (4) |72.68±0.82 (2) |51.58±1.11 (2.5) |**95.34**±0.37 (1) |82.65±0.76 (3) |59.90±1.02 (3.5) \n", + "BOHB [[5]] |7.85 |51.92±1.05 (8.5) |67.57±1.21 (5.5) |54.12±0.90 (10.5) |70.69±0.90 (6.5) |68.34±0.76 (9.5) |50.33±1.04 (9.5) |41.38±1.12 (6.5) |87.34±0.59 (6) |51.80±1.04 (10.5) |48.03±0.99 (5.5) \n", + "SimpleCNAPS [[14],[7]] |8.75 |54.80±1.20 (6) |62.00±1.30 (8.5) |49.20±0.90 (13) |66.50±1.00 (10.5) |71.60±0.70 (5.5) |56.60±1.00 (7) |37.50±1.20 (11) |82.10±0.90 (12) |63.10±1.10 (6.5) |45.80±1.00 (7.5) \n", + "TransductiveCNAPS [[14],[8]]|8.6 |54.10±1.10 (6) |62.90±1.30 (8.5) |48.40±0.90 (13) |67.30±0.90 (10.5) |72.50±0.70 (5.5) |58.00±1.00 (5.5) |37.70±1.10 (11) |82.80±0.80 (12) |61.80±1.10 (6.5) |45.80±1.00 (7.5) \n", + "TSA_resnet18 [[12]] |3.8 |59.50±1.10 (4) |78.20±1.20 (4) |72.20±1.00 (4) |74.90±0.90 (4.5) |77.30±0.70 (3) |67.60±0.90 (4) |44.70±1.00 (4) |90.90±0.60 (4) |82.50±0.80 (3) |59.00±1.00 (3.5) \n", + "TSA_resnet34 [[12]] |2.25 |63.73±0.99 (2.5) |**82.58**±1.11 (2) |**80.13**±1.01 (1.5) |83.39±0.80 (2) |79.61±0.68 (2) |71.03±0.84 (3) |51.38±1.17 (2.5) |94.05±0.45 (2.5) |81.71±0.95 (3) |**61.67**±0.95 (1.5) \n", + "PMF-DINOSmall [[15]] |**1.5** |**75.51**±0.72 (1) |**82.81**±1.10 (2) |78.38±1.09 (3) |**85.18**±0.77 (1) |**86.95**±0.60 (1) |**74.47**±0.83 (1) |**55.16**±1.09 (1) |94.66±0.48 (2.5) |**90.04**±0.81 (1) |**62.60**±0.96 (1.5) \n" ] } ], @@ -9534,7 +9893,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 80, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -9549,24 +9908,25 @@ "text": [ "Method |Avg rank |ILSVRC (test) |Omniglot |Aircraft |Birds |Textures |QuickDraw |Fungi |VGG Flower |Traffic signs |MSCOCO \n", "---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------\n", - "k-NN [[1]] |15.85 |38.55±0.94 (15.5) |74.60±1.08 (17) |64.98±0.82 (18) |66.35±0.92 (13.5) |63.58±0.79 (14.5) |44.88±1.05 (18) |37.12±1.06 (14.5) |83.47±0.61 (14.5) |40.11±1.10 (17) |29.55±0.96 (16) \n", - "Finetune [[1]] |13.1 |43.08±1.08 (13.5) |71.11±1.37 (18) |72.03±1.07 (14.5) |59.82±1.15 (16) |69.14±0.85 (8.5) |47.05±1.16 (17) |38.16±1.04 (14.5) |85.28±0.69 (13) |66.74±1.23 (2) |35.17±1.08 (14) \n", - "MatchingNet [[1]] |15.4 |36.08±1.00 (17) |78.25±1.01 (15.5) |69.17±0.96 (16.5) |56.40±1.00 (17) |61.80±0.74 (16) |60.81±1.03 (14.5) |33.70±1.04 (17) |81.90±0.72 (16) |55.57±1.08 (8.5) |28.79±0.96 (16) \n", - "ProtoNet [[1]] |13.5 |44.50±1.05 (13.5) |79.56±1.12 (15.5) |71.14±0.86 (14.5) |67.01±1.02 (13.5) |65.18±0.84 (12.5) |64.88±0.89 (13) |40.26±1.13 (13) |86.85±0.71 (12) |46.48±1.00 (15) |39.87±1.06 (12.5) \n", - "fo-MAML [[1]] |15.25 |37.83±1.01 (15.5) |83.92±0.95 (13.5) |76.41±0.69 (12) |62.43±1.08 (15) |64.16±0.83 (14.5) |59.73±1.10 (16) |33.54±1.11 (17) |79.94±0.84 (17) |42.91±1.31 (16) |29.37±1.08 (16) \n", - "RelationNet [[1]] |16.8 |30.89±0.93 (18) |86.57±0.79 (12) |69.71±0.83 (16.5) |54.14±0.99 (18) |56.56±0.73 (18) |61.75±0.97 (14.5) |32.56±1.08 (17) |76.08±0.76 (18) |37.48±0.93 (18) |27.41±0.89 (18) \n", - "fo-Proto-MAML [[1]] |11.6 |46.52±1.05 (12) |82.69±0.97 (13.5) |75.23±0.76 (13) |69.88±1.02 (11.5) |68.25±0.81 (10.5) |66.84±0.94 (12) |41.99±1.17 (12) |88.72±0.67 (10) |52.42±1.08 (11.5) |41.74±1.13 (10) \n", - "CNAPs [[2]] |10.25 |50.80±1.10 (10.5) |91.70±0.50 (8) |83.70±0.60 (7.5) |73.60±0.90 (10) |59.50±0.70 (17) |74.70±0.80 (11) |50.20±1.10 (7.5) |88.90±0.50 (10) |56.50±1.10 (8.5) |39.40±1.00 (12.5) \n", - "SUR [[6]] |7.65 |56.10±1.10 (7) |93.10±0.50 (5.5) |84.60±0.70 (5.5) |70.60±1.00 (11.5) |71.00±0.80 (6.5) |81.30±0.60 (4) |64.20±1.10 (3.5) |82.80±0.80 (14.5) |53.40±1.00 (11.5) |50.10±1.00 (7) \n", - "SUR-pnf [[6]] |8.2 |56.00±1.10 (7) |90.00±0.60 (10.5) |79.70±0.80 (10.5) |75.90±0.90 (7.5) |72.50±0.70 (4.5) |76.70±0.70 (8.5) |49.80±1.10 (7.5) |90.00±0.60 (7.5) |52.20±0.80 (11.5) |50.20±1.10 (7) \n", - "SimpleCNAPS [[14],[7]] |7.45 |56.50±1.10 (7) |91.90±0.60 (8) |83.80±0.60 (7.5) |76.10±0.90 (7.5) |70.00±0.80 (8.5) |78.30±0.70 (6.5) |49.10±1.20 (7.5) |91.30±0.60 (6) |59.20±1.00 (6) |42.40±1.10 (10) \n", - "TransductiveCNAPS [[14],[8]]|6.05 |**57.90**±1.10 (2.5) |94.30±0.40 (3.5) |84.70±0.50 (5.5) |78.80±0.70 (3.5) |66.20±0.80 (12.5) |77.90±0.60 (6.5) |48.90±1.20 (7.5) |**92.30**±0.40 (3) |59.70±1.10 (6) |42.50±1.10 (10) \n", - "URT [[9]] |6.05 |55.70±1.00 (7) |94.40±0.40 (3.5) |85.80±0.60 (4) |76.30±0.80 (7.5) |71.80±0.70 (4.5) |**82.50**±0.60 (2) |63.50±1.00 (3.5) |88.20±0.60 (10) |51.10±1.10 (14) |52.20±1.10 (4.5) \n", - "URT-pf [[9]] |7.55 |55.50±1.10 (7) |90.20±0.60 (10.5) |79.80±0.70 (10.5) |77.50±0.80 (5) |73.50±0.70 (3) |75.80±0.70 (10) |48.10±0.90 (10.5) |**91.90**±0.50 (3) |52.00±1.40 (11.5) |52.10±1.00 (4.5) \n", - "FLUTE [[10]] |5.9 |51.80±1.10 (10.5) |93.20±0.50 (5.5) |87.20±0.50 (3) |79.20±0.80 (3.5) |68.80±0.80 (10.5) |79.50±0.70 (5) |58.10±1.10 (5) |**91.60**±0.60 (3) |58.40±1.10 (6) |50.00±1.00 (7) \n", - "URL [[11]] |2.15 |**57.51**±1.08 (2.5) |**94.51**±0.41 (1.5) |88.59±0.46 (2) |**80.54**±0.69 (1.5) |**76.17**±0.67 (1.5) |**81.94**±0.56 (2) |**68.75**±0.95 (1.5) |**92.11**±0.48 (3) |63.34±1.19 (3.5) |54.03±0.96 (2.5) \n", - "TSA [[12]] |**1.65** |**57.35**±1.05 (2.5) |**94.96**±0.38 (1.5) |**89.33**±0.44 (1) |**81.42**±0.74 (1.5) |**76.74**±0.72 (1.5) |**82.01**±0.57 (2) |**67.40**±0.99 (1.5) |**92.18**±0.52 (3) |**83.55**±0.90 (1) |**55.75**±1.06 (1) \n", - "TriM [[13]] |6.6 |**58.60**±1.00 (2.5) |92.00±0.60 (8) |82.80±0.70 (9) |75.30±0.80 (7.5) |71.20±0.80 (6.5) |77.30±0.70 (8.5) |48.50±1.00 (10.5) |90.50±0.50 (7.5) |63.00±1.00 (3.5) |52.80±1.10 (2.5) \n" + "k-NN [[1]] |16.85 |38.55±0.94 (16.5) |74.60±1.08 (18) |64.98±0.82 (19) |66.35±0.92 (14.5) |63.58±0.79 (15.5) |44.88±1.05 (19) |37.12±1.06 (15.5) |83.47±0.61 (15.5) |40.11±1.10 (18) |29.55±0.96 (17) \n", + "Finetune [[1]] |14.1 |43.08±1.08 (14.5) |71.11±1.37 (19) |72.03±1.07 (15.5) |59.82±1.15 (17) |69.14±0.85 (9.5) |47.05±1.16 (18) |38.16±1.04 (15.5) |85.28±0.69 (14) |66.74±1.23 (3) |35.17±1.08 (15) \n", + "MatchingNet [[1]] |16.4 |36.08±1.00 (18) |78.25±1.01 (16.5) |69.17±0.96 (17.5) |56.40±1.00 (18) |61.80±0.74 (17) |60.81±1.03 (15.5) |33.70±1.04 (18) |81.90±0.72 (17) |55.57±1.08 (9.5) |28.79±0.96 (17) \n", + "ProtoNet [[1]] |14.5 |44.50±1.05 (14.5) |79.56±1.12 (16.5) |71.14±0.86 (15.5) |67.01±1.02 (14.5) |65.18±0.84 (13.5) |64.88±0.89 (14) |40.26±1.13 (14) |86.85±0.71 (13) |46.48±1.00 (16) |39.87±1.06 (13.5) \n", + "fo-MAML [[1]] |16.25 |37.83±1.01 (16.5) |83.92±0.95 (14.5) |76.41±0.69 (13) |62.43±1.08 (16) |64.16±0.83 (15.5) |59.73±1.10 (17) |33.54±1.11 (18) |79.94±0.84 (18) |42.91±1.31 (17) |29.37±1.08 (17) \n", + "RelationNet [[1]] |17.8 |30.89±0.93 (19) |86.57±0.79 (13) |69.71±0.83 (17.5) |54.14±0.99 (19) |56.56±0.73 (19) |61.75±0.97 (15.5) |32.56±1.08 (18) |76.08±0.76 (19) |37.48±0.93 (19) |27.41±0.89 (19) \n", + "fo-Proto-MAML [[1]] |12.6 |46.52±1.05 (13) |82.69±0.97 (14.5) |75.23±0.76 (14) |69.88±1.02 (12.5) |68.25±0.81 (11.5) |66.84±0.94 (13) |41.99±1.17 (13) |88.72±0.67 (11) |52.42±1.08 (12.5) |41.74±1.13 (11) \n", + "CNAPs [[2]] |11.2 |50.80±1.10 (11.5) |91.70±0.50 (8.5) |83.70±0.60 (8.5) |73.60±0.90 (11) |59.50±0.70 (18) |74.70±0.80 (12) |50.20±1.10 (8.5) |88.90±0.50 (11) |56.50±1.10 (9.5) |39.40±1.00 (13.5) \n", + "SUR [[6]] |8.45 |56.10±1.10 (8) |93.10±0.50 (5.5) |84.60±0.70 (6.5) |70.60±1.00 (12.5) |71.00±0.80 (7.5) |81.30±0.60 (4) |64.20±1.10 (4.5) |82.80±0.80 (15.5) |53.40±1.00 (12.5) |50.10±1.00 (8) \n", + "SUR-pnf [[6]] |9.2 |56.00±1.10 (8) |90.00±0.60 (11.5) |79.70±0.80 (11.5) |75.90±0.90 (8.5) |72.50±0.70 (5.5) |76.70±0.70 (9.5) |49.80±1.10 (8.5) |90.00±0.60 (8.5) |52.20±0.80 (12.5) |50.20±1.10 (8) \n", + "SimpleCNAPS [[14],[7]] |8.4 |56.50±1.10 (8) |91.90±0.60 (8.5) |83.80±0.60 (8.5) |76.10±0.90 (8.5) |70.00±0.80 (9.5) |78.30±0.70 (7.5) |49.10±1.20 (8.5) |91.30±0.60 (7) |59.20±1.00 (7) |42.40±1.10 (11) \n", + "TransductiveCNAPS [[14],[8]]|6.95 |57.90±1.10 (3.5) |94.30±0.40 (3.5) |84.70±0.50 (6.5) |78.80±0.70 (4.5) |66.20±0.80 (13.5) |77.90±0.60 (7.5) |48.90±1.20 (8.5) |92.30±0.40 (4) |59.70±1.10 (7) |42.50±1.10 (11) \n", + "URT [[9]] |6.85 |55.70±1.00 (8) |94.40±0.40 (3.5) |85.80±0.60 (5) |76.30±0.80 (8.5) |71.80±0.70 (5.5) |**82.50**±0.60 (2) |63.50±1.00 (4.5) |88.20±0.60 (11) |51.10±1.10 (15) |52.20±1.10 (5.5) \n", + "URT-pf [[9]] |8.55 |55.50±1.10 (8) |90.20±0.60 (11.5) |79.80±0.70 (11.5) |77.50±0.80 (6) |73.50±0.70 (4) |75.80±0.70 (11) |48.10±0.90 (11.5) |91.90±0.50 (4) |52.00±1.40 (12.5) |52.10±1.00 (5.5) \n", + "FLUTE [[10]] |6.75 |51.80±1.10 (11.5) |93.20±0.50 (5.5) |87.20±0.50 (4) |79.20±0.80 (4.5) |68.80±0.80 (11.5) |79.50±0.70 (5.5) |58.10±1.10 (6) |91.60±0.60 (4) |58.40±1.10 (7) |50.00±1.00 (8) \n", + "URL [[11]] |2.95 |57.51±1.08 (3.5) |**94.51**±0.41 (1.5) |88.59±0.46 (3) |80.54±0.69 (2.5) |76.17±0.67 (2.5) |**81.94**±0.56 (2) |68.75±0.95 (2.5) |92.11±0.48 (4) |63.34±1.19 (4.5) |54.03±0.96 (3.5) \n", + "TSA [[12]] |2.4 |57.35±1.05 (3.5) |**94.96**±0.38 (1.5) |**89.33**±0.44 (1.5) |81.42±0.74 (2.5) |76.74±0.72 (2.5) |**82.01**±0.57 (2) |67.40±0.99 (2.5) |92.18±0.52 (4) |83.55±0.90 (2) |55.75±1.06 (2) \n", + "TriM [[13]] |7.55 |58.60±1.00 (3.5) |92.00±0.60 (8.5) |82.80±0.70 (10) |75.30±0.80 (8.5) |71.20±0.80 (7.5) |77.30±0.70 (9.5) |48.50±1.00 (11.5) |90.50±0.50 (8.5) |63.00±1.00 (4.5) |52.80±1.10 (3.5) \n", + "PMF-DINOSmall [[15]] |**2.25** |**73.52**±0.80 (1) |92.17±0.57 (8.5) |**89.49**±0.52 (1.5) |**91.04**±0.37 (1) |**85.73**±0.62 (1) |79.43±0.67 (5.5) |**74.99**±0.94 (1) |**95.30**±0.44 (1) |**89.85**±0.76 (1) |**59.69**±1.02 (1) \n" ] } ], @@ -9594,7 +9954,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 81, "metadata": { "id": "dc5MOCw1n02N" }, @@ -9615,7 +9975,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 82, "metadata": { "id": "76-TdhVTZfdZ" }, @@ -9643,7 +10003,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 83, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -9670,6 +10030,7 @@ "[12]: #12-li-et-al-2021b\n", "[13]: #13-liu-et-al-2021b\n", "[14]: #14-bateni-et-al-2022b\n", + "[15]: #15-hu-et-al-2022\n", "\n", "###### \\[1\\] Triantafillou et al. (2020)\n", "\n", @@ -9739,6 +10100,11 @@ "###### \\[14\\] Bateni et al. (2022b)\n", "\n", "Bateni Peyman, Jarred Barber, Raghav Goyal, Vaden Masrani, Jan-Willem van de Meent, Leonid Sigal, and Frank Wood.; [_Beyond Simple Meta-Learning: Multi-Purpose Models for Multi-Domain, Active and Continual Few-Shot Learning._](https://arxiv.org/abs/2201.05151); arXiv 2022.\n", + "\n", + "\n", + "###### \\[15\\] Hu et al. (2022)\n", + "\n", + "Shell Xu Hu, Da Li, Jan Stühmer, Minyoung Kim and Timothy Hospedales.; [_Pushing the Limits of Simple Pipelines for Few-Shot Learning: External Data and Fine-Tuning Make a Difference._](https://arxiv.org/abs/2204.07305); CVPR 2022.\n", "\n" ] } @@ -9758,7 +10124,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 84, "metadata": { "id": "aOAZQlucj59E" }, @@ -9783,14 +10149,13 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 85, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "_tAgydyHX_R_", - "outputId": "d3838676-d1d2-48be-95c2-583667298cf7", - "scrolled": false + "outputId": "d3838676-d1d2-48be-95c2-583667298cf7" }, "outputs": [ { @@ -9803,44 +10168,46 @@ "\n", "Method |Avg rank |ILSVRC (test) |Omniglot |Aircraft |Birds |Textures |QuickDraw |Fungi |VGG Flower |Traffic signs |MSCOCO \n", "---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------\n", - "k-NN [[1]] |13.6 |41.03±1.01 (14) |37.07±1.15 (15) |46.81±0.89 (14) |50.13±1.00 (14.5) |66.36±0.75 (12) |32.06±1.08 (15) |36.16±1.02 (12) |83.10±0.68 (11) |44.59±1.19 (14) |30.38±0.99 (14.5) \n", - "Finetune [[1]] |9.45 |45.78±1.10 (12) |60.85±1.58 (10.5) |68.69±1.26 (4) |57.31±1.26 (13) |69.05±0.90 (8.5) |42.60±1.17 (12.5) |38.20±1.02 (10) |85.51±0.68 (8) |66.79±1.31 (4) |34.86±0.97 (12) \n", - "MatchingNet [[1]] |12.55 |45.00±1.10 (12) |52.27±1.28 (13) |48.97±0.93 (12) |62.21±0.95 (11.5) |64.15±0.85 (14) |42.87±1.09 (12.5) |33.97±1.00 (13) |80.13±0.71 (14) |47.80±1.14 (11.5) |34.99±1.00 (12) \n", - "ProtoNet [[1]] |9.75 |50.50±1.08 (9.5) |59.98±1.35 (10.5) |53.10±1.00 (9.5) |68.79±1.01 (7.5) |66.56±0.83 (12) |48.96±1.08 (10) |39.71±1.11 (8) |85.27±0.77 (8) |47.12±1.10 (13) |41.00±1.10 (9.5) \n", - "fo-MAML [[1]] |11.25 |45.51±1.11 (12) |55.55±1.54 (12) |56.24±1.11 (7.5) |63.61±1.06 (11.5) |68.04±0.81 (8.5) |43.96±1.29 (12.5) |32.10±1.10 (14) |81.74±0.83 (13) |50.93±1.51 (9.5) |35.30±1.23 (12) \n", - "RelationNet [[1]] |14.55 |34.69±1.01 (15) |45.35±1.36 (14) |40.73±0.83 (15) |49.51±1.05 (14.5) |52.97±0.69 (15) |43.30±1.08 (12.5) |30.55±1.04 (15) |68.76±0.83 (15) |33.67±1.05 (15) |29.15±1.01 (14.5) \n", - "fo-Proto-MAML [[1]] |8.25 |49.53±1.05 (9.5) |63.37±1.33 (7.5) |55.95±0.99 (7.5) |68.66±0.96 (7.5) |66.49±0.83 (12) |51.52±1.00 (8.5) |39.96±1.14 (5.5) |87.15±0.69 (5) |48.83±1.09 (11.5) |43.74±1.12 (8) \n", - "ALFA+fo-Proto-MAML [[3]] |6.1 |52.80±1.11 (7.5) |61.87±1.51 (7.5) |63.43±1.10 (5) |69.75±1.05 (5.5) |70.78±0.88 (6) |59.17±1.16 (4.5) |41.49±1.17 (5.5) |85.96±0.77 (8) |60.78±1.29 (7) |48.11±1.14 (4.5) \n", - "ProtoNet (large) [[4]] |6.25 |53.69±1.07 (5) |68.50±1.27 (4.5) |58.04±0.96 (6) |74.07±0.92 (3.5) |68.76±0.77 (8.5) |53.30±1.06 (7) |40.73±1.15 (5.5) |86.96±0.73 (5) |58.11±1.05 (8) |41.70±1.08 (9.5) \n", - "CTX [[4]] |1.75 |**62.76**±0.99 (1.5) |**82.21**±1.00 (1.5) |**79.49**±0.89 (1.5) |80.63±0.88 (2) |75.57±0.64 (3) |**72.68**±0.82 (1) |**51.58**±1.11 (1.5) |**95.34**±0.37 (1) |**82.65**±0.76 (2) |59.90±1.02 (2.5) \n", - "BOHB [[5]] |6.85 |51.92±1.05 (7.5) |67.57±1.21 (4.5) |54.12±0.90 (9.5) |70.69±0.90 (5.5) |68.34±0.76 (8.5) |50.33±1.04 (8.5) |41.38±1.12 (5.5) |87.34±0.59 (5) |51.80±1.04 (9.5) |48.03±0.99 (4.5) \n", - "SimpleCNAPS [[14],[7]] |7.75 |54.80±1.20 (5) |62.00±1.30 (7.5) |49.20±0.90 (12) |66.50±1.00 (9.5) |71.60±0.70 (4.5) |56.60±1.00 (6) |37.50±1.20 (10) |82.10±0.90 (11) |63.10±1.10 (5.5) |45.80±1.00 (6.5) \n", - "TransductiveCNAPS [[14],[8]]|7.6 |54.10±1.10 (5) |62.90±1.30 (7.5) |48.40±0.90 (12) |67.30±0.90 (9.5) |72.50±0.70 (4.5) |58.00±1.00 (4.5) |37.70±1.10 (10) |82.80±0.80 (11) |61.80±1.10 (5.5) |45.80±1.00 (6.5) \n", - "TSA_resnet18 [[12]] |2.8 |59.50±1.10 (3) |78.20±1.20 (3) |72.20±1.00 (3) |74.90±0.90 (3.5) |77.30±0.70 (2) |67.60±0.90 (3) |44.70±1.00 (3) |90.90±0.60 (3) |**82.50**±0.80 (2) |59.00±1.00 (2.5) \n", - "TSA_resnet34 [[12]] |**1.5** |**63.73**±0.99 (1.5) |**82.58**±1.11 (1.5) |**80.13**±1.01 (1.5) |**83.39**±0.80 (1) |**79.61**±0.68 (1) |71.03±0.84 (2) |**51.38**±1.17 (1.5) |94.05±0.45 (2) |**81.71**±0.95 (2) |**61.67**±0.95 (1) \n", + "k-NN [[1]] |14.6 |41.03±1.01 (15) |37.07±1.15 (16) |46.81±0.89 (15) |50.13±1.00 (15.5) |66.36±0.75 (13) |32.06±1.08 (16) |36.16±1.02 (13) |83.10±0.68 (12) |44.59±1.19 (15) |30.38±0.99 (15.5) \n", + "Finetune [[1]] |10.45 |45.78±1.10 (13) |60.85±1.58 (11.5) |68.69±1.26 (5) |57.31±1.26 (14) |69.05±0.90 (9.5) |42.60±1.17 (13.5) |38.20±1.02 (11) |85.51±0.68 (9) |66.79±1.31 (5) |34.86±0.97 (13) \n", + "MatchingNet [[1]] |13.55 |45.00±1.10 (13) |52.27±1.28 (14) |48.97±0.93 (13) |62.21±0.95 (12.5) |64.15±0.85 (15) |42.87±1.09 (13.5) |33.97±1.00 (14) |80.13±0.71 (15) |47.80±1.14 (12.5) |34.99±1.00 (13) \n", + "ProtoNet [[1]] |10.75 |50.50±1.08 (10.5) |59.98±1.35 (11.5) |53.10±1.00 (10.5) |68.79±1.01 (8.5) |66.56±0.83 (13) |48.96±1.08 (11) |39.71±1.11 (9) |85.27±0.77 (9) |47.12±1.10 (14) |41.00±1.10 (10.5) \n", + "fo-MAML [[1]] |12.25 |45.51±1.11 (13) |55.55±1.54 (13) |56.24±1.11 (8.5) |63.61±1.06 (12.5) |68.04±0.81 (9.5) |43.96±1.29 (13.5) |32.10±1.10 (15) |81.74±0.83 (14) |50.93±1.51 (10.5) |35.30±1.23 (13) \n", + "RelationNet [[1]] |15.55 |34.69±1.01 (16) |45.35±1.36 (15) |40.73±0.83 (16) |49.51±1.05 (15.5) |52.97±0.69 (16) |43.30±1.08 (13.5) |30.55±1.04 (16) |68.76±0.83 (16) |33.67±1.05 (16) |29.15±1.01 (15.5) \n", + "fo-Proto-MAML [[1]] |9.25 |49.53±1.05 (10.5) |63.37±1.33 (8.5) |55.95±0.99 (8.5) |68.66±0.96 (8.5) |66.49±0.83 (13) |51.52±1.00 (9.5) |39.96±1.14 (6.5) |87.15±0.69 (6) |48.83±1.09 (12.5) |43.74±1.12 (9) \n", + "ALFA+fo-Proto-MAML [[3]] |7.1 |52.80±1.11 (8.5) |61.87±1.51 (8.5) |63.43±1.10 (6) |69.75±1.05 (6.5) |70.78±0.88 (7) |59.17±1.16 (5.5) |41.49±1.17 (6.5) |85.96±0.77 (9) |60.78±1.29 (8) |48.11±1.14 (5.5) \n", + "ProtoNet (large) [[4]] |7.25 |53.69±1.07 (6) |68.50±1.27 (5.5) |58.04±0.96 (7) |74.07±0.92 (4.5) |68.76±0.77 (9.5) |53.30±1.06 (8) |40.73±1.15 (6.5) |86.96±0.73 (6) |58.11±1.05 (9) |41.70±1.08 (10.5) \n", + "CTX [[4]] |2.5 |62.76±0.99 (2.5) |**82.21**±1.00 (2) |**79.49**±0.89 (1.5) |80.63±0.88 (3) |75.57±0.64 (4) |72.68±0.82 (2) |51.58±1.11 (2.5) |**95.34**±0.37 (1) |82.65±0.76 (3) |59.90±1.02 (3.5) \n", + "BOHB [[5]] |7.85 |51.92±1.05 (8.5) |67.57±1.21 (5.5) |54.12±0.90 (10.5) |70.69±0.90 (6.5) |68.34±0.76 (9.5) |50.33±1.04 (9.5) |41.38±1.12 (6.5) |87.34±0.59 (6) |51.80±1.04 (10.5) |48.03±0.99 (5.5) \n", + "SimpleCNAPS [[14],[7]] |8.75 |54.80±1.20 (6) |62.00±1.30 (8.5) |49.20±0.90 (13) |66.50±1.00 (10.5) |71.60±0.70 (5.5) |56.60±1.00 (7) |37.50±1.20 (11) |82.10±0.90 (12) |63.10±1.10 (6.5) |45.80±1.00 (7.5) \n", + "TransductiveCNAPS [[14],[8]]|8.6 |54.10±1.10 (6) |62.90±1.30 (8.5) |48.40±0.90 (13) |67.30±0.90 (10.5) |72.50±0.70 (5.5) |58.00±1.00 (5.5) |37.70±1.10 (11) |82.80±0.80 (12) |61.80±1.10 (6.5) |45.80±1.00 (7.5) \n", + "TSA_resnet18 [[12]] |3.8 |59.50±1.10 (4) |78.20±1.20 (4) |72.20±1.00 (4) |74.90±0.90 (4.5) |77.30±0.70 (3) |67.60±0.90 (4) |44.70±1.00 (4) |90.90±0.60 (4) |82.50±0.80 (3) |59.00±1.00 (3.5) \n", + "TSA_resnet34 [[12]] |2.25 |63.73±0.99 (2.5) |**82.58**±1.11 (2) |**80.13**±1.01 (1.5) |83.39±0.80 (2) |79.61±0.68 (2) |71.03±0.84 (3) |51.38±1.17 (2.5) |94.05±0.45 (2.5) |81.71±0.95 (3) |**61.67**±0.95 (1.5) \n", + "PMF-DINOSmall [[15]] |**1.5** |**75.51**±0.72 (1) |**82.81**±1.10 (2) |78.38±1.09 (3) |**85.18**±0.77 (1) |**86.95**±0.60 (1) |**74.47**±0.83 (1) |**55.16**±1.09 (1) |94.66±0.48 (2.5) |**90.04**±0.81 (1) |**62.60**±0.96 (1.5) \n", "\n", "## Training on all datasets\n", "\n", "Method |Avg rank |ILSVRC (test) |Omniglot |Aircraft |Birds |Textures |QuickDraw |Fungi |VGG Flower |Traffic signs |MSCOCO \n", "---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------\n", - "k-NN [[1]] |15.85 |38.55±0.94 (15.5) |74.60±1.08 (17) |64.98±0.82 (18) |66.35±0.92 (13.5) |63.58±0.79 (14.5) |44.88±1.05 (18) |37.12±1.06 (14.5) |83.47±0.61 (14.5) |40.11±1.10 (17) |29.55±0.96 (16) \n", - "Finetune [[1]] |13.1 |43.08±1.08 (13.5) |71.11±1.37 (18) |72.03±1.07 (14.5) |59.82±1.15 (16) |69.14±0.85 (8.5) |47.05±1.16 (17) |38.16±1.04 (14.5) |85.28±0.69 (13) |66.74±1.23 (2) |35.17±1.08 (14) \n", - "MatchingNet [[1]] |15.4 |36.08±1.00 (17) |78.25±1.01 (15.5) |69.17±0.96 (16.5) |56.40±1.00 (17) |61.80±0.74 (16) |60.81±1.03 (14.5) |33.70±1.04 (17) |81.90±0.72 (16) |55.57±1.08 (8.5) |28.79±0.96 (16) \n", - "ProtoNet [[1]] |13.5 |44.50±1.05 (13.5) |79.56±1.12 (15.5) |71.14±0.86 (14.5) |67.01±1.02 (13.5) |65.18±0.84 (12.5) |64.88±0.89 (13) |40.26±1.13 (13) |86.85±0.71 (12) |46.48±1.00 (15) |39.87±1.06 (12.5) \n", - "fo-MAML [[1]] |15.25 |37.83±1.01 (15.5) |83.92±0.95 (13.5) |76.41±0.69 (12) |62.43±1.08 (15) |64.16±0.83 (14.5) |59.73±1.10 (16) |33.54±1.11 (17) |79.94±0.84 (17) |42.91±1.31 (16) |29.37±1.08 (16) \n", - "RelationNet [[1]] |16.8 |30.89±0.93 (18) |86.57±0.79 (12) |69.71±0.83 (16.5) |54.14±0.99 (18) |56.56±0.73 (18) |61.75±0.97 (14.5) |32.56±1.08 (17) |76.08±0.76 (18) |37.48±0.93 (18) |27.41±0.89 (18) \n", - "fo-Proto-MAML [[1]] |11.6 |46.52±1.05 (12) |82.69±0.97 (13.5) |75.23±0.76 (13) |69.88±1.02 (11.5) |68.25±0.81 (10.5) |66.84±0.94 (12) |41.99±1.17 (12) |88.72±0.67 (10) |52.42±1.08 (11.5) |41.74±1.13 (10) \n", - "CNAPs [[2]] |10.25 |50.80±1.10 (10.5) |91.70±0.50 (8) |83.70±0.60 (7.5) |73.60±0.90 (10) |59.50±0.70 (17) |74.70±0.80 (11) |50.20±1.10 (7.5) |88.90±0.50 (10) |56.50±1.10 (8.5) |39.40±1.00 (12.5) \n", - "SUR [[6]] |7.65 |56.10±1.10 (7) |93.10±0.50 (5.5) |84.60±0.70 (5.5) |70.60±1.00 (11.5) |71.00±0.80 (6.5) |81.30±0.60 (4) |64.20±1.10 (3.5) |82.80±0.80 (14.5) |53.40±1.00 (11.5) |50.10±1.00 (7) \n", - "SUR-pnf [[6]] |8.2 |56.00±1.10 (7) |90.00±0.60 (10.5) |79.70±0.80 (10.5) |75.90±0.90 (7.5) |72.50±0.70 (4.5) |76.70±0.70 (8.5) |49.80±1.10 (7.5) |90.00±0.60 (7.5) |52.20±0.80 (11.5) |50.20±1.10 (7) \n", - "SimpleCNAPS [[14],[7]] |7.45 |56.50±1.10 (7) |91.90±0.60 (8) |83.80±0.60 (7.5) |76.10±0.90 (7.5) |70.00±0.80 (8.5) |78.30±0.70 (6.5) |49.10±1.20 (7.5) |91.30±0.60 (6) |59.20±1.00 (6) |42.40±1.10 (10) \n", - "TransductiveCNAPS [[14],[8]]|6.05 |**57.90**±1.10 (2.5) |94.30±0.40 (3.5) |84.70±0.50 (5.5) |78.80±0.70 (3.5) |66.20±0.80 (12.5) |77.90±0.60 (6.5) |48.90±1.20 (7.5) |**92.30**±0.40 (3) |59.70±1.10 (6) |42.50±1.10 (10) \n", - "URT [[9]] |6.05 |55.70±1.00 (7) |94.40±0.40 (3.5) |85.80±0.60 (4) |76.30±0.80 (7.5) |71.80±0.70 (4.5) |**82.50**±0.60 (2) |63.50±1.00 (3.5) |88.20±0.60 (10) |51.10±1.10 (14) |52.20±1.10 (4.5) \n", - "URT-pf [[9]] |7.55 |55.50±1.10 (7) |90.20±0.60 (10.5) |79.80±0.70 (10.5) |77.50±0.80 (5) |73.50±0.70 (3) |75.80±0.70 (10) |48.10±0.90 (10.5) |**91.90**±0.50 (3) |52.00±1.40 (11.5) |52.10±1.00 (4.5) \n", - "FLUTE [[10]] |5.9 |51.80±1.10 (10.5) |93.20±0.50 (5.5) |87.20±0.50 (3) |79.20±0.80 (3.5) |68.80±0.80 (10.5) |79.50±0.70 (5) |58.10±1.10 (5) |**91.60**±0.60 (3) |58.40±1.10 (6) |50.00±1.00 (7) \n", - "URL [[11]] |2.15 |**57.51**±1.08 (2.5) |**94.51**±0.41 (1.5) |88.59±0.46 (2) |**80.54**±0.69 (1.5) |**76.17**±0.67 (1.5) |**81.94**±0.56 (2) |**68.75**±0.95 (1.5) |**92.11**±0.48 (3) |63.34±1.19 (3.5) |54.03±0.96 (2.5) \n", - "TSA [[12]] |**1.65** |**57.35**±1.05 (2.5) |**94.96**±0.38 (1.5) |**89.33**±0.44 (1) |**81.42**±0.74 (1.5) |**76.74**±0.72 (1.5) |**82.01**±0.57 (2) |**67.40**±0.99 (1.5) |**92.18**±0.52 (3) |**83.55**±0.90 (1) |**55.75**±1.06 (1) \n", - "TriM [[13]] |6.6 |**58.60**±1.00 (2.5) |92.00±0.60 (8) |82.80±0.70 (9) |75.30±0.80 (7.5) |71.20±0.80 (6.5) |77.30±0.70 (8.5) |48.50±1.00 (10.5) |90.50±0.50 (7.5) |63.00±1.00 (3.5) |52.80±1.10 (2.5) \n", + "k-NN [[1]] |16.85 |38.55±0.94 (16.5) |74.60±1.08 (18) |64.98±0.82 (19) |66.35±0.92 (14.5) |63.58±0.79 (15.5) |44.88±1.05 (19) |37.12±1.06 (15.5) |83.47±0.61 (15.5) |40.11±1.10 (18) |29.55±0.96 (17) \n", + "Finetune [[1]] |14.1 |43.08±1.08 (14.5) |71.11±1.37 (19) |72.03±1.07 (15.5) |59.82±1.15 (17) |69.14±0.85 (9.5) |47.05±1.16 (18) |38.16±1.04 (15.5) |85.28±0.69 (14) |66.74±1.23 (3) |35.17±1.08 (15) \n", + "MatchingNet [[1]] |16.4 |36.08±1.00 (18) |78.25±1.01 (16.5) |69.17±0.96 (17.5) |56.40±1.00 (18) |61.80±0.74 (17) |60.81±1.03 (15.5) |33.70±1.04 (18) |81.90±0.72 (17) |55.57±1.08 (9.5) |28.79±0.96 (17) \n", + "ProtoNet [[1]] |14.5 |44.50±1.05 (14.5) |79.56±1.12 (16.5) |71.14±0.86 (15.5) |67.01±1.02 (14.5) |65.18±0.84 (13.5) |64.88±0.89 (14) |40.26±1.13 (14) |86.85±0.71 (13) |46.48±1.00 (16) |39.87±1.06 (13.5) \n", + "fo-MAML [[1]] |16.25 |37.83±1.01 (16.5) |83.92±0.95 (14.5) |76.41±0.69 (13) |62.43±1.08 (16) |64.16±0.83 (15.5) |59.73±1.10 (17) |33.54±1.11 (18) |79.94±0.84 (18) |42.91±1.31 (17) |29.37±1.08 (17) \n", + "RelationNet [[1]] |17.8 |30.89±0.93 (19) |86.57±0.79 (13) |69.71±0.83 (17.5) |54.14±0.99 (19) |56.56±0.73 (19) |61.75±0.97 (15.5) |32.56±1.08 (18) |76.08±0.76 (19) |37.48±0.93 (19) |27.41±0.89 (19) \n", + "fo-Proto-MAML [[1]] |12.6 |46.52±1.05 (13) |82.69±0.97 (14.5) |75.23±0.76 (14) |69.88±1.02 (12.5) |68.25±0.81 (11.5) |66.84±0.94 (13) |41.99±1.17 (13) |88.72±0.67 (11) |52.42±1.08 (12.5) |41.74±1.13 (11) \n", + "CNAPs [[2]] |11.2 |50.80±1.10 (11.5) |91.70±0.50 (8.5) |83.70±0.60 (8.5) |73.60±0.90 (11) |59.50±0.70 (18) |74.70±0.80 (12) |50.20±1.10 (8.5) |88.90±0.50 (11) |56.50±1.10 (9.5) |39.40±1.00 (13.5) \n", + "SUR [[6]] |8.45 |56.10±1.10 (8) |93.10±0.50 (5.5) |84.60±0.70 (6.5) |70.60±1.00 (12.5) |71.00±0.80 (7.5) |81.30±0.60 (4) |64.20±1.10 (4.5) |82.80±0.80 (15.5) |53.40±1.00 (12.5) |50.10±1.00 (8) \n", + "SUR-pnf [[6]] |9.2 |56.00±1.10 (8) |90.00±0.60 (11.5) |79.70±0.80 (11.5) |75.90±0.90 (8.5) |72.50±0.70 (5.5) |76.70±0.70 (9.5) |49.80±1.10 (8.5) |90.00±0.60 (8.5) |52.20±0.80 (12.5) |50.20±1.10 (8) \n", + "SimpleCNAPS [[14],[7]] |8.4 |56.50±1.10 (8) |91.90±0.60 (8.5) |83.80±0.60 (8.5) |76.10±0.90 (8.5) |70.00±0.80 (9.5) |78.30±0.70 (7.5) |49.10±1.20 (8.5) |91.30±0.60 (7) |59.20±1.00 (7) |42.40±1.10 (11) \n", + "TransductiveCNAPS [[14],[8]]|6.95 |57.90±1.10 (3.5) |94.30±0.40 (3.5) |84.70±0.50 (6.5) |78.80±0.70 (4.5) |66.20±0.80 (13.5) |77.90±0.60 (7.5) |48.90±1.20 (8.5) |92.30±0.40 (4) |59.70±1.10 (7) |42.50±1.10 (11) \n", + "URT [[9]] |6.85 |55.70±1.00 (8) |94.40±0.40 (3.5) |85.80±0.60 (5) |76.30±0.80 (8.5) |71.80±0.70 (5.5) |**82.50**±0.60 (2) |63.50±1.00 (4.5) |88.20±0.60 (11) |51.10±1.10 (15) |52.20±1.10 (5.5) \n", + "URT-pf [[9]] |8.55 |55.50±1.10 (8) |90.20±0.60 (11.5) |79.80±0.70 (11.5) |77.50±0.80 (6) |73.50±0.70 (4) |75.80±0.70 (11) |48.10±0.90 (11.5) |91.90±0.50 (4) |52.00±1.40 (12.5) |52.10±1.00 (5.5) \n", + "FLUTE [[10]] |6.75 |51.80±1.10 (11.5) |93.20±0.50 (5.5) |87.20±0.50 (4) |79.20±0.80 (4.5) |68.80±0.80 (11.5) |79.50±0.70 (5.5) |58.10±1.10 (6) |91.60±0.60 (4) |58.40±1.10 (7) |50.00±1.00 (8) \n", + "URL [[11]] |2.95 |57.51±1.08 (3.5) |**94.51**±0.41 (1.5) |88.59±0.46 (3) |80.54±0.69 (2.5) |76.17±0.67 (2.5) |**81.94**±0.56 (2) |68.75±0.95 (2.5) |92.11±0.48 (4) |63.34±1.19 (4.5) |54.03±0.96 (3.5) \n", + "TSA [[12]] |2.4 |57.35±1.05 (3.5) |**94.96**±0.38 (1.5) |**89.33**±0.44 (1.5) |81.42±0.74 (2.5) |76.74±0.72 (2.5) |**82.01**±0.57 (2) |67.40±0.99 (2.5) |92.18±0.52 (4) |83.55±0.90 (2) |55.75±1.06 (2) \n", + "TriM [[13]] |7.55 |58.60±1.00 (3.5) |92.00±0.60 (8.5) |82.80±0.70 (10) |75.30±0.80 (8.5) |71.20±0.80 (7.5) |77.30±0.70 (9.5) |48.50±1.00 (11.5) |90.50±0.50 (8.5) |63.00±1.00 (4.5) |52.80±1.10 (3.5) \n", + "PMF-DINOSmall [[15]] |**2.25** |**73.52**±0.80 (1) |92.17±0.57 (8.5) |**89.49**±0.52 (1.5) |**91.04**±0.37 (1) |**85.73**±0.62 (1) |79.43±0.67 (5.5) |**74.99**±0.94 (1) |**95.30**±0.44 (1) |**89.85**±0.76 (1) |**59.69**±1.02 (1) \n", "\n", "## References\n", "\n", @@ -9858,6 +10225,7 @@ "[12]: #12-li-et-al-2021b\n", "[13]: #13-liu-et-al-2021b\n", "[14]: #14-bateni-et-al-2022b\n", + "[15]: #15-hu-et-al-2022\n", "\n", "###### \\[1\\] Triantafillou et al. (2020)\n", "\n", @@ -9929,6 +10297,11 @@ "Bateni Peyman, Jarred Barber, Raghav Goyal, Vaden Masrani, Jan-Willem van de Meent, Leonid Sigal, and Frank Wood.; [_Beyond Simple Meta-Learning: Multi-Purpose Models for Multi-Domain, Active and Continual Few-Shot Learning._](https://arxiv.org/abs/2201.05151); arXiv 2022.\n", "\n", "\n", + "###### \\[15\\] Hu et al. (2022)\n", + "\n", + "Shell Xu Hu, Da Li, Jan Stühmer, Minyoung Kim and Timothy Hospedales.; [_Pushing the Limits of Simple Pipelines for Few-Shot Learning: External Data and Fine-Tuning Make a Difference._](https://arxiv.org/abs/2204.07305); CVPR 2022.\n", + "\n", + "\n", "\n" ] } @@ -9936,6 +10309,13 @@ "source": [ "print(export_md())" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -9961,9 +10341,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.8.15" } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/README.md b/README.md index 5598d8c..d73292c 100644 --- a/README.md +++ b/README.md @@ -137,44 +137,46 @@ The tables below were generated by Method |Avg rank |ILSVRC (test) |Omniglot |Aircraft |Birds |Textures |QuickDraw |Fungi |VGG Flower |Traffic signs |MSCOCO ---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|--------------------------- -k-NN [[1]] |13.6 |41.03±1.01 (14) |37.07±1.15 (15) |46.81±0.89 (14) |50.13±1.00 (14.5) |66.36±0.75 (12) |32.06±1.08 (15) |36.16±1.02 (12) |83.10±0.68 (11) |44.59±1.19 (14) |30.38±0.99 (14.5) -Finetune [[1]] |9.45 |45.78±1.10 (12) |60.85±1.58 (10.5) |68.69±1.26 (4) |57.31±1.26 (13) |69.05±0.90 (8.5) |42.60±1.17 (12.5) |38.20±1.02 (10) |85.51±0.68 (8) |66.79±1.31 (4) |34.86±0.97 (12) -MatchingNet [[1]] |12.55 |45.00±1.10 (12) |52.27±1.28 (13) |48.97±0.93 (12) |62.21±0.95 (11.5) |64.15±0.85 (14) |42.87±1.09 (12.5) |33.97±1.00 (13) |80.13±0.71 (14) |47.80±1.14 (11.5) |34.99±1.00 (12) -ProtoNet [[1]] |9.75 |50.50±1.08 (9.5) |59.98±1.35 (10.5) |53.10±1.00 (9.5) |68.79±1.01 (7.5) |66.56±0.83 (12) |48.96±1.08 (10) |39.71±1.11 (8) |85.27±0.77 (8) |47.12±1.10 (13) |41.00±1.10 (9.5) -fo-MAML [[1]] |11.25 |45.51±1.11 (12) |55.55±1.54 (12) |56.24±1.11 (7.5) |63.61±1.06 (11.5) |68.04±0.81 (8.5) |43.96±1.29 (12.5) |32.10±1.10 (14) |81.74±0.83 (13) |50.93±1.51 (9.5) |35.30±1.23 (12) -RelationNet [[1]] |14.55 |34.69±1.01 (15) |45.35±1.36 (14) |40.73±0.83 (15) |49.51±1.05 (14.5) |52.97±0.69 (15) |43.30±1.08 (12.5) |30.55±1.04 (15) |68.76±0.83 (15) |33.67±1.05 (15) |29.15±1.01 (14.5) -fo-Proto-MAML [[1]] |8.25 |49.53±1.05 (9.5) |63.37±1.33 (7.5) |55.95±0.99 (7.5) |68.66±0.96 (7.5) |66.49±0.83 (12) |51.52±1.00 (8.5) |39.96±1.14 (5.5) |87.15±0.69 (5) |48.83±1.09 (11.5) |43.74±1.12 (8) -ALFA+fo-Proto-MAML [[3]] |6.1 |52.80±1.11 (7.5) |61.87±1.51 (7.5) |63.43±1.10 (5) |69.75±1.05 (5.5) |70.78±0.88 (6) |59.17±1.16 (4.5) |41.49±1.17 (5.5) |85.96±0.77 (8) |60.78±1.29 (7) |48.11±1.14 (4.5) -ProtoNet (large) [[4]] |6.25 |53.69±1.07 (5) |68.50±1.27 (4.5) |58.04±0.96 (6) |74.07±0.92 (3.5) |68.76±0.77 (8.5) |53.30±1.06 (7) |40.73±1.15 (5.5) |86.96±0.73 (5) |58.11±1.05 (8) |41.70±1.08 (9.5) -CTX [[4]] |1.75 |**62.76**±0.99 (1.5) |**82.21**±1.00 (1.5) |**79.49**±0.89 (1.5) |80.63±0.88 (2) |75.57±0.64 (3) |**72.68**±0.82 (1) |**51.58**±1.11 (1.5) |**95.34**±0.37 (1) |**82.65**±0.76 (2) |59.90±1.02 (2.5) -BOHB [[5]] |6.85 |51.92±1.05 (7.5) |67.57±1.21 (4.5) |54.12±0.90 (9.5) |70.69±0.90 (5.5) |68.34±0.76 (8.5) |50.33±1.04 (8.5) |41.38±1.12 (5.5) |87.34±0.59 (5) |51.80±1.04 (9.5) |48.03±0.99 (4.5) -Simple CNAPS [[14],[7]] |7.75 |54.80±1.20 (5) |62.00±1.30 (7.5) |49.20±0.90 (12) |66.50±1.00 (9.5) |71.60±0.70 (4.5) |56.60±1.00 (6) |37.50±1.20 (10) |82.10±0.90 (11) |63.10±1.10 (5.5) |45.80±1.00 (6.5) -Transductive CNAPS [[14],[8]] |7.6 |54.10±1.10 (5) |62.90±1.30 (7.5) |48.40±0.90 (12) |67.30±0.90 (9.5) |72.50±0.70 (4.5) |58.00±1.00 (4.5) |37.70±1.10 (10) |82.80±0.80 (11) |61.80±1.10 (5.5) |45.80±1.00 (6.5) -TSA_resnet18 [[12]] |2.8 |59.50±1.10 (3) |78.20±1.20 (3) |72.20±1.00 (3) |74.90±0.90 (3.5) |77.30±0.70 (2) |67.60±0.90 (3) |44.70±1.00 (3) |90.90±0.60 (3) |**82.50**±0.80 (2) |59.00±1.00 (2.5) -TSA_resnet34 [[12]] |**1.5** |**63.73**±0.99 (1.5) |**82.58**±1.11 (1.5) |**80.13**±1.01 (1.5) |**83.39**±0.80 (1) |**79.61**±0.68 (1) |71.03±0.84 (2) |**51.38**±1.17 (1.5) |94.05±0.45 (2) |**81.71**±0.95 (2) |**61.67**±0.95 (1) +k-NN [[1]] |14.6 |41.03±1.01 (15) |37.07±1.15 (16) |46.81±0.89 (15) |50.13±1.00 (15.5) |66.36±0.75 (13) |32.06±1.08 (16) |36.16±1.02 (13) |83.10±0.68 (12) |44.59±1.19 (15) |30.38±0.99 (15.5) +Finetune [[1]] |10.45 |45.78±1.10 (13) |60.85±1.58 (11.5) |68.69±1.26 (5) |57.31±1.26 (14) |69.05±0.90 (9.5) |42.60±1.17 (13.5) |38.20±1.02 (11) |85.51±0.68 (9) |66.79±1.31 (5) |34.86±0.97 (13) +MatchingNet [[1]] |13.55 |45.00±1.10 (13) |52.27±1.28 (14) |48.97±0.93 (13) |62.21±0.95 (12.5) |64.15±0.85 (15) |42.87±1.09 (13.5) |33.97±1.00 (14) |80.13±0.71 (15) |47.80±1.14 (12.5) |34.99±1.00 (13) +ProtoNet [[1]] |10.75 |50.50±1.08 (10.5) |59.98±1.35 (11.5) |53.10±1.00 (10.5) |68.79±1.01 (8.5) |66.56±0.83 (13) |48.96±1.08 (11) |39.71±1.11 (9) |85.27±0.77 (9) |47.12±1.10 (14) |41.00±1.10 (10.5) +fo-MAML [[1]] |12.25 |45.51±1.11 (13) |55.55±1.54 (13) |56.24±1.11 (8.5) |63.61±1.06 (12.5) |68.04±0.81 (9.5) |43.96±1.29 (13.5) |32.10±1.10 (15) |81.74±0.83 (14) |50.93±1.51 (10.5) |35.30±1.23 (13) +RelationNet [[1]] |15.55 |34.69±1.01 (16) |45.35±1.36 (15) |40.73±0.83 (16) |49.51±1.05 (15.5) |52.97±0.69 (16) |43.30±1.08 (13.5) |30.55±1.04 (16) |68.76±0.83 (16) |33.67±1.05 (16) |29.15±1.01 (15.5) +fo-Proto-MAML [[1]] |9.25 |49.53±1.05 (10.5) |63.37±1.33 (8.5) |55.95±0.99 (8.5) |68.66±0.96 (8.5) |66.49±0.83 (13) |51.52±1.00 (9.5) |39.96±1.14 (6.5) |87.15±0.69 (6) |48.83±1.09 (12.5) |43.74±1.12 (9) +ALFA+fo-Proto-MAML [[3]] |7.1 |52.80±1.11 (8.5) |61.87±1.51 (8.5) |63.43±1.10 (6) |69.75±1.05 (6.5) |70.78±0.88 (7) |59.17±1.16 (5.5) |41.49±1.17 (6.5) |85.96±0.77 (9) |60.78±1.29 (8) |48.11±1.14 (5.5) +ProtoNet (large) [[4]] |7.25 |53.69±1.07 (6) |68.50±1.27 (5.5) |58.04±0.96 (7) |74.07±0.92 (4.5) |68.76±0.77 (9.5) |53.30±1.06 (8) |40.73±1.15 (6.5) |86.96±0.73 (6) |58.11±1.05 (9) |41.70±1.08 (10.5) +CTX [[4]] |2.5 |62.76±0.99 (2.5) |**82.21**±1.00 (2) |**79.49**±0.89 (1.5) |80.63±0.88 (3) |75.57±0.64 (4) |72.68±0.82 (2) |51.58±1.11 (2.5) |**95.34**±0.37 (1) |82.65±0.76 (3) |59.90±1.02 (3.5) +BOHB [[5]] |7.85 |51.92±1.05 (8.5) |67.57±1.21 (5.5) |54.12±0.90 (10.5) |70.69±0.90 (6.5) |68.34±0.76 (9.5) |50.33±1.04 (9.5) |41.38±1.12 (6.5) |87.34±0.59 (6) |51.80±1.04 (10.5) |48.03±0.99 (5.5) +SimpleCNAPS [[14],[7]] |8.75 |54.80±1.20 (6) |62.00±1.30 (8.5) |49.20±0.90 (13) |66.50±1.00 (10.5) |71.60±0.70 (5.5) |56.60±1.00 (7) |37.50±1.20 (11) |82.10±0.90 (12) |63.10±1.10 (6.5) |45.80±1.00 (7.5) +TransductiveCNAPS [[14],[8]]|8.6 |54.10±1.10 (6) |62.90±1.30 (8.5) |48.40±0.90 (13) |67.30±0.90 (10.5) |72.50±0.70 (5.5) |58.00±1.00 (5.5) |37.70±1.10 (11) |82.80±0.80 (12) |61.80±1.10 (6.5) |45.80±1.00 (7.5) +TSA_resnet18 [[12]] |3.8 |59.50±1.10 (4) |78.20±1.20 (4) |72.20±1.00 (4) |74.90±0.90 (4.5) |77.30±0.70 (3) |67.60±0.90 (4) |44.70±1.00 (4) |90.90±0.60 (4) |82.50±0.80 (3) |59.00±1.00 (3.5) +TSA_resnet34 [[12]] |2.25 |63.73±0.99 (2.5) |**82.58**±1.11 (2) |**80.13**±1.01 (1.5) |83.39±0.80 (2) |79.61±0.68 (2) |71.03±0.84 (3) |51.38±1.17 (2.5) |94.05±0.45 (2.5) |81.71±0.95 (3) |**61.67**±0.95 (1.5) +PMF-DINOSmall [[15]] |**1.5** |**75.51**±0.72 (1) |**82.81**±1.10 (2) |78.38±1.09 (3) |**85.18**±0.77 (1) |**86.95**±0.60 (1) |**74.47**±0.83 (1) |**55.16**±1.09 (1) |94.66±0.48 (2.5) |**90.04**±0.81 (1) |**62.60**±0.96 (1.5) ## Training on all datasets Method |Avg rank |ILSVRC (test) |Omniglot |Aircraft |Birds |Textures |QuickDraw |Fungi |VGG Flower |Traffic signs |MSCOCO ---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|---------------------------|--------------------------- -k-NN [[1]] |15.85 |38.55±0.94 (15.5) |74.60±1.08 (17) |64.98±0.82 (18) |66.35±0.92 (13.5) |63.58±0.79 (14.5) |44.88±1.05 (18) |37.12±1.06 (14.5) |83.47±0.61 (14.5) |40.11±1.10 (17) |29.55±0.96 (16) -Finetune [[1]] |13.1 |43.08±1.08 (13.5) |71.11±1.37 (18) |72.03±1.07 (14.5) |59.82±1.15 (16) |69.14±0.85 (8.5) |47.05±1.16 (17) |38.16±1.04 (14.5) |85.28±0.69 (13) |66.74±1.23 (2) |35.17±1.08 (14) -MatchingNet [[1]] |15.4 |36.08±1.00 (17) |78.25±1.01 (15.5) |69.17±0.96 (16.5) |56.40±1.00 (17) |61.80±0.74 (16) |60.81±1.03 (14.5) |33.70±1.04 (17) |81.90±0.72 (16) |55.57±1.08 (8.5) |28.79±0.96 (16) -ProtoNet [[1]] |13.5 |44.50±1.05 (13.5) |79.56±1.12 (15.5) |71.14±0.86 (14.5) |67.01±1.02 (13.5) |65.18±0.84 (12.5) |64.88±0.89 (13) |40.26±1.13 (13) |86.85±0.71 (12) |46.48±1.00 (15) |39.87±1.06 (12.5) -fo-MAML [[1]] |15.25 |37.83±1.01 (15.5) |83.92±0.95 (13.5) |76.41±0.69 (12) |62.43±1.08 (15) |64.16±0.83 (14.5) |59.73±1.10 (16) |33.54±1.11 (17) |79.94±0.84 (17) |42.91±1.31 (16) |29.37±1.08 (16) -RelationNet [[1]] |16.8 |30.89±0.93 (18) |86.57±0.79 (12) |69.71±0.83 (16.5) |54.14±0.99 (18) |56.56±0.73 (18) |61.75±0.97 (14.5) |32.56±1.08 (17) |76.08±0.76 (18) |37.48±0.93 (18) |27.41±0.89 (18) -fo-Proto-MAML [[1]] |11.6 |46.52±1.05 (12) |82.69±0.97 (13.5) |75.23±0.76 (13) |69.88±1.02 (11.5) |68.25±0.81 (10.5) |66.84±0.94 (12) |41.99±1.17 (12) |88.72±0.67 (10) |52.42±1.08 (11.5) |41.74±1.13 (10) -CNAPs [[2]] |10.25 |50.80±1.10 (10.5) |91.70±0.50 (8) |83.70±0.60 (7.5) |73.60±0.90 (10) |59.50±0.70 (17) |74.70±0.80 (11) |50.20±1.10 (7.5) |88.90±0.50 (10) |56.50±1.10 (8.5) |39.40±1.00 (12.5) -SUR [[6]] |7.65 |56.10±1.10 (7) |93.10±0.50 (5.5) |84.60±0.70 (5.5) |70.60±1.00 (11.5) |71.00±0.80 (6.5) |81.30±0.60 (4) |64.20±1.10 (3.5) |82.80±0.80 (14.5) |53.40±1.00 (11.5) |50.10±1.00 (7) -SUR-pnf [[6]] |8.2 |56.00±1.10 (7) |90.00±0.60 (10.5) |79.70±0.80 (10.5) |75.90±0.90 (7.5) |72.50±0.70 (4.5) |76.70±0.70 (8.5) |49.80±1.10 (7.5) |90.00±0.60 (7.5) |52.20±0.80 (11.5) |50.20±1.10 (7) -Simple CNAPS [[14],[7]] |7.45 |56.50±1.10 (7) |91.90±0.60 (8) |83.80±0.60 (7.5) |76.10±0.90 (7.5) |70.00±0.80 (8.5) |78.30±0.70 (6.5) |49.10±1.20 (7.5) |91.30±0.60 (6) |59.20±1.00 (6) |42.40±1.10 (10) -Transductive CNAPS [[14],[8]] |6.05 |**57.90**±1.10 (2.5) |94.30±0.40 (3.5) |84.70±0.50 (5.5) |78.80±0.70 (3.5) |66.20±0.80 (12.5) |77.90±0.60 (6.5) |48.90±1.20 (7.5) |**92.30**±0.40 (3) |59.70±1.10 (6) |42.50±1.10 (10) -URT [[9]] |6.05 |55.70±1.00 (7) |94.40±0.40 (3.5) |85.80±0.60 (4) |76.30±0.80 (7.5) |71.80±0.70 (4.5) |**82.50**±0.60 (2) |63.50±1.00 (3.5) |88.20±0.60 (10) |51.10±1.10 (14) |52.20±1.10 (4.5) -URT-pf [[9]] |7.55 |55.50±1.10 (7) |90.20±0.60 (10.5) |79.80±0.70 (10.5) |77.50±0.80 (5) |73.50±0.70 (3) |75.80±0.70 (10) |48.10±0.90 (10.5) |**91.90**±0.50 (3) |52.00±1.40 (11.5) |52.10±1.00 (4.5) -FLUTE [[10]] |5.9 |51.80±1.10 (10.5) |93.20±0.50 (5.5) |87.20±0.50 (3) |79.20±0.80 (3.5) |68.80±0.80 (10.5) |79.50±0.70 (5) |58.10±1.10 (5) |**91.60**±0.60 (3) |58.40±1.10 (6) |50.00±1.00 (7) -URL [[11]] |2.15 |**57.51**±1.08 (2.5) |**94.51**±0.41 (1.5) |88.59±0.46 (2) |**80.54**±0.69 (1.5) |**76.17**±0.67 (1.5) |**81.94**±0.56 (2) |**68.75**±0.95 (1.5) |**92.11**±0.48 (3) |63.34±1.19 (3.5) |54.03±0.96 (2.5) -TSA [[12]] |**1.65** |**57.35**±1.05 (2.5) |**94.96**±0.38 (1.5) |**89.33**±0.44 (1) |**81.42**±0.74 (1.5) |**76.74**±0.72 (1.5) |**82.01**±0.57 (2) |**67.40**±0.99 (1.5) |**92.18**±0.52 (3) |**83.55**±0.90 (1) |**55.75**±1.06 (1) -TriM [[13]] |6.6 |**58.60**±1.00 (2.5) |92.00±0.60 (8) |82.80±0.70 (9) |75.30±0.80 (7.5) |71.20±0.80 (6.5) |77.30±0.70 (8.5) |48.50±1.00 (10.5) |90.50±0.50 (7.5) |63.00±1.00 (3.5) |52.80±1.10 (2.5) +k-NN [[1]] |16.85 |38.55±0.94 (16.5) |74.60±1.08 (18) |64.98±0.82 (19) |66.35±0.92 (14.5) |63.58±0.79 (15.5) |44.88±1.05 (19) |37.12±1.06 (15.5) |83.47±0.61 (15.5) |40.11±1.10 (18) |29.55±0.96 (17) +Finetune [[1]] |14.1 |43.08±1.08 (14.5) |71.11±1.37 (19) |72.03±1.07 (15.5) |59.82±1.15 (17) |69.14±0.85 (9.5) |47.05±1.16 (18) |38.16±1.04 (15.5) |85.28±0.69 (14) |66.74±1.23 (3) |35.17±1.08 (15) +MatchingNet [[1]] |16.4 |36.08±1.00 (18) |78.25±1.01 (16.5) |69.17±0.96 (17.5) |56.40±1.00 (18) |61.80±0.74 (17) |60.81±1.03 (15.5) |33.70±1.04 (18) |81.90±0.72 (17) |55.57±1.08 (9.5) |28.79±0.96 (17) +ProtoNet [[1]] |14.5 |44.50±1.05 (14.5) |79.56±1.12 (16.5) |71.14±0.86 (15.5) |67.01±1.02 (14.5) |65.18±0.84 (13.5) |64.88±0.89 (14) |40.26±1.13 (14) |86.85±0.71 (13) |46.48±1.00 (16) |39.87±1.06 (13.5) +fo-MAML [[1]] |16.25 |37.83±1.01 (16.5) |83.92±0.95 (14.5) |76.41±0.69 (13) |62.43±1.08 (16) |64.16±0.83 (15.5) |59.73±1.10 (17) |33.54±1.11 (18) |79.94±0.84 (18) |42.91±1.31 (17) |29.37±1.08 (17) +RelationNet [[1]] |17.8 |30.89±0.93 (19) |86.57±0.79 (13) |69.71±0.83 (17.5) |54.14±0.99 (19) |56.56±0.73 (19) |61.75±0.97 (15.5) |32.56±1.08 (18) |76.08±0.76 (19) |37.48±0.93 (19) |27.41±0.89 (19) +fo-Proto-MAML [[1]] |12.6 |46.52±1.05 (13) |82.69±0.97 (14.5) |75.23±0.76 (14) |69.88±1.02 (12.5) |68.25±0.81 (11.5) |66.84±0.94 (13) |41.99±1.17 (13) |88.72±0.67 (11) |52.42±1.08 (12.5) |41.74±1.13 (11) +CNAPs [[2]] |11.2 |50.80±1.10 (11.5) |91.70±0.50 (8.5) |83.70±0.60 (8.5) |73.60±0.90 (11) |59.50±0.70 (18) |74.70±0.80 (12) |50.20±1.10 (8.5) |88.90±0.50 (11) |56.50±1.10 (9.5) |39.40±1.00 (13.5) +SUR [[6]] |8.45 |56.10±1.10 (8) |93.10±0.50 (5.5) |84.60±0.70 (6.5) |70.60±1.00 (12.5) |71.00±0.80 (7.5) |81.30±0.60 (4) |64.20±1.10 (4.5) |82.80±0.80 (15.5) |53.40±1.00 (12.5) |50.10±1.00 (8) +SUR-pnf [[6]] |9.2 |56.00±1.10 (8) |90.00±0.60 (11.5) |79.70±0.80 (11.5) |75.90±0.90 (8.5) |72.50±0.70 (5.5) |76.70±0.70 (9.5) |49.80±1.10 (8.5) |90.00±0.60 (8.5) |52.20±0.80 (12.5) |50.20±1.10 (8) +SimpleCNAPS [[14],[7]] |8.4 |56.50±1.10 (8) |91.90±0.60 (8.5) |83.80±0.60 (8.5) |76.10±0.90 (8.5) |70.00±0.80 (9.5) |78.30±0.70 (7.5) |49.10±1.20 (8.5) |91.30±0.60 (7) |59.20±1.00 (7) |42.40±1.10 (11) +TransductiveCNAPS [[14],[8]]|6.95 |57.90±1.10 (3.5) |94.30±0.40 (3.5) |84.70±0.50 (6.5) |78.80±0.70 (4.5) |66.20±0.80 (13.5) |77.90±0.60 (7.5) |48.90±1.20 (8.5) |92.30±0.40 (4) |59.70±1.10 (7) |42.50±1.10 (11) +URT [[9]] |6.85 |55.70±1.00 (8) |94.40±0.40 (3.5) |85.80±0.60 (5) |76.30±0.80 (8.5) |71.80±0.70 (5.5) |**82.50**±0.60 (2) |63.50±1.00 (4.5) |88.20±0.60 (11) |51.10±1.10 (15) |52.20±1.10 (5.5) +URT-pf [[9]] |8.55 |55.50±1.10 (8) |90.20±0.60 (11.5) |79.80±0.70 (11.5) |77.50±0.80 (6) |73.50±0.70 (4) |75.80±0.70 (11) |48.10±0.90 (11.5) |91.90±0.50 (4) |52.00±1.40 (12.5) |52.10±1.00 (5.5) +FLUTE [[10]] |6.75 |51.80±1.10 (11.5) |93.20±0.50 (5.5) |87.20±0.50 (4) |79.20±0.80 (4.5) |68.80±0.80 (11.5) |79.50±0.70 (5.5) |58.10±1.10 (6) |91.60±0.60 (4) |58.40±1.10 (7) |50.00±1.00 (8) +URL [[11]] |2.95 |57.51±1.08 (3.5) |**94.51**±0.41 (1.5) |88.59±0.46 (3) |80.54±0.69 (2.5) |76.17±0.67 (2.5) |**81.94**±0.56 (2) |68.75±0.95 (2.5) |92.11±0.48 (4) |63.34±1.19 (4.5) |54.03±0.96 (3.5) +TSA [[12]] |2.4 |57.35±1.05 (3.5) |**94.96**±0.38 (1.5) |**89.33**±0.44 (1.5) |81.42±0.74 (2.5) |76.74±0.72 (2.5) |**82.01**±0.57 (2) |67.40±0.99 (2.5) |92.18±0.52 (4) |83.55±0.90 (2) |55.75±1.06 (2) +TriM [[13]] |7.55 |58.60±1.00 (3.5) |92.00±0.60 (8.5) |82.80±0.70 (10) |75.30±0.80 (8.5) |71.20±0.80 (7.5) |77.30±0.70 (9.5) |48.50±1.00 (11.5) |90.50±0.50 (8.5) |63.00±1.00 (4.5) |52.80±1.10 (3.5) +PMF-DINOSmall [[15]] |**2.25** |**73.52**±0.80 (1) |92.17±0.57 (8.5) |**89.49**±0.52 (1.5) |**91.04**±0.37 (1) |**85.73**±0.62 (1) |79.43±0.67 (5.5) |**74.99**±0.94 (1) |**95.30**±0.44 (1) |**89.85**±0.76 (1) |**59.69**±1.02 (1) ## References @@ -192,6 +194,7 @@ TriM [[13]] |6.6 |**58.60**±1.00 (2. [12]: #12-li-et-al-2021b [13]: #13-liu-et-al-2021b [14]: #14-bateni-et-al-2022b +[15]: #15-hu-et-al-2022 ###### \[1\] Triantafillou et al. (2020) @@ -263,6 +266,11 @@ Yanbin Liu, Juho Lee, Linchao Zhu, Ling Chen, Humphrey Shi, Yi Yang; [_A Multi-M Bateni Peyman, Jarred Barber, Raghav Goyal, Vaden Masrani, Jan-Willem van de Meent, Leonid Sigal, and Frank Wood.; [_Beyond Simple Meta-Learning: Multi-Purpose Models for Multi-Domain, Active and Continual Few-Shot Learning._](https://arxiv.org/abs/2201.05151); arXiv 2022. +###### \[15\] Hu et al. (2022) + +Shell Xu Hu, Da Li, Jan Stühmer, Minyoung Kim and Timothy Hospedales.; [_Pushing the Limits of Simple Pipelines for Few-Shot Learning: External Data and Fine-Tuning Make a Difference._](https://arxiv.org/abs/2204.07305); CVPR 2022. + + # User instructions