new version of time distributed (#232)

zamba/models/config.py

@@ -35,7 +35,7 @@
 GPUS_AVAILABLE = torch.cuda.device_count()
 
 WEIGHT_LOOKUP = {
-    "time_distributed": "s3://drivendata-client-zamba/data/results/zamba_classification_retraining/td_full_set/version_0/",
+    "time_distributed": "s3://drivendata-client-zamba/data/results/zamba_classification_retraining/td_full_set/version_1/",
     "european": "s3://drivendata-client-zamba/data/results/zamba_v2_classification/european_td_dev_base/version_0/",
     "slowfast": "s3://drivendata-client-zamba/data/results/zamba_v2_classification/experiments/slowfast_small_set_full_size_mdlite/version_2/",
     "blank_nonblank": "s3://drivendata-client-zamba/data/results/zamba_classification_retraining/td_full_set_bnb/version_0/",

zamba/models/official_models/time_distributed/config.yaml

@@ -9,7 +9,7 @@ train_config:
   early_stopping_config:
     mode: max
     monitor: val_macro_f1
-    patience: 5
+    patience: 4
     verbose: true
   model_name: time_distributed
   scheduler_config:
@@ -46,4 +46,4 @@ video_loader_config:
   total_frames: 16
 predict_config:
   model_name: time_distributed
-public_checkpoint: time_distributed_845b088aa1.ckpt
+public_checkpoint: time_distributed_1d483fc723.ckpt

zamba/models/official_models/time_distributed/predict_configuration.yaml

@@ -1,19 +1,19 @@
-inference_start_time: '2022-09-08T22:11:19.578412'
+inference_start_time: '2022-09-26T16:57:18.856926'
 model_class: TimeDistributedEfficientNet
 predict_config:
   batch_size: 3
-  checkpoint: version_0/time_distributed.ckpt
+  checkpoint: data/results/zamba_classification_retraining/td_full_set/version_1/time_distributed.ckpt
   data_dir: ''
   dry_run: false
   gpus: 1
   model_cache_dir: /home/ubuntu/.cache/zamba
   model_name: time_distributed
   num_workers: 2
   output_class_names: false
-  overwrite: false
+  overwrite: true
   proba_threshold: null
   save: true
-  save_dir: version_0
+  save_dir: data/results/zamba_classification_retraining/td_full_set/version_1
   skip_load_validation: true
   weight_download_region: us
 species:

zamba/models/official_models/time_distributed/train_configuration.yaml

@@ -1,4 +1,4 @@
-git_hash: 3e8bbdd821af094a2183fa78fdc74f78834f71fb
+git_hash: b939a93807cbf3f407b42656f9a943b6192f250b
 model_class: TimeDistributedEfficientNet
 species:
 - aardvark
@@ -33,9 +33,9 @@ species:
 - rodent
 - small_cat
 - wild_dog_jackal
-starting_learning_rate: 0.001
+starting_learning_rate: 0.0010964781961431851
 train_config:
-  auto_lr_find: false
+  auto_lr_find: true
   backbone_finetune_config:
     backbone_initial_ratio_lr: 0.01
     multiplier: 1
@@ -50,7 +50,7 @@ train_config:
   early_stopping_config:
     mode: max
     monitor: val_macro_f1
-    patience: 5
+    patience: 4
     verbose: true
   from_scratch: true
   gpus: 1
@@ -71,7 +71,7 @@ train_config:
   skip_load_validation: true
   split_proportions: null
   weight_download_region: us
-training_start_time: '2022-08-28T14:35:02.087643'
+training_start_time: '2022-09-15T19:48:49.775436'
 video_loader_config:
   cache_dir: data/zamba-cache
   cleanup_cache: false

zamba/models/official_models/time_distributed/val_metrics.json

@@ -1,62 +1,62 @@
 {
-  "val_loss": 0.03829624876379967,
-  "val_macro_f1": 0.40069426903010186,
-  "val_top_1_accuracy": 0.803049860414853,
-  "val_top_3_accuracy": 0.9375619127667877,
-  "val_top_5_accuracy": 0.9676102422477711,
-  "val_top_10_accuracy": 0.9901539939363011,
-  "species/val_accuracy/aardvark": 0.9997598535106414,
+  "val_loss": 0.041939787566661835,
+  "val_macro_f1": 0.4418218489172663,
+  "val_top_1_accuracy": 0.810944676252514,
+  "val_top_3_accuracy": 0.9386125536577312,
+  "val_top_5_accuracy": 0.9660793083781106,
+  "val_top_10_accuracy": 0.9892534446012068,
+  "species/val_accuracy/aardvark": 0.9996397802659622,
   "species/val_f1/aardvark": 0.0,
   "species/val_precision/aardvark": 0.0,
   "species/val_recall/aardvark": 0.0,
-  "species/val_accuracy/antelope_duiker": 0.9017800858523699,
-  "species/val_f1/antelope_duiker": 0.7739394776841233,
-  "species/val_precision/antelope_duiker": 0.928239973483593,
-  "species/val_recall/antelope_duiker": 0.6636255924170616,
-  "species/val_accuracy/badger": 0.9989793774202264,
+  "species/val_accuracy/antelope_duiker": 0.9121964398282952,
+  "species/val_f1/antelope_duiker": 0.8127280875856328,
+  "species/val_precision/antelope_duiker": 0.8841064215071737,
+  "species/val_recall/antelope_duiker": 0.7520142180094787,
+  "species/val_accuracy/badger": 0.9985891393750188,
   "species/val_f1/badger": 0.0,
   "species/val_precision/badger": 0.0,
   "species/val_recall/badger": 0.0,
   "species/val_accuracy/bat": 0.9993395971542641,
   "species/val_f1/bat": 0.0,
   "species/val_precision/bat": 0.0,
   "species/val_recall/bat": 0.0,
-  "species/val_accuracy/bird": 0.9872422177528293,
-  "species/val_f1/bird": 0.5540398740818467,
-  "species/val_precision/bird": 0.8461538461538461,
-  "species/val_recall/bird": 0.4118564742589704,
-  "species/val_accuracy/blank": 0.8529703118902531,
-  "species/val_f1/blank": 0.8342021528671045,
-  "species/val_precision/blank": 0.7676780262911969,
-  "species/val_recall/blank": 0.9133496405010748,
-  "species/val_accuracy/cattle": 0.9993696154654339,
-  "species/val_f1/cattle": 0.6865671641791045,
-  "species/val_precision/cattle": 0.9583333333333334,
+  "species/val_accuracy/bird": 0.9860715036172065,
+  "species/val_f1/bird": 0.6239870340356565,
+  "species/val_precision/bird": 0.6492411467116358,
+  "species/val_recall/bird": 0.6006240249609984,
+  "species/val_accuracy/blank": 0.8495182061057245,
+  "species/val_f1/blank": 0.8130245048674051,
+  "species/val_precision/blank": 0.8182432432432433,
+  "species/val_recall/blank": 0.8078719146097398,
+  "species/val_accuracy/cattle": 0.9993395971542641,
+  "species/val_f1/cattle": 0.6764705882352942,
+  "species/val_precision/cattle": 0.92,
   "species/val_recall/cattle": 0.5348837209302325,
   "species/val_accuracy/cheetah": 1.0,
   "species/val_f1/cheetah": 0.0,
   "species/val_precision/cheetah": 0.0,
   "species/val_recall/cheetah": 0.0,
-  "species/val_accuracy/chimpanzee_bonobo": 0.9897637558910936,
-  "species/val_f1/chimpanzee_bonobo": 0.8472906403940887,
-  "species/val_precision/chimpanzee_bonobo": 0.9497991967871486,
-  "species/val_recall/chimpanzee_bonobo": 0.7647534357316087,
-  "species/val_accuracy/civet_genet": 0.9918950559841503,
-  "species/val_f1/civet_genet": 0.41810344827586204,
-  "species/val_precision/civet_genet": 0.9603960396039604,
-  "species/val_recall/civet_genet": 0.26721763085399447,
-  "species/val_accuracy/elephant": 0.9961876744814336,
-  "species/val_f1/elephant": 0.9057164068299925,
-  "species/val_precision/elephant": 0.9370199692780338,
-  "species/val_recall/elephant": 0.8764367816091954,
-  "species/val_accuracy/equid": 0.9993095788430942,
-  "species/val_f1/equid": 0.8130081300813008,
-  "species/val_precision/equid": 0.9259259259259259,
-  "species/val_recall/equid": 0.7246376811594203,
-  "species/val_accuracy/forest_buffalo": 0.9992195239095848,
-  "species/val_f1/forest_buffalo": 0.23529411764705882,
-  "species/val_precision/forest_buffalo": 0.6666666666666666,
-  "species/val_recall/forest_buffalo": 0.14285714285714285,
+  "species/val_accuracy/chimpanzee_bonobo": 0.9917449644283013,
+  "species/val_f1/chimpanzee_bonobo": 0.8830284985112719,
+  "species/val_precision/chimpanzee_bonobo": 0.9317773788150808,
+  "species/val_recall/chimpanzee_bonobo": 0.8391269199676638,
+  "species/val_accuracy/civet_genet": 0.9936361180319995,
+  "species/val_f1/civet_genet": 0.6357388316151203,
+  "species/val_precision/civet_genet": 0.8447488584474886,
+  "species/val_recall/civet_genet": 0.509641873278237,
+  "species/val_accuracy/elephant": 0.9964878575931319,
+  "species/val_f1/elephant": 0.9132690882134915,
+  "species/val_precision/elephant": 0.9433384379785605,
+  "species/val_recall/elephant": 0.8850574712643678,
+  "species/val_accuracy/equid": 0.9995797436436226,
+  "species/val_f1/equid": 0.8939393939393939,
+  "species/val_precision/equid": 0.9365079365079365,
+  "species/val_recall/equid": 0.855072463768116,
+  "species/val_accuracy/forest_buffalo": 0.9993996337766037,
+  "species/val_f1/forest_buffalo": 0.5,
+  "species/val_precision/forest_buffalo": 0.8333333333333334,
+  "species/val_recall/forest_buffalo": 0.35714285714285715,
   "species/val_accuracy/fox": 0.9998198901329811,
   "species/val_f1/fox": 0.0,
   "species/val_precision/fox": 0.0,
@@ -66,71 +66,71 @@
   "species/val_precision/giraffe": 0.0,
   "species/val_recall/giraffe": 0.0,
   "species/val_accuracy/gorilla": 0.9984690661303395,
-  "species/val_f1/gorilla": 0.5405405405405406,
-  "species/val_precision/gorilla": 0.8823529411764706,
-  "species/val_recall/gorilla": 0.38961038961038963,
-  "species/val_accuracy/hare_rabbit": 0.9996998168883019,
-  "species/val_f1/hare_rabbit": 0.2857142857142857,
+  "species/val_f1/gorilla": 0.6222222222222222,
+  "species/val_precision/gorilla": 0.7241379310344828,
+  "species/val_recall/gorilla": 0.5454545454545454,
+  "species/val_accuracy/hare_rabbit": 0.999669798577132,
+  "species/val_f1/hare_rabbit": 0.15384615384615385,
   "species/val_precision/hare_rabbit": 1.0,
-  "species/val_recall/hare_rabbit": 0.16666666666666666,
-  "species/val_accuracy/hippopotamus": 0.9993095788430942,
-  "species/val_f1/hippopotamus": 0.684931506849315,
-  "species/val_precision/hippopotamus": 0.9615384615384616,
-  "species/val_recall/hippopotamus": 0.5319148936170213,
-  "species/val_accuracy/hog": 0.9867018881517726,
-  "species/val_f1/hog": 0.7520984890878567,
-  "species/val_precision/hog": 0.8028673835125448,
-  "species/val_recall/hog": 0.7073684210526315,
-  "species/val_accuracy/human": 0.9912046348272446,
-  "species/val_f1/human": 0.8837762792542643,
-  "species/val_precision/human": 0.8737254901960785,
-  "species/val_recall/human": 0.8940609951845907,
-  "species/val_accuracy/hyena": 0.9996397802659622,
-  "species/val_f1/hyena": 0.6,
-  "species/val_precision/hyena": 0.9,
-  "species/val_recall/hyena": 0.45,
+  "species/val_recall/hare_rabbit": 0.08333333333333333,
+  "species/val_accuracy/hippopotamus": 0.9993696154654339,
+  "species/val_f1/hippopotamus": 0.7341772151898734,
+  "species/val_precision/hippopotamus": 0.90625,
+  "species/val_recall/hippopotamus": 0.6170212765957447,
+  "species/val_accuracy/hog": 0.9900039023804521,
+  "species/val_f1/hog": 0.8040023543260743,
+  "species/val_precision/hog": 0.9118825100133512,
+  "species/val_recall/hog": 0.7189473684210527,
+  "species/val_accuracy/human": 0.9937261729655089,
+  "species/val_f1/human": 0.9156237383932175,
+  "species/val_precision/human": 0.9212022745735174,
+  "species/val_recall/human": 0.9101123595505618,
+  "species/val_accuracy/hyena": 0.9996998168883019,
+  "species/val_f1/hyena": 0.7368421052631577,
+  "species/val_precision/hyena": 0.7777777777777778,
+  "species/val_recall/hyena": 0.7,
   "species/val_accuracy/large_flightless_bird": 0.9999099450664906,
   "species/val_f1/large_flightless_bird": 0.0,
   "species/val_precision/large_flightless_bird": 0.0,
   "species/val_recall/large_flightless_bird": 0.0,
-  "species/val_accuracy/leopard": 0.9993395971542641,
-  "species/val_f1/leopard": 0.8405797101449275,
+  "species/val_accuracy/leopard": 0.999189505598415,
+  "species/val_f1/leopard": 0.7969924812030075,
   "species/val_precision/leopard": 1.0,
-  "species/val_recall/leopard": 0.725,
+  "species/val_recall/leopard": 0.6625,
   "species/val_accuracy/lion": 0.9999699816888302,
   "species/val_f1/lion": 0.0,
   "species/val_precision/lion": 0.0,
   "species/val_recall/lion": 0.0,
-  "species/val_accuracy/mongoose": 0.9909644883378861,
-  "species/val_f1/mongoose": 0.23017902813299232,
-  "species/val_precision/mongoose": 0.9,
-  "species/val_recall/mongoose": 0.13196480938416422,
-  "species/val_accuracy/monkey_prosimian": 0.9673400774472428,
-  "species/val_f1/monkey_prosimian": 0.8695443645083932,
-  "species/val_precision/monkey_prosimian": 0.9168141592920354,
-  "species/val_recall/monkey_prosimian": 0.8269099201824401,
-  "species/val_accuracy/pangolin": 0.9986491759973584,
-  "species/val_f1/pangolin": 0.08163265306122448,
-  "species/val_precision/pangolin": 0.6666666666666666,
-  "species/val_recall/pangolin": 0.043478260869565216,
-  "species/val_accuracy/porcupine": 0.9945666856782638,
-  "species/val_f1/porcupine": 0.4986149584487534,
-  "species/val_precision/porcupine": 0.9278350515463918,
-  "species/val_recall/porcupine": 0.3409090909090909,
-  "species/val_accuracy/reptile": 0.9992795605319245,
+  "species/val_accuracy/mongoose": 0.991474799627773,
+  "species/val_f1/mongoose": 0.4083333333333333,
+  "species/val_precision/mongoose": 0.7050359712230215,
+  "species/val_recall/mongoose": 0.2873900293255132,
+  "species/val_accuracy/monkey_prosimian": 0.9689010296280731,
+  "species/val_f1/monkey_prosimian": 0.8764607679465777,
+  "species/val_precision/monkey_prosimian": 0.9185203699075232,
+  "species/val_recall/monkey_prosimian": 0.8380843785632839,
+  "species/val_accuracy/pangolin": 0.998709212619698,
+  "species/val_f1/pangolin": 0.1568627450980392,
+  "species/val_precision/pangolin": 0.8,
+  "species/val_recall/pangolin": 0.08695652173913043,
+  "species/val_accuracy/porcupine": 0.995016960345811,
+  "species/val_f1/porcupine": 0.576530612244898,
+  "species/val_precision/porcupine": 0.8828125,
+  "species/val_recall/porcupine": 0.42803030303030304,
+  "species/val_accuracy/reptile": 0.9987392309308678,
   "species/val_f1/reptile": 0.0,
   "species/val_precision/reptile": 0.0,
   "species/val_recall/reptile": 0.0,
-  "species/val_accuracy/rodent": 0.9759853510641491,
-  "species/val_f1/rodent": 0.12280701754385966,
-  "species/val_precision/rodent": 0.9180327868852459,
-  "species/val_recall/rodent": 0.06580493537015276,
-  "species/val_accuracy/small_cat": 0.999189505598415,
+  "species/val_accuracy/rodent": 0.9764356257316963,
+  "species/val_f1/rodent": 0.3375527426160337,
+  "species/val_precision/rodent": 0.5988023952095808,
+  "species/val_recall/rodent": 0.23501762632197415,
+  "species/val_accuracy/small_cat": 0.9991594872872452,
   "species/val_f1/small_cat": 0.0,
   "species/val_precision/small_cat": 0.0,
   "species/val_recall/small_cat": 0.0,
-  "species/val_accuracy/wild_dog_jackal": 0.9993696154654339,
-  "species/val_f1/wild_dog_jackal": 0.36363636363636365,
-  "species/val_precision/wild_dog_jackal": 0.5,
-  "species/val_recall/wild_dog_jackal": 0.2857142857142857
+  "species/val_accuracy/wild_dog_jackal": 0.9993395971542641,
+  "species/val_f1/wild_dog_jackal": 0.26666666666666666,
+  "species/val_precision/wild_dog_jackal": 0.4444444444444444,
+  "species/val_recall/wild_dog_jackal": 0.19047619047619047
 }