Nlq fixes and fp16 support

.gitignore

@@ -1 +1,2 @@
 /MQ/runs/
+EgoTracks/run.sh

EgoTracks/README.md

@@ -0,0 +1,59 @@
+# EgoTracks 
+
+## Install packages
+```sh
+pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.8/index.html
+
+pip install torch==1.8.1+cu101 torchvision==0.9.1+cu101 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html
+```
+
+Then install EgoTracks as package:
+```
+python setup.py install
+```
+
+## Download clips
+Please follow [Start here](https://github.com/EGO4D/docs/blob/main/docs/start-here.md) for instructions on how to access the dataset by accepting the terms of our license agreement.
+And then use the following command to download clips and annotations for EgoTracks:
+```
+ego4d --output_directory ./ --datasets egotracks clips --benchmark EM --version v2
+```
+
+## Preprocess - extract frames from exported clips
+Replace the following paths for reading and outputing to the correct place in ExtractFramesWorkflowParams in tools/preprocess/extract_ego4d_clip_frames.py: 
+* clip_dir: path to the downloaded clip directory
+* annotation_path: path to the annotation file we would like to extract frames. If you want the test set only, only extract frames for the test set. If you also would like training part, then need to extract for training set as well.
+* output_dir: path to the directory where to the save the extracted frame images. 
+
+And then extract frames from video clips:
+```sh
+python tools/preprocess/extract_ego4d_clip_frames.py 
+```
+
+## Finetuning on EgoTracks
+We used STARK (Res50) as pre-trained models, so download the model weights from https://drive.google.com/drive/folders/1fSgll53ZnVKeUn22W37Nijk-b9LGhMdN
+Or you can use our trained checkpoint at https://drive.google.com/file/d/14vZmWxYSGJXZGxD5U1LthvvTR_eRzWCw/view?usp=share_link
+
+Change the following paths in the tracking/config/stark_defaults.py:
+* cfg.DATA.EGO4DLTT_ANNOTATION_PATH = "your_path/train_v1.json"
+* cfg.DATA.EGO4DLTT_DATA_DIR = "your_path_to_extract_frames" - This is the same as the output_dir in preprocess
+
+And change the model weights path and output directory in train.sh, and then run:
+```sh
+bash train.sh
+```
+
+
+## Infer challenge set and submit challenge result
+We use "{clip_uid}\_{query_set_id}\_{object_title}" as unique name for each sequence (object). One could use the EGO4DLTTrackingDataset from tracking/dataset/eval_datasets/ego4d_lt_tracking_dataset.py for loading images and sequence name.
+An example of how to run test and generate submission file is in tools/eval_datasets/eval_ego4d_lt_tracking.py and tools/train_net.py result2submission.
+
+Change the following paths in the tracking/config/stark_defaults.py:
+* cfg.EVAL.EGO4DLT.ANNOTATION_PATH = "your_path/challenge_test_v1_unannotated.json"
+* cfg.EVAL.EGO4DLT.DATA_DIR = "your_path_to_extract_frames" 
+
+And change the model weights path and output directory in test.sh, and then run:
+```sh
+bash test.sh
+```
+

EgoTracks/configs/STARK/stark_st_R101.yaml

@@ -0,0 +1,103 @@
+_BASE_: "./stark_st_base.yaml"
+MODEL_TYPE: STARK
+DATA:
+  SAMPLER_MODE: trident_pro
+  MAX_SAMPLE_INTERVAL:
+  - 200
+  MEAN:
+  - 0.485
+  - 0.456
+  - 0.406
+  SEARCH:
+    CENTER_JITTER: 4.5
+    FACTOR: 5.0
+    SCALE_JITTER: 0.5
+    SIZE: 320
+    NUMBER: 1
+  STD:
+  - 0.229
+  - 0.224
+  - 0.225
+  TEMPLATE:
+    CENTER_JITTER: 0
+    FACTOR: 2.0
+    SCALE_JITTER: 0
+    SIZE: 128
+    NUMBER: 2
+  TRAIN:
+    DATASETS_NAME:
+    - LASOT
+    - GOT10K_vottrain
+    - COCO17
+    - TRACKINGNET
+    DATASETS_RATIO:
+    - 1
+    - 1
+    - 1
+    - 1
+    SAMPLE_PER_EPOCH: 600000
+  VAL:
+    DATASETS_NAME:
+    - GOT10K_votval
+    DATASETS_RATIO:
+    - 1
+    SAMPLE_PER_EPOCH: 100000
+MODEL:
+  HEAD_TYPE: CORNER
+  NLAYER_HEAD: 3
+  BACKBONE:
+    DILATION: false
+    OUTPUT_LAYERS:
+    - layer3
+    STRIDE: 16
+    TYPE: resnet101
+  HIDDEN_DIM: 256
+  NUM_OBJECT_QUERIES: 1
+  POSITION_EMBEDDING: sine
+  PREDICT_MASK: false
+  TRANSFORMER:
+    DEC_LAYERS: 6
+    DIM_FEEDFORWARD: 2048
+    DIVIDE_NORM: false
+    DROPOUT: 0.1
+    ENC_LAYERS: 6
+    NHEADS: 8
+    PRE_NORM: false
+TRAIN_STAGE_1:
+  BACKBONE_MULTIPLIER: 0.1
+  BATCH_SIZE: 16
+  DEEP_SUPERVISION: false
+  EPOCH: 50
+  FREEZE_BACKBONE_BN: true
+  FREEZE_LAYERS:
+  - conv1
+  - layer1
+  GRAD_CLIP_NORM: 0.1
+  LR: 0.0001
+  LR_DROP_EPOCH: 40
+  NUM_WORKER: 4
+  OPTIMIZER: ADAMW
+  PRINT_INTERVAL: 50
+  VAL_EPOCH_INTERVAL: 10
+  WEIGHT_DECAY: 0.0001
+  LOSS_FUNCTIONS: ["giou", "l1"]
+  LOSS_WEIGHTS: [2.0, 5.0]
+TRAIN_STAGE_2:
+  BACKBONE_MULTIPLIER: 0.1
+  BATCH_SIZE: 16
+  DEEP_SUPERVISION: false
+  EPOCH: 5
+  FREEZE_BACKBONE_BN: true
+  FREEZE_LAYERS:
+  - conv1
+  - layer1
+  GRAD_CLIP_NORM: 0.1
+  LR: 0.0001
+  LR_DROP_EPOCH: 40
+  NUM_WORKER: 4
+  OPTIMIZER: ADAMW
+  PRINT_INTERVAL: 50
+  VAL_EPOCH_INTERVAL: 1
+  WEIGHT_DECAY: 0.0001
+  LOSS_FUNCTIONS: ["cls"]
+  LOSS_WEIGHTS: [1.0]

EgoTracks/configs/STARK/stark_st_base.yaml

@@ -0,0 +1,116 @@
+MODEL_TYPE: STARK
+DATA:
+  SAMPLER_MODE: trident_pro
+  MAX_SAMPLE_INTERVAL:
+  - 200
+  MEAN:
+  - 0.485
+  - 0.456
+  - 0.406
+  SEARCH:
+    CENTER_JITTER: 4.5
+    FACTOR: 5.0
+    SCALE_JITTER: 0.5
+    SIZE: 320
+    NUMBER: 1
+  STD:
+  - 0.229
+  - 0.224
+  - 0.225
+  TEMPLATE:
+    CENTER_JITTER: 0
+    FACTOR: 2.0
+    SCALE_JITTER: 0
+    SIZE: 128
+    NUMBER: 2
+  TRAIN:
+    DATASETS_NAME:
+    - LASOT
+    - GOT10K_vottrain
+    - COCO17
+    - TRACKINGNET
+    DATASETS_RATIO:
+    - 1
+    - 1
+    - 1
+    - 1
+    SAMPLE_PER_EPOCH: 600000
+  VAL:
+    DATASETS_NAME:
+    - GOT10K_votval
+    DATASETS_RATIO:
+    - 1
+    SAMPLE_PER_EPOCH: 100000
+MODEL:
+  HEAD_TYPE: CORNER
+  NLAYER_HEAD: 3
+  BACKBONE:
+    DILATION: false
+    OUTPUT_LAYERS:
+    - layer3
+    STRIDE: 16
+    TYPE: resnet50
+  HIDDEN_DIM: 256
+  NUM_OBJECT_QUERIES: 1
+  POSITION_EMBEDDING: sine
+  PREDICT_MASK: false
+  TRANSFORMER:
+    DEC_LAYERS: 6
+    DIM_FEEDFORWARD: 2048
+    DIVIDE_NORM: false
+    DROPOUT: 0.1
+    ENC_LAYERS: 6
+    NHEADS: 8
+    PRE_NORM: false
+TRAIN_STAGE_1:
+  BACKBONE_MULTIPLIER: 0.1
+  BATCH_SIZE: 16
+  DEEP_SUPERVISION: false
+  EPOCH: 50
+  FREEZE_BACKBONE_BN: true
+  FREEZE_LAYERS:
+  - conv1
+  - layer1
+  GRAD_CLIP_NORM: 0.1
+  LR: 0.0001
+  LR_DROP_EPOCH: 40
+  NUM_WORKER: 4
+  OPTIMIZER: ADAMW
+  PRINT_INTERVAL: 50
+  VAL_EPOCH_INTERVAL: 10
+  WEIGHT_DECAY: 0.0001
+  LOSS_FUNCTIONS: ["giou", "l1"]
+  LOSS_WEIGHTS: [2.0, 5.0]
+TRAIN_STAGE_2:
+  BACKBONE_MULTIPLIER: 0.1
+  BATCH_SIZE: 16
+  DEEP_SUPERVISION: false
+  EPOCH: 5
+  FREEZE_BACKBONE_BN: true
+  FREEZE_LAYERS:
+  - conv1
+  - layer1
+  GRAD_CLIP_NORM: 0.1
+  LR: 0.0001
+  LR_DROP_EPOCH: 4
+  NUM_WORKER: 4
+  OPTIMIZER: ADAMW
+  PRINT_INTERVAL: 50
+  VAL_EPOCH_INTERVAL: 1
+  WEIGHT_DECAY: 0.0001
+  LOSS_FUNCTIONS: ["cls"]
+  LOSS_WEIGHTS: [1.0]
+TEST:
+  SEARCH_FACTOR: 5.0
+  SEARCH_SIZE: 320
+  TEMPLATE_FACTOR: 2.0
+  TEMPLATE_SIZE: 128
+  TEST_BATCHSIZE: 8
+  IS_SEARCH_LOCAL: True
+  UPDATE_INTERVALS:
+    LASOT: [200]
+    GOT10K_TEST: [200]
+    TRACKINGNET: [25]
+    VOT20: [10]
+    VOT20LT: [200]
+    EGO4DVQTracking: [1]

EgoTracks/setup.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+from setuptools import find_packages, setup
+
+PROJECTS = {
+    "tracking.tools": "tools",
+}
+
+setup(
+    name="egotracks",
+    version="1.0",
+    # author="",
+    # url="unknown",
+    # description="EgoTracks",
+    # python_requires=">=3.7",
+    # install_requires=[
+    #     "matplotlib",
+    #     "detectron2",
+    #     "opencv-python",
+    #     "pandas",
+    #     "torchvision>=0.4.2",
+    #     "scikit-learn",
+    #     "iopath",
+    # ],
+    packages=find_packages(exclude=("tests", "tools")) + list(PROJECTS.keys()),
+    package_dir=PROJECTS,
+    package_data={"tracking.tools": ["**"]},
+)

EgoTracks/test.sh

@@ -0,0 +1,4 @@
+python tools/train_net.py \
+--num-gpus 8 --eval-only \
+MODEL.WEIGHTS /checkpoint/haotang/experiments/EgoTracks/STARKST_ep0001.pth.tar \
+OUTPUT_DIR /checkpoint/haotang/experiments/EgoTracks/res/test_challenge_set_small_v1_5FPS_multi_gpu_test

EgoTracks/tools/eval_datasets/build.py

@@ -0,0 +1,18 @@
+from .eval_ego4d_lt_tracking import (
+    eval_ego4d_lt_tracking,
+)
+
+EVAL_FUNCTIONS = {
+    "EGO4DLTTracking": eval_ego4d_lt_tracking,
+}
+CALCULATE_METRICS_FUNCTIONS = {
+    "EGO4DLTTracking": None,
+}
+
+
+def build_eval_function(dataset_name):
+    return EVAL_FUNCTIONS[dataset_name]
+
+
+def build_calculate_metrics_function(dataset_name):
+    return CALCULATE_METRICS_FUNCTIONS[dataset_name]