first commit

.gitignore

@@ -0,0 +1,3 @@
+*.swp
+*.pyc
+__pycache__

config.py

@@ -0,0 +1,273 @@
+import torch
+from yacs.config import CfgNode as CN
+
+_C = CN()
+_C.BASE_CONFIG = ""  # will use this file as base config and cover self's unique changes
+_C.DEVICE = None
+_C.RANK = 0
+_C.WORLD_SIZE = 1
+_C.LOCAL_RANK = 0
+_C.SEED = None
+_C.OUTPUT = ""
+
+_C.EVALUATOR = ""
+_C.EVALUATION_DOMAIN = None  # support 'float'/'bn_merged'/'quant'
+
+_C.MODEL = CN()
+_C.MODEL.ARCH = ""
+_C.MODEL.CHECKPOINT = ""
+_C.MODEL.PRETRAINED = ""
+_C.MODEL.NUM_CLASSES = 0
+_C.MODEL.INPUTSHAPE = [-1, -1]  # h, w
+
+_C.TRAIN = CN()
+_C.TRAIN.USE_DDP = False
+_C.TRAIN.SYNC_BN = False
+_C.TRAIN.LINEAR_EVAL = False
+_C.TRAIN.WARMUP_FC = False # load from a linear evaluation model
+_C.TRAIN.RESUME = ""
+_C.TRAIN.EPOCHS = 0
+_C.TRAIN.START_EPOCH = 0
+_C.TRAIN.PRETRAIN = ""
+_C.TRAIN.DATASET = ""
+_C.TRAIN.LABEL_SMOOTHING = 0.0
+_C.TRAIN.BATCH_SIZE = 1  # per-gpu
+_C.TRAIN.NUM_WORKERS = 0
+_C.TRAIN.PRINT_FREQ = 1
+
+_C.TRAIN.LR_SCHEDULER = CN()
+_C.TRAIN.LR_SCHEDULER.TYPE = None  # support cosine / step / multiStep
+_C.TRAIN.LR_SCHEDULER.WARMUP_EPOCHS = 0
+_C.TRAIN.LR_SCHEDULER.WARMUP_LR = 0.0
+_C.TRAIN.LR_SCHEDULER.BASE_LR = 0.0
+_C.TRAIN.LR_SCHEDULER.FC_LR = 0.0 # specific learning rate for final fc layer
+_C.TRAIN.LR_SCHEDULER.MIN_LR = 0.0
+_C.TRAIN.LR_SCHEDULER.SPECIFIC_LRS = []
+_C.TRAIN.LR_SCHEDULER.DECAY_MILESTONES = []
+_C.TRAIN.LR_SCHEDULER.DECAY_EPOCH = 0
+_C.TRAIN.LR_SCHEDULER.DECAY_RATE = 0.1
+
+_C.TRAIN.OPTIMIZER = CN()
+_C.TRAIN.OPTIMIZER.NAME = ""
+_C.TRAIN.OPTIMIZER.EPS = None
+_C.TRAIN.OPTIMIZER.BETAS = None  # (0.9, 0.999)
+_C.TRAIN.OPTIMIZER.MOMENTUM = 0.0
+_C.TRAIN.OPTIMIZER.WEIGHT_DECAY = 0.0
+
+_C.TRAIN.LOSS = CN()
+_C.TRAIN.LOSS.CRITERION = CN()
+_C.TRAIN.LOSS.REGULARIZER = CN()
+_C.TRAIN.LOSS.LAMBDA = 0.0  # a ratio controls the importance of regularizer
+_C.TRAIN.LOSS.CRITERION.NAME = ""  # support CrossEntropy / LpLoss
+_C.TRAIN.LOSS.CRITERION.LPLOSS = CN()
+_C.TRAIN.LOSS.CRITERION.LPLOSS.P = 2.0
+_C.TRAIN.LOSS.CRITERION.LPLOSS.REDUCTION = "none"
+_C.TRAIN.LOSS.REGULARIZER.NAME = ""  # support PACT/SLIMMING
+
+_C.TRAIN.RUNNER = CN()
+_C.TRAIN.RUNNER.NAME = ""  # support default
+
+_C.TRAIN.METER = CN()
+_C.TRAIN.METER.NAME = ""  # support ACC / MAP / MIOU
+_C.TRAIN.METER.ACC = CN()
+_C.TRAIN.METER.ACC.TOPK = []
+_C.TRAIN.METER.MAP = CN()
+_C.TRAIN.METER.MIOU = CN()
+
+_C.AUG = CN()
+_C.AUG.TRAIN = CN()
+_C.AUG.TRAIN.RANDOMRESIZEDCROP = CN()
+_C.AUG.TRAIN.RANDOMRESIZEDCROP.ENABLE = False
+# _C.AUG.TRAIN.RANDOMRESIZEDCROP.SIZE = MODEL.INPUT_SHAPE
+_C.AUG.TRAIN.RANDOMRESIZEDCROP.SCALE = (0.08, 1.0)
+_C.AUG.TRAIN.RANDOMRESIZEDCROP.INTERPOLATION = "bilinear"
+_C.AUG.TRAIN.RESIZE = CN()
+_C.AUG.TRAIN.RESIZE.ENABLE = False
+_C.AUG.TRAIN.RESIZE.SIZE = (-1, -1)  # h, w
+_C.AUG.TRAIN.RESIZE.KEEP_RATIO = True
+_C.AUG.TRAIN.RESIZE.INTERPOLATION = "bilinear"
+_C.AUG.TRAIN.HORIZONTAL_FLIP = CN()
+_C.AUG.TRAIN.HORIZONTAL_FLIP.PROB = 0.0
+_C.AUG.TRAIN.VERTICAL_FLIP = CN()
+_C.AUG.TRAIN.VERTICAL_FLIP.PROB = 0.0
+_C.AUG.TRAIN.RANDOMCROP = CN()
+_C.AUG.TRAIN.RANDOMCROP.ENABLE = False
+# _C.AUG.TRAIN.RANDOMCROP.SIZE = MODEL.INPUT_SHAPE
+_C.AUG.TRAIN.RANDOMCROP.PADDING = 0
+_C.AUG.TRAIN.CENTERCROP = CN()
+_C.AUG.TRAIN.CENTERCROP.ENABLE = False
+_C.AUG.TRAIN.COLOR_JITTER = CN()
+_C.AUG.TRAIN.COLOR_JITTER.PROB = 0.0
+_C.AUG.TRAIN.COLOR_JITTER.BRIGHTNESS = 0.4
+_C.AUG.TRAIN.COLOR_JITTER.CONTRAST = 0.4
+_C.AUG.TRAIN.COLOR_JITTER.SATURATION = 0.2
+_C.AUG.TRAIN.COLOR_JITTER.HUE = 0.1
+_C.AUG.TRAIN.AUTO_AUGMENT = CN()
+_C.AUG.TRAIN.AUTO_AUGMENT.ENABLE = False
+_C.AUG.TRAIN.AUTO_AUGMENT.POLICY = 0.0
+_C.AUG.TRAIN.RANDOMERASE = CN()
+_C.AUG.TRAIN.RANDOMERASE.PROB = 0.0
+_C.AUG.TRAIN.RANDOMERASE.MODE = "const"
+_C.AUG.TRAIN.RANDOMERASE.MAX_COUNT = None
+_C.AUG.TRAIN.MIX = CN()  # mixup & cutmix
+_C.AUG.TRAIN.MIX.PROB = 0.0
+_C.AUG.TRAIN.MIX.MODE = "batch"
+_C.AUG.TRAIN.MIX.SWITCH_MIXUP_CUTMIX_PROB = 0.0
+_C.AUG.TRAIN.MIX.MIXUP_ALPHA = 0.0
+_C.AUG.TRAIN.MIX.CUTMIX_ALPHA = 0.0
+_C.AUG.TRAIN.MIX.CUTMIX_MIXMAX = None
+_C.AUG.TRAIN.NORMLIZATION = CN()
+_C.AUG.TRAIN.NORMLIZATION.MEAN = []
+_C.AUG.TRAIN.NORMLIZATION.STD = []
+_C.AUG.EVALUATION = CN()
+_C.AUG.EVALUATION.RESIZE = CN()
+_C.AUG.EVALUATION.RESIZE.ENABLE = False
+_C.AUG.EVALUATION.RESIZE.SIZE = (-1, -1)  # h, w
+_C.AUG.EVALUATION.RESIZE.KEEP_RATIO = True
+_C.AUG.EVALUATION.RESIZE.INTERPOLATION = "bilinear"
+_C.AUG.EVALUATION.CENTERCROP = CN()
+_C.AUG.EVALUATION.CENTERCROP.ENABLE = False
+_C.AUG.EVALUATION.NORMLIZATION = CN()
+_C.AUG.EVALUATION.NORMLIZATION.MEAN = []
+_C.AUG.EVALUATION.NORMLIZATION.STD = []
+
+_C.QUANT = CN()
+_C.QUANT.TYPE = ""  # support 'qat' / ptq
+_C.QUANT.BIT_ASSIGNER = CN()
+_C.QUANT.BIT_ASSIGNER.NAME = None  # support 'HAWQ'
+_C.QUANT.BIT_ASSIGNER.W_BIT_CHOICES = [2, 4, 8]
+_C.QUANT.BIT_ASSIGNER.A_BIT_CHOICES = [2, 4, 8, 16]
+
+_C.QUANT.BIT_ASSIGNER.HAWQ = CN()
+_C.QUANT.BIT_ASSIGNER.HAWQ.EIGEN_TYPE = "avg"  # support 'max' / 'avg'
+_C.QUANT.BIT_ASSIGNER.HAWQ.SENSITIVITY_CALC_ITER_NUM = 50
+_C.QUANT.BIT_ASSIGNER.HAWQ.LIMITATION = CN()
+_C.QUANT.BIT_ASSIGNER.HAWQ.LIMITATION.BIT_ASCEND_SORT = False
+_C.QUANT.BIT_ASSIGNER.HAWQ.LIMITATION.BIT_WIDTH_COEFF = 1e10
+_C.QUANT.BIT_ASSIGNER.HAWQ.LIMITATION.BOPS_COEFF = 1e10
+_C.QUANT.BIT_CONFIG = (
+    []
+)  # a mapping, key is layer_name, value is {"w":w_bit, "a":a_bit}
+_C.QUANT.FOLD_BN = False
+_C.QUANT.W = CN()
+_C.QUANT.W.QUANTIZER = None  # support "LSQ" / "DOREFA"
+_C.QUANT.W.BIT = 8
+_C.QUANT.W.BIT_RANGE = [2,9] # left include, right exclude, default 2~8
+_C.QUANT.W.SYMMETRY = True
+_C.QUANT.W.GRANULARITY = (
+    "channelwise"  # support "layerwise"/"channelwise" currently, default is channelwise
+)
+_C.QUANT.W.OBSERVER_METHOD = CN()
+_C.QUANT.W.OBSERVER_METHOD.NAME = (
+    "MINMAX"  # support "MINMAX"/"MSE" currently, default is MINMAX
+)
+_C.QUANT.W.OBSERVER_METHOD.ALPHA = 0.0001  # support percentile
+_C.QUANT.W.OBSERVER_METHOD.BINS = 2049  # support kl_histogram
+_C.QUANT.A = CN()
+_C.QUANT.A.BIT = 8
+_C.QUANT.A.BIT_RANGE = [4,9] # left include, right exclude, default 4~8
+_C.QUANT.A.QUANTIZER = None  # support "LSQ" / "DOREFA"
+_C.QUANT.A.SYMMETRY = False
+_C.QUANT.A.GRANULARITY = (
+    "layerwise"  # support "layerwise"/"channelwise" currently, default is layerwise
+)
+_C.QUANT.A.OBSERVER_METHOD = CN()
+_C.QUANT.A.OBSERVER_METHOD.NAME = (
+    "MINMAX"  # support "MINMAX"/"MSE" currently, default is MINMAX
+)
+_C.QUANT.A.OBSERVER_METHOD.ALPHA = 0.0001  # support percentile
+_C.QUANT.A.OBSERVER_METHOD.BINS = 2049  # support kl_histogram
+_C.QUANT.CALIBRATION = CN()
+_C.QUANT.CALIBRATION.PATH = ""
+_C.QUANT.CALIBRATION.TYPE = ""  # support tarfile / python_module
+_C.QUANT.CALIBRATION.MODULE_PATH = ""  # the import path of calibration dataset
+_C.QUANT.CALIBRATION.SIZE = 0
+_C.QUANT.CALIBRATION.BATCHSIZE = 1
+_C.QUANT.CALIBRATION.NUM_WORKERS = 0
+
+_C.QUANT.FINETUNE = CN()
+_C.QUANT.FINETUNE.ENABLE = False
+_C.QUANT.FINETUNE.METHOD = ""
+_C.QUANT.FINETUNE.BATCHSIZE = 32
+_C.QUANT.FINETUNE.ITERS_W = 0
+_C.QUANT.FINETUNE.ITERS_A = 0
+
+_C.QUANT.FINETUNE.BRECQ = CN()
+_C.QUANT.FINETUNE.BRECQ.KEEP_GPU = True
+
+
+_C.SSL = CN()
+_C.SSL.TYPE = None # support mocov2
+_C.SSL.SETTING = CN()
+_C.SSL.SETTING.DIM = 128 # output dimension for the MLP head
+_C.SSL.SETTING.HIDDEN_DIM = 2048 # hidden dimension for the MLP head
+
+_C.SSL.SETTING.T = 0.07 # temperature for InfoNCE loss
+_C.SSL.SETTING.MOCO_K = 65536 # size of memory bank for MoCo
+_C.SSL.SETTING.MOMENTUM = 0.999 # MoCo momentum of updating key encoder
+_C.SSL.SETTING.MLP = True # whether to use MLP head, default True
+
+
+def get_config(args):
+    """Get a yacs CfgNode object with default values."""
+    # Return a clone so that the defaults will not be altered
+    # This is for the "local variable" use pattern
+    config = _C.clone()
+
+    config.defrost()
+    if hasattr(args, "checkpoint"):
+        config.MODEL.CHECKPOINT = args.checkpoint
+    if hasattr(args, "pretrained"):
+        config.MODEL.PRETRAINED = args.pretrained
+    if hasattr(args, "calibration"):
+        config.QUANT.CALIBRATION.PATH = args.calibration
+    if hasattr(args, "batch_size"):
+        config.QUANT.CALIBRATION.BATCHSIZE = args.batch_size
+    if hasattr(args, "num_workers"):
+        config.QUANT.CALIBRATION.NUM_WORKERS = args.num_workers
+        config.TRAIN.NUM_WORKERS = args.num_workers
+    if hasattr(args, "eval_domain"):
+        config.EVALUATION_DOMAIN = args.eval_domain
+    if hasattr(args, "print_freq"):
+        config.TRAIN.PRINT_FREQ = args.print_freq
+    if hasattr(args, "output"):
+        config.OUTPUT = args.output
+
+    config.DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+    # get config depend chain files recursively
+    config_depends = []
+
+    tmp_config = _C.clone()
+    tmp_config.defrost()
+    tmp_config.merge_from_file(args.config)
+    config_depends.append(args.config)
+    while tmp_config.BASE_CONFIG:
+        next_config = tmp_config.BASE_CONFIG
+        config_depends.append(next_config)
+        tmp_config.BASE_CONFIG = ""
+        tmp_config.merge_from_file(next_config)
+    # tmp_config's merge order is reversed so can't use it directly
+
+    for conf_path in reversed(config_depends):
+        config.merge_from_file(conf_path)
+
+    config.freeze()
+
+    return config
+
+
+def update_config(config, key, value):
+    config.defrost()
+    keys = key.split(".")
+
+    def _set_config_attr(cfg, keys, value):
+        if len(keys) > 1:
+            cfg = getattr(cfg, keys[0].upper())
+            _set_config_attr(cfg, keys[1:], value)
+        else:
+            setattr(cfg, keys[0].upper(), value)
+
+    _set_config_attr(config, keys, value)
+    config.freeze()
+    return config

configs/aircraft/resnet18_finetune_aircraft.yaml

@@ -0,0 +1,106 @@
+SEED: 100
+MODEL:
+    #ARCH: cifar_resnet18
+    ARCH: resnet18
+    #INPUTSHAPE: [32, 32]
+    INPUTSHAPE: [224, 224] # imagenet transfer
+    #PRETRAINED:  checkpoints/resnet18.pth
+    #PRETRAINED: train_log/imagenet/simsiam_imagenet_r18_imagenet_baseline/checkpoint.pth.tar
+    #PRETRAINED: train_log/imagenet/RQbyol_randombit_4_16_add_float_r18_imagenet_baseline/checkpoint.pth.tar
+    #PRETRAINED: train_log/imagenet/RQbyol_randombit_w_2_8_f_4_8+floatloss_add_float_r18_imagenet_baseline/checkpoint.pth.tar
+    #PRETRAINED: /data/train_log_NEW/resnet18_linear_eval_aircraft+floatloss/checkpoint.pth.tar
+    #PRETRAINED: /data/train_log_NEW/resnet18_linear_eval_aircraft_simsiam/checkpoint.pth.tar
+    #CHECKPOINT: /data/train_log_NEW/resnet18_finetune_aircraft/checkpoint.pth.tar 
+    CHECKPOINT: /data/train_log_NEW/resnet18_finetune_aircraft_simsiam/checkpoint.pth.tar 
+    #CHECKPOINT: train_log_NEW/resnet18_finetune_baseline_wbound/checkpoint.pth.tar 
+    #CHECKPOINT: train_log_NEW/resnet18_finetune_baseline2/checkpoint.pth.tar 
+    NUM_CLASSES: 100
+TRAIN:
+    EPOCHS: 50
+    USE_DDP: True
+    LINEAR_EVAL: False
+    WARMUP_FC: True
+    DATASET: aircraft
+    BATCH_SIZE: 64 # per-gpu
+    OPTIMIZER: 
+        NAME: sgd
+        MOMENTUM: 0.9
+        WEIGHT_DECAY: 0.0001 # 1e-5
+    LR_SCHEDULER:
+        WARMUP_EPOCHS: 0
+        WARMUP_LR: 0.0002 # 1e-4
+        BASE_LR: 0.001 # 1e-2
+        #BASE_LR: 0.1 
+        MIN_LR: 0.
+        #TYPE: cosine
+        TYPE: multiStep
+        DECAY_RATE: 0.1
+        DECAY_MILESTONES : [30, 40]
+    LOSS: 
+        CRITERION:
+            NAME: CrossEntropy
+        #REGULARIZER:
+        #    NAME: PACT
+        LAMBDA: 0.0001
+    METER:
+        NAME: ACC
+        ACC:
+            TOPK: [1, 5]
+    RUNNER:
+        NAME: default
+AUG:
+    TRAIN:
+        HORIZONTAL_FLIP:
+            PROB: 0.5
+        RANDOMRESIZEDCROP:
+            ENABLE: True
+            SCALE: (0.08, 1.0)
+            INTERPOLATION: bilinear
+        NORMLIZATION:
+            MEAN: [0.485, 0.456, 0.406]
+            STD: [0.229, 0.224, 0.225]
+    EVALUATION:
+        RESIZE: 
+            ENABLE: True
+            SIZE: [256, 256]
+        CENTERCROP: 
+            ENABLE: True
+        NORMLIZATION:
+            MEAN: [0.485, 0.456, 0.406]
+            STD: [0.229, 0.224, 0.225]
+QUANT:
+    TYPE: ptq
+    CALIBRATION:
+        TYPE: tar
+        PATH: calibrations/cifar10_train_1000.tar
+        SIZE: 1000
+        BATCHSIZE: 128
+    W:
+        BIT: 2
+        SYMMETRY: True
+        QUANTIZER: uniform
+        GRANULARITY : channelwise
+        OBSERVER_METHOD:
+            NAME: MINMAX
+    A:
+        BIT: 4
+        SYMMETRY: False
+        QUANTIZER: uniform
+        GRANULARITY : layerwise
+        OBSERVER_METHOD:
+            NAME: MINMAX
+    BIT_CONFIG: [{
+        "conv1": {"w": 8, "a": 8},
+        "layer1.0.conv2": {"a": 0},
+        "layer1.1.conv2": {"a": 0},
+        "layer2.0.conv2": {"a": 0},
+        "layer2.0.downsample": {"a": 0},
+        "layer2.1.conv2": {"a": 0},
+        "layer3.0.downsample": {"a": 0},
+        "layer3.0.conv2": {"a": 0},
+        "layer3.1.conv2": {"a": 0},
+        "layer4.0.downsample": {"a": 0},
+        "layer4.0.conv2": {"a": 0},
+        "layer4.1.conv2": {"a": 0},
+        "fc": {"a": 0},
+    }]