From 667e804d04ec88e5a2d289bdb540273bb65df726 Mon Sep 17 00:00:00 2001 From: Hanwen Xiong Date: Tue, 2 May 2023 18:13:16 -0700 Subject: [PATCH] Added PyTorch SegNet and Tensorflow MobileNetV2, ResNet50 models; pylint cleanup; Model refactoring and code updates - Further cleaned up code to minimize errors from static analysis (pylint) - CI/CD Jenkins pipeline refactored to incorporate proper code quality checks - Model refactoring, code updates and added artifacts for SalsaNext W4A8 - Releasing fourth package release of aimet model zoo (installable wheel file binaries) - Added code, documentation and artifacts for the following new TensorFlow models: MobileNetV2, ResNet50 - Added code, documentation and artifacts for the following new PyTorch models: SegNet Signed-off-by: Hanwen Xiong --- .pylintrc | 10 +- Jenkins/Dockerfile.tf-torch-cpu | 209 +----- Jenkins/Jenkinsfile | 33 +- Jenkins/dobuildntest.sh | 24 +- Jenkins/jenkins_threshold_configs.json | 2 +- Jenkins/opencv_320_python38.patch | 4 + NightlyTests/CMakeLists.txt | 30 + NightlyTests/torch/CMakeLists.txt | 34 + NightlyTests/torch/conftest.py | 34 + NightlyTests/torch/hrnet.yaml | 158 +++++ NightlyTests/torch/pytest.ini | 4 + .../torch/test_image_classification.py | 39 + NightlyTests/torch/test_object_detection.py | 38 + NightlyTests/torch/test_pose_estimation.py | 39 + .../torch/test_semantic_segmentation.py | 42 ++ NightlyTests/torch/test_super_resolution.py | 39 + README.md | 53 +- aimet_zoo_tensorflow/common/__init__.py | 0 aimet_zoo_tensorflow/common/downloader.py | 170 +++-- .../common/object_detection/__init__.py | 0 aimet_zoo_tensorflow/common/utils/__init__.py | 0 .../evaluators/efficientnet_quanteval.py | 37 +- .../mobiledetedgetpu/dataloader/__init__.py | 0 .../mobiledetedgetpu/model/__init__.py | 0 .../model/model_cards/__init__.py | 0 .../model/model_definition.py | 22 +- .../evaluators/mobilenet_v2_140_quanteval.py | 67 +- .../mobilenet_v2_tf2/MobileNetV2_TF2.md | 61 ++ .../mobilenet_v2_tf2/__init__.py | 4 + .../mobilenet_v2_tf2/evaluators/__init__.py | 0 .../mobilenet_v2_tf2/evaluators/eval_func.py | 85 +++ .../evaluators/mobilenet_v2_tf2_quanteval.py | 59 ++ .../mobilenet_v2_tf2/evaluators/preprocess.py | 485 +++++++++++++ .../mobilenet_v2_tf2/model/__init__.py | 0 .../model/model_cards/mobilenetv2_w8a8.json | 24 + .../model/model_definition.py | 100 +++ .../mobilenet_v2_tf2/requirements.txt | 1 + .../evaluators/pose_estimation_quanteval.py | 94 +-- .../evaluators/resnet50_v1_quanteval.py | 50 +- .../resnet50_tf2/ResNet50_TF2.md | 61 ++ aimet_zoo_tensorflow/resnet50_tf2/__init__.py | 4 + .../resnet50_tf2/evaluators/__init__.py | 0 .../resnet50_tf2/evaluators/eval_func.py | 85 +++ .../resnet50_tf2/evaluators/preprocess.py | 485 +++++++++++++ .../evaluators/resnet50_tf2_quanteval.py | 59 ++ .../resnet50_tf2/model/__init__.py | 0 .../model/model_cards/resnet50_w8a8.json | 24 + .../resnet50_tf2/model/model_definition.py | 99 +++ .../resnet50_tf2/requirements.txt | 1 + .../evaluators/retinanet_quanteval.py | 48 +- .../srgan/evaluators/srgan_quanteval.py | 45 +- .../ssd_mobilenet_v2/dataloader/__init__.py | 0 .../ssd_mobilenet_v2/model/__init__.py | 0 .../model/model_cards/__init__.py | 0 .../model/model_definition.py | 24 +- aimet_zoo_torch/abpn/__init__.py | 3 +- aimet_zoo_torch/abpn/evaluators/__init__.py | 0 .../abpn/evaluators/abpn_quanteval.py | 74 +- .../abpn/model/model_cards/__init__.py | 0 .../abpn/model/model_definition.py | 95 ++- aimet_zoo_torch/bert/__init__.py | 2 +- aimet_zoo_torch/bert/dataloader/__init__.py | 2 +- .../bert/dataloader/utils/__init__.py | 0 aimet_zoo_torch/bert/evaluators/__init__.py | 0 .../bert/evaluators/bert_quanteval.py | 2 - aimet_zoo_torch/bert/model/__init__.py | 0 .../bert/model/baseline_models/__init__.py | 0 .../model/baseline_models/bert/__init__.py | 0 .../bert/model/model_cards/__init__.py | 0 .../model/model_cards/bert_w8a8_cola.json | 2 +- .../model/model_cards/bert_w8a8_mnli.json | 2 +- .../model/model_cards/bert_w8a8_mrpc.json | 2 +- .../model/model_cards/bert_w8a8_qnli.json | 2 +- .../bert/model/model_cards/bert_w8a8_rte.json | 2 +- .../model/model_cards/bert_w8a8_squad.json | 2 +- .../model/model_cards/bert_w8a8_sst2.json | 2 +- .../model/model_cards/bert_w8a8_stsb.json | 2 +- .../bert/model/model_definition.py | 3 +- aimet_zoo_torch/bert/model/utils/__init__.py | 0 aimet_zoo_torch/common/downloader.py | 188 +++-- .../common/super_resolution/blocks.py | 1 + .../common/super_resolution/imresize.py | 2 + .../common/super_resolution/inference.py | 1 + .../common/super_resolution/models.py | 1 + .../common/super_resolution/psnr.py | 1 + .../common/super_resolution/utils.py | 1 + .../common/utils/image_net_data_loader.py | 114 ++- aimet_zoo_torch/common/utils/utils.py | 15 +- aimet_zoo_torch/deeplabv3/__init__.py | 3 +- .../deeplabv3/dataloader/__init__.py | 3 +- .../deeplabv3/dataloader/ade20k_dataloader.py | 1 + .../dataloader/dataloaders_and_eval_func.py | 51 +- .../deeplabv3/dataloader/segbase.py | 1 + aimet_zoo_torch/deeplabv3/dataloader/utils.py | 1 + .../evaluators/deeplabv3_quanteval.py | 109 ++- .../deeplabv3/model/dataloaders/__init__.py | 64 +- .../model/dataloaders/custom_transforms.py | 1 + .../model/dataloaders/datasets/cityscapes.py | 1 + .../model/dataloaders/datasets/coco.py | 1 + .../model/dataloaders/datasets/combine_dbs.py | 1 + .../model/dataloaders/datasets/pascal.py | 1 + .../model/dataloaders/datasets/pascalbc.py | 1 + .../model/dataloaders/datasets/sbd.py | 1 + .../deeplabv3/model/dataloaders/utils.py | 1 + .../deeplabv3/model/model_cards/__init__.py | 0 .../deeplabv3/model/model_definition.py | 79 ++- .../deeplabv3/model/modeling/aspp.py | 69 +- .../model/modeling/backbone/__init__.py | 17 +- .../deeplabv3/model/modeling/backbone/drn.py | 411 +++++++---- .../model/modeling/backbone/mobilenet.py | 54 +- .../model/modeling/backbone/resnet.py | 130 +++- .../model/modeling/backbone/xception.py | 366 ++++++++-- .../deeplabv3/model/modeling/decoder.py | 40 +- .../deeplabv3/model/modeling/deeplab.py | 39 +- .../model/modeling/sync_batchnorm/__init__.py | 9 +- .../modeling/sync_batchnorm/batchnorm.py | 1 + .../model/modeling/sync_batchnorm/comm.py | 1 + .../modeling/sync_batchnorm/replicate.py | 1 + .../model/modeling/sync_batchnorm/unittest.py | 1 + aimet_zoo_torch/deeplabv3/model/train.py | 1 + .../deeplabv3/model/utils/__init__.py | 0 .../model/utils/calculate_weights.py | 1 + aimet_zoo_torch/deeplabv3/model/utils/loss.py | 1 + .../deeplabv3/model/utils/lr_scheduler.py | 1 + .../deeplabv3/model/utils/metrics.py | 1 + .../deeplabv3/model/utils/saver.py | 1 + .../deeplabv3/model/utils/summaries.py | 1 + .../evaluators/deepspeech2_quanteval.py | 51 +- .../distilbert/dataloader/utils/__init__.py | 0 .../distilbert/evaluators/__init__.py | 0 .../evaluators/distilbert_quanteval.py | 10 +- aimet_zoo_torch/distilbert/model/__init__.py | 0 .../model/baseline_models/__init__.py | 0 .../distilbert/model/model_cards/__init__.py | 0 .../model_cards/distilbert_w8a8_cola.json | 2 +- .../model_cards/distilbert_w8a8_mnli.json | 2 +- .../model_cards/distilbert_w8a8_mrpc.json | 2 +- .../model_cards/distilbert_w8a8_qnli.json | 2 +- .../model_cards/distilbert_w8a8_qqp.json | 2 +- .../model_cards/distilbert_w8a8_rte.json | 2 +- .../model_cards/distilbert_w8a8_squad.json | 2 +- .../model_cards/distilbert_w8a8_sst2.json | 2 +- .../model_cards/distilbert_w8a8_stsb.json | 2 +- .../distilbert/model/model_definition.py | 94 ++- .../distilbert/model/utils/__init__.py | 0 aimet_zoo_torch/efficientnetlite0/__init__.py | 3 +- .../efficientnetlite0/dataloader/__init__.py | 3 +- .../dataloader/dataloaders_and_eval_func.py | 45 +- .../evaluators/efficientnetlite0_quanteval.py | 94 ++- .../model/model_cards/__init__.py | 0 .../model/model_definition.py | 80 ++- aimet_zoo_torch/ffnet/__init__.py | 3 +- aimet_zoo_torch/ffnet/dataloader/__init__.py | 3 +- .../ffnet/dataloader/base_loader.py | 1 + .../ffnet/dataloader/cityscapes/cityscapes.py | 1 + .../cityscapes/cityscapes_labels.py | 1 + .../cityscapes/dataloader/base_loader.py | 1 + .../cityscapes/dataloader/get_dataloaders.py | 1 + .../cityscapes/dataloader/sampler.py | 1 + .../cityscapes/dataloader/transforms.py | 1 + .../dataloader/cityscapes/utils/attr_dict.py | 1 + .../ffnet/dataloader/cityscapes/utils/misc.py | 1 + .../cityscapes/utils/my_data_parallel.py | 1 + .../cityscapes/utils/progress_bar.py | 4 +- .../cityscapes/utils/trnval_utils.py | 1 + .../dataloader/dataloaders_and_eval_func.py | 24 +- .../ffnet/evaluators/ffnet_quanteval.py | 140 ++-- aimet_zoo_torch/ffnet/model/__init__.py | 1 + aimet_zoo_torch/ffnet/model/config.py | 2 + .../ffnet/model/ffnet_NS_mobile.py | 1 + .../ffnet/model/ffnet_N_gpu_large.py | 1 + .../ffnet/model/ffnet_S_gpu_large.py | 1 + .../ffnet/model/ffnet_S_gpu_small.py | 1 + aimet_zoo_torch/ffnet/model/ffnet_S_mobile.py | 1 + aimet_zoo_torch/ffnet/model/ffnet_blocks.py | 1 + .../ffnet/model/ffnet_gpu_large.py | 1 + .../ffnet/model/ffnet_gpu_small.py | 1 + .../ffnet/model/model_cards/__init__.py | 0 .../ffnet/model/model_definition.py | 75 +- aimet_zoo_torch/ffnet/model/model_registry.py | 1 + aimet_zoo_torch/ffnet/model/resnet.py | 1 + aimet_zoo_torch/ffnet/model/utils.py | 1 + aimet_zoo_torch/gpt2/__init__.py | 1 + aimet_zoo_torch/gpt2/dataloader/__init__.py | 1 + aimet_zoo_torch/gpt2/evaluators/__init__.py | 0 .../gpt2/evaluators/gpt2_quanteval.py | 15 +- aimet_zoo_torch/gpt2/model/__init__.py | 0 .../gpt2/model/model_cards/__init__.py | 0 .../gpt2/model/model_cards/gpt2_w8a8.json | 2 +- .../gpt2/model/model_definition.py | 61 +- aimet_zoo_torch/gpunet0/evaluator/__init__.py | 0 .../gpunet0/evaluator/gpunet0_quanteval.py | 88 ++- aimet_zoo_torch/gpunet0/model/__init__.py | 0 .../gpunet0/model/model_cards/__init__.py | 0 .../gpunet0/model/model_definition.py | 79 ++- aimet_zoo_torch/gpunet0/model/src/__init__.py | 0 .../gpunet0/model/src/configs/__init__.py | 0 .../src/configs/batch1/GV100/__init__.py | 0 .../model/src/configs/batch1/__init__.py | 0 .../gpunet0/model/src/evaluate_model.py | 1 + .../gpunet0/model/src/models/__init__.py | 0 .../model/src/models/gpunet_builder.py | 1 + .../model/src/models/gpunet_modules.py | 1 + .../hrnet_image_classification/__init__.py | 1 + .../dataloader/dataloaders_and_eval_func.py | 14 +- .../dataloader/list/__init__.py | 0 .../dataloader/list/cityscapes/__init__.py | 0 .../dataloader/list/lip/__init__.py | 0 .../hrnet_image_classification_quanteval.py | 64 +- .../model/experiments/__init__.py | 0 .../model/lib/__init__.py | 0 .../model/lib/config/__init__.py | 1 + .../model/lib/core/__init__.py | 0 .../model/lib/utils/__init__.py | 0 .../model/model_cards/__init__.py | 0 .../model/model_definition.py | 2 +- .../dataloader/dataloader_and_eval_func.py | 4 +- .../evaluators/hrnet_posenet_quanteval.py | 8 +- .../models/model_cards/__init__.py | 0 .../dataloader/list/__init__.py | 0 .../dataloader/list/cityscapes/__init__.py | 0 .../dataloader/list/lip/__init__.py | 0 .../evaluators/experiments/__init__.py | 0 .../experiments/cityscapes/__init__.py | 0 .../evaluators/experiments/lip/__init__.py | 0 .../experiments/pascal_ctx/__init__.py | 0 .../evaluators/hrnet_sem_seg_quanteval.py | 8 +- .../model/core/__init__.py | 0 .../model/datasets/cityscapes.py | 2 +- .../model/model_cards/__init__.py | 0 .../sync_bn/inplace_abn/src/__init__.py | 0 aimet_zoo_torch/inverseform/__init__.py | 3 +- .../inverseform/dataloader/__init__.py | 3 +- .../inverseform/dataloader/data/cityscapes.py | 3 +- .../dataloader/data/cityscapes_labels.py | 1 + .../dataloader/dataloaders_and_eval_func.py | 3 +- .../dataloader/datasets/base_loader.py | 9 +- .../dataloader/datasets/get_dataloaders.py | 1 + .../dataloader/datasets/sampler.py | 1 + .../inverseform/dataloader/helper.py | 1 + .../dataloader/transforms/joint_transforms.py | 1 + .../dataloader/transforms/transforms.py | 1 + .../evaluators/inverseform_quanteval.py | 106 ++- .../inverseform/model/model_cards/__init__.py | 0 .../inverseform/model/model_definition.py | 95 ++- .../inverseform/model/models/InverseForm.py | 1 + .../inverseform/model/models/__init__.py | 10 +- .../inverseform/model/models/attnscale.py | 2 + .../inverseform/model/models/basic.py | 2 + .../inverseform/model/models/bn_helper.py | 2 + .../inverseform/model/models/hrnetv2.py | 2 + .../inverseform/model/models/lighthrnet.py | 1 + .../inverseform/model/models/loss/utils.py | 1 + .../inverseform/model/models/model_loader.py | 2 + .../inverseform/model/models/mscale.py | 2 + .../inverseform/model/models/mscale2.py | 2 + .../inverseform/model/models/mynn.py | 2 + .../inverseform/model/models/ocr_utils.py | 2 + .../inverseform/model/models/ocrnet.py | 2 + .../inverseform/model/models/utils.py | 2 + .../inverseform/model/utils/attr_dict.py | 1 + .../model/utils/cityscapes_labels.py | 1 + .../inverseform/model/utils/config.py | 1 + .../inverseform/model/utils/misc.py | 1 + .../model/utils/my_data_parallel.py | 1 + .../inverseform/model/utils/progress_bar.py | 12 +- .../inverseform/model/utils/trnval_utils.py | 1 + aimet_zoo_torch/minilm/__init__.py | 3 +- aimet_zoo_torch/minilm/dataloader/__init__.py | 3 +- .../minilm/dataloader/utils/__init__.py | 0 aimet_zoo_torch/minilm/evaluators/__init__.py | 0 .../minilm/evaluators/minilm_quanteval.py | 50 +- aimet_zoo_torch/minilm/model/__init__.py | 0 .../minilm/model/model_cards/__init__.py | 0 .../model/model_cards/minilm_w8a8_cola.json | 2 +- .../model/model_cards/minilm_w8a8_mnli.json | 2 +- .../model/model_cards/minilm_w8a8_mrpc.json | 2 +- .../model/model_cards/minilm_w8a8_qnli.json | 2 +- .../model/model_cards/minilm_w8a8_qqp.json | 2 +- .../model/model_cards/minilm_w8a8_rte.json | 2 +- .../model/model_cards/minilm_w8a8_squad.json | 2 +- .../model/model_cards/minilm_w8a8_sst2.json | 2 +- .../model/model_cards/minilm_w8a8_stsb.json | 2 +- .../minilm/model/model_definition.py | 97 ++- .../minilm/model/utils/__init__.py | 0 aimet_zoo_torch/mobilebert/__init__.py | 3 +- .../mobilebert/dataloader/__init__.py | 3 +- .../mobilebert/dataloader/utils/__init__.py | 0 .../mobilebert/evaluators/__init__.py | 0 .../evaluators/mobilebert_quanteval.py | 48 +- aimet_zoo_torch/mobilebert/model/__init__.py | 0 .../model/baseline_models/__init__.py | 0 .../mobilebert/model/model_cards/__init__.py | 0 .../model_cards/mobilebert_w8a8_cola.json | 2 +- .../model_cards/mobilebert_w8a8_mnli.json | 2 +- .../model_cards/mobilebert_w8a8_mrpc.json | 2 +- .../model_cards/mobilebert_w8a8_qnli.json | 2 +- .../model_cards/mobilebert_w8a8_qqp.json | 2 +- .../model_cards/mobilebert_w8a8_rte.json | 2 +- .../model_cards/mobilebert_w8a8_squad.json | 2 +- .../model_cards/mobilebert_w8a8_sst2.json | 2 +- .../model_cards/mobilebert_w8a8_stsb.json | 2 +- .../mobilebert/model/model_definition.py | 94 ++- .../mobilebert/model/utils/__init__.py | 0 aimet_zoo_torch/mobilenetv2/__init__.py | 3 +- .../mobilenetv2/dataloader/__init__.py | 3 +- .../dataloader/dataloaders_and_eval_func.py | 68 +- .../evaluators/mobilenetv2_quanteval.py | 72 +- .../mobilenetv2/model/MobileNetV2.py | 1 + .../mobilenetv2/model/model_cards/__init__.py | 0 .../mobilenetv2/model/model_definition.py | 101 ++- aimet_zoo_torch/mobilevit/__init__.py | 1 + .../mobilevit/dataloader/__init__.py | 1 + .../mobilevit/dataloader/utils/__init__.py | 0 .../mobilevit/evaluators/__init__.py | 0 .../evaluators/mobilevit_quanteval.py | 8 +- aimet_zoo_torch/mobilevit/model/__init__.py | 0 .../mobilevit/model/huggingface/__init__.py | 0 .../huggingface/baseline_models/__init__.py | 0 .../baseline_models/mobilevit/__init__.py | 0 .../mobilevit/model/model_cards/__init__.py | 0 .../model/model_cards/mobilevit_w8a8.json | 2 +- .../mobilevit/model/model_definition.py | 20 +- .../evaluators/pose_estimation_quanteval.py | 92 +-- aimet_zoo_torch/quicksrnet/__init__.py | 3 +- .../quicksrnet/dataloader/imresize.py | 1 + .../quicksrnet/dataloader/utils.py | 45 +- .../evaluators/quicksrnet_quanteval.py | 74 +- aimet_zoo_torch/quicksrnet/model/blocks.py | 1 + .../quicksrnet/model/downloader.py | 1 + aimet_zoo_torch/quicksrnet/model/helpers.py | 1 + aimet_zoo_torch/quicksrnet/model/imresize.py | 1 + aimet_zoo_torch/quicksrnet/model/inference.py | 1 + .../quicksrnet/model/model_cards/__init__.py | 0 .../quicksrnet/model/model_definition.py | 107 ++- aimet_zoo_torch/quicksrnet/model/models.py | 1 + .../rangenet/evaluators/__init__.py | 1 + .../rangenet/evaluators/rangenet_quanteval.py | 11 +- .../rangenet/models/model_cards/__init__.py | 0 .../rangenet/models/model_definition.py | 3 +- .../train/tasks/semantic/config/__init__.py | 0 .../tasks/semantic/config/arch/__init__.py | 0 .../tasks/semantic/config/labels/__init__.py | 0 .../train/tasks/semantic/dataset/__init__.py | 0 aimet_zoo_torch/regnet/__init__.py | 3 +- .../dataloader/dataloaders_and_eval_func.py | 15 +- .../regnet/evaluator/regnet_quanteval.py | 56 +- .../regnet/model/model_cards/__init__.py | 0 .../regnet/model/model_definition.py | 68 +- aimet_zoo_torch/resnet/__init__.py | 3 +- .../dataloader/dataloaders_and_eval_func.py | 11 +- .../resnet/evaluator/resnet_quanteval.py | 19 +- .../resnet/model/model_cards/__init__.py | 0 .../resnet/model/model_definition.py | 81 ++- aimet_zoo_torch/resnext/__init__.py | 3 +- .../dataloader/dataloaders_and_eval_func.py | 24 +- .../resnext/evaluator/resnext_quanteval.py | 60 +- .../resnext/model/model_cards/__init__.py | 0 .../model/model_cards/resnext101_w8a8.json | 4 +- .../resnext/model/model_definition.py | 81 ++- aimet_zoo_torch/roberta/__init__.py | 1 + .../roberta/dataloader/__init__.py | 3 +- .../roberta/dataloader/utils/__init__.py | 0 .../roberta/evaluators/__init__.py | 0 .../roberta/evaluators/roberta_quanteval.py | 50 +- .../roberta/model/baseline_models/__init__.py | 0 .../roberta/model/model_cards/__init__.py | 0 .../model/model_cards/roberta_w8a8_cola.json | 2 +- .../model/model_cards/roberta_w8a8_mnli.json | 2 +- .../model/model_cards/roberta_w8a8_mrpc.json | 2 +- .../model/model_cards/roberta_w8a8_qnli.json | 2 +- .../model/model_cards/roberta_w8a8_qqp.json | 2 +- .../model/model_cards/roberta_w8a8_rte.json | 2 +- .../model/model_cards/roberta_w8a8_sst2.json | 2 +- .../model/model_cards/roberta_w8a8_stsb.json | 2 +- .../roberta/model/model_definition.py | 95 ++- .../roberta/model/utils/__init__.py | 0 .../salsaNext/evaluators/evaluation_func.py | 309 -------- .../evaluators/salsaNext_quanteval.py | 636 ----------------- aimet_zoo_torch/salsaNext/salsaNext.md | 171 ----- aimet_zoo_torch/salsanext/SalsaNext.md | 206 ++++++ aimet_zoo_torch/salsanext/__init__.py | 0 .../dataloader/sematickitti_parser.py | 452 ++++++++++++ .../salsanext/evaluators/__init__.py | 0 .../salsanext/evaluators/evaluation_func.py | 383 ++++++++++ .../evaluators/salsanext_quanteval.py | 596 ++++++++++++++++ aimet_zoo_torch/salsanext/models/SalsaNext.py | 248 +++++++ aimet_zoo_torch/salsanext/models/__init__.py | 0 .../salsanext/models/arch_cfg.yaml | 66 ++ .../salsanext/models/common/__init__.py | 0 .../salsanext/models/common/avgmeter.py | 44 ++ .../salsanext/models/common/laserscan.py | 343 +++++++++ .../salsanext/models/common/laserscanvis.py | 248 +++++++ .../salsanext/models/common/logger.py | 79 +++ .../salsanext/models/common/summary.py | 116 +++ .../models/common/sync_batchnorm/__init__.py | 0 .../models/common/sync_batchnorm/batchnorm.py | 369 ++++++++++ .../models/common/sync_batchnorm/comm.py | 139 ++++ .../models/common/sync_batchnorm/replicate.py | 96 +++ .../salsanext/models/common/visualization.py | 138 ++++ .../salsanext/models/common/warmupLR.py | 76 ++ .../salsanext/models/data_cfg.yaml | 212 ++++++ .../models/model_cards/salsanext_w4a8.json | 24 + .../models/model_cards/salsanext_w8a8.json | 24 + .../salsanext/models/model_definition.py | 119 ++++ .../salsanext/models/tasks/__init__.py | 1 + .../models/tasks/semantic/__init__.py | 6 + .../config/labels/semantic-kitti-all.yaml | 224 ++++++ .../config/labels/semantic-kitti.yaml | 212 ++++++ .../tasks/semantic/dataset/kitti/__init__.py | 0 .../tasks/semantic/dataset/kitti/parser.py | 462 ++++++++++++ .../models/tasks/semantic/evaluate_iou.py | 267 +++++++ .../salsanext/models/tasks/semantic/infer.py | 170 +++++ .../tasks/semantic/modules/Lovasz_Softmax.py | 148 ++++ .../tasks/semantic/modules}/SalsaNext.py | 38 +- .../tasks/semantic/modules/SalsaNextAdf.py | 271 +++++++ .../models/tasks/semantic/modules/__init__.py | 0 .../models/tasks/semantic/modules/adf.py | 453 ++++++++++++ .../models/tasks/semantic/modules/ioueval.py | 106 +++ .../models/tasks/semantic/modules/trainer.py | 670 ++++++++++++++++++ .../models/tasks/semantic/postproc/KNN.py | 168 +++++ .../tasks/semantic/postproc/__init__.py | 6 + .../salsanext/models/tasks/semantic/train.py | 203 ++++++ .../models/tasks/semantic/visualize.py | 182 +++++ aimet_zoo_torch/segnet/SegNet.md | 57 ++ aimet_zoo_torch/segnet/__init__.py | 2 + aimet_zoo_torch/segnet/dataloader/__init__.py | 0 .../dataloader/dataloaders_and_eval_func.py | 58 ++ aimet_zoo_torch/segnet/evaluator/__init__.py | 0 .../segnet/evaluator/segnet_quanteval.py | 83 +++ aimet_zoo_torch/segnet/model/__init__.py | 0 .../segnet/model/datasets/camvid.py | 45 ++ .../segnet/model/model_cards/__init__.py | 0 .../segnet/model/model_cards/segnet_w4a8.json | 25 + .../segnet/model/model_cards/segnet_w8a8.json | 25 + .../segnet/model/model_definition.py | 107 +++ aimet_zoo_torch/segnet/model/models/segnet.py | 177 +++++ aimet_zoo_torch/segnet/requirements.txt | 2 + aimet_zoo_torch/sesr/__init__.py | 3 +- aimet_zoo_torch/sesr/evaluators/__init__.py | 0 .../sesr/evaluators/sesr_quanteval.py | 74 +- .../sesr/model/model_cards/__init__.py | 0 .../sesr/model/model_definition.py | 106 ++- .../srgan/evaluators/srgan_quanteval.py | 27 +- aimet_zoo_torch/ssd_mobilenetv2/__init__.py | 6 +- .../dataloader/datasets/collation.py | 1 + .../dataloader/datasets/generate_vocdata.py | 1 + .../dataloader/datasets/open_images.py | 1 + .../dataloader/datasets/voc_dataset.py | 1 + .../evaluators/ssd_mobilenetv2_quanteval.py | 570 ++++++++------- .../model/model_cards/__init__.py | 0 .../ssd_mobilenetv2/model/model_definition.py | 1 + .../model/vision/nn/alexnet.py | 1 + .../model/vision/nn/mobilenet.py | 1 + .../model/vision/nn/mobilenet_v2.py | 1 + .../model/vision/nn/mobilenetv3.py | 1 + .../model/vision/nn/multibox_loss.py | 1 + .../model/vision/nn/scaled_l2_norm.py | 1 + .../model/vision/nn/squeezenet.py | 1 + .../ssd_mobilenetv2/model/vision/nn/vgg.py | 1 + .../model/vision/prunning/prunner.py | 1 + .../ssd/config/mobilenetv1_ssd_config.py | 1 + .../ssd/config/squeezenet_ssd_config.py | 1 + .../model/vision/ssd/config/vgg_ssd_config.py | 1 + .../model/vision/ssd/data_preprocessing.py | 1 + .../model/vision/ssd/fpn_mobilenetv1_ssd.py | 1 + .../model/vision/ssd/fpn_ssd.py | 1 + .../model/vision/ssd/mobilenet_v2_ssd_lite.py | 1 + .../model/vision/ssd/mobilenetv1_ssd.py | 1 + .../model/vision/ssd/mobilenetv1_ssd_lite.py | 1 + .../model/vision/ssd/mobilenetv3_ssd_lite.py | 1 + .../model/vision/ssd/predictor.py | 1 + .../model/vision/ssd/squeezenet_ssd_lite.py | 1 + .../ssd_mobilenetv2/model/vision/ssd/ssd.py | 1 + .../model/vision/ssd/vgg_ssd.py | 1 + .../model/vision/transforms/transforms.py | 1 + .../model/vision/utils/__init__.py | 1 + .../model/vision/utils/box_utils.py | 1 + .../model/vision/utils/box_utils_numpy.py | 1 + .../model/vision/utils/measurements.py | 1 + .../model/vision/utils/misc.py | 1 + .../model/vision/utils/model_book.py | 1 + aimet_zoo_torch/ssd_res50/__init__.py | 0 .../ssd_res50/dataloader/__init__.py | 0 .../ssd_res50/evaluators/__init__.py | 0 .../evaluators/ssd_res50_quanteval.py | 109 ++- aimet_zoo_torch/ssd_res50/model/__init__.py | 0 .../ssd_res50/model/model_cards/__init__.py | 0 .../model/model_cards/ssd_res50_w8a8.json | 2 +- aimet_zoo_torch/vit/__init__.py | 1 + aimet_zoo_torch/vit/dataloader/__init__.py | 1 + .../vit/dataloader/utils/__init__.py | 0 aimet_zoo_torch/vit/evaluators/__init__.py | 0 aimet_zoo_torch/vit/model/__init__.py | 0 .../vit/model/huggingface/__init__.py | 0 .../huggingface/baseline_models/__init__.py | 0 .../baseline_models/vit/__init__.py | 0 .../vit/model/model_cards/__init__.py | 0 .../vit/model/model_cards/vit_w8a8.json | 2 +- aimet_zoo_torch/vit/model/model_definition.py | 27 +- aimet_zoo_torch/xlsr/__init__.py | 3 +- aimet_zoo_torch/xlsr/evaluators/__init__.py | 0 .../xlsr/evaluators/xlsr_quanteval.py | 74 +- .../xlsr/model/model_cards/__init__.py | 0 .../xlsr/model/model_definition.py | 85 ++- aimet_zoo_torch/yolox/__init__.py | 4 +- aimet_zoo_torch/yolox/dataloader/__init__.py | 2 +- .../yolox/dataloader/data/__init__.py | 4 +- .../yolox/dataloader/data/data_augment.py | 1 + .../yolox/dataloader/data/dataloading.py | 1 + .../dataloader/data/datasets/__init__.py | 2 +- .../yolox/dataloader/data/datasets/coco.py | 1 + .../data/datasets/datasets_wrapper.py | 1 + .../yolox/dataloader/data/samplers.py | 1 + .../yolox/dataloader/dataloaders.py | 5 +- aimet_zoo_torch/yolox/evaluators/__init__.py | 0 .../yolox/evaluators/coco_evaluator.py | 1 + .../yolox/evaluators/yolox_quanteval.py | 95 ++- aimet_zoo_torch/yolox/model/__init__.py | 2 +- .../yolox/model/model_cards/__init__.py | 0 .../yolox/model/model_definition.py | 75 +- .../yolox/model/yolo_x/__init__.py | 0 .../yolox/model/yolo_x/models/__init__.py | 4 +- .../yolox/model/yolo_x/models/build.py | 1 + .../yolox/model/yolo_x/models/darknet.py | 1 + .../model/yolo_x/models/network_blocks.py | 1 + .../yolox/model/yolo_x/models/yolo_head.py | 1 + .../yolox/model/yolo_x/models/yolo_pafpn.py | 1 + .../yolox/model/yolo_x/models/yolox.py | 1 + .../yolox/model/yolo_x/utils/__init__.py | 4 +- .../yolox/model/yolo_x/utils/boxes.py | 1 + aimet_zoo_torch/yolox/model/yolox_model.py | 1 + packaging/aimet_zoo_tensorflow_pyproject.toml | 2 +- packaging/aimet_zoo_torch_pyproject.toml | 2 +- packaging/pylint_model_zoo.cmake | 41 +- packaging/version.txt | 2 +- 536 files changed, 14662 insertions(+), 3816 deletions(-) create mode 100644 Jenkins/opencv_320_python38.patch create mode 100644 NightlyTests/CMakeLists.txt create mode 100644 NightlyTests/torch/CMakeLists.txt create mode 100644 NightlyTests/torch/conftest.py create mode 100644 NightlyTests/torch/hrnet.yaml create mode 100644 NightlyTests/torch/pytest.ini create mode 100644 NightlyTests/torch/test_image_classification.py create mode 100644 NightlyTests/torch/test_object_detection.py create mode 100644 NightlyTests/torch/test_pose_estimation.py create mode 100644 NightlyTests/torch/test_semantic_segmentation.py create mode 100644 NightlyTests/torch/test_super_resolution.py create mode 100644 aimet_zoo_tensorflow/common/__init__.py create mode 100644 aimet_zoo_tensorflow/common/object_detection/__init__.py create mode 100644 aimet_zoo_tensorflow/common/utils/__init__.py create mode 100644 aimet_zoo_tensorflow/mobiledetedgetpu/dataloader/__init__.py create mode 100644 aimet_zoo_tensorflow/mobiledetedgetpu/model/__init__.py create mode 100644 aimet_zoo_tensorflow/mobiledetedgetpu/model/model_cards/__init__.py create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/MobileNetV2_TF2.md create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/__init__.py create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/__init__.py create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/eval_func.py create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/preprocess.py create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/model/__init__.py create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/model/model_cards/mobilenetv2_w8a8.json create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/model/model_definition.py create mode 100644 aimet_zoo_tensorflow/mobilenet_v2_tf2/requirements.txt create mode 100755 aimet_zoo_tensorflow/resnet50_tf2/ResNet50_TF2.md create mode 100755 aimet_zoo_tensorflow/resnet50_tf2/__init__.py create mode 100755 aimet_zoo_tensorflow/resnet50_tf2/evaluators/__init__.py create mode 100755 aimet_zoo_tensorflow/resnet50_tf2/evaluators/eval_func.py create mode 100755 aimet_zoo_tensorflow/resnet50_tf2/evaluators/preprocess.py create mode 100755 aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py create mode 100755 aimet_zoo_tensorflow/resnet50_tf2/model/__init__.py create mode 100755 aimet_zoo_tensorflow/resnet50_tf2/model/model_cards/resnet50_w8a8.json create mode 100644 aimet_zoo_tensorflow/resnet50_tf2/model/model_definition.py create mode 100644 aimet_zoo_tensorflow/resnet50_tf2/requirements.txt create mode 100644 aimet_zoo_tensorflow/ssd_mobilenet_v2/dataloader/__init__.py create mode 100644 aimet_zoo_tensorflow/ssd_mobilenet_v2/model/__init__.py create mode 100644 aimet_zoo_tensorflow/ssd_mobilenet_v2/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/abpn/evaluators/__init__.py create mode 100644 aimet_zoo_torch/abpn/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/bert/dataloader/utils/__init__.py create mode 100644 aimet_zoo_torch/bert/evaluators/__init__.py create mode 100644 aimet_zoo_torch/bert/model/__init__.py create mode 100644 aimet_zoo_torch/bert/model/baseline_models/__init__.py create mode 100644 aimet_zoo_torch/bert/model/baseline_models/bert/__init__.py create mode 100644 aimet_zoo_torch/bert/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/bert/model/utils/__init__.py create mode 100644 aimet_zoo_torch/deeplabv3/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/deeplabv3/model/utils/__init__.py create mode 100644 aimet_zoo_torch/distilbert/dataloader/utils/__init__.py create mode 100644 aimet_zoo_torch/distilbert/evaluators/__init__.py create mode 100644 aimet_zoo_torch/distilbert/model/__init__.py create mode 100644 aimet_zoo_torch/distilbert/model/baseline_models/__init__.py create mode 100644 aimet_zoo_torch/distilbert/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/distilbert/model/utils/__init__.py create mode 100644 aimet_zoo_torch/efficientnetlite0/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/ffnet/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/gpt2/evaluators/__init__.py create mode 100644 aimet_zoo_torch/gpt2/model/__init__.py create mode 100644 aimet_zoo_torch/gpt2/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/gpunet0/evaluator/__init__.py create mode 100644 aimet_zoo_torch/gpunet0/model/__init__.py create mode 100644 aimet_zoo_torch/gpunet0/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/gpunet0/model/src/__init__.py create mode 100644 aimet_zoo_torch/gpunet0/model/src/configs/__init__.py create mode 100644 aimet_zoo_torch/gpunet0/model/src/configs/batch1/GV100/__init__.py create mode 100644 aimet_zoo_torch/gpunet0/model/src/configs/batch1/__init__.py create mode 100644 aimet_zoo_torch/gpunet0/model/src/models/__init__.py create mode 100644 aimet_zoo_torch/hrnet_image_classification/dataloader/list/__init__.py create mode 100644 aimet_zoo_torch/hrnet_image_classification/dataloader/list/cityscapes/__init__.py create mode 100644 aimet_zoo_torch/hrnet_image_classification/dataloader/list/lip/__init__.py create mode 100644 aimet_zoo_torch/hrnet_image_classification/model/experiments/__init__.py create mode 100644 aimet_zoo_torch/hrnet_image_classification/model/lib/__init__.py create mode 100644 aimet_zoo_torch/hrnet_image_classification/model/lib/core/__init__.py create mode 100644 aimet_zoo_torch/hrnet_image_classification/model/lib/utils/__init__.py create mode 100644 aimet_zoo_torch/hrnet_image_classification/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/hrnet_posenet/models/model_cards/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/cityscapes/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/lip/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/cityscapes/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/lip/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/pascal_ctx/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/model/core/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/hrnet_semantic_segmentation/model/models/sync_bn/inplace_abn/src/__init__.py create mode 100644 aimet_zoo_torch/inverseform/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/minilm/dataloader/utils/__init__.py create mode 100644 aimet_zoo_torch/minilm/evaluators/__init__.py create mode 100644 aimet_zoo_torch/minilm/model/__init__.py create mode 100644 aimet_zoo_torch/minilm/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/minilm/model/utils/__init__.py create mode 100644 aimet_zoo_torch/mobilebert/dataloader/utils/__init__.py create mode 100644 aimet_zoo_torch/mobilebert/evaluators/__init__.py create mode 100644 aimet_zoo_torch/mobilebert/model/__init__.py create mode 100644 aimet_zoo_torch/mobilebert/model/baseline_models/__init__.py create mode 100644 aimet_zoo_torch/mobilebert/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/mobilebert/model/utils/__init__.py create mode 100644 aimet_zoo_torch/mobilenetv2/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/mobilevit/dataloader/utils/__init__.py create mode 100644 aimet_zoo_torch/mobilevit/evaluators/__init__.py create mode 100644 aimet_zoo_torch/mobilevit/model/__init__.py create mode 100644 aimet_zoo_torch/mobilevit/model/huggingface/__init__.py create mode 100644 aimet_zoo_torch/mobilevit/model/huggingface/baseline_models/__init__.py create mode 100644 aimet_zoo_torch/mobilevit/model/huggingface/baseline_models/mobilevit/__init__.py create mode 100644 aimet_zoo_torch/mobilevit/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/quicksrnet/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/rangenet/models/model_cards/__init__.py create mode 100644 aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/__init__.py create mode 100644 aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/arch/__init__.py create mode 100644 aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/labels/__init__.py create mode 100644 aimet_zoo_torch/rangenet/models/train/tasks/semantic/dataset/__init__.py create mode 100644 aimet_zoo_torch/regnet/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/resnet/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/resnext/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/roberta/dataloader/utils/__init__.py create mode 100644 aimet_zoo_torch/roberta/evaluators/__init__.py create mode 100644 aimet_zoo_torch/roberta/model/baseline_models/__init__.py create mode 100644 aimet_zoo_torch/roberta/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/roberta/model/utils/__init__.py delete mode 100755 aimet_zoo_torch/salsaNext/evaluators/evaluation_func.py delete mode 100755 aimet_zoo_torch/salsaNext/evaluators/salsaNext_quanteval.py delete mode 100644 aimet_zoo_torch/salsaNext/salsaNext.md create mode 100644 aimet_zoo_torch/salsanext/SalsaNext.md create mode 100644 aimet_zoo_torch/salsanext/__init__.py create mode 100644 aimet_zoo_torch/salsanext/dataloader/sematickitti_parser.py create mode 100644 aimet_zoo_torch/salsanext/evaluators/__init__.py create mode 100755 aimet_zoo_torch/salsanext/evaluators/evaluation_func.py create mode 100755 aimet_zoo_torch/salsanext/evaluators/salsanext_quanteval.py create mode 100644 aimet_zoo_torch/salsanext/models/SalsaNext.py create mode 100644 aimet_zoo_torch/salsanext/models/__init__.py create mode 100644 aimet_zoo_torch/salsanext/models/arch_cfg.yaml create mode 100644 aimet_zoo_torch/salsanext/models/common/__init__.py create mode 100644 aimet_zoo_torch/salsanext/models/common/avgmeter.py create mode 100644 aimet_zoo_torch/salsanext/models/common/laserscan.py create mode 100644 aimet_zoo_torch/salsanext/models/common/laserscanvis.py create mode 100644 aimet_zoo_torch/salsanext/models/common/logger.py create mode 100644 aimet_zoo_torch/salsanext/models/common/summary.py create mode 100644 aimet_zoo_torch/salsanext/models/common/sync_batchnorm/__init__.py create mode 100644 aimet_zoo_torch/salsanext/models/common/sync_batchnorm/batchnorm.py create mode 100644 aimet_zoo_torch/salsanext/models/common/sync_batchnorm/comm.py create mode 100644 aimet_zoo_torch/salsanext/models/common/sync_batchnorm/replicate.py create mode 100644 aimet_zoo_torch/salsanext/models/common/visualization.py create mode 100644 aimet_zoo_torch/salsanext/models/common/warmupLR.py create mode 100644 aimet_zoo_torch/salsanext/models/data_cfg.yaml create mode 100644 aimet_zoo_torch/salsanext/models/model_cards/salsanext_w4a8.json create mode 100644 aimet_zoo_torch/salsanext/models/model_cards/salsanext_w8a8.json create mode 100644 aimet_zoo_torch/salsanext/models/model_definition.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/__init__.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/__init__.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/config/labels/semantic-kitti-all.yaml create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/config/labels/semantic-kitti.yaml create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/dataset/kitti/__init__.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/dataset/kitti/parser.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/evaluate_iou.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/infer.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/modules/Lovasz_Softmax.py rename aimet_zoo_torch/{salsaNext/models => salsanext/models/tasks/semantic/modules}/SalsaNext.py (76%) create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/modules/SalsaNextAdf.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/modules/__init__.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/modules/adf.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/modules/ioueval.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/modules/trainer.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/postproc/KNN.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/postproc/__init__.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/train.py create mode 100644 aimet_zoo_torch/salsanext/models/tasks/semantic/visualize.py create mode 100644 aimet_zoo_torch/segnet/SegNet.md create mode 100644 aimet_zoo_torch/segnet/__init__.py create mode 100644 aimet_zoo_torch/segnet/dataloader/__init__.py create mode 100644 aimet_zoo_torch/segnet/dataloader/dataloaders_and_eval_func.py create mode 100644 aimet_zoo_torch/segnet/evaluator/__init__.py create mode 100644 aimet_zoo_torch/segnet/evaluator/segnet_quanteval.py create mode 100644 aimet_zoo_torch/segnet/model/__init__.py create mode 100644 aimet_zoo_torch/segnet/model/datasets/camvid.py create mode 100644 aimet_zoo_torch/segnet/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/segnet/model/model_cards/segnet_w4a8.json create mode 100644 aimet_zoo_torch/segnet/model/model_cards/segnet_w8a8.json create mode 100644 aimet_zoo_torch/segnet/model/model_definition.py create mode 100644 aimet_zoo_torch/segnet/model/models/segnet.py create mode 100644 aimet_zoo_torch/segnet/requirements.txt create mode 100644 aimet_zoo_torch/sesr/evaluators/__init__.py create mode 100644 aimet_zoo_torch/sesr/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/ssd_mobilenetv2/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/ssd_res50/__init__.py create mode 100644 aimet_zoo_torch/ssd_res50/dataloader/__init__.py create mode 100644 aimet_zoo_torch/ssd_res50/evaluators/__init__.py create mode 100644 aimet_zoo_torch/ssd_res50/model/__init__.py create mode 100644 aimet_zoo_torch/ssd_res50/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/vit/dataloader/utils/__init__.py create mode 100644 aimet_zoo_torch/vit/evaluators/__init__.py create mode 100644 aimet_zoo_torch/vit/model/__init__.py create mode 100644 aimet_zoo_torch/vit/model/huggingface/__init__.py create mode 100644 aimet_zoo_torch/vit/model/huggingface/baseline_models/__init__.py create mode 100644 aimet_zoo_torch/vit/model/huggingface/baseline_models/vit/__init__.py create mode 100644 aimet_zoo_torch/vit/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/xlsr/evaluators/__init__.py create mode 100644 aimet_zoo_torch/xlsr/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/yolox/evaluators/__init__.py create mode 100644 aimet_zoo_torch/yolox/model/model_cards/__init__.py create mode 100644 aimet_zoo_torch/yolox/model/yolo_x/__init__.py diff --git a/.pylintrc b/.pylintrc index 08b79d0..4ba02ef 100644 --- a/.pylintrc +++ b/.pylintrc @@ -54,7 +54,15 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=print-statement, +disable=bad-option-value, + useless-option-value, + unrecognized-option, + unknown-option-value, + consider-using-f-string, + f-string-without-interpolation, + unbalanced-tuple-unpacking, + unspecified-encoding, + print-statement, parameter-unpacking, unpacking-in-except, old-raise-syntax, diff --git a/Jenkins/Dockerfile.tf-torch-cpu b/Jenkins/Dockerfile.tf-torch-cpu index b9221fd..6451d48 100644 --- a/Jenkins/Dockerfile.tf-torch-cpu +++ b/Jenkins/Dockerfile.tf-torch-cpu @@ -15,219 +15,16 @@ # Docker image file to build and test AIMET for both Tensorflow and PyTorch in a CPU environment -FROM ubuntu:bionic +FROM artifacts.codelinaro.org/codelinaro-aimet/aimet:latest.tf-torch-cpu ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update > /dev/null && \ - apt-get install --no-install-recommends -y \ - # Bare minimum Packages - ca-certificates \ - git \ - ssh \ - sudo \ - wget \ - xterm \ - xauth > /dev/null && \ - rm -rf /var/lib/apt/lists/* - -# Install certificates -RUN sudo update-ca-certificates - -# Modified version of bash.bashrc that adjusts the prompt -### COPY bash.bashrc /etc/ -### RUN chmod 644 /etc/bash.bashrc - -### COPY profile.global /usr/local/etc/ -### RUN chmod 555 /usr/local/etc/profile.global - -# Add sudo support -RUN echo "%users ALL = (ALL) NOPASSWD: ALL" >> /etc/sudoers - RUN apt-get update -y > /dev/null && \ apt-get install --no-install-recommends -y \ - # Python - python3.8 \ - python3.8-dev \ - python3-pip \ - python3-setuptools \ - build-essential \ - # lmdb dependency - libffi-dev && \ + python3-pip && \ rm -rf /var/lib/apt/lists/* -# Register the version in alternatives -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 -# Set python 3.8 as the default python -RUN update-alternatives --set python3 /usr/bin/python3.8 - -# Python 2 pip installation -RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* && \ - python2.7 -m pip --no-cache-dir install --upgrade \ - pip==20.3.4 \ - restkit - # Upgrade Python3 pip and install some more packages RUN python3 -m pip --no-cache-dir install --upgrade \ pip \ - setuptools==41.0.1 \ - wheel==0.33.4 - -# Ubuntu packages for tensorflow and pytorch aimet -RUN dpkg --add-architecture i386 -RUN apt-get update > /dev/null && \ - apt-get install --no-install-recommends -y \ - build-essential \ - emacs \ - environment-modules \ - less \ - libavcodec-dev \ - libavformat-dev \ - libgtest-dev \ - libgtk2.0-dev \ - libsox-dev \ - libsox-fmt-all \ - libstdc++6:i386 \ - libswscale-dev \ - libxtst6 \ - lsb-release \ - meld \ - nano \ - pandoc \ - pkg-config \ - python3-tk \ - sox \ - tree \ - vim && \ - rm -rf /var/lib/apt/lists/* - -# Python3 Packages -RUN python3 -m pip --no-cache-dir install \ - astroid==2.5.3 \ - attrs==19.1.0 \ - behave==1.2.6 \ - bert-tensorflow \ - blosc==1.10.1 \ - cffi==1.12.3 \ - click \ - cumm==0.2.8 \ - cython==0.29.12 \ - dataclasses \ - Deprecated \ - docutils==0.16 \ - grpcio \ - grpcio-tools \ - h5py==2.10.0 \ - ipykernel \ - Jinja2==3.0.3 \ - jupyter \ - keras==2.2.4 \ - lmdb==1.2.1 \ - matplotlib>=3 \ - mock \ - nbsphinx \ - numpy==1.19.5 \ - onnx==1.12.0 \ - onnxsim \ - onnxruntime \ - onnxruntime-extensions \ - opencv-python \ - Pillow==9.3.0 \ - pluggy==0.12.0 \ - progressbar2 \ - protobuf==3.20.1 \ - psutil \ - ptflops \ - pybind11 \ - pyDOE2 \ - pylint==2.3.1 \ - pymoo \ - pytest==4.6.5 \ - pytest-cov==2.6.1 \ - pytorch-ignite \ - PyYAML \ - scikit-learn==1.1.3 \ - scipy==1.8.1 \ - spconv==2.1.20 \ - sphinx==2.1.1 \ - sphinx-jinja==1.1.1 \ - sphinx-autodoc-typehints==1.6.0 \ - tensorboard==2.4.0 \ - tensorboardX==2.4 \ - tensorflow-cpu==2.4.3 \ - tensorflow-hub \ - tensorflow-model-optimization \ - tensorlayer==2.2.1 \ - timm==0.4.12 \ - torch==1.9.1+cpu -f https://download.pytorch.org/whl/torch_stable.html \ - torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html \ - torchtext==0.10.1 \ - torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html \ - tqdm \ - transformers==4.11.3 \ - wget && \ - python3 -m ipykernel.kernelspec - -RUN cd /tmp && \ - wget https://github.com/Kitware/CMake/releases/download/v3.19.3/cmake-3.19.3-Linux-x86_64.sh && \ - mkdir /opt/cmake && \ - sh cmake-3.19.3-Linux-x86_64.sh --prefix=/opt/cmake --skip-license - -RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -RUN ln -s /opt/cmake/bin/ctest /usr/local/bin/ctest -RUN ln -s /opt/cmake/bin/cpack /usr/local/bin/cpack - -ENV PATH=/usr/local/bin:$PATH - -# Opencv -# Ref: https://docs.opencv.org/3.2.0/d7/d9f/tutorial_linux_install.html -COPY opencv_320_python38.patch /tmp -RUN wget -q https://github.com/Itseez/opencv/archive/3.2.0.tar.gz -O /tmp/3.2.0.tar.gz > /dev/null && \ - tar -C /tmp -xvf /tmp/3.2.0.tar.gz > /dev/null && \ - patch /tmp/opencv-3.2.0/modules/python/src2/cv2.cpp /tmp/opencv_320_python38.patch && \ - cd /tmp/opencv-3.2.0 && mkdir release && cd release && \ - cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=release -DWITH_FFMPEG=OFF -DBUILD_TESTS=OFF -DWITH_CUDA=OFF -DBUILD_PERF_TESTS=OFF -DWITH_IPP=OFF -DENABLE_PRECOMPILED_HEADERS=OFF .. > /dev/null && \ - make -j16 > /dev/null && \ - make -j16 install > /dev/null && \ - rm -rf /tmp/opencv-3.2.0* - -EXPOSE 25000 -RUN apt-get update && apt-get install -y openssh-server && rm -rf /var/lib/apt/lists/* -RUN mkdir /var/run/sshd - -RUN apt-get update && apt-get install -y liblapacke liblapacke-dev && rm -rf /var/lib/apt/lists/* - -RUN apt-get update && apt-get install -y libjpeg8-dev && \ - rm -rf /var/lib/apt/lists/* - -# Set up symlink to point to the correct python version -RUN ln -sf /usr/bin/python3.8 /usr/bin/python -RUN ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib - -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ - sed -i 's/Port 22/Port 25000/' /etc/ssh/sshd_config - -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# Clone the tensorflow repo to enable development -RUN cd / && git clone --depth 1 --single-branch --branch v2.4.3 https://github.com/tensorflow/tensorflow.git - -RUN python3 -m pip install git-pylint-commit-hook osqp - -# NOTE: We need to pin the holoviews version to this since the latest version has a circular dependency on bokeh 2.0.0 through the panel package -RUN python3 -m pip install holoviews==1.12.7 netron jsonschema pandas==1.4.3 - -RUN python3 -m pip install bokeh==1.2.0 hvplot==0.4.0 - -# Remove existing Pillow & Pillow-SIMD and replace with correct version of Pillow-SIMD. -RUN python3 -m pip uninstall -y Pillow Pillow-SIMD -RUN python3 -m pip --no-cache-dir install Pillow-SIMD==9.0.0.post1 - -RUN apt-get update && apt-get install -y gnupg2 -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add - && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main" >> /etc/apt/sources.list -RUN apt-get update --fix-missing -y && apt upgrade -y && apt-get install -y clang-11 clang-format clang-tidy-11 && \ - rm -rf /var/lib/apt/lists/* - -# Create a version-less symbolic link for clang-tidy -RUN ln -s /usr/bin/run-clang-tidy-11.py /usr/bin/run-clang-tidy.py \ No newline at end of file + pylint==2.17.2 diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile index b18494c..6f87af0 100644 --- a/Jenkins/Jenkinsfile +++ b/Jenkins/Jenkinsfile @@ -20,6 +20,8 @@ pipeline { PROJECT_NAME = "${params.PROJECT_NAME}" PROJECT_BRANCH = "${params.PROJECT_BRANCH}" WORKSPACE_ROOT = "${workspace}" + // Bypass the pre-built images until we have model-zoo-dev images + USE_LINARO = "" } stages { @@ -54,9 +56,9 @@ pipeline { cleanWs() unstash 'AIMETBuildTree' script { - env.AIMET_ZOO_VARIANT_TF_CPU = "tf-torch-cpu" + env.AIMET_ZOO_VARIANT = "tf-torch-cpu" } - echo "*** Running SETUP stage for ${env.AIMET_ZOO_VARIANT_TF_CPU} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***" + echo "*** Running SETUP stage for ${env.AIMET_ZOO_VARIANT} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***" } } @@ -65,7 +67,7 @@ pipeline { echo 'Building and Creating Package...' script { //TODO Change option back to "-bp" as soon as issue is fixed - runStage(env.AIMET_ZOO_VARIANT_TF_CPU, "-bp") + runStage(env.AIMET_ZOO_VARIANT, "-bp") } } } @@ -74,7 +76,7 @@ pipeline { steps { echo 'Running code violations...' script { - runStage(env.AIMET_ZOO_VARIANT_TF_CPU, "-v") + runStage(env.AIMET_ZOO_VARIANT, "-v") } } post { @@ -117,17 +119,28 @@ pipeline { } } -def runStage(AIMET_ZOO_VARIANT, options) { - - echo "*** Running stage ${options} for ${AIMET_ZOO_VARIANT} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***" - +def didDockerFileChange(AIMET_ZOO_VARIANT) { def changedFiles = pullRequest.files.collect { it.getFilename() } print changedFiles - if (!changedFiles.contains("Jenkins/Dockerfile.${AIMET_ZOO_VARIANT}".toString())) { - print "Jenkins/Dockerfile.${AIMET_ZOO_VARIANT} not found in changed file list, so using Linaro Docker image for ${AIMET_ZOO_VARIANT}" + if (changedFiles.contains("Jenkins/Dockerfile.${AIMET_ZOO_VARIANT}".toString())) { + print "Jenkins/Dockerfile.${AIMET_ZOO_VARIANT} changed in PR, so building docker image locally." + return true } + print "Jenkins/Dockerfile.${AIMET_ZOO_VARIANT} NOT changed in PR, so using pre-built docker image." + return false +} + +def runStage(AIMET_ZOO_VARIANT, options) { + + echo "*** Running stage ${options} for ${AIMET_ZOO_VARIANT} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***" + + + if (didDockerFileChange(env.AIMET_ZOO_VARIANT)) { + env.USE_LINARO="" + } + sh """ AIMET_ZOO_VARIANT=${AIMET_ZOO_VARIANT} bash -l -c "cd ${REPO_NAME} && ./Jenkins/buildntest.sh -e AIMET_ZOO_VARIANT ${options} ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}" """ diff --git a/Jenkins/dobuildntest.sh b/Jenkins/dobuildntest.sh index 8ab595b..6cde691 100755 --- a/Jenkins/dobuildntest.sh +++ b/Jenkins/dobuildntest.sh @@ -223,9 +223,29 @@ if [ $run_code_violation -eq 1 ]; then cd $buildFolder echo -e "\n********** Stage 4: Code violation checks **********\n" + + extra_opts="" + # Add build options based on variant + echo -e "Verify aimet variant: $AIMET_ZOO_VARIANT" + if [ -n "$AIMET_ZOO_VARIANT" ]; then + if [[ "$AIMET_ZOO_VARIANT" == *"tf"* ]]; then + extra_opts+=" -DENABLE_TENSORFLOW=ON" + fi + if [[ "$AIMET_ZOO_VARIANT" == *"torch"* ]]; then + extra_opts+=" -DENABLE_TORCH=ON" + fi + if [[ "$AIMET_ZOO_VARIANT" != *"tf"* ]]; then + extra_opts+=" -DENABLE_TENSORFLOW=OFF" + fi + if [[ "$AIMET_ZOO_VARIANT" != *"torch"* ]]; then + extra_opts+=" -DENABLE_TORCH=OFF" + fi + fi + echo -e "Extra Options added: " ${extra_opts} + cmake ${extra_opts} .. + make pylintmodelzoo check_stage $? "Code Violations" "true" fi -exit $EXIT_CODE - +exit $EXIT_CODE \ No newline at end of file diff --git a/Jenkins/jenkins_threshold_configs.json b/Jenkins/jenkins_threshold_configs.json index 38ebcf6..5a1125c 100644 --- a/Jenkins/jenkins_threshold_configs.json +++ b/Jenkins/jenkins_threshold_configs.json @@ -2,7 +2,7 @@ "pylint_fail_thresholds" : { "high_priority" : "10", "normal_priority" : "10", - "low_priority" : "10" + "low_priority" : "33" } } diff --git a/Jenkins/opencv_320_python38.patch b/Jenkins/opencv_320_python38.patch new file mode 100644 index 0000000..14360b1 --- /dev/null +++ b/Jenkins/opencv_320_python38.patch @@ -0,0 +1,4 @@ +730c730 +< char* str = PyString_AsString(obj); +--- +> const char* str = PyString_AsString(obj); diff --git a/NightlyTests/CMakeLists.txt b/NightlyTests/CMakeLists.txt new file mode 100644 index 0000000..09515f0 --- /dev/null +++ b/NightlyTests/CMakeLists.txt @@ -0,0 +1,30 @@ + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +add_custom_target(AcceptanceTests) + +if (ENABLE_TORCH) + add_dependencies(AcceptanceTests + AcceptanceTests.Torch) + +endif (ENABLE_TORCH) + +#if (ENABLE_TENSORFLOW) +# add_dependencies(AcceptanceTests +# AcceptanceTests.Tensorflow) +# +#endif (ENABLE_TENSORFLOW) + +if (ENABLE_TORCH) + add_subdirectory(torch) +endif (ENABLE_TORCH) + +#if (ENABLE_TENSORFLOW) +# add_subdirectory(tensorflow) +#endif (ENABLE_TENSORFLOW) diff --git a/NightlyTests/torch/CMakeLists.txt b/NightlyTests/torch/CMakeLists.txt new file mode 100644 index 0000000..77ce71b --- /dev/null +++ b/NightlyTests/torch/CMakeLists.txt @@ -0,0 +1,34 @@ +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +if (ENABLE_CUDA) + set(CUDA_FLAG -m "not blah") + set(USE_CUDA True) +else (ENABLE_CUDA) + set(CUDA_FLAG -m "not cuda") + set(USE_CUDA False) +endif (ENABLE_CUDA) + +#add_custom_target(AcceptanceTests.TorchDependencies +# COMMAND ${CMAKE_COMMAND} -E env +# "${AIMET_PYTHONPATH}" +# "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/dependencies.py ${CMAKE_CURRENT_SOURCE_DIR}/resnet18_eval_scores.csv +# ${USE_CUDA}) + +add_custom_target( AcceptanceTests.Torch ) + +file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py") + foreach(filename ${files}) + get_filename_component( testname "${filename}" NAME_WE ) + add_custom_target(AcceptanceTests.Torch.${testname} + VERBATIM COMMAND ${CMAKE_COMMAND} -E env + "${AIMET_PYTHONPATH}" + ${Python3_EXECUTABLE} -m pytest -s ${filename} ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml) + + add_dependencies( AcceptanceTests.Torch.${testname} AcceptanceTests.TorchDependencies ) + add_dependencies( AcceptanceTests.Torch AcceptanceTests.Torch.${testname} ) + endforeach( filename ) diff --git a/NightlyTests/torch/conftest.py b/NightlyTests/torch/conftest.py new file mode 100644 index 0000000..2bd96ef --- /dev/null +++ b/NightlyTests/torch/conftest.py @@ -0,0 +1,34 @@ +import pytest +from pathlib import Path + + +@pytest.fixture(scope='module') +def data_folder(tmp_path_factory): + """ + This fixture will return a shortcut path for saved data. When there's no + such shortcut path, a tmp path will be generated. Use this with discretion + because anything stored in a tmp path will be gone. Set DEPENDENCY_DATA_PATH + only when you have permanent files stored for testing + """ + if is_cache_env_set(): + dependency_path = Path(os.getenv('DEPENDENCY_DATA_PATH')) + else: + dependency_path = None + + data_path = dependency_path if (dependency_path and dependency_path.exists()) else tmp_path_factory.mktemp('data') + + yield data_path + + +@pytest.fixture(autouse=True) +def dataset_path(data_path): + """this fixture return the dataset paths for acceptance tests""" + + dataset_path = { + "image_classification":str(data_path)+"ILSVRC2012_PyTorch_reduced/val", + "object_detection":str(data_path)+"tiny_coco/val_2017", + "pose_estimation":str(data_path)+"tiny_coco/val_2017", + "semantic_segmentation":str(data_path)+"cityscapes", + "super_reslution":str(data_path)+"/super_resolution_data/Set5/image_SRF_4_HR" + } + return dataset_path diff --git a/NightlyTests/torch/hrnet.yaml b/NightlyTests/torch/hrnet.yaml new file mode 100644 index 0000000..f1cbd61 --- /dev/null +++ b/NightlyTests/torch/hrnet.yaml @@ -0,0 +1,158 @@ +############################################################################### + +# MIT License + +# Copyright (c) 2019 Leo Xiao + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +############################################################################### + +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2022 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ + + +AUTO_RESUME: true +CUDNN: + BENCHMARK: true + DETERMINISTIC: false + ENABLED: true +DATA_DIR: '' +GPUS: (0,) +OUTPUT_DIR: 'output' +LOG_DIR: 'log' +WORKERS: 24 +PRINT_FREQ: 100 + +DATASET: + COLOR_RGB: true + DATASET: 'coco' + DATA_FORMAT: jpg + FLIP: true + NUM_JOINTS_HALF_BODY: 8 + PROB_HALF_BODY: 0.3 + ROT_FACTOR: 45 + SCALE_FACTOR: 0.35 + TEST_SET: 'val2017' + TRAIN_SET: 'train2017' +MODEL: + INIT_WEIGHTS: true + NAME: pose_hrnet + NUM_JOINTS: 17 + TARGET_TYPE: gaussian + IMAGE_SIZE: + - 192 + - 256 + HEATMAP_SIZE: + - 48 + - 64 + SIGMA: 2 + EXTRA: + PRETRAINED_LAYERS: + - 'conv1' + - 'bn1' + - 'conv2' + - 'bn2' + - 'layer1' + - 'transition1' + - 'stage2' + - 'transition2' + - 'stage3' + - 'transition3' + - 'stage4' + FINAL_CONV_KERNEL: 1 + STAGE2: + NUM_MODULES: 1 + NUM_BRANCHES: 2 + BLOCK: BASIC + NUM_BLOCKS: + - 4 + - 4 + NUM_CHANNELS: + - 32 + - 64 + FUSE_METHOD: SUM + STAGE3: + NUM_MODULES: 4 + NUM_BRANCHES: 3 + BLOCK: BASIC + NUM_BLOCKS: + - 4 + - 4 + - 4 + NUM_CHANNELS: + - 32 + - 64 + - 128 + FUSE_METHOD: SUM + STAGE4: + NUM_MODULES: 3 + NUM_BRANCHES: 4 + BLOCK: BASIC + NUM_BLOCKS: + - 4 + - 4 + - 4 + - 4 + NUM_CHANNELS: + - 32 + - 64 + - 128 + - 256 + FUSE_METHOD: SUM +LOSS: + USE_TARGET_WEIGHT: true +TRAIN: + BATCH_SIZE_PER_GPU: 32 + SHUFFLE: true + BEGIN_EPOCH: 0 + END_EPOCH: 210 + OPTIMIZER: adam + LR: 0.001 + LR_FACTOR: 0.1 + LR_STEP: + - 170 + - 200 + WD: 0.0001 + GAMMA1: 0.99 + GAMMA2: 0.0 + MOMENTUM: 0.9 + NESTEROV: false +TEST: + BATCH_SIZE_PER_GPU: 32 + COCO_BBOX_FILE: './COCO_test-dev2017_detections_AP_H_609_person.json' + BBOX_THRE: 1.0 + IMAGE_THRE: 0.0 + IN_VIS_THRE: 0.2 + MODEL_FILE: '' + NMS_THRE: 1.0 + OKS_THRE: 0.9 + USE_GT_BBOX: true + FLIP_TEST: true + POST_PROCESS: true + SHIFT_HEATMAP: true +DEBUG: + DEBUG: true + SAVE_BATCH_IMAGES_GT: true + SAVE_BATCH_IMAGES_PRED: true + SAVE_HEATMAPS_GT: true + SAVE_HEATMAPS_PRED: true diff --git a/NightlyTests/torch/pytest.ini b/NightlyTests/torch/pytest.ini new file mode 100644 index 0000000..360046b --- /dev/null +++ b/NightlyTests/torch/pytest.ini @@ -0,0 +1,4 @@ +# content of pytest.ini +[pytest] +markers = + cuda: test that require CUDA to be installed \ No newline at end of file diff --git a/NightlyTests/torch/test_image_classification.py b/NightlyTests/torch/test_image_classification.py new file mode 100644 index 0000000..4a52f78 --- /dev/null +++ b/NightlyTests/torch/test_image_classification.py @@ -0,0 +1,39 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" acceptance test for image classification""" +import pytest +import torch +from aimet_zoo_torch.resnet.evaluator import resnet_quanteval + + +@pytest.fixture() +def model_config(): + """model config fixture""" + model_config_dict = { + "resnet18": "resnet18_w8a8", + } + return model_config_dict + + +@pytest.mark.cuda +# pylint:disable = redefined-outer-name +def test_quanteval_resnet18(model_config, dataset_path): + """resnet18 image classification test""" + torch.cuda.empty_cache() + resnet_quanteval.main( + [ + "--model-config", + model_config["resnet18"], + "--dataset-path", + dataset_path["image_classification"], + ] + ) diff --git a/NightlyTests/torch/test_object_detection.py b/NightlyTests/torch/test_object_detection.py new file mode 100644 index 0000000..6ca1e58 --- /dev/null +++ b/NightlyTests/torch/test_object_detection.py @@ -0,0 +1,38 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" acceptance test for object detection""" +import pytest +import torch + +from aimet_zoo_torch.yolox.evaluators import yolox_quanteval + + +@pytest.fixture() +def model_config(): + model_config_dict = { + "yolox": "yolox_s", + } + return model_config_dict + + +@pytest.mark.cuda +def test_quaneval_yolox(model_config, dataset_path): + torch.cuda.empty_cache() + + yolox_quanteval.main( + [ + "--model-config", + model_config["yolox"], + "--dataset-path", + dataset_path["object_detection"], + ] + ) diff --git a/NightlyTests/torch/test_pose_estimation.py b/NightlyTests/torch/test_pose_estimation.py new file mode 100644 index 0000000..072a89f --- /dev/null +++ b/NightlyTests/torch/test_pose_estimation.py @@ -0,0 +1,39 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" acceptance test for pose estimation""" +import pytest +import torch +from aimet_zoo_torch.hrnet_posenet.evaluators import hrnet_posenet_quanteval + + +@pytest.fixture() +def model_config(): + """model config fixture""" + model_config_dict = { + "hrnet_posenet": "hrnet_posenet_w8a8", + } + return model_config_dict + + +@pytest.mark.cuda +def test_quaneval_hrnet_posenet(model_config, dataset_path): + """hrnet_posenet pose estimation test""" + torch.cuda.empty_cache() + + accuracy = hrnet_posenet_quanteval.main( + [ + "--model-config", + model_config["hrnet_posenet"], + "--dataset-path", + dataset_path["pose_estimation"], + ] + ) diff --git a/NightlyTests/torch/test_semantic_segmentation.py b/NightlyTests/torch/test_semantic_segmentation.py new file mode 100644 index 0000000..bf34c0f --- /dev/null +++ b/NightlyTests/torch/test_semantic_segmentation.py @@ -0,0 +1,42 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" acceptance test for semantic segmentation""" +import pytest +import torch + +from aimet_zoo_torch.hrnet_semantic_segmentation.evaluators import ( + hrnet_sem_seg_quanteval, +) + + +@pytest.fixture() +def model_config(): + """model config fixture""" + model_config_dict = { + "hrnet_sem_seg": "hrnet_sem_seg_w8a8", + } + return model_config_dict + + +@pytest.mark.cuda +#pylint:disable = redefined-outer-name +def test_quaneval_hrnet_sem_seg(model_config, dataset_path): + """acceptance test of hrnet for semantic segmentation""" + torch.cuda.empty_cache() + hrnet_sem_seg_quanteval.main( + [ + "--model-config", + model_config["hrnet_sem_seg"], + "--dataset-path", + dataset_path["semantic_segmentation"], + ] + ) diff --git a/NightlyTests/torch/test_super_resolution.py b/NightlyTests/torch/test_super_resolution.py new file mode 100644 index 0000000..e01ae93 --- /dev/null +++ b/NightlyTests/torch/test_super_resolution.py @@ -0,0 +1,39 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" acceptance test for super resolution""" +import pytest +import torch +from aimet_zoo_torch.quicksrnet.evaluators import quicksrnet_quanteval + + +@pytest.fixture() +def model_config(): + """model config fixture""" + model_config_dict = { + "quicksrnet": "quicksrnet_small_1.5x_w8a8", + } + return model_config_dict + + +@pytest.mark.cuda +# pylint:disable = redefined-outer-name +def test_quaneval_quicksrnet(model_config, dataset_path): + """quicksrnet super resolution acceptance test""" + torch.cuda.empty_cache() + quicksrnet_quanteval.main( + [ + "--model-config", + model_config["quicksrnet"], + "--dataset-path", + dataset_path["super_resolution"], + ] + ) diff --git a/README.md b/README.md index 7f5cedd..29d5920 100755 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ An original FP32 source model is quantized either using post-training quantizati EfficientNet-lite0 GitHub Repo Pretrained Model - Quantized Model + Quantized Model (ImageNet) Top-1 Accuracy 75.40% 75.36% @@ -197,8 +197,8 @@ An original FP32 source model is quantized either using post-training quantizati HRNET-Posenet Based on Ref. - FP32 Model - Quantized Model + FP32 Model + Quantized Model (COCO) mAP 0.765 0.763 @@ -248,13 +248,13 @@ An original FP32 source model is quantized either using post-training quantizati QuickSRNet - - See Tarballs + See Tarballs See Example Average PSNR Results TBD - Semantic Segmentation + Semantic Segmentation DeepLabV3+ GitHub Repo Pretrained Model @@ -268,7 +268,7 @@ An original FP32 source model is quantized either using post-training quantizati HRNet-W48 GitHub Repo Original model weight not available - Quantized Model + Quantized Model (Cityscapes) mIOU 81.04% 80.65% @@ -313,14 +313,24 @@ An original FP32 source model is quantized either using post-training quantizati 46.8% - SalsaNext + SalsaNext GitHub Repo Pretrained Model - Quantized Model + Quantized Model (Semantic kitti) mIOU 55.8% 54.9% - TBD + 55.1% + + + SegNet + GitHub Repo + Pretrained Model + Quantized Model + (CamVid dataset) mIOU + 50.48% + 50.59% + 50.58% Speech Recognition @@ -469,7 +479,7 @@ An original FP32 source model is quantized either using post-training quantizati W4A8[7] - Image Classification + Image Classification ResNet-50 (v1) GitHub Repo Pretrained Model @@ -480,6 +490,17 @@ An original FP32 source model is quantized either using post-training quantizati 74.96% TBD + + ResNet-50-tf2 + GitHub Repo + Pretrained Model + Quantized Model + 2.4 + (ImageNet) Top-1 Accuracy + 74.9% + 74.8% + TBD + MobileNet-v2-1.4 GitHub Repo @@ -491,6 +512,17 @@ An original FP32 source model is quantized either using post-training quantizati 74.21% TBD + + MobileNet-v2-tf2 + GitHub Repo + Pretrained Model + See Example + 2.4 + (ImageNet) Top-1 Accuracy + 71.6% + 71.0% + TBD + EfficientNet Lite GitHub Repo @@ -593,3 +625,4 @@ AIMET Model Zoo is a project maintained by Qualcomm Innovation Center, Inc. ## License Please see the [LICENSE file](LICENSE.pdf) for details. + diff --git a/aimet_zoo_tensorflow/common/__init__.py b/aimet_zoo_tensorflow/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/common/downloader.py b/aimet_zoo_tensorflow/common/downloader.py index acb7d00..0edf829 100644 --- a/aimet_zoo_tensorflow/common/downloader.py +++ b/aimet_zoo_tensorflow/common/downloader.py @@ -7,34 +7,37 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +""" Downloader and downloader progress bar class for downloading and loading weights and encodings""" - -import gdown import os -import progressbar -import shutil -import tarfile +import tarfile from pathlib import Path +import shutil from shutil import copy2 from urllib.request import urlretrieve +import progressbar +import gdown # pylint: disable=import-error -class Downloader(): +class Downloader: """ Parent class for inheritance of utility methods used for downloading and loading weights and encodings """ - def __init__(self, - url_pre_opt_weights: str = None, - url_post_opt_weights: str = None, - url_adaround_encodings: str = None, - url_aimet_encodings: str = None, - url_aimet_config: str = None, - tar_url_pre_opt_weights: str= None, - tar_url_post_opt_weights: str = None, - url_zipped_checkpoint: str = None, - model_dir: str = '', - model_config: str = ''): - + # pylint: disable=too-many-instance-attributes, too-many-arguments + # 16 is reasonable in this case. + def __init__( + self, + url_pre_opt_weights: str = None, + url_post_opt_weights: str = None, + url_adaround_encodings: str = None, + url_aimet_encodings: str = None, + url_aimet_config: str = None, + tar_url_pre_opt_weights: str = None, + tar_url_post_opt_weights: str = None, + url_zipped_checkpoint: str = None, + model_dir: str = "", + model_config: str = "", + ): """ :param url_pre_opt_weights: url hosting pre optimization weights as a state dict :param url_post_opt_weights: url hosting post optimization weights as a state dict @@ -53,46 +56,87 @@ def __init__(self, self.tar_url_pre_opt_weights = tar_url_pre_opt_weights self.tar_url_post_opt_weights = tar_url_post_opt_weights self.url_zipped_checkpoint = url_zipped_checkpoint - self._download_storage_path = Path(model_dir + '/weights/' + model_config + '/') if model_config else Path(model_dir + '/weights/') - self.path_pre_opt_weights = os.path.join(self._download_storage_path, "pre_opt_weights") if self.url_pre_opt_weights else None - self.path_post_opt_weights = os.path.join(self._download_storage_path, "post_opt_weights") if self.url_post_opt_weights else None - self.path_adaround_encodings = os.path.join(self._download_storage_path,"adaround_encodings") if self.url_adaround_encodings else None - self.path_aimet_encodings = os.path.join(self._download_storage_path, "aimet_encodings") if self.url_aimet_encodings else None - self.path_aimet_config = os.path.join(self._download_storage_path,"aimet_config") if self.url_aimet_config else None - self.path_zipped_checkpoint = os.path.join(self._download_storage_path, "zipped_checkpoint.zip") if self.url_zipped_checkpoint else None - self.extract_dir = self.path_zipped_checkpoint.split(".zip")[0] if self.path_zipped_checkpoint else None - - def _download_from_url(self, - src: str, - dst: str): + self._download_storage_path = ( + Path(model_dir + "/weights/" + model_config + "/") + if model_config + else Path(model_dir + "/weights/") + ) + self.path_pre_opt_weights = ( + os.path.join(self._download_storage_path, "pre_opt_weights") + if self.url_pre_opt_weights + else None + ) + self.path_post_opt_weights = ( + os.path.join(self._download_storage_path, "post_opt_weights") + if self.url_post_opt_weights + else None + ) + self.path_adaround_encodings = ( + os.path.join(self._download_storage_path, "adaround_encodings") + if self.url_adaround_encodings + else None + ) + self.path_aimet_encodings = ( + os.path.join(self._download_storage_path, "aimet_encodings") + if self.url_aimet_encodings + else None + ) + self.path_aimet_config = ( + os.path.join(self._download_storage_path, "aimet_config") + if self.url_aimet_config + else None + ) + self.path_zipped_checkpoint = ( + os.path.join(self._download_storage_path, "zipped_checkpoint.zip") + if self.url_zipped_checkpoint + else None + ) + self.extract_dir = ( + self.path_zipped_checkpoint.split(".zip")[0] + if self.path_zipped_checkpoint + else None + ) + + def _download_from_url(self, src: str, dst: str): """Receives a source URL or path and a storage destination path, evaluates the source, fetches the file, and stores at the destination""" if not os.path.exists(self._download_storage_path): os.makedirs(self._download_storage_path) if src is None: - return 'Skipping download, URL not provided on model definition' - if src.startswith('https://drive.google.com'): + return "Skipping download, URL not provided on model definition" + if src.startswith("https://drive.google.com"): gdown.download(url=src, output=dst, quiet=True, verify=False) - elif src.startswith('http'): + elif src.startswith("http"): urlretrieve(src, dst) else: - assert os.path.exists(src), 'URL passed is not an http, assumed it to be a system path, but such path does not exist' + assert os.path.exists( + src + ), "URL passed is not an http, assumed it to be a system path, but such path does not exist" copy2(src, dst) + return None def _download_pre_opt_weights(self): """downloads pre optimization weights""" - self._download_from_url(src=self.url_pre_opt_weights, dst=self.path_pre_opt_weights) + self._download_from_url( + src=self.url_pre_opt_weights, dst=self.path_pre_opt_weights + ) def _download_post_opt_weights(self): """downloads post optimization weights""" - self._download_from_url(src=self.url_post_opt_weights, dst=self.path_post_opt_weights) + self._download_from_url( + src=self.url_post_opt_weights, dst=self.path_post_opt_weights + ) def _download_adaround_encodings(self): """downloads adaround encodings""" - self._download_from_url(src=self.url_adaround_encodings, dst=self.path_adaround_encodings) + self._download_from_url( + src=self.url_adaround_encodings, dst=self.path_adaround_encodings + ) def _download_aimet_encodings(self): """downloads aimet encodings""" - self._download_from_url(src=self.url_aimet_encodings, dst=self.path_aimet_encodings) + self._download_from_url( + src=self.url_aimet_encodings, dst=self.path_aimet_encodings + ) def _download_aimet_config(self): """downloads aimet configuration""" @@ -105,42 +149,53 @@ def _download_tar_post_opt_weights(self): self._download_tar_decompress(tar_url=self.tar_url_post_opt_weights) def _download_compressed_checkpoint(self): - """ download a zipped checkpoint file and unzip it """ - self._download_from_url(src=self.url_zipped_checkpoint, dst=self.path_zipped_checkpoint) - format = "".join(self.url_zipped_checkpoint.split('/')[-1].split('.')[1:][::-1]) + """download a zipped checkpoint file and unzip it""" + self._download_from_url( + src=self.url_zipped_checkpoint, dst=self.path_zipped_checkpoint + ) + file_format = "".join(self.url_zipped_checkpoint.split("/")[-1].split(".")[1:][::-1]) if not os.path.exists(self.extract_dir): os.makedirs(self.extract_dir) - shutil.unpack_archive(filename=self.path_zipped_checkpoint, extract_dir=self.extract_dir, format=format) - - def _download_tar_decompress(self, tar_url): - """"download tarball and decompress into downloaded_weights folder""" + shutil.unpack_archive( + filename=self.path_zipped_checkpoint, + extract_dir=self.extract_dir, + format=file_format, + ) + + def _download_tar_decompress(self, tar_url): + """ "download tarball and decompress into downloaded_weights folder""" if not os.path.exists(self._download_storage_path): os.mkdir(self._download_storage_path) - download_tar_name = str(self._download_storage_path) +"/downloaded_weights.tar.gz" - urlretrieve(tar_url,download_tar_name,DownloadProgressBar()) + download_tar_name = ( + str(self._download_storage_path) + "/downloaded_weights.tar.gz" + ) + urlretrieve(tar_url, download_tar_name, DownloadProgressBar()) with tarfile.open(download_tar_name) as pth_weights: - pth_weights.extractall(self._download_storage_path) - folder_name=pth_weights.getnames()[0] - download_path = str(self._download_storage_path)+'/'+str(folder_name) - new_download_path = str(self._download_storage_path)+'/downloaded_weights' + pth_weights.extractall(self._download_storage_path) + folder_name = pth_weights.getnames()[0] + download_path = str(self._download_storage_path) + "/" + str(folder_name) + new_download_path = str(self._download_storage_path) + "/downloaded_weights" if os.path.exists(new_download_path): shutil.rmtree(new_download_path) - os.rename(download_path,new_download_path) + os.rename(download_path, new_download_path) -class DownloadProgressBar(): +class DownloadProgressBar: """Downloading progress bar to show status of downloading""" + def __init__(self): self.dpb = None def __call__(self, b_num, b_size, size): widgets = [ - '\x1b[33mDownloading weights \x1b[39m', - progressbar.Percentage(), - progressbar.Bar(marker='\x1b[32m#\x1b[39m'), + "\x1b[33mDownloading weights \x1b[39m", + progressbar.Percentage(), + progressbar.Bar(marker="\x1b[32m#\x1b[39m"), ] if not self.dpb: - self.dpb=progressbar.ProgressBar(widgets=widgets, maxval=size,redirect_stdout=True) + self.dpb = progressbar.ProgressBar( + widgets=widgets, maxval=size, redirect_stdout=True + ) self.dpb.start() processed = b_num * b_size @@ -148,4 +203,3 @@ def __call__(self, b_num, b_size, size): self.dpb.update(processed) else: self.dpb.finish() - diff --git a/aimet_zoo_tensorflow/common/object_detection/__init__.py b/aimet_zoo_tensorflow/common/object_detection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/common/utils/__init__.py b/aimet_zoo_tensorflow/common/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/efficientnet/evaluators/efficientnet_quanteval.py b/aimet_zoo_tensorflow/efficientnet/evaluators/efficientnet_quanteval.py index c3a5d09..f61b47d 100755 --- a/aimet_zoo_tensorflow/efficientnet/evaluators/efficientnet_quanteval.py +++ b/aimet_zoo_tensorflow/efficientnet/evaluators/efficientnet_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3.6 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -9,25 +9,27 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """Quanteval Evaluation script for efficientnet""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet import os import argparse import urllib import tarfile -import aimet_common.defs import numpy as np -import eval_ckpt_main -import model_builder_factory +import aimet_common.defs from aimet_tensorflow.batch_norm_fold import fold_all_batch_norms from aimet_tensorflow.quantsim import QuantizationSimModel +import eval_ckpt_main +import model_builder_factory # import tensorflow as tf import tensorflow.compat.v1 as tf - tf.disable_v2_behavior() class EvalCkptDriver(eval_ckpt_main.EvalCkptDriver): - """Wrap evaluation with quantsim evaluation """ + """Wrap evaluation with quantsim evaluation""" + def build_dataset(self, filenames, labels, is_training): """Wrap build_dataset function to create an initializable iterator rather than a one shot iterator.""" make_one_shot_iterator = tf.data.Dataset.make_one_shot_iterator @@ -38,7 +40,8 @@ def build_dataset(self, filenames, labels, is_training): tf.data.Dataset.make_one_shot_iterator = make_one_shot_iterator return r - #pylint: disable=W0613 + + # pylint: disable=W0613 def run_inference( self, ckpt_path, image_files, labels, enable_ema=True, export_ckpt=None ): @@ -71,15 +74,14 @@ def run_inference( # Return the top 5 predictions (idx and prob) for each image. return prediction_idx, prediction_prob # Fold all BatchNorms before QuantSim - #pylint: disable=W0612 - sess, folded_pairs = fold_all_batch_norms( - sess, ["IteratorGetNext"], ["logits"] - ) + # pylint: disable=W0612 + sess, folded_pairs = fold_all_batch_norms(sess, ["IteratorGetNext"], ["logits"]) with sess.graph.as_default(): checkpoint = ckpt_path saver = tf.train.Saver() saver.restore(sess, checkpoint) sess.run("MakeIterator") + # Define an eval function to use during compute encodings def eval_func(sess, iterations): sess.run("MakeIterator") @@ -89,18 +91,18 @@ def eval_func(sess, iterations): # Select the right quant_scheme if self.quant_scheme == "range_learning_tf": quant_scheme = ( - aimet_common.defs.QuantScheme.training_range_learning_with_tf_init) + aimet_common.defs.QuantScheme.training_range_learning_with_tf_init + ) elif self.quant_scheme == "range_learning_tf_enhanced": quant_scheme = ( - aimet_common.defs.QuantScheme.training_range_learning_with_tf_enhanced_init) + aimet_common.defs.QuantScheme.training_range_learning_with_tf_enhanced_init + ) elif self.quant_scheme == "tf": quant_scheme = aimet_common.defs.QuantScheme.post_training_tf elif self.quant_scheme == "tf_enhanced": quant_scheme = aimet_common.defs.QuantScheme.post_training_tf_enhanced else: - raise ValueError( - "Got unrecognized quant_scheme: " + - self.quant_scheme) + raise ValueError("Got unrecognized quant_scheme: " + self.quant_scheme) # Create QuantizationSimModel sim = QuantizationSimModel( @@ -142,7 +144,7 @@ def run_evaluation(args): include_background_label=args.include_background_label, advprop_preprocessing=args.advprop_preprocessing, ) - #pylint: disable=W0201 + # pylint: disable=W0201 driver.quant_scheme = args.quant_scheme driver.round_mode = args.round_mode driver.default_output_bw = args.default_output_bw @@ -180,6 +182,7 @@ def download_weights(): class ModelConfig: """hardcoded model configurations""" + def __init__(self, args): self.model_name = "efficientnet-lite0" if args.model_to_eval == "fp32": diff --git a/aimet_zoo_tensorflow/mobiledetedgetpu/dataloader/__init__.py b/aimet_zoo_tensorflow/mobiledetedgetpu/dataloader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobiledetedgetpu/model/__init__.py b/aimet_zoo_tensorflow/mobiledetedgetpu/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobiledetedgetpu/model/model_cards/__init__.py b/aimet_zoo_tensorflow/mobiledetedgetpu/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobiledetedgetpu/model/model_definition.py b/aimet_zoo_tensorflow/mobiledetedgetpu/model/model_definition.py index e8c23d5..c99f7b2 100644 --- a/aimet_zoo_tensorflow/mobiledetedgetpu/model/model_definition.py +++ b/aimet_zoo_tensorflow/mobiledetedgetpu/model/model_definition.py @@ -7,14 +7,16 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - -import os, json, pathlib +"""Class for downloading and setting up of optmized and original mobiledetedgetpu model for AIMET model zoo""" +import os +import json +import pathlib import tensorflow.compat.v1 as tf -tf.disable_v2_behavior() -from aimet_tensorflow.quantsim import QuantizationSimModel -from aimet_tensorflow.batch_norm_fold import fold_all_batch_norms +from aimet_tensorflow.quantsim import QuantizationSimModel # pylint: disable=import-error +from aimet_tensorflow.batch_norm_fold import fold_all_batch_norms # pylint: disable=import-error from aimet_zoo_tensorflow.common.downloader import Downloader +tf.disable_v2_behavior() class MobileDet(Downloader): @@ -40,12 +42,17 @@ def __init__(self, model_config=None): model_config=model_config, ) self.input_shape = tuple( - x if x != None else 1 for x in self.cfg["input_shape"] + x if x is not None else 1 for x in self.cfg["input_shape"] ) self.starting_op_names = self.cfg["model_args"]["starting_op_names"] self.output_op_names = self.cfg["model_args"]["output_op_names"] - def from_pretrained(self, quantized=False): + @classmethod + def from_pretrained(cls, quantized=False): + #pylint:disable = unused-argument + """load pretrained model + for tensorflow models, get_session is used instead + """ return "For TF 1.X based models, use get_session()" def get_quantsim(self, quantized=False): @@ -84,6 +91,7 @@ def get_session(self, quantized=False): self._download_adaround_encodings() self._download_compressed_checkpoint() meta_graph = None + #pylint:disable = unused-variable for root, dirs, files in os.walk(self.extract_dir): for file in files: if file.endswith(".meta"): diff --git a/aimet_zoo_tensorflow/mobilenet_v2/evaluators/mobilenet_v2_140_quanteval.py b/aimet_zoo_tensorflow/mobilenet_v2/evaluators/mobilenet_v2_140_quanteval.py index b95949c..09d8f1b 100644 --- a/aimet_zoo_tensorflow/mobilenet_v2/evaluators/mobilenet_v2_140_quanteval.py +++ b/aimet_zoo_tensorflow/mobilenet_v2/evaluators/mobilenet_v2_140_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3.6 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -9,6 +9,7 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """mobilenevt v2 quantsim evaluation script""" +# pylint:disable = wrong-import-order import os import argparse import urllib @@ -29,12 +30,7 @@ from preprocessing import preprocessing_factory -def wrap_preprocessing( - preprocessing, - height, - width, - num_classes, - labels_offset): +def wrap_preprocessing(preprocessing, height, width, num_classes, labels_offset): """Wrap preprocessing function to do parsing of TFrecords.""" def parse(serialized_example): @@ -57,7 +53,8 @@ def parse(serialized_example): return parse -#pylint: disable=W0612 + +# pylint: disable=W0612 def run_evaluation(args): """define evaluation and build graph definition for evaluation""" # Build graph definition @@ -83,20 +80,19 @@ def run_evaluation(args): images, labels = iterator.get_next() network_fn = nets_factory.get_network_fn( - args.model_name, num_classes=( - 1001 - args.labels_offset), is_training=False) + args.model_name, num_classes=(1001 - args.labels_offset), is_training=False + ) with tf.device("/cpu:0"): - images = tf.placeholder_with_default(images, shape=( - None, args.image_size, args.image_size, 3), name="input") + images = tf.placeholder_with_default( + images, shape=(None, args.image_size, args.image_size, 3), name="input" + ) labels = tf.placeholder_with_default( labels, shape=(None, 1001 - args.labels_offset), name="labels" ) logits, end_points = network_fn(images) confidences = tf.nn.softmax(logits, axis=1, name="confidences") - categorical_preds = tf.argmax( - confidences, axis=1, name="categorical_preds") - categorical_labels = tf.argmax( - labels, axis=1, name="categorical_labels") + categorical_preds = tf.argmax(confidences, axis=1, name="categorical_preds") + categorical_labels = tf.argmax(labels, axis=1, name="categorical_labels") correct_predictions = tf.equal(categorical_labels, categorical_preds) top1_acc = tf.reduce_mean( tf.cast(correct_predictions, tf.float32), name="top1-acc" @@ -153,18 +149,18 @@ def eval_func(session, iterations): # Select the right quant_scheme if args.quant_scheme == "range_learning_tf": quant_scheme = ( - aimet_common.defs.QuantScheme.training_range_learning_with_tf_init) + aimet_common.defs.QuantScheme.training_range_learning_with_tf_init + ) elif args.quant_scheme == "range_learning_tf_enhanced": quant_scheme = ( - aimet_common.defs.QuantScheme.training_range_learning_with_tf_enhanced_init) + aimet_common.defs.QuantScheme.training_range_learning_with_tf_enhanced_init + ) elif args.quant_scheme == "tf": quant_scheme = aimet_common.defs.QuantScheme.post_training_tf elif args.quant_scheme == "tf_enhanced": quant_scheme = aimet_common.defs.QuantScheme.post_training_tf_enhanced else: - raise ValueError( - "Got unrecognized quant_scheme: " + - args.quant_scheme) + raise ValueError("Got unrecognized quant_scheme: " + args.quant_scheme) # Create QuantizationSimModel sim = QuantizationSimModel( session=sess, @@ -210,6 +206,7 @@ def download_weights(): class ModelConfig: """Hardcoded model configurations""" + def __init__(self, args): self.model_name = "mobilenet_v2_140" if args.model_to_eval == "fp32": @@ -230,12 +227,30 @@ def __init__(self, args): def parse_args(args): """Parse the arguments.""" - parser = argparse.ArgumentParser(description="Evaluation script for an MobileNetv2 network.") - parser.add_argument("--dataset-path", help="Imagenet validation dataset Tf-records directory.") + parser = argparse.ArgumentParser( + description="Evaluation script for an MobileNetv2 network." + ) + parser.add_argument( + "--dataset-path", help="Imagenet validation dataset Tf-records directory." + ) parser.add_argument("--batch-size", help="Batch size.", type=int, default=32) - parser.add_argument("--default-output-bw", help="Default output bitwidth for quantization.", type=int, default=8,) - parser.add_argument("--default-param-bw", help="Default parameter bitwidth for quantization.", type=int, default=8,) - parser.add_argument("--model-to-eval", help="which model to evaluate. There are two options: fp32 or int8 ", default="int8", choices={"fp32", "int8"}, + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--model-to-eval", + help="which model to evaluate. There are two options: fp32 or int8 ", + default="int8", + choices={"fp32", "int8"}, ) return parser.parse_args(args) diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/MobileNetV2_TF2.md b/aimet_zoo_tensorflow/mobilenet_v2_tf2/MobileNetV2_TF2.md new file mode 100644 index 0000000..d7047f2 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/MobileNetV2_TF2.md @@ -0,0 +1,61 @@ +# TensorFlow MobileNetV2 with TensorFlow 2.4 + +## Setup AI Model Efficiency Toolkit (AIMET) +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.26/packaging/install.md) before proceeding further. This evaluation was run using [AIMET 1.26 for TensorFlow 2.4](https://github.com/quic/aimet/releases/tag/1.26) i.e. please set `release_tag="1.26"` and `AIMET_VARIANT="tf_gpu"` in the above instructions. + +## Additional Dependencies +pip install numpy==1.19.5 + +## Model checkpoint and dataset +The TF2 pretrained mobilenetv2 model is directly imported from package tensorflow.keras.applications + +## Dataset +- ImageNet can be downloaded from here: + - http://www.image-net.org/ +- The directory where the data is located should contains subdirectories, each containing images for a class +- The ImageNet validation dataset should be organized in the following way +```bash +< path to ImageNet validation dataset > +├── n01440764 +├── n01443537 +├── ... +``` + +## Usage +- To run evaluation with QuantSim in AIMET, use the following: +```bash +python aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py \ + --dataset-path \ + --batch-size \ + --model-config +``` +Available model configurations are: + +- mobilenetv2_w8a8 + +- Example: python aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py --dataset-path --batch_size 4 --model-config mobilenetv2_w8a8 + +## Quantization configuration +In the evaluation script included, we have used the default config file, which configures the quantizer ops with the following assumptions: +- Weight quantization: 8 bits, symmetric quantization +- Bias parameters are not quantized +- Activation quantization: 8 bits, asymmetric quantization +- Model inputs are quantized + +## Results +Below are the *top1 accuracy* results of the TensorFlow 2.4 mobilenetv2 model for the imagenet dataset: + + + + + + + + + + + + + + +
Model ConfigurationTop1 (%)
Mobilenet_V2_FP3271.6
Mobilenet_V2 + simple PTQ(w8a8)71.0
diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/__init__.py b/aimet_zoo_tensorflow/mobilenet_v2_tf2/__init__.py new file mode 100644 index 0000000..dc25590 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/__init__.py @@ -0,0 +1,4 @@ +''' +import mobilenetv2 model from tensorflow built-in API +''' +from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2 diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/__init__.py b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/eval_func.py b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/eval_func.py new file mode 100644 index 0000000..89e661d --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/eval_func.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +'''get evaluation function''' + +import os +import numpy as np +from tqdm import tqdm +import tensorflow as tf +from tensorflow.keras.applications.mobilenet_v2 import preprocess_input +from aimet_zoo_tensorflow.mobilenet_v2_tf2.evaluators.preprocess import image_dataset_from_directory + +def get_eval_func(dataset_dir, batch_size, num_iterations=50000): + ''' + :param dataset_dir: data path + :param batch_size: batch size in evaluation and calibration + :param num_iterations: number of images used + :return evaluation function + ''' + def func_wrapper(model, iterations=num_iterations): + ''' + :param model: FP32 model or sim.model + :param iterations: number of images used + ''' + # do center crop + def crop(image): + ratio = tf.cast(224 / 256, tf.float32) + image = tf.image.central_crop(image, ratio) + return image + + # get validation dataset + validation_dir = os.path.join(dataset_dir, 'val') + + # get validation dataset, AIMET is using TF2.4 at the moment and will upgrade TF to 2.10 + tf_version = tf.version.VERSION + tf_sub_version = int(tf_version.split(".")[1]) + validation_ds = None + if tf_sub_version >= 10: + validation_ds = tf.keras.preprocessing.image_dataset_from_directory( + directory=validation_dir, + labels='inferred', + label_mode='categorical', + batch_size=batch_size, + shuffle=False, + crop_to_aspect_ratio=True, + image_size=(256, 256), + interpolation="area", + ) + else: + validation_ds = image_dataset_from_directory( + directory=validation_dir, + labels='inferred', + label_mode='categorical', + batch_size=batch_size, + shuffle=False, + crop_to_aspect_ratio=True, + image_size=(256, 256), + interpolation="area" + ) + + # compute accuracy + top1 = 0 + total = 0 + for (img, label) in tqdm(validation_ds): + img = crop(img) + x = preprocess_input(img) + preds = model.predict(x, batch_size=batch_size) + label = np.where(label)[1] + _, indices = tf.math.top_k(preds, k=1) + indices = np.squeeze(indices) + cnt = tf.reduce_sum(tf.cast(label==indices, tf.float32)).numpy() + top1 += cnt + total += len(label) + if total >= iterations: + break + + return top1 / total + + return func_wrapper diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py new file mode 100644 index 0000000..8a4efe4 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +''' do TF2 mobilenetv2 quantization and evaluation''' +import argparse +import tensorflow as tf +from aimet_zoo_tensorflow.mobilenet_v2_tf2.model.model_definition import MobilenetV2 +from aimet_zoo_tensorflow.mobilenet_v2_tf2.evaluators.eval_func import get_eval_func + +def arguments(): + ''' + parses command line arguments + ''' + parser = argparse.ArgumentParser(description='Arguments for evaluating model') + parser.add_argument('--dataset-path', help='path to image evaluation dataset', type=str) + parser.add_argument('--model-config', help='model configuration to be tested', type=str) + parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) + parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) + parser.add_argument('--batch-size', help='batch_size for loading data', type=int, default=16) + parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) + args = parser.parse_args() + return args + +def main(): + """ Run evaluation """ + gpu_devices = tf.config.experimental.list_physical_devices("GPU") + for device in gpu_devices: + tf.config.experimental.set_memory_growth(device, True) + + args = arguments() + + # Evaluation function + eval_func = get_eval_func(dataset_dir=args.dataset_path, + batch_size=args.batch_size, + num_iterations=50000) + + # Models + model = MobilenetV2(model_config = args.model_config) + model.from_pretrained(quantized=True) + sim = model.get_quantsim(quantized=True) + + # Evaluate original + print("start evaluating FP32 accuracy") + fp32_acc = eval_func(model.model) + print(f'FP32 top1 accuracy: {fp32_acc:0.3f}') + + # Evaluate optimized + print("start evaluating quantized accuracy") + quant_acc = eval_func(sim.model) + print(f'Quantized top1 accuracy: {quant_acc:0.3f}') + +if __name__ == '__main__': + main() diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/preprocess.py b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/preprocess.py new file mode 100644 index 0000000..810c682 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/preprocess.py @@ -0,0 +1,485 @@ +#pylint: skip-file +# ============================================================================== +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import numpy as np +import tensorflow as tf +from tensorflow.python.keras.preprocessing import dataset_utils + +ALLOWLIST_FORMATS = (".bmp", ".gif", ".jpeg", ".jpg", ".png", ".JPEG") + +ResizeMethod = tf.image.ResizeMethod + +_TF_INTERPOLATION_METHODS = { + "bilinear": ResizeMethod.BILINEAR, + "nearest": ResizeMethod.NEAREST_NEIGHBOR, + "bicubic": ResizeMethod.BICUBIC, + "area": ResizeMethod.AREA, + "lanczos3": ResizeMethod.LANCZOS3, + "lanczos5": ResizeMethod.LANCZOS5, + "gaussian": ResizeMethod.GAUSSIAN, + "mitchellcubic": ResizeMethod.MITCHELLCUBIC, +} + +def paths_and_labels_to_dataset( + image_paths, + image_size, + num_channels, + labels, + label_mode, + num_classes, + interpolation, + crop_to_aspect_ratio=False, +): + """Constructs a dataset of images and labels.""" + # TODO(fchollet): consider making num_parallel_calls settable + path_ds = tf.data.Dataset.from_tensor_slices(image_paths) + args = (image_size, num_channels, interpolation, crop_to_aspect_ratio) + img_ds = path_ds.map( + lambda x: load_image(x, *args), num_parallel_calls=tf.data.AUTOTUNE + ) + if label_mode: + label_ds = dataset_utils.labels_to_dataset( + labels, label_mode, num_classes + ) + img_ds = tf.data.Dataset.zip((img_ds, label_ds)) + return img_ds + +def load_image( + path, image_size, num_channels, interpolation, crop_to_aspect_ratio=False +): + """Load an image from a path and resize it.""" + + img = tf.io.read_file(path) + + img = tf.image.decode_image( + img, channels=num_channels, expand_animations=False + ) + + # crop_to_aspect_ratio = False + if crop_to_aspect_ratio: + # img = image_utils.smart_resize( + # img, image_size, interpolation=interpolation + # ) + img = smart_resize( + img, image_size, interpolation=interpolation + ) + else: + img = tf.image.resize(img, image_size, method=interpolation) + img.set_shape((image_size[0], image_size[1], num_channels)) + return img + +def image_dataset_from_directory( + directory, + labels="inferred", + label_mode="int", + class_names=None, + color_mode="rgb", + batch_size=32, + image_size=(256, 256), + shuffle=True, + seed=None, + validation_split=None, + subset=None, + interpolation="bilinear", + follow_links=False, + crop_to_aspect_ratio=False, + **kwargs, +): + """Generates a `tf.data.Dataset` from image files in a directory. + If your directory structure is: + ``` + main_directory/ + ...class_a/ + ......a_image_1.jpg + ......a_image_2.jpg + ...class_b/ + ......b_image_1.jpg + ......b_image_2.jpg + ``` + Then calling `image_dataset_from_directory(main_directory, + labels='inferred')` will return a `tf.data.Dataset` that yields batches of + images from the subdirectories `class_a` and `class_b`, together with labels + 0 and 1 (0 corresponding to `class_a` and 1 corresponding to `class_b`). + Supported image formats: jpeg, png, bmp, gif. + Animated gifs are truncated to the first frame. + Args: + directory: Directory where the data is located. + If `labels` is "inferred", it should contain + subdirectories, each containing images for a class. + Otherwise, the directory structure is ignored. + labels: Either "inferred" + (labels are generated from the directory structure), + None (no labels), + or a list/tuple of integer labels of the same size as the number of + image files found in the directory. Labels should be sorted according + to the alphanumeric order of the image file paths + (obtained via `os.walk(directory)` in Python). + label_mode: String describing the encoding of `labels`. Options are: + - 'int': means that the labels are encoded as integers + (e.g. for `sparse_categorical_crossentropy` loss). + - 'categorical' means that the labels are + encoded as a categorical vector + (e.g. for `categorical_crossentropy` loss). + - 'binary' means that the labels (there can be only 2) + are encoded as `float32` scalars with values 0 or 1 + (e.g. for `binary_crossentropy`). + - None (no labels). + class_names: Only valid if "labels" is "inferred". This is the explicit + list of class names (must match names of subdirectories). Used + to control the order of the classes + (otherwise alphanumerical order is used). + color_mode: One of "grayscale", "rgb", "rgba". Default: "rgb". + Whether the images will be converted to + have 1, 3, or 4 channels. + batch_size: Size of the batches of data. Default: 32. + If `None`, the data will not be batched + (the dataset will yield individual samples). + image_size: Size to resize images to after they are read from disk, + specified as `(height, width)`. Defaults to `(256, 256)`. + Since the pipeline processes batches of images that must all have + the same size, this must be provided. + shuffle: Whether to shuffle the data. Default: True. + If set to False, sorts the data in alphanumeric order. + seed: Optional random seed for shuffling and transformations. + validation_split: Optional float between 0 and 1, + fraction of data to reserve for validation. + subset: Subset of the data to return. + One of "training", "validation" or "both". + Only used if `validation_split` is set. + When `subset="both"`, the utility returns a tuple of two datasets + (the training and validation datasets respectively). + interpolation: String, the interpolation method used when resizing images. + Defaults to `bilinear`. Supports `bilinear`, `nearest`, `bicubic`, + `area`, `lanczos3`, `lanczos5`, `gaussian`, `mitchellcubic`. + follow_links: Whether to visit subdirectories pointed to by symlinks. + Defaults to False. + crop_to_aspect_ratio: If True, resize the images without aspect + ratio distortion. When the original aspect ratio differs from the target + aspect ratio, the output image will be cropped so as to return the + largest possible window in the image (of size `image_size`) that matches + the target aspect ratio. By default (`crop_to_aspect_ratio=False`), + aspect ratio may not be preserved. + **kwargs: Legacy keyword arguments. + Returns: + A `tf.data.Dataset` object. + - If `label_mode` is None, it yields `float32` tensors of shape + `(batch_size, image_size[0], image_size[1], num_channels)`, + encoding images (see below for rules regarding `num_channels`). + - Otherwise, it yields a tuple `(images, labels)`, where `images` + has shape `(batch_size, image_size[0], image_size[1], num_channels)`, + and `labels` follows the format described below. + Rules regarding labels format: + - if `label_mode` is `int`, the labels are an `int32` tensor of shape + `(batch_size,)`. + - if `label_mode` is `binary`, the labels are a `float32` tensor of + 1s and 0s of shape `(batch_size, 1)`. + - if `label_mode` is `categorical`, the labels are a `float32` tensor + of shape `(batch_size, num_classes)`, representing a one-hot + encoding of the class index. + Rules regarding number of channels in the yielded images: + - if `color_mode` is `grayscale`, + there's 1 channel in the image tensors. + - if `color_mode` is `rgb`, + there are 3 channels in the image tensors. + - if `color_mode` is `rgba`, + there are 4 channels in the image tensors. + """ + if "smart_resize" in kwargs: + crop_to_aspect_ratio = kwargs.pop("smart_resize") + if kwargs: + raise TypeError(f"Unknown keywords argument(s): {tuple(kwargs.keys())}") + if labels not in ("inferred", None): + if not isinstance(labels, (list, tuple)): + raise ValueError( + "`labels` argument should be a list/tuple of integer labels, " + "of the same size as the number of image files in the target " + "directory. If you wish to infer the labels from the " + "subdirectory " + 'names in the target directory, pass `labels="inferred"`. ' + "If you wish to get a dataset that only contains images " + f"(no labels), pass `labels=None`. Received: labels={labels}" + ) + if class_names: + raise ValueError( + "You can only pass `class_names` if " + f'`labels="inferred"`. Received: labels={labels}, and ' + f"class_names={class_names}" + ) + if label_mode not in {"int", "categorical", "binary", None}: + raise ValueError( + '`label_mode` argument must be one of "int", ' + '"categorical", "binary", ' + f"or None. Received: label_mode={label_mode}" + ) + if labels is None or label_mode is None: + labels = None + label_mode = None + if color_mode == "rgb": + num_channels = 3 + elif color_mode == "rgba": + num_channels = 4 + elif color_mode == "grayscale": + num_channels = 1 + else: + raise ValueError( + '`color_mode` must be one of {"rgb", "rgba", "grayscale"}. ' + f"Received: color_mode={color_mode}" + ) + # interpolation = image_utils.get_interpolation(interpolation) + interpolation = get_interpolation(interpolation) + # dataset_utils.check_validation_split_arg( + # validation_split, subset, shuffle, seed + # ) + + if seed is None: + seed = np.random.randint(1e6) + image_paths, labels, class_names = dataset_utils.index_directory( + directory, + labels, + formats=ALLOWLIST_FORMATS, + class_names=class_names, + shuffle=shuffle, + seed=seed, + follow_links=follow_links, + ) + + if label_mode == "binary" and len(class_names) != 2: + raise ValueError( + 'When passing `label_mode="binary"`, there must be exactly 2 ' + f"class_names. Received: class_names={class_names}" + ) + + if subset == "both": + ( + image_paths_train, + labels_train, + ) = dataset_utils.get_training_or_validation_split( + image_paths, labels, validation_split, "training" + ) + ( + image_paths_val, + labels_val, + ) = dataset_utils.get_training_or_validation_split( + image_paths, labels, validation_split, "validation" + ) + if not image_paths_train: + raise ValueError( + f"No training images found in directory {directory}. " + f"Allowed formats: {ALLOWLIST_FORMATS}" + ) + if not image_paths_val: + raise ValueError( + f"No validation images found in directory {directory}. " + f"Allowed formats: {ALLOWLIST_FORMATS}" + ) + train_dataset = paths_and_labels_to_dataset( + image_paths=image_paths_train, + image_size=image_size, + num_channels=num_channels, + labels=labels_train, + label_mode=label_mode, + num_classes=len(class_names), + interpolation=interpolation, + crop_to_aspect_ratio=crop_to_aspect_ratio, + ) + val_dataset = paths_and_labels_to_dataset( + image_paths=image_paths_val, + image_size=image_size, + num_channels=num_channels, + labels=labels_val, + label_mode=label_mode, + num_classes=len(class_names), + interpolation=interpolation, + crop_to_aspect_ratio=crop_to_aspect_ratio, + ) + train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE) + val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE) + if batch_size is not None: + if shuffle: + # Shuffle locally at each iteration + train_dataset = train_dataset.shuffle( + buffer_size=batch_size * 8, seed=seed + ) + train_dataset = train_dataset.batch(batch_size) + val_dataset = val_dataset.batch(batch_size) + else: + if shuffle: + train_dataset = train_dataset.shuffle( + buffer_size=1024, seed=seed + ) + + # Users may need to reference `class_names`. + train_dataset.class_names = class_names + val_dataset.class_names = class_names + # Include file paths for images as attribute. + train_dataset.file_paths = image_paths_train + val_dataset.file_paths = image_paths_val + dataset = [train_dataset, val_dataset] + else: + image_paths, labels = dataset_utils.get_training_or_validation_split( + image_paths, labels, validation_split, subset + ) + if not image_paths: + raise ValueError( + f"No images found in directory {directory}. " + f"Allowed formats: {ALLOWLIST_FORMATS}" + ) + + dataset = paths_and_labels_to_dataset( + image_paths=image_paths, + image_size=image_size, + num_channels=num_channels, + labels=labels, + label_mode=label_mode, + num_classes=len(class_names), + interpolation=interpolation, + crop_to_aspect_ratio=crop_to_aspect_ratio, + ) + dataset = dataset.prefetch(tf.data.AUTOTUNE) + if batch_size is not None: + if shuffle: + # Shuffle locally at each iteration + dataset = dataset.shuffle(buffer_size=batch_size * 8, seed=seed) + dataset = dataset.batch(batch_size) + else: + if shuffle: + dataset = dataset.shuffle(buffer_size=1024, seed=seed) + + # Users may need to reference `class_names`. + dataset.class_names = class_names + # Include file paths for images as attribute. + dataset.file_paths = image_paths + return dataset + + +def smart_resize(x, size, interpolation="bilinear"): + """Resize images to a target size without aspect ratio distortion. + Warning: `tf.keras.preprocessing.image.smart_resize` is not recommended for + new code. Prefer `tf.keras.layers.Resizing`, which provides the same + functionality as a preprocessing layer and adds `tf.RaggedTensor` support. + See the [preprocessing layer guide]( + https://www.tensorflow.org/guide/keras/preprocessing_layers) + for an overview of preprocessing layers. + TensorFlow image datasets typically yield images that have each a different + size. However, these images need to be batched before they can be + processed by Keras layers. To be batched, images need to share the same + height and width. + You could simply do: + ```python + size = (200, 200) + ds = ds.map(lambda img: tf.image.resize(img, size)) + ``` + However, if you do this, you distort the aspect ratio of your images, since + in general they do not all have the same aspect ratio as `size`. This is + fine in many cases, but not always (e.g. for GANs this can be a problem). + Note that passing the argument `preserve_aspect_ratio=True` to `resize` + will preserve the aspect ratio, but at the cost of no longer respecting the + provided target size. Because `tf.image.resize` doesn't crop images, + your output images will still have different sizes. + This calls for: + ```python + size = (200, 200) + ds = ds.map(lambda img: smart_resize(img, size)) + ``` + Your output images will actually be `(200, 200)`, and will not be distorted. + Instead, the parts of the image that do not fit within the target size + get cropped out. + The resizing process is: + 1. Take the largest centered crop of the image that has the same aspect + ratio as the target size. For instance, if `size=(200, 200)` and the input + image has size `(340, 500)`, we take a crop of `(340, 340)` centered along + the width. + 2. Resize the cropped image to the target size. In the example above, + we resize the `(340, 340)` crop to `(200, 200)`. + Args: + x: Input image or batch of images (as a tensor or NumPy array). Must be in + format `(height, width, channels)` or `(batch_size, height, width, + channels)`. + size: Tuple of `(height, width)` integer. Target size. + interpolation: String, interpolation to use for resizing. Defaults to + `'bilinear'`. Supports `bilinear`, `nearest`, `bicubic`, `area`, + `lanczos3`, `lanczos5`, `gaussian`, `mitchellcubic`. + Returns: + Array with shape `(size[0], size[1], channels)`. If the input image was a + NumPy array, the output is a NumPy array, and if it was a TF tensor, + the output is a TF tensor. + """ + if len(size) != 2: + raise ValueError( + f"Expected `size` to be a tuple of 2 integers, but got: {size}." + ) + img = tf.convert_to_tensor(x) + if img.shape.rank is not None: + if img.shape.rank < 3 or img.shape.rank > 4: + raise ValueError( + "Expected an image array with shape `(height, width, " + "channels)`, or `(batch_size, height, width, channels)`, but " + f"got input with incorrect rank, of shape {img.shape}." + ) + shape = tf.shape(img) + height, width = shape[-3], shape[-2] + target_height, target_width = size + if img.shape.rank is not None: + static_num_channels = img.shape[-1] + else: + static_num_channels = None + + crop_height = tf.cast( + tf.cast(width * target_height, "float32") / target_width, "int32" + ) + crop_width = tf.cast( + tf.cast(height * target_width, "float32") / target_height, "int32" + ) + + # Set back to input height / width if crop_height / crop_width is not + # smaller. + crop_height = tf.minimum(height, crop_height) + crop_width = tf.minimum(width, crop_width) + + crop_box_hstart = tf.cast( + tf.cast(height - crop_height, "float32") / 2, "int32" + ) + crop_box_wstart = tf.cast( + tf.cast(width - crop_width, "float32") / 2, "int32" + ) + + if img.shape.rank == 4: + crop_box_start = tf.stack([0, crop_box_hstart, crop_box_wstart, 0]) + crop_box_size = tf.stack([-1, crop_height, crop_width, -1]) + else: + crop_box_start = tf.stack([crop_box_hstart, crop_box_wstart, 0]) + crop_box_size = tf.stack([crop_height, crop_width, -1]) + + img = tf.slice(img, crop_box_start, crop_box_size) + img = tf.image.resize(images=img, size=size, method=interpolation) + # Apparent bug in resize_images_v2 may cause shape to be lost + if img.shape.rank is not None: + if img.shape.rank == 4: + img.set_shape((None, None, None, static_num_channels)) + if img.shape.rank == 3: + img.set_shape((None, None, static_num_channels)) + if isinstance(x, np.ndarray): + return img.numpy() + return img + +def get_interpolation(interpolation): + interpolation = interpolation.lower() + if interpolation not in _TF_INTERPOLATION_METHODS: + raise NotImplementedError( + "Value not recognized for `interpolation`: {}. Supported values " + "are: {}".format(interpolation, _TF_INTERPOLATION_METHODS.keys()) + ) + return _TF_INTERPOLATION_METHODS[interpolation] \ No newline at end of file diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/model/__init__.py b/aimet_zoo_tensorflow/mobilenet_v2_tf2/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/model/model_cards/mobilenetv2_w8a8.json b/aimet_zoo_tensorflow/mobilenet_v2_tf2/model/model_cards/mobilenetv2_w8a8.json new file mode 100644 index 0000000..5fc69f4 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/model/model_cards/mobilenetv2_w8a8.json @@ -0,0 +1,24 @@ +{ + "name": "mobilenetv2", + "framework": "tensorflow2.x", + "task": "image classification", + "input_shape": [null, 256, 256, 3], + "dataset": "imagenet", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf", + "techniques": "simple PTQ" + } + }, + "artifacts": { + "url_pre_opt_weights": null, + "url_post_opt_weights": null, + "url_adaround_encodings": null, + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/tensorflow2-mobilenetv2/mobilenetv2_w8a8.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.25/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} \ No newline at end of file diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/model/model_definition.py b/aimet_zoo_tensorflow/mobilenet_v2_tf2/model/model_definition.py new file mode 100644 index 0000000..a8ff0f4 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/model/model_definition.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +''' Define MobilenetV2 model and do Quantsim''' +import os +import json +from aimet_tensorflow.keras.quantsim import QuantizationSimModel # pylint: disable=import-error +from aimet_tensorflow.keras.batch_norm_fold import fold_all_batch_norms # pylint: disable=import-error +from aimet_zoo_tensorflow.mobilenet_v2_tf2 import MobileNetV2 +from aimet_zoo_tensorflow.common.downloader import Downloader + +class MobilenetV2(Downloader): + """MobilenetV2 parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + # pylint: disable=unused-argument + def __init__(self, model_config = None, **kwargs): + """ + :param model_config: named model config from which to obtain model artifacts and arguments. + If provided, overwrites the other arguments passed to this object + """ + parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + self.cfg = False + if model_config: + config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + if os.path.exists(config_filepath): + with open(config_filepath, encoding='UTF-8') as f_in: + self.cfg = json.load(f_in) + if self.cfg: + Downloader.__init__(self, + url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], + url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], + url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], + url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], + url_aimet_config = self.cfg['artifacts']['url_aimet_config'], + model_dir = parent_dir, + model_config = model_config) + self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) + + self.model = MobileNetV2(input_shape=None, + alpha=1.0, + include_top=True, + weights="imagenet", + input_tensor=None, + pooling=None, + classes=1000) + + def from_pretrained(self, quantized=False): + """download config file and encodings from model cards""" + if not self.cfg: + raise NotImplementedError('There are no pretrained weights available for the model_config passed') + self._download_pre_opt_weights() + self._download_post_opt_weights() + self._download_aimet_config() + self._download_aimet_encodings() + self._download_adaround_encodings() + if quantized: + # bn folding to weights + _, self.model = fold_all_batch_norms(self.model) + + + def get_quantsim(self, quantized=False): + """get quantsim object with pre-loaded encodings""" + if not self.cfg: + raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + if quantized: + self.from_pretrained(quantized=True) + else: + self.from_pretrained(quantized=False) + + kwargs = { + 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], + 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], + 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], + 'config_file': self.path_aimet_config, + } + + sim = QuantizationSimModel(self.model, **kwargs) + + # load encoding file back to sim + if self.path_aimet_encodings and quantized: + sim.load_encodings_to_sim(self.path_aimet_encodings) + # This is the temporary solution for slow speed issue after loading_encoding_to_sim + # it will be removed once official solution available + # pylint: disable=protected-access + op_mode = sim._param_op_mode_after_analysis(sim.quant_scheme) + # pylint: disable=protected-access + sim._set_op_mode_parameters(op_mode) + print('load_encodings_to_sim finished!') + + # load adaround encoding file back to sim + if self.path_adaround_encodings and quantized: + sim.set_and_freeze_param_encodings(self.path_adaround_encodings) + print('set_and_freeze_param_encodings finished!') + + return sim diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/requirements.txt b/aimet_zoo_tensorflow/mobilenet_v2_tf2/requirements.txt new file mode 100644 index 0000000..1e99151 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/requirements.txt @@ -0,0 +1 @@ +numpy==1.19.5 diff --git a/aimet_zoo_tensorflow/pose_estimation/evaluators/pose_estimation_quanteval.py b/aimet_zoo_tensorflow/pose_estimation/evaluators/pose_estimation_quanteval.py index a81edf3..2d700fc 100755 --- a/aimet_zoo_tensorflow/pose_estimation/evaluators/pose_estimation_quanteval.py +++ b/aimet_zoo_tensorflow/pose_estimation/evaluators/pose_estimation_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3.6 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W0622,I1101 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W0622,I1101 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -9,6 +9,7 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """Quantsim evaluation script for pose estimation""" +# pylint: disable=wrong-import-order import os import math import argparse @@ -44,9 +45,18 @@ def non_maximum_suppression(map, thresh): map_down = np.zeros(map_s.shape) map_down[:, :-1] = map_s[:, 1:] - peaks_binary = np.logical_and.reduce((map_s >= map_left, map_s >= map_right, map_s >= map_up, map_s >= map_down, - map_s > thresh)) - peaks = zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]) # note reverse + peaks_binary = np.logical_and.reduce( + ( + map_s >= map_left, + map_s >= map_right, + map_s >= map_up, + map_s >= map_down, + map_s > thresh, + ) + ) + peaks = zip( + np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0] + ) # note reverse peaks_with_score = [x + (map[x[1], x[0]],) for x in peaks] return peaks_with_score @@ -92,8 +102,7 @@ def decode_output(data, stride, padding, input_shape, image_shape): output = cv2.resize( output, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC ) - output = output[: input_shape[0] - padding[2], - : input_shape[1] - padding[3], :] + output = output[: input_shape[0] - padding[2], : input_shape[1] - padding[3], :] output = cv2.resize( output, (image_shape[1], image_shape[0]), interpolation=cv2.INTER_CUBIC ) @@ -152,19 +161,16 @@ def run_session(session, output_names, input_name, image, fast=False): ) else: image_encoded, pad = encode_input(image, scale, stride, padValue) - image_encoded_ = preprocess( - image_encoded, [ - "addchannel", "normalize", "bgr"]) + image_encoded_ = preprocess(image_encoded, ["addchannel", "normalize", "bgr"]) paf, heatmap = session.run( - output_names, feed_dict={ - session.graph.get_tensor_by_name(input_name): image_encoded_}, ) + output_names, + feed_dict={session.graph.get_tensor_by_name(input_name): image_encoded_}, + ) if fast: paf = cv2.resize(paf[0], (image.shape[1], image.shape[0])) - heatmap = cv2.resize( - heatmap[0], dsize=( - image.shape[1], image.shape[0])) + heatmap = cv2.resize(heatmap[0], dsize=(image.shape[1], image.shape[0])) else: paf = paf.transpose((0, 3, 1, 2)) heatmap = heatmap.transpose((0, 3, 1, 2)) @@ -212,22 +218,21 @@ def get_limb_consistancy(paf, start_keypoint, end_keypoint, image_h, div_num=10) ) ) - vec_paf_x = np.array([paf[vec_paf[k][1], vec_paf[k][0], 0] - for k in range(div_num)]) - vec_paf_y = np.array([paf[vec_paf[k][1], vec_paf[k][0], 1] - for k in range(div_num)]) + vec_paf_x = np.array([paf[vec_paf[k][1], vec_paf[k][0], 0] for k in range(div_num)]) + vec_paf_y = np.array([paf[vec_paf[k][1], vec_paf[k][0], 1] for k in range(div_num)]) - vec_sims = np.multiply( - vec_paf_x, vec_key[0]) + np.multiply(vec_paf_y, vec_key[1]) + vec_sims = np.multiply(vec_paf_x, vec_key[0]) + np.multiply(vec_paf_y, vec_key[1]) vec_sims_prior = vec_sims.mean() + min(0.5 * image_h / vec_key_norm - 1, 0) return vec_sims, vec_sims_prior -#pylint: disable=C1801 + +# pylint: disable=len-as-condition def connect_keypoints(image_shape, keypoints, paf, limbs, limbsInds): """connect keypoints""" thre2 = 0.05 connections = [] + #pylint: disable=consider-using-enumerate for k in range(len(limbsInds)): paf_limb = paf[:, :, limbsInds[k]] limb_strs = keypoints[limbs[k][0]] @@ -275,9 +280,8 @@ def create_skeletons(keypoints, connections, limbs): # the second last number in each row is the score of the overall # configuration skeletons = -1 * np.ones((0, 20)) - keypoints_flatten = np.array( - [item for sublist in keypoints for item in sublist]) - + keypoints_flatten = np.array([item for sublist in keypoints for item in sublist]) + #pylint: disable=consider-using-enumerate for k in range(len(limbs)): if connections[k] != []: detected_str = connections[k][:, 0] @@ -287,6 +291,7 @@ def create_skeletons(keypoints, connections, limbs): for i in range(len(connections[k])): found = 0 subset_idx = [-1, -1] + #pylint: disable=consider-using-enumerate for j in range(len(skeletons)): if ( skeletons[j][limb_str] == detected_str[i] @@ -330,12 +335,15 @@ def create_skeletons(keypoints, connections, limbs): row[limb_str] = detected_str[i] row[limb_end] = detected_end[i] row[-1] = 2 - row[-2] = (sum(keypoints_flatten[connections[k] - [i, :2].astype(int), 2]) + connections[k][i][2]) + row[-2] = ( + sum(keypoints_flatten[connections[k][i, :2].astype(int), 2]) + + connections[k][i][2] + ) skeletons = np.vstack([skeletons, row]) # delete some rows of subset which has few parts occur deleteIdx = [] + #pylint: disable=C0200 for i in range(len(skeletons)): if skeletons[i][-1] < 4 or skeletons[i][-2] / skeletons[i][-1] < 0.4: deleteIdx.append(i) @@ -395,23 +403,21 @@ def estimate_pose(image_shape, heatmap, paf): keypoints = get_keypoints(heatmap) - connections = connect_keypoints( - image_shape, keypoints, paf, limbs, limbsInd) + connections = connect_keypoints(image_shape, keypoints, paf, limbs, limbsInd) skeletons = create_skeletons(keypoints, connections, limbs) - return skeletons, np.array( - [item for sublist in keypoints for item in sublist]) + return skeletons, np.array([item for sublist in keypoints for item in sublist]) def parse_results(skeletons, points): """parse results""" - coco_indices = [0, -1, 6, 8, 10, 5, 7, 9, - 12, 14, 16, 11, 13, 15, 2, 1, 4, 3] + coco_indices = [0, -1, 6, 8, 10, 5, 7, 9, 12, 14, 16, 11, 13, 15, 2, 1, 4, 3] skeletons_out, scores = [], [] for score, keypoints in zip(skeletons["scores"], skeletons["keypoints"]): skeleton = [] + #pylint: disable=C0200 for p in range(len(keypoints)): if p == 1: continue @@ -433,6 +439,7 @@ def parse_results(skeletons, points): class COCOWrapper: """Coco wrapper class""" + def __init__(self, coco_path, num_imgs=None): self.coco_path = coco_path self.num_imgs = num_imgs @@ -465,13 +472,13 @@ def evaluate_json(self, obj): cocoEval.accumulate() cocoEval.summarize() return cocoEval.stats[0::5] -#pylint: disable=R0201 + + # pylint: disable=R0201 def get_results_json(self, results, imgs): """get results json""" results_obj = [] for img, result in list(zip(imgs, results)): - for score, skeleton in list( - zip(result["scores"], result["skeletons"])): + for score, skeleton in list(zip(result["scores"], result["skeletons"])): obj = { "image_id": img["id"], "category_id": 1, @@ -493,7 +500,7 @@ def get_results_json(self, results, imgs): @property def cocoGT(self): - """coco ground truth """ + """coco ground truth""" annType = "keypoints" prefix = "person_keypoints" print("Initializing demo for *%s* results." % (annType)) @@ -506,12 +513,12 @@ def cocoGT(self): cocoGT = COCO(annFile) if not cocoGT: - raise AttributeError( - "COCO ground truth demo failed to initialize!") + raise AttributeError("COCO ground truth demo failed to initialize!") return cocoGT -#pylint: disable=W0612 + +# pylint: disable=W0612 def evaluate_session( session, coco_path, input_name, output_names, num_imgs=None, fast=False ): @@ -525,8 +532,7 @@ def evaluate_session( for i, img in enumerate(imgs): image = cv2.imread(image_path + img["file_name"]) # B,G,R order - heatmap, paf = run_session( - session, output_names, input_name, image, fast) + heatmap, paf = run_session(session, output_names, input_name, image, fast) skeletons, keypoints = estimate_pose(image.shape, heatmap, paf) results.append(parse_results(skeletons, keypoints)) @@ -578,15 +584,14 @@ def download_weights(): if not os.path.exists("./pose_estimation_tensorflow"): url_checkpoint = "https://github.com/quic/aimet-model-zoo/releases/download/pose_estimation/pose_estimation_tensorflow.tar.gz" - urllib.request.urlretrieve( - url_checkpoint, - "pose_estimation_tensorflow.tar.gz") + urllib.request.urlretrieve(url_checkpoint, "pose_estimation_tensorflow.tar.gz") with tarfile.open("pose_estimation_tensorflow.tar.gz") as pth_weights: pth_weights.extractall("./pose_estimation_tensorflow/") class ModelConfig: """hardcoded model configurations""" + def __init__(self, args): """ hardcode values added on parsearg arguments @@ -626,6 +631,7 @@ def pose_estimation_quanteval(args): ) if not tf.test.gpu_device_name(): + #pylint:disable = broad-exception-raised raise Exception(" GPU not available") partial_eval = partial( diff --git a/aimet_zoo_tensorflow/resnet50/evaluators/resnet50_v1_quanteval.py b/aimet_zoo_tensorflow/resnet50/evaluators/resnet50_v1_quanteval.py index ed97135..dc1eae6 100644 --- a/aimet_zoo_tensorflow/resnet50/evaluators/resnet50_v1_quanteval.py +++ b/aimet_zoo_tensorflow/resnet50/evaluators/resnet50_v1_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -9,6 +9,8 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """Quantsim evaluation for resnet50""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet import ast import tarfile import urllib.request @@ -29,8 +31,6 @@ from nets import nets_factory - - os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" @@ -49,12 +49,7 @@ def download_weights(): urllib.request.urlretrieve(URL, "default_config.json") -def wrap_preprocessing( - preprocessing, - height, - width, - num_classes, - labels_offset): +def wrap_preprocessing(preprocessing, height, width, num_classes, labels_offset): """Wrap preprocessing function to do parsing of TFrecords.""" def parse(serialized_example): @@ -77,9 +72,10 @@ def parse(serialized_example): return parse -#pylint: disable=W0612 + +# pylint: disable=W0612 def run_evaluation(args): - """run evaluatioin and build graph definition for evaluation """ + """run evaluatioin and build graph definition for evaluation""" # Build graph definition with tf.Graph().as_default(): # Create iterator @@ -103,20 +99,19 @@ def run_evaluation(args): images, labels = iterator.get_next() network_fn = nets_factory.get_network_fn( - args.model_name, num_classes=( - 1001 - args.labels_offset), is_training=False) + args.model_name, num_classes=(1001 - args.labels_offset), is_training=False + ) with tf.device("/cpu:0"): - images = tf.placeholder_with_default(images, shape=( - None, args.image_size, args.image_size, 3), name="input") + images = tf.placeholder_with_default( + images, shape=(None, args.image_size, args.image_size, 3), name="input" + ) labels = tf.placeholder_with_default( labels, shape=(None, 1001 - args.labels_offset), name="labels" ) logits, end_points = network_fn(images) confidences = tf.nn.softmax(logits, axis=1, name="confidences") - categorical_preds = tf.argmax( - confidences, axis=1, name="categorical_preds") - categorical_labels = tf.argmax( - labels, axis=1, name="categorical_labels") + categorical_preds = tf.argmax(confidences, axis=1, name="categorical_preds") + categorical_labels = tf.argmax(labels, axis=1, name="categorical_labels") correct_predictions = tf.equal(categorical_labels, categorical_preds) top1_acc = tf.reduce_mean( tf.cast(correct_predictions, tf.float32), name="top1-acc" @@ -241,10 +236,8 @@ def eval_quantized_model(sess, args, logits): print() print("Evaluation Summary") - print( - f"Optimized Model | FP32 Environment | Accuracy: {acc_optim_fp32}") - print( - f"Optimized Model | INT8 Environment | Accuracy: {acc_optim_int8}") + print(f"Optimized Model | FP32 Environment | Accuracy: {acc_optim_fp32}") + print(f"Optimized Model | INT8 Environment | Accuracy: {acc_optim_int8}") if args.eval_quantized: eval_quantized_model(sess, args, logits) @@ -262,14 +255,8 @@ def parse_args(args): parser = argparse.ArgumentParser( description="Evaluation script for an Resnet 50 network." ) - parser.add_argument( - "--dataset-path", - help="Imagenet eval dataset directory.") - parser.add_argument( - "--batch-size", - help="Batch size.", - type=int, - default=32) + parser.add_argument("--dataset-path", help="Imagenet eval dataset directory.") + parser.add_argument("--batch-size", help="Batch size.", type=int, default=32) parser.add_argument( "--default-output-bw", help="Default output bitwidth for quantization.", @@ -292,6 +279,7 @@ def parse_args(args): class ModelConfig: """hardcoded model configuration""" + def __init__(self, args): self.model_name = "resnet_v1_50" self.labels_offset = 1 diff --git a/aimet_zoo_tensorflow/resnet50_tf2/ResNet50_TF2.md b/aimet_zoo_tensorflow/resnet50_tf2/ResNet50_TF2.md new file mode 100755 index 0000000..70a1baf --- /dev/null +++ b/aimet_zoo_tensorflow/resnet50_tf2/ResNet50_TF2.md @@ -0,0 +1,61 @@ +# Tensorflow ResNet50 for TensorFlow 2.4 + +## Setup AI Model Efficiency Toolkit (AIMET) +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.25/packaging/install.md) before proceeding further. This evaluation was run using [AIMET 1.25 for TensorFlow 2.4](https://github.com/quic/aimet/releases/tag/1.25) i.e. please set `release_tag="1.25"` and `AIMET_VARIANT="tf_gpu"` in the above instructions. + +## Additional Dependencies +pip install numpy==1.19.5 + +## Model checkpoint and dataset +The TF2 pretrained resnet50 model is directly imported from package tensorflow.keras.applications + +## Dataset +- ImageNet can be downloaded from here: + - http://www.image-net.org/ +- The directory where the data is located should contains subdirectories, each containing images for a class +- The ImageNet validation dataset should be organized in the following way +```bash +< path to ImageNet validation dataset > +├── n01440764 +├── n01443537 +├── ... +``` + +## Usage +- To run evaluation with QuantSim in AIMET, use the following: +```bash +python aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py \ + --dataset-path \ + --batch-size \ + --model-config +``` +Available model configurations are: + +- resnet50_w8a8 + +- Example : python aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py --dataset-path --batch-size 4 --model-config resnet50_w8a8 + +## Quantization configuration +In the evaluation script included, we have used the default config file, which configures the quantizer ops with the following assumptions: +- Weight quantization: 8 bits, symmetric quantization +- Bias parameters are not quantized +- Activation quantization: 8 bits, asymmetric quantization +- Model inputs are quantized + +## Results +Below are the *top1 accuracy* results of the TensorFlow 2.4 resnet50 model for the imagenet dataset: + + + + + + + + + + + + + + +
Model ConfigurationTop1 (%)
Resnet50_FP3274.9
Resnet50 + simple PTQ(w8a8)74.8
diff --git a/aimet_zoo_tensorflow/resnet50_tf2/__init__.py b/aimet_zoo_tensorflow/resnet50_tf2/__init__.py new file mode 100755 index 0000000..c42065e --- /dev/null +++ b/aimet_zoo_tensorflow/resnet50_tf2/__init__.py @@ -0,0 +1,4 @@ +''' +import resnet50 model from tensorflow built-in API +''' +from tensorflow.keras.applications.resnet import ResNet50 diff --git a/aimet_zoo_tensorflow/resnet50_tf2/evaluators/__init__.py b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/resnet50_tf2/evaluators/eval_func.py b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/eval_func.py new file mode 100755 index 0000000..2abe299 --- /dev/null +++ b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/eval_func.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +'''get evaluation function''' + +import os +import numpy as np +from tqdm import tqdm +import tensorflow as tf +from tensorflow.keras.applications.resnet import preprocess_input +from aimet_zoo_tensorflow.resnet50_tf2.evaluators.preprocess import image_dataset_from_directory + +def get_eval_func(dataset_dir, batch_size, num_iterations=50000): + ''' + :param dataset_dir: data path + :param batch_size: batch size in evaluation and calibration + :param num_iterations: number of images used + :return evaluation function + ''' + def func_wrapper(model, iterations=num_iterations): + ''' + :param model: FP32 model or sim.model + :param iterations: number of images used + ''' + # do center crop + def crop(image): + ratio = tf.cast(224 / 256, tf.float32) + image = tf.image.central_crop(image, ratio) + return image + + # get validation dataset + validation_dir = os.path.join(dataset_dir, 'val') + + # get validation dataset, AIMET is using TF2.4 at the moment and will upgrade TF to 2.10 + tf_version = tf.version.VERSION + tf_sub_version = int(tf_version.split(".")[1]) + validation_ds = None + if tf_sub_version >= 10: + validation_ds = tf.keras.preprocessing.image_dataset_from_directory( + directory=validation_dir, + labels='inferred', + label_mode='categorical', + batch_size=batch_size, + shuffle=False, + crop_to_aspect_ratio=True, + image_size=(256, 256), + interpolation="area", + ) + else: + validation_ds = image_dataset_from_directory( + directory=validation_dir, + labels='inferred', + label_mode='categorical', + batch_size=batch_size, + shuffle=False, + crop_to_aspect_ratio=True, + image_size=(256, 256), + interpolation="area" + ) + + # compute accuracy + top1 = 0 + total = 0 + for (img, label) in tqdm(validation_ds): + img = crop(img) + x = preprocess_input(img) + preds = model.predict(x, batch_size=batch_size) + label = np.where(label)[1] + _, indices = tf.math.top_k(preds, k=1) + indices = np.squeeze(indices) + cnt = tf.reduce_sum(tf.cast(label==indices, tf.float32)).numpy() + top1 += cnt + total += len(label) + if total >= iterations: + break + + return top1 / total + + return func_wrapper diff --git a/aimet_zoo_tensorflow/resnet50_tf2/evaluators/preprocess.py b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/preprocess.py new file mode 100755 index 0000000..4fd6927 --- /dev/null +++ b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/preprocess.py @@ -0,0 +1,485 @@ +#pylint: skip-file +# ============================================================================== +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import numpy as np +import tensorflow as tf +from tensorflow.python.keras.preprocessing import dataset_utils + +ALLOWLIST_FORMATS = (".bmp", ".gif", ".jpeg", ".jpg", ".png", ".JPEG") + +ResizeMethod = tf.image.ResizeMethod + +_TF_INTERPOLATION_METHODS = { + "bilinear": ResizeMethod.BILINEAR, + "nearest": ResizeMethod.NEAREST_NEIGHBOR, + "bicubic": ResizeMethod.BICUBIC, + "area": ResizeMethod.AREA, + "lanczos3": ResizeMethod.LANCZOS3, + "lanczos5": ResizeMethod.LANCZOS5, + "gaussian": ResizeMethod.GAUSSIAN, + "mitchellcubic": ResizeMethod.MITCHELLCUBIC, +} + +def paths_and_labels_to_dataset( + image_paths, + image_size, + num_channels, + labels, + label_mode, + num_classes, + interpolation, + crop_to_aspect_ratio=False, +): + """Constructs a dataset of images and labels.""" + # TODO(fchollet): consider making num_parallel_calls settable + path_ds = tf.data.Dataset.from_tensor_slices(image_paths) + args = (image_size, num_channels, interpolation, crop_to_aspect_ratio) + img_ds = path_ds.map( + lambda x: load_image(x, *args), num_parallel_calls=tf.data.AUTOTUNE + ) + if label_mode: + label_ds = dataset_utils.labels_to_dataset( + labels, label_mode, num_classes + ) + img_ds = tf.data.Dataset.zip((img_ds, label_ds)) + return img_ds + +def load_image( + path, image_size, num_channels, interpolation, crop_to_aspect_ratio=False +): + """Load an image from a path and resize it.""" + + img = tf.io.read_file(path) + + img = tf.image.decode_image( + img, channels=num_channels, expand_animations=False + ) + + # crop_to_aspect_ratio = False + if crop_to_aspect_ratio: + # img = image_utils.smart_resize( + # img, image_size, interpolation=interpolation + # ) + img = smart_resize( + img, image_size, interpolation=interpolation + ) + else: + img = tf.image.resize(img, image_size, method=interpolation) + img.set_shape((image_size[0], image_size[1], num_channels)) + return img + +def image_dataset_from_directory( + directory, + labels="inferred", + label_mode="int", + class_names=None, + color_mode="rgb", + batch_size=32, + image_size=(256, 256), + shuffle=True, + seed=None, + validation_split=None, + subset=None, + interpolation="bilinear", + follow_links=False, + crop_to_aspect_ratio=False, + **kwargs, +): + """Generates a `tf.data.Dataset` from image files in a directory. + If your directory structure is: + ``` + main_directory/ + ...class_a/ + ......a_image_1.jpg + ......a_image_2.jpg + ...class_b/ + ......b_image_1.jpg + ......b_image_2.jpg + ``` + Then calling `image_dataset_from_directory(main_directory, + labels='inferred')` will return a `tf.data.Dataset` that yields batches of + images from the subdirectories `class_a` and `class_b`, together with labels + 0 and 1 (0 corresponding to `class_a` and 1 corresponding to `class_b`). + Supported image formats: jpeg, png, bmp, gif. + Animated gifs are truncated to the first frame. + Args: + directory: Directory where the data is located. + If `labels` is "inferred", it should contain + subdirectories, each containing images for a class. + Otherwise, the directory structure is ignored. + labels: Either "inferred" + (labels are generated from the directory structure), + None (no labels), + or a list/tuple of integer labels of the same size as the number of + image files found in the directory. Labels should be sorted according + to the alphanumeric order of the image file paths + (obtained via `os.walk(directory)` in Python). + label_mode: String describing the encoding of `labels`. Options are: + - 'int': means that the labels are encoded as integers + (e.g. for `sparse_categorical_crossentropy` loss). + - 'categorical' means that the labels are + encoded as a categorical vector + (e.g. for `categorical_crossentropy` loss). + - 'binary' means that the labels (there can be only 2) + are encoded as `float32` scalars with values 0 or 1 + (e.g. for `binary_crossentropy`). + - None (no labels). + class_names: Only valid if "labels" is "inferred". This is the explicit + list of class names (must match names of subdirectories). Used + to control the order of the classes + (otherwise alphanumerical order is used). + color_mode: One of "grayscale", "rgb", "rgba". Default: "rgb". + Whether the images will be converted to + have 1, 3, or 4 channels. + batch_size: Size of the batches of data. Default: 32. + If `None`, the data will not be batched + (the dataset will yield individual samples). + image_size: Size to resize images to after they are read from disk, + specified as `(height, width)`. Defaults to `(256, 256)`. + Since the pipeline processes batches of images that must all have + the same size, this must be provided. + shuffle: Whether to shuffle the data. Default: True. + If set to False, sorts the data in alphanumeric order. + seed: Optional random seed for shuffling and transformations. + validation_split: Optional float between 0 and 1, + fraction of data to reserve for validation. + subset: Subset of the data to return. + One of "training", "validation" or "both". + Only used if `validation_split` is set. + When `subset="both"`, the utility returns a tuple of two datasets + (the training and validation datasets respectively). + interpolation: String, the interpolation method used when resizing images. + Defaults to `bilinear`. Supports `bilinear`, `nearest`, `bicubic`, + `area`, `lanczos3`, `lanczos5`, `gaussian`, `mitchellcubic`. + follow_links: Whether to visit subdirectories pointed to by symlinks. + Defaults to False. + crop_to_aspect_ratio: If True, resize the images without aspect + ratio distortion. When the original aspect ratio differs from the target + aspect ratio, the output image will be cropped so as to return the + largest possible window in the image (of size `image_size`) that matches + the target aspect ratio. By default (`crop_to_aspect_ratio=False`), + aspect ratio may not be preserved. + **kwargs: Legacy keyword arguments. + Returns: + A `tf.data.Dataset` object. + - If `label_mode` is None, it yields `float32` tensors of shape + `(batch_size, image_size[0], image_size[1], num_channels)`, + encoding images (see below for rules regarding `num_channels`). + - Otherwise, it yields a tuple `(images, labels)`, where `images` + has shape `(batch_size, image_size[0], image_size[1], num_channels)`, + and `labels` follows the format described below. + Rules regarding labels format: + - if `label_mode` is `int`, the labels are an `int32` tensor of shape + `(batch_size,)`. + - if `label_mode` is `binary`, the labels are a `float32` tensor of + 1s and 0s of shape `(batch_size, 1)`. + - if `label_mode` is `categorical`, the labels are a `float32` tensor + of shape `(batch_size, num_classes)`, representing a one-hot + encoding of the class index. + Rules regarding number of channels in the yielded images: + - if `color_mode` is `grayscale`, + there's 1 channel in the image tensors. + - if `color_mode` is `rgb`, + there are 3 channels in the image tensors. + - if `color_mode` is `rgba`, + there are 4 channels in the image tensors. + """ + if "smart_resize" in kwargs: + crop_to_aspect_ratio = kwargs.pop("smart_resize") + if kwargs: + raise TypeError(f"Unknown keywords argument(s): {tuple(kwargs.keys())}") + if labels not in ("inferred", None): + if not isinstance(labels, (list, tuple)): + raise ValueError( + "`labels` argument should be a list/tuple of integer labels, " + "of the same size as the number of image files in the target " + "directory. If you wish to infer the labels from the " + "subdirectory " + 'names in the target directory, pass `labels="inferred"`. ' + "If you wish to get a dataset that only contains images " + f"(no labels), pass `labels=None`. Received: labels={labels}" + ) + if class_names: + raise ValueError( + "You can only pass `class_names` if " + f'`labels="inferred"`. Received: labels={labels}, and ' + f"class_names={class_names}" + ) + if label_mode not in {"int", "categorical", "binary", None}: + raise ValueError( + '`label_mode` argument must be one of "int", ' + '"categorical", "binary", ' + f"or None. Received: label_mode={label_mode}" + ) + if labels is None or label_mode is None: + labels = None + label_mode = None + if color_mode == "rgb": + num_channels = 3 + elif color_mode == "rgba": + num_channels = 4 + elif color_mode == "grayscale": + num_channels = 1 + else: + raise ValueError( + '`color_mode` must be one of {"rgb", "rgba", "grayscale"}. ' + f"Received: color_mode={color_mode}" + ) + # interpolation = image_utils.get_interpolation(interpolation) + interpolation = get_interpolation(interpolation) + # dataset_utils.check_validation_split_arg( + # validation_split, subset, shuffle, seed + # ) + + if seed is None: + seed = np.random.randint(1e6) + image_paths, labels, class_names = dataset_utils.index_directory( + directory, + labels, + formats=ALLOWLIST_FORMATS, + class_names=class_names, + shuffle=shuffle, + seed=seed, + follow_links=follow_links, + ) + + if label_mode == "binary" and len(class_names) != 2: + raise ValueError( + 'When passing `label_mode="binary"`, there must be exactly 2 ' + f"class_names. Received: class_names={class_names}" + ) + + if subset == "both": + ( + image_paths_train, + labels_train, + ) = dataset_utils.get_training_or_validation_split( + image_paths, labels, validation_split, "training" + ) + ( + image_paths_val, + labels_val, + ) = dataset_utils.get_training_or_validation_split( + image_paths, labels, validation_split, "validation" + ) + if not image_paths_train: + raise ValueError( + f"No training images found in directory {directory}. " + f"Allowed formats: {ALLOWLIST_FORMATS}" + ) + if not image_paths_val: + raise ValueError( + f"No validation images found in directory {directory}. " + f"Allowed formats: {ALLOWLIST_FORMATS}" + ) + train_dataset = paths_and_labels_to_dataset( + image_paths=image_paths_train, + image_size=image_size, + num_channels=num_channels, + labels=labels_train, + label_mode=label_mode, + num_classes=len(class_names), + interpolation=interpolation, + crop_to_aspect_ratio=crop_to_aspect_ratio, + ) + val_dataset = paths_and_labels_to_dataset( + image_paths=image_paths_val, + image_size=image_size, + num_channels=num_channels, + labels=labels_val, + label_mode=label_mode, + num_classes=len(class_names), + interpolation=interpolation, + crop_to_aspect_ratio=crop_to_aspect_ratio, + ) + train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE) + val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE) + if batch_size is not None: + if shuffle: + # Shuffle locally at each iteration + train_dataset = train_dataset.shuffle( + buffer_size=batch_size * 8, seed=seed + ) + train_dataset = train_dataset.batch(batch_size) + val_dataset = val_dataset.batch(batch_size) + else: + if shuffle: + train_dataset = train_dataset.shuffle( + buffer_size=1024, seed=seed + ) + + # Users may need to reference `class_names`. + train_dataset.class_names = class_names + val_dataset.class_names = class_names + # Include file paths for images as attribute. + train_dataset.file_paths = image_paths_train + val_dataset.file_paths = image_paths_val + dataset = [train_dataset, val_dataset] + else: + image_paths, labels = dataset_utils.get_training_or_validation_split( + image_paths, labels, validation_split, subset + ) + if not image_paths: + raise ValueError( + f"No images found in directory {directory}. " + f"Allowed formats: {ALLOWLIST_FORMATS}" + ) + + dataset = paths_and_labels_to_dataset( + image_paths=image_paths, + image_size=image_size, + num_channels=num_channels, + labels=labels, + label_mode=label_mode, + num_classes=len(class_names), + interpolation=interpolation, + crop_to_aspect_ratio=crop_to_aspect_ratio, + ) + dataset = dataset.prefetch(tf.data.AUTOTUNE) + if batch_size is not None: + if shuffle: + # Shuffle locally at each iteration + dataset = dataset.shuffle(buffer_size=batch_size * 8, seed=seed) + dataset = dataset.batch(batch_size) + else: + if shuffle: + dataset = dataset.shuffle(buffer_size=1024, seed=seed) + + # Users may need to reference `class_names`. + dataset.class_names = class_names + # Include file paths for images as attribute. + dataset.file_paths = image_paths + return dataset + + +def smart_resize(x, size, interpolation="bilinear"): + """Resize images to a target size without aspect ratio distortion. + Warning: `tf.keras.preprocessing.image.smart_resize` is not recommended for + new code. Prefer `tf.keras.layers.Resizing`, which provides the same + functionality as a preprocessing layer and adds `tf.RaggedTensor` support. + See the [preprocessing layer guide]( + https://www.tensorflow.org/guide/keras/preprocessing_layers) + for an overview of preprocessing layers. + TensorFlow image datasets typically yield images that have each a different + size. However, these images need to be batched before they can be + processed by Keras layers. To be batched, images need to share the same + height and width. + You could simply do: + ```python + size = (200, 200) + ds = ds.map(lambda img: tf.image.resize(img, size)) + ``` + However, if you do this, you distort the aspect ratio of your images, since + in general they do not all have the same aspect ratio as `size`. This is + fine in many cases, but not always (e.g. for GANs this can be a problem). + Note that passing the argument `preserve_aspect_ratio=True` to `resize` + will preserve the aspect ratio, but at the cost of no longer respecting the + provided target size. Because `tf.image.resize` doesn't crop images, + your output images will still have different sizes. + This calls for: + ```python + size = (200, 200) + ds = ds.map(lambda img: smart_resize(img, size)) + ``` + Your output images will actually be `(200, 200)`, and will not be distorted. + Instead, the parts of the image that do not fit within the target size + get cropped out. + The resizing process is: + 1. Take the largest centered crop of the image that has the same aspect + ratio as the target size. For instance, if `size=(200, 200)` and the input + image has size `(340, 500)`, we take a crop of `(340, 340)` centered along + the width. + 2. Resize the cropped image to the target size. In the example above, + we resize the `(340, 340)` crop to `(200, 200)`. + Args: + x: Input image or batch of images (as a tensor or NumPy array). Must be in + format `(height, width, channels)` or `(batch_size, height, width, + channels)`. + size: Tuple of `(height, width)` integer. Target size. + interpolation: String, interpolation to use for resizing. Defaults to + `'bilinear'`. Supports `bilinear`, `nearest`, `bicubic`, `area`, + `lanczos3`, `lanczos5`, `gaussian`, `mitchellcubic`. + Returns: + Array with shape `(size[0], size[1], channels)`. If the input image was a + NumPy array, the output is a NumPy array, and if it was a TF tensor, + the output is a TF tensor. + """ + if len(size) != 2: + raise ValueError( + f"Expected `size` to be a tuple of 2 integers, but got: {size}." + ) + img = tf.convert_to_tensor(x) + if img.shape.rank is not None: + if img.shape.rank < 3 or img.shape.rank > 4: + raise ValueError( + "Expected an image array with shape `(height, width, " + "channels)`, or `(batch_size, height, width, channels)`, but " + f"got input with incorrect rank, of shape {img.shape}." + ) + shape = tf.shape(img) + height, width = shape[-3], shape[-2] + target_height, target_width = size + if img.shape.rank is not None: + static_num_channels = img.shape[-1] + else: + static_num_channels = None + + crop_height = tf.cast( + tf.cast(width * target_height, "float32") / target_width, "int32" + ) + crop_width = tf.cast( + tf.cast(height * target_width, "float32") / target_height, "int32" + ) + + # Set back to input height / width if crop_height / crop_width is not + # smaller. + crop_height = tf.minimum(height, crop_height) + crop_width = tf.minimum(width, crop_width) + + crop_box_hstart = tf.cast( + tf.cast(height - crop_height, "float32") / 2, "int32" + ) + crop_box_wstart = tf.cast( + tf.cast(width - crop_width, "float32") / 2, "int32" + ) + + if img.shape.rank == 4: + crop_box_start = tf.stack([0, crop_box_hstart, crop_box_wstart, 0]) + crop_box_size = tf.stack([-1, crop_height, crop_width, -1]) + else: + crop_box_start = tf.stack([crop_box_hstart, crop_box_wstart, 0]) + crop_box_size = tf.stack([crop_height, crop_width, -1]) + + img = tf.slice(img, crop_box_start, crop_box_size) + img = tf.image.resize(images=img, size=size, method=interpolation) + # Apparent bug in resize_images_v2 may cause shape to be lost + if img.shape.rank is not None: + if img.shape.rank == 4: + img.set_shape((None, None, None, static_num_channels)) + if img.shape.rank == 3: + img.set_shape((None, None, static_num_channels)) + if isinstance(x, np.ndarray): + return img.numpy() + return img + +def get_interpolation(interpolation): + interpolation = interpolation.lower() + if interpolation not in _TF_INTERPOLATION_METHODS: + raise NotImplementedError( + "Value not recognized for `interpolation`: {}. Supported values " + "are: {}".format(interpolation, _TF_INTERPOLATION_METHODS.keys()) + ) + return _TF_INTERPOLATION_METHODS[interpolation] diff --git a/aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py new file mode 100755 index 0000000..edd2d1e --- /dev/null +++ b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2022 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +''' do TF2 resnet50 quantization and evaluation''' +import argparse +import tensorflow as tf +from aimet_zoo_tensorflow.resnet50_tf2.model.model_definition import Resnet50 +from aimet_zoo_tensorflow.resnet50_tf2.evaluators.eval_func import get_eval_func + +def arguments(): + ''' + parses command line arguments + ''' + parser = argparse.ArgumentParser(description='Arguments for evaluating model') + parser.add_argument('--dataset-path', help='path to image evaluation dataset', type=str) + parser.add_argument('--model-config', help='model configuration to be tested', type=str) + parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) + parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) + parser.add_argument('--batch-size', help='batch_size for loading data', type=int, default=16) + parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) + args = parser.parse_args() + return args + +def main(): + """ Run evaluation """ + args = arguments() + + gpu_devices = tf.config.experimental.list_physical_devices("GPU") + for device in gpu_devices: + tf.config.experimental.set_memory_growth(device, True) + + # Evaluation function + eval_func = get_eval_func(dataset_dir=args.dataset_path, + batch_size=args.batch_size, + num_iterations=50000) + + # Models + model = Resnet50(model_config = args.model_config) + model.from_pretrained(quantized=True) + sim = model.get_quantsim(quantized=True) + + # Evaluate original + print("start evaluating FP32 accuracy") + fp32_acc = eval_func(model.model) + print(f'FP32 top1 accuracy: {fp32_acc:0.3f}') + + # Evaluate optimized + print("start evaluating quantized accuracy") + quant_acc = eval_func(sim.model) + print(f'Quantized top1 accuracy: {quant_acc:0.3f}') + +if __name__ == '__main__': + main() diff --git a/aimet_zoo_tensorflow/resnet50_tf2/model/__init__.py b/aimet_zoo_tensorflow/resnet50_tf2/model/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/resnet50_tf2/model/model_cards/resnet50_w8a8.json b/aimet_zoo_tensorflow/resnet50_tf2/model/model_cards/resnet50_w8a8.json new file mode 100755 index 0000000..5afdd3f --- /dev/null +++ b/aimet_zoo_tensorflow/resnet50_tf2/model/model_cards/resnet50_w8a8.json @@ -0,0 +1,24 @@ +{ + "name": "resnet50", + "framework": "tensorflow2.x", + "task": "image classification", + "input_shape": [null, 256, 256, 3], + "dataset": "imagenet", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf", + "techniques": null + } + }, + "artifacts": { + "url_pre_opt_weights": null, + "url_post_opt_weights": null, + "url_adaround_encodings": null, + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/tensorflow2-resnet50/resnet50_w8a8.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.25/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} diff --git a/aimet_zoo_tensorflow/resnet50_tf2/model/model_definition.py b/aimet_zoo_tensorflow/resnet50_tf2/model/model_definition.py new file mode 100644 index 0000000..2e53d93 --- /dev/null +++ b/aimet_zoo_tensorflow/resnet50_tf2/model/model_definition.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +''' Define Resnet50 model and do Quantsim''' +import os +import json +from aimet_tensorflow.keras.quantsim import QuantizationSimModel # pylint:disable = import-error +from aimet_tensorflow.keras.batch_norm_fold import fold_all_batch_norms # pylint:disable = import-error +from aimet_zoo_tensorflow.resnet50_tf2 import ResNet50 +from aimet_zoo_tensorflow.common.downloader import Downloader + +class Resnet50(Downloader): + """Resnet50 parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + # pylint: disable=unused-argument + def __init__(self, model_config = None, **kwargs): + """ + :param model_config: named model config from which to obtain model artifacts and arguments. + If provided, overwrites the other arguments passed to this object + """ + parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + self.cfg = False + if model_config: + config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + if os.path.exists(config_filepath): + with open(config_filepath, encoding='UTF-8') as f_in: + self.cfg = json.load(f_in) + if self.cfg: + Downloader.__init__(self, + url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], + url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], + url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], + url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], + url_aimet_config = self.cfg['artifacts']['url_aimet_config'], + model_dir = parent_dir, + model_config = model_config) + self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) + + self.model = ResNet50(include_top=True, + weights="imagenet", + input_tensor=None, + input_shape=None, + pooling=None, + classes=1000) + + def from_pretrained(self, quantized=False): + """download config file and encodings from model cards""" + if not self.cfg: + raise NotImplementedError('There are no pretrained weights available for the model_config passed') + self._download_pre_opt_weights() + self._download_post_opt_weights() + self._download_aimet_config() + self._download_aimet_encodings() + self._download_adaround_encodings() + if quantized: + # bn folding to weights + _, self.model = fold_all_batch_norms(self.model) + + + def get_quantsim(self, quantized=False): + """get quantsim object with pre-loaded encodings""" + if not self.cfg: + raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + if quantized: + self.from_pretrained(quantized=True) + else: + self.from_pretrained(quantized=False) + + kwargs = { + 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], + 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], + 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], + 'config_file': self.path_aimet_config, + } + + sim = QuantizationSimModel(self.model, **kwargs) + + # load encoding file back to sim + if self.path_aimet_encodings and quantized: + sim.load_encodings_to_sim(self.path_aimet_encodings) + # This is the temporary solution for slow speed issue after loading_encoding_to_sim + # it will be removed once official solution available + # pylint: disable=protected-access + op_mode = sim._param_op_mode_after_analysis(sim.quant_scheme) + # pylint: disable=protected-access + sim._set_op_mode_parameters(op_mode) + print('load_encodings_to_sim finished!') + + # load adaround encoding file back to sim + if self.path_adaround_encodings and quantized: + sim.set_and_freeze_param_encodings(self.path_adaround_encodings) + print('set_and_freeze_param_encodings finished!') + + return sim diff --git a/aimet_zoo_tensorflow/resnet50_tf2/requirements.txt b/aimet_zoo_tensorflow/resnet50_tf2/requirements.txt new file mode 100644 index 0000000..1e99151 --- /dev/null +++ b/aimet_zoo_tensorflow/resnet50_tf2/requirements.txt @@ -0,0 +1 @@ +numpy==1.19.5 diff --git a/aimet_zoo_tensorflow/retinanet/evaluators/retinanet_quanteval.py b/aimet_zoo_tensorflow/retinanet/evaluators/retinanet_quanteval.py index 6299362..7b0f63e 100644 --- a/aimet_zoo_tensorflow/retinanet/evaluators/retinanet_quanteval.py +++ b/aimet_zoo_tensorflow/retinanet/evaluators/retinanet_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -9,6 +9,8 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """Quantsim evaluation script for retinanet""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet from glob import glob import urllib.request import argparse @@ -26,7 +28,6 @@ from keras import backend as K - os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" @@ -46,8 +47,10 @@ def download_weights(): URL = "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.22/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" urllib.request.urlretrieve(URL, "default_config.json") -#pylint: disable=W0613 -#pylint: disable=W0612 + +# pylint: disable=W0613 +# pylint: disable=W0612 + def quantize_retinanet(model_path, cocopath, action): """ @@ -93,6 +96,7 @@ def quantize_retinanet(model_path, cocopath, action): "filtered_detections/map/TensorArrayStack_2/TensorArrayGatherV3:0", ] selected_ops = ["P" + str(i) + "/BiasAdd" for i in range(3, 8)] + #pylint:disable = use-maxsplit-arg sim = quantsim.QuantizationSimModel( session, [in_tensor.split(":")[0]], @@ -119,9 +123,11 @@ def forward_pass(session2: tf.Session, args): "filtered_detections/map/TensorArrayStack_2/TensorArrayGatherV3:0", ] selected_ops = ["P" + str(i) + "/BiasAdd" for i in range(3, 8)] + #pylint:disable = use-maxsplit-arg session, folded_pairs = fold_all_batch_norms( session, [in_tensor.split(":")[0]], selected_ops ) + #pylint:disable = use-maxsplit-arg sim = quantsim.QuantizationSimModel( session, [in_tensor.split(":")[0]], @@ -148,6 +154,7 @@ def forward_pass(session2: tf.Session, args): "filtered_detections/map/TensorArrayStack_1/TensorArrayGatherV3:0", "filtered_detections/map/TensorArrayStack_2/TensorArrayGatherV3:0", ] + #pylint:disable = use-maxsplit-arg selected_ops = ["P" + str(i) + "/BiasAdd" for i in range(3, 8)] session, folded_pairs = fold_all_batch_norms( session, [in_tensor.split(":")[0]], selected_ops @@ -168,7 +175,7 @@ def forward_pass(session2: tf.Session, args): save_checkpoint(sim, "./optimized_int8/model.ckpt", "model") else: - raise Exception( + raise ValueError( "--action must be one of: original_fp32, original_int8, optimized_fp32, optimized_int8" ) @@ -195,20 +202,16 @@ def evaluate(generator, action, threshold=0.05): with tf.Session() as new_sess: if action == "original_fp32": - saver = tf.train.import_meta_graph( - "./original_fp32/model.ckpt.meta") + saver = tf.train.import_meta_graph("./original_fp32/model.ckpt.meta") saver.restore(new_sess, "./original_fp32/model.ckpt") elif action == "original_int8": - new_quantsim = load_checkpoint( - "./original_int8/model.ckpt", "model") + new_quantsim = load_checkpoint("./original_int8/model.ckpt", "model") new_sess = new_quantsim.session elif action == "optimized_fp32": - saver = tf.train.import_meta_graph( - "./optimized_fp32/model.ckpt.meta") + saver = tf.train.import_meta_graph("./optimized_fp32/model.ckpt.meta") saver.restore(new_sess, "./optimized_fp32/model.ckpt") elif action == "optimized_int8": - new_quantsim = load_checkpoint( - "./optimized_int8/model.ckpt", "model") + new_quantsim = load_checkpoint("./optimized_int8/model.ckpt", "model") new_sess = new_quantsim.session model = TFRunWrapper(new_sess, in_tensor, out_tensor) @@ -225,7 +228,7 @@ def create_generator(args, preprocess_image): common_args = { "preprocess_image": preprocess_image, } - + #pylint:disable = import-outside-toplevel from keras_retinanet.preprocessing.coco import CocoGenerator validation_generator = CocoGenerator( @@ -253,20 +256,16 @@ def parse_args(args): "--action", help="action to perform - eval_quantized|eval_original", default="eval_quantized", - choices={ - "original_fp32", - "original_int8", - "optimized_fp32", - "optimized_int8"}, + choices={"original_fp32", "original_int8", "optimized_fp32", "optimized_int8"}, ) return parser.parse_args(args) - class TFRunWrapper: """The coco_eval in keras-retinanet repository needs a model as input for prediction - We have a TF back-end session - so we wrap it in a Wrapper and implement - predict to call session run""" + We have a TF back-end session - so we wrap it in a Wrapper and implement + predict to call session run""" + def __init__(self, tf_session, in_tensor, out_tensor): self.sess = tf_session self.in_tensor = in_tensor @@ -274,13 +273,12 @@ def __init__(self, tf_session, in_tensor, out_tensor): def predict_on_batch(self, input_name): """predict on batch""" - return self.sess.run( - self.out_tensor, feed_dict={ - self.in_tensor: input_name}) + return self.sess.run(self.out_tensor, feed_dict={self.in_tensor: input_name}) class ModelConfig: """Hardcoded model configuration""" + def __init__(self, args): self.model_path = "./" self.score_threshold = 0.05 diff --git a/aimet_zoo_tensorflow/srgan/evaluators/srgan_quanteval.py b/aimet_zoo_tensorflow/srgan/evaluators/srgan_quanteval.py index 9c80ac7..661cd1c 100644 --- a/aimet_zoo_tensorflow/srgan/evaluators/srgan_quanteval.py +++ b/aimet_zoo_tensorflow/srgan/evaluators/srgan_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -9,6 +9,8 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """Quantsim evaluation script for srgan""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet from functools import partial import argparse import tarfile @@ -31,7 +33,6 @@ from skimage.metrics import peak_signal_noise_ratio as psnr - os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" @@ -47,12 +48,8 @@ def make_dataset(filenames): def evaluate_session( - sess, - image_files, - input_name, - output_name, - mode="y_channel", - output_dir=None): + sess, image_files, input_name, output_name, mode="y_channel", output_dir=None +): """ :param sess: a tensorflow session on which we run evaluation :param image_files: a sequence containing sequence of image filenames as strings @@ -70,8 +67,8 @@ def evaluate_session( print("Testing on Y channel...") else: raise ValueError( - "evaluation mode not supported!" - "Must be one of `RGB` or `y_channel`") + "evaluation mode not supported!Must be one of `RGB` or `y_channel`" + ) # batch size needed to align with input shape (?, ?, ?, 3) batch_size = 1 @@ -123,10 +120,8 @@ def evaluate_session( sr_img = np.expand_dims(sr_img, axis=-1) hr_img = np.expand_dims(hr_img, axis=-1) - sr_img = sr_img[:, crop_border:- - crop_border, crop_border:-crop_border, :] - hr_img = hr_img[:, crop_border:- - crop_border, crop_border:-crop_border, :] + sr_img = sr_img[:, crop_border:-crop_border, crop_border:-crop_border, :] + hr_img = hr_img[:, crop_border:-crop_border, crop_border:-crop_border, :] psnr_value = psnr(hr_img[0], sr_img[0], data_range=255) ssim_value = ssim( @@ -177,9 +172,7 @@ def parse_args(): "--default-output-bw", help="Default bitwidth (4-31) to use for quantizing layer inputs and outputs", default=16, - choices=range( - 4, - 32), + choices=range(4, 32), type=int, ) parser.add_argument( @@ -216,16 +209,8 @@ def main(args): srgan_generator.load_weights("weights/srgan/gan_generator.h5") # sort files by filenames, assuming names match in both paths - lr_images_files = sorted( - glob.glob( - os.path.join( - args.dataset_path, - "*LR.png"))) - hr_images_files = sorted( - glob.glob( - os.path.join( - args.dataset_path, - "*HR.png"))) + lr_images_files = sorted(glob.glob(os.path.join(args.dataset_path, "*LR.png"))) + hr_images_files = sorted(glob.glob(os.path.join(args.dataset_path, "*HR.png"))) # check if number of images align if len(lr_images_files) != len(hr_images_files): @@ -245,7 +230,8 @@ def main(args): ### ===========Original Model=============### # Evaluate original model on gpu psnr_vals, ssim_vals = evaluate_session( - gen_sess, image_files, srgan_generator.input.name, srgan_generator.output.name) + gen_sess, image_files, srgan_generator.input.name, srgan_generator.output.name + ) psnr_val_orig_fp32 = np.mean(psnr_vals) ssim_val_orig_fp32 = np.mean(ssim_vals) @@ -287,7 +273,8 @@ def main(args): # Evaluate optimized model on gpu psnr_vals, ssim_vals = evaluate_session( - gen_sess, image_files, srgan_generator.input.name, srgan_generator.output.name) + gen_sess, image_files, srgan_generator.input.name, srgan_generator.output.name + ) psnr_val_optim_fp32 = np.mean(psnr_vals) ssim_val_optim_fp32 = np.mean(ssim_vals) diff --git a/aimet_zoo_tensorflow/ssd_mobilenet_v2/dataloader/__init__.py b/aimet_zoo_tensorflow/ssd_mobilenet_v2/dataloader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/ssd_mobilenet_v2/model/__init__.py b/aimet_zoo_tensorflow/ssd_mobilenet_v2/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/ssd_mobilenet_v2/model/model_cards/__init__.py b/aimet_zoo_tensorflow/ssd_mobilenet_v2/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/ssd_mobilenet_v2/model/model_definition.py b/aimet_zoo_tensorflow/ssd_mobilenet_v2/model/model_definition.py index f758a45..538b17c 100644 --- a/aimet_zoo_tensorflow/ssd_mobilenet_v2/model/model_definition.py +++ b/aimet_zoo_tensorflow/ssd_mobilenet_v2/model/model_definition.py @@ -7,14 +7,16 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - -import os, json -import tensorflow.compat.v1 as tf +"""Class for downloading and setting up of optmized and original ssd-mobilenetv2 model for AIMET model zoo""" +import os +import json import pathlib -tf.disable_v2_behavior() -from aimet_tensorflow.quantsim import QuantizationSimModel -from aimet_tensorflow.batch_norm_fold import fold_all_batch_norms +import tensorflow.compat.v1 as tf + +from aimet_tensorflow.quantsim import QuantizationSimModel # pylint: disable=import-error +from aimet_tensorflow.batch_norm_fold import fold_all_batch_norms # pylint: disable=import-error from aimet_zoo_tensorflow.common.downloader import Downloader +tf.disable_v2_behavior() class SSDMobileNetV2(Downloader): @@ -40,12 +42,17 @@ def __init__(self, model_config=None): model_config=model_config, ) self.input_shape = tuple( - x if x != None else 1 for x in self.cfg["input_shape"] + x if x is not None else 1 for x in self.cfg["input_shape"] ) self.starting_op_names = self.cfg["model_args"]["starting_op_names"] self.output_op_names = self.cfg["model_args"]["output_op_names"] - def from_pretrained(self, quantized=False): + @classmethod + def from_pretrained(cls, quantized=False): + #pylint:disable = unused-argument + """load pretrained model + for tensorflow models, get_session is used instead + """ return "For TF 1.X based models, use get_session()" def get_quantsim(self, quantized=False): @@ -84,6 +91,7 @@ def get_session(self, quantized=False): self._download_adaround_encodings() self._download_compressed_checkpoint() meta_graph = None + # pylint:disable = unused-variable for root, dirs, files in os.walk(self.extract_dir): for file in files: if file.endswith(".meta"): diff --git a/aimet_zoo_torch/abpn/__init__.py b/aimet_zoo_torch/abpn/__init__.py index 05a23a1..c2e37aa 100644 --- a/aimet_zoo_torch/abpn/__init__.py +++ b/aimet_zoo_torch/abpn/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import ABPN \ No newline at end of file +""" loading ABPN downloader class """ +from .model.model_definition import ABPN diff --git a/aimet_zoo_torch/abpn/evaluators/__init__.py b/aimet_zoo_torch/abpn/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/abpn/evaluators/abpn_quanteval.py b/aimet_zoo_torch/abpn/evaluators/abpn_quanteval.py index c245959..5589ae2 100644 --- a/aimet_zoo_torch/abpn/evaluators/abpn_quanteval.py +++ b/aimet_zoo_torch/abpn/evaluators/abpn_quanteval.py @@ -8,25 +8,52 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET evaluation code for QuickSRNet ''' +""" AIMET evaluation code for QuickSRNet """ import argparse from aimet_zoo_torch.abpn import ABPN from aimet_zoo_torch.common.super_resolution.psnr import evaluate_average_psnr -from aimet_zoo_torch.common.super_resolution.utils import load_dataset, pass_calibration_data +from aimet_zoo_torch.common.super_resolution.utils import ( + load_dataset, + pass_calibration_data, +) from aimet_zoo_torch.common.super_resolution.inference import run_model # add arguments def arguments(): """parses command line arguments""" - parser = argparse.ArgumentParser(description='Arguments for evaluating model') - parser.add_argument('--dataset-path', help='path to image evaluation dataset', type=str, required=True) - parser.add_argument('--model-config', help='model configuration to be tested', type=str, required=True) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) - parser.add_argument('--batch-size',help='batch_size for loading data',type=int,default=16) - parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) + parser = argparse.ArgumentParser(description="Arguments for evaluating model") + parser.add_argument( + "--dataset-path", + help="path to image evaluation dataset", + type=str, + required=True, + ) + parser.add_argument( + "--model-config", + help="model configuration to be tested", + type=str, + required=True, + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--batch-size", help="batch_size for loading data", type=int, default=16 + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU", type=bool, default=True + ) args = parser.parse_args() return args @@ -35,20 +62,24 @@ def main(): """executes evaluation""" args = arguments() - model_fp32 = ABPN(model_config = args.model_config) + model_fp32 = ABPN(model_config=args.model_config) model_fp32.from_pretrained(quantized=False) sim_fp32 = model_fp32.get_quantsim(quantized=False) - model_int8 = ABPN(model_config = args.model_config) + model_int8 = ABPN(model_config=args.model_config) model_int8.from_pretrained(quantized=True) sim_int8 = model_int8.get_quantsim(quantized=True) IMAGES_LR, IMAGES_HR = load_dataset(args.dataset_path, model_fp32.scaling_factor) - sim_fp32.compute_encodings(forward_pass_callback=pass_calibration_data, - forward_pass_callback_args=(IMAGES_LR, args.use_cuda)) - sim_int8.compute_encodings(forward_pass_callback=pass_calibration_data, - forward_pass_callback_args=(IMAGES_LR, args.use_cuda)) + sim_fp32.compute_encodings( + forward_pass_callback=pass_calibration_data, + forward_pass_callback_args=(IMAGES_LR, args.use_cuda), + ) + sim_int8.compute_encodings( + forward_pass_callback=pass_calibration_data, + forward_pass_callback_args=(IMAGES_LR, args.use_cuda), + ) # Run model inference on test images and get super-resolved images IMAGES_SR_original_fp32 = run_model(model_fp32, IMAGES_LR, args.use_cuda) @@ -58,13 +89,14 @@ def main(): # Get the average PSNR for all test-images avg_psnr = evaluate_average_psnr(IMAGES_SR_original_fp32, IMAGES_HR) - print(f'Original Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Original Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_original_int8, IMAGES_HR) - print(f'Original Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Original Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_optimized_fp32, IMAGES_HR) - print(f'Optimized Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Optimized Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_optimized_int8, IMAGES_HR) - print(f'Optimized Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Optimized Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}") -if __name__ == '__main__': - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/abpn/model/model_cards/__init__.py b/aimet_zoo_torch/abpn/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/abpn/model/model_definition.py b/aimet_zoo_torch/abpn/model/model_definition.py index 999053e..2d62011 100644 --- a/aimet_zoo_torch/abpn/model/model_definition.py +++ b/aimet_zoo_torch/abpn/model/model_definition.py @@ -7,10 +7,12 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - -import torch -import json +"""Class for downloading and setting up of optmized and original ABPN model for AIMET model zoo""" +#pylint:disable = import-error +#ignoring this due to aimet not setup in environment import os +import json +import torch from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim from aimet_zoo_torch.common.downloader import Downloader from aimet_zoo_torch.common.super_resolution.models import ABPNRelease @@ -18,51 +20,69 @@ class ABPN(ABPNRelease, Downloader): """ABPN parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + def __init__(self, model_config=None, num_channels=28, scaling_factor=2, **kwargs): """ :param model_config: named model config from which to obtain model artifacts and arguments. - If provided, overwrites the other arguments passed to this object + If provided, overwrites the other arguments passed to this object :param scaling_factor: scaling factor for LR-to-HR upscaling (2x, 3x, 4x... or 1.5x) - :param num_channels: number of feature channels for convolutional layers + :param num_channels: number of feature channels for convolutional layers """ - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) - self.scaling_factor = self.cfg['model_args']['scaling_factor'] if self.cfg else scaling_factor - self.num_channels = self.cfg['model_args']['num_channels'] if self.cfg else num_channels + self.scaling_factor = ( + self.cfg["model_args"]["scaling_factor"] if self.cfg else scaling_factor + ) + self.num_channels = ( + self.cfg["model_args"]["num_channels"] if self.cfg else num_channels + ) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) - ABPNRelease.__init__(self, scaling_factor = self.cfg['model_args']['scaling_factor'] if self.cfg else scaling_factor, - num_channels = self.cfg['model_args']['num_channels'] if self.cfg else num_channels, - **kwargs) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + ABPNRelease.__init__( + self, + scaling_factor=self.cfg["model_args"]["scaling_factor"] + if self.cfg + else scaling_factor, + num_channels=self.cfg["model_args"]["num_channels"] + if self.cfg + else num_channels, + **kwargs + ) def from_pretrained(self, quantized=False): """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() self._download_aimet_encodings() self._download_adaround_encodings() if quantized: - state_dict = torch.load(self.path_post_opt_weights)['state_dict'] + state_dict = torch.load(self.path_post_opt_weights)["state_dict"] self.load_state_dict(state_dict) self.cuda() else: - state_dict = torch.load(self.path_pre_opt_weights)['state_dict'] + state_dict = torch.load(self.path_pre_opt_weights)["state_dict"] self.load_state_dict(state_dict) self.cuda() self.eval() @@ -70,23 +90,32 @@ def from_pretrained(self, quantized=False): def get_quantsim(self, quantized=False): """get quantsim object with pre-loaded encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self, **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) sim.model.eval() - return sim \ No newline at end of file + return sim diff --git a/aimet_zoo_torch/bert/__init__.py b/aimet_zoo_torch/bert/__init__.py index 076873f..378110d 100644 --- a/aimet_zoo_torch/bert/__init__.py +++ b/aimet_zoo_torch/bert/__init__.py @@ -1,2 +1,2 @@ """ package for getting bert original model and quantized model""" -from .model.model_definition import Bert +from .model.model_definition import Bert diff --git a/aimet_zoo_torch/bert/dataloader/__init__.py b/aimet_zoo_torch/bert/dataloader/__init__.py index b047e99..229cc2a 100644 --- a/aimet_zoo_torch/bert/dataloader/__init__.py +++ b/aimet_zoo_torch/bert/dataloader/__init__.py @@ -1,2 +1,2 @@ """ datasets and eval function are defined and loaded""" -from .dataloaders import get_datasets,get_num_labels,eval_function +from .dataloaders import get_datasets, get_num_labels, eval_function diff --git a/aimet_zoo_torch/bert/dataloader/utils/__init__.py b/aimet_zoo_torch/bert/dataloader/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/bert/evaluators/__init__.py b/aimet_zoo_torch/bert/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/bert/evaluators/bert_quanteval.py b/aimet_zoo_torch/bert/evaluators/bert_quanteval.py index bf091fc..f7bc340 100644 --- a/aimet_zoo_torch/bert/evaluators/bert_quanteval.py +++ b/aimet_zoo_torch/bert/evaluators/bert_quanteval.py @@ -28,8 +28,6 @@ import argparse import logging -from transformers.utils.versions import require_version -from datasets import load_dataset, load_metric from aimet_zoo_torch.bert import Bert from aimet_zoo_torch.bert.dataloader import get_datasets, eval_function diff --git a/aimet_zoo_torch/bert/model/__init__.py b/aimet_zoo_torch/bert/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/bert/model/baseline_models/__init__.py b/aimet_zoo_torch/bert/model/baseline_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/bert/model/baseline_models/bert/__init__.py b/aimet_zoo_torch/bert/model/baseline_models/bert/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/bert/model/model_cards/__init__.py b/aimet_zoo_torch/bert/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_cola.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_cola.json index df94d47..ca091dd 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_cola.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_cola.json @@ -39,7 +39,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/cola_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/cola_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mnli.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mnli.json index 7e8a47e..8e88bae 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mnli.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mnli.json @@ -39,7 +39,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/mnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/mnli_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mrpc.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mrpc.json index 6f35e6e..587ae70 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mrpc.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mrpc.json @@ -39,7 +39,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/mrpc_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/mrpc_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qnli.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qnli.json index 12f82bf..ea85270 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qnli.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qnli.json @@ -39,7 +39,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/qnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/qnli_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_rte.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_rte.json index 31eaf55..671a6a2 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_rte.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_rte.json @@ -39,7 +39,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/rte_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/rte_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_squad.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_squad.json index cd3187a..5abdb66 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_squad.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_squad.json @@ -46,7 +46,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/squad_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/squad_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_sst2.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_sst2.json index a76c37f..af8e539 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_sst2.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_sst2.json @@ -39,7 +39,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/sst2_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/sst2_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_stsb.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_stsb.json index 822f508..c15cb66 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_stsb.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_stsb.json @@ -39,7 +39,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/stsb_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_bert/stsb_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/bert/model/model_definition.py b/aimet_zoo_torch/bert/model/model_definition.py index e9db592..776bf19 100644 --- a/aimet_zoo_torch/bert/model/model_definition.py +++ b/aimet_zoo_torch/bert/model/model_definition.py @@ -12,6 +12,7 @@ import json import os import sys +import pathlib from collections import defaultdict import torch @@ -31,7 +32,7 @@ class Bert(Downloader): """model Bert configuration class""" - + #pylint:disable = import-outside-toplevel def __init__(self, model_config=None): if model_config == "bert_w8a8_squad": from aimet_zoo_torch.bert.model.utils.utils_qa_dataclass import ( diff --git a/aimet_zoo_torch/bert/model/utils/__init__.py b/aimet_zoo_torch/bert/model/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/common/downloader.py b/aimet_zoo_torch/common/downloader.py index c969e2b..1a31a6d 100644 --- a/aimet_zoo_torch/common/downloader.py +++ b/aimet_zoo_torch/common/downloader.py @@ -7,33 +7,39 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +""" Downloader and downlowder progress bar class for downloading and loading weights and encodings""" -from urllib.request import urlretrieve -from pathlib import Path import os -from shutil import copy2 -import gdown -import progressbar -import tarfile +import tarfile +from pathlib import Path import shutil -import urllib.request +from shutil import copy2 +from urllib.request import urlretrieve +import urllib.request +import progressbar +import gdown# pylint: disable=import-error -class Downloader(): + + +class Downloader: """ Parent class for inheritance of utility methods used for downloading and loading weights and encodings """ - def __init__(self, - url_pre_opt_weights: str = None, - url_post_opt_weights: str = None, - url_adaround_encodings: str = None, - url_aimet_encodings: str = None, - url_aimet_config: str = None, - tar_url_pre_opt_weights: str= None, - tar_url_post_opt_weights: str = None, - url_zipped_checkpoint: str = None, - model_dir: str = '', - model_config: str = ''): - + # pylint: disable=too-many-instance-attributes + # 16 is reasonable in this case. + def __init__( + self, + url_pre_opt_weights: str = None, + url_post_opt_weights: str = None, + url_adaround_encodings: str = None, + url_aimet_encodings: str = None, + url_aimet_config: str = None, + tar_url_pre_opt_weights: str = None, + tar_url_post_opt_weights: str = None, + url_zipped_checkpoint: str = None, + model_dir: str = "", + model_config: str = "", + ):# pylint: disable=too-many-arguments """ :param url_pre_opt_weights: url hosting pre optimization weights as a state dict :param url_post_opt_weights: url hosting post optimization weights as a state dict @@ -52,49 +58,94 @@ def __init__(self, self.tar_url_pre_opt_weights = tar_url_pre_opt_weights self.tar_url_post_opt_weights = tar_url_post_opt_weights self.url_zipped_checkpoint = url_zipped_checkpoint - self._download_storage_path = Path(model_dir + '/weights/' + model_config + '/') if model_config else Path(model_dir + '/weights/') - self.path_pre_opt_weights = str(self._download_storage_path) + "/pre_opt_weights" if self.url_pre_opt_weights else None - self.path_post_opt_weights = str(self._download_storage_path) + "/post_opt_weights" if self.url_post_opt_weights else None - self.path_adaround_encodings = str(self._download_storage_path) + "/adaround_encodings" if self.url_adaround_encodings else None - self.path_aimet_encodings = str(self._download_storage_path) + "/aimet_encodings" if self.url_aimet_encodings else None - self.path_aimet_config = str(self._download_storage_path) + "/aimet_config" if self.url_aimet_config else None - self.path_zipped_checkpoint = str(self._download_storage_path) + "/zipped_checkpoint.zip" if self.url_zipped_checkpoint else None - self.extract_dir = self.path_zipped_checkpoint.split(".zip")[0] if self.path_zipped_checkpoint else None - - def _download_from_url(self, - src: str, - dst: str,show_progress=False): + self._download_storage_path = ( + Path(model_dir + "/weights/" + model_config + "/") + if model_config + else Path(model_dir + "/weights/") + ) + self.path_pre_opt_weights = ( + str(self._download_storage_path) + "/pre_opt_weights" + if self.url_pre_opt_weights + else None + ) + self.path_post_opt_weights = ( + str(self._download_storage_path) + "/post_opt_weights" + if self.url_post_opt_weights + else None + ) + self.path_adaround_encodings = ( + str(self._download_storage_path) + "/adaround_encodings" + if self.url_adaround_encodings + else None + ) + self.path_aimet_encodings = ( + str(self._download_storage_path) + "/aimet_encodings" + if self.url_aimet_encodings + else None + ) + self.path_aimet_config = ( + str(self._download_storage_path) + "/aimet_config" + if self.url_aimet_config + else None + ) + self.path_zipped_checkpoint = ( + str(self._download_storage_path) + "/zipped_checkpoint.zip" + if self.url_zipped_checkpoint + else None + ) + self.extract_dir = ( + self.path_zipped_checkpoint.split(".zip")[0] + if self.path_zipped_checkpoint + else None + ) + + def _download_from_url(self, src: str, dst: str, show_progress=False): """Receives a source URL or path and a storage destination path, evaluates the source, fetches the file, and stores at the destination""" if not os.path.exists(self._download_storage_path): os.makedirs(self._download_storage_path) if src is None: - return 'Skipping download, URL not provided on model definition' - if src.startswith('https://drive.google.com'): + return "Skipping download, URL not provided on model definition" + if src.startswith("https://drive.google.com"): gdown.download(url=src, output=dst, quiet=True, verify=False) - elif src.startswith('http'): + elif src.startswith("http"): if show_progress: urlretrieve(src, dst, DownloadProgressBar()) else: - urlretrieve(src,dst) + urlretrieve(src, dst) else: - assert os.path.exists(src), 'URL passed is not an http, assumed it to be a system path, but such path does not exist' + assert os.path.exists( + src + ), "URL passed is not an http, assumed it to be a system path, but such path does not exist" copy2(src, dst) + return None - def _download_pre_opt_weights(self,show_progress=False): + def _download_pre_opt_weights(self, show_progress=False): """downloads pre optimization weights""" - self._download_from_url(src=self.url_pre_opt_weights, dst=self.path_pre_opt_weights,show_progress=show_progress) + self._download_from_url( + src=self.url_pre_opt_weights, + dst=self.path_pre_opt_weights, + show_progress=show_progress, + ) - def _download_post_opt_weights(self,show_progress=False): + def _download_post_opt_weights(self, show_progress=False): """downloads post optimization weights""" - self._download_from_url(src=self.url_post_opt_weights, dst=self.path_post_opt_weights,show_progress=show_progress) + self._download_from_url( + src=self.url_post_opt_weights, + dst=self.path_post_opt_weights, + show_progress=show_progress, + ) def _download_adaround_encodings(self): """downloads adaround encodings""" - self._download_from_url(src=self.url_adaround_encodings, dst=self.path_adaround_encodings) + self._download_from_url( + src=self.url_adaround_encodings, dst=self.path_adaround_encodings + ) def _download_aimet_encodings(self): """downloads aimet encodings""" - self._download_from_url(src=self.url_aimet_encodings, dst=self.path_aimet_encodings) + self._download_from_url( + src=self.url_aimet_encodings, dst=self.path_aimet_encodings + ) def _download_aimet_config(self): """downloads aimet configuration""" @@ -107,42 +158,53 @@ def _download_tar_post_opt_weights(self): self._download_tar_decompress(tar_url=self.tar_url_post_opt_weights) def _download_compressed_checkpoint(self): - """ download a zipped checkpoint file and unzip it """ - self._download_from_url(src=self.url_zipped_checkpoint, dst=self.path_zipped_checkpoint) - format = "".join(self.url_zipped_checkpoint.split('/')[-1].split('.')[1:][::-1]) + """download a zipped checkpoint file and unzip it""" + self._download_from_url( + src=self.url_zipped_checkpoint, dst=self.path_zipped_checkpoint + ) + file_format = "".join(self.url_zipped_checkpoint.split("/")[-1].split(".")[1:][::-1]) if not os.path.exists(self.extract_dir): os.makedirs(self.extract_dir) - shutil.unpack_archive(filename=self.path_zipped_checkpoint, extract_dir=self.extract_dir, format=format) - - def _download_tar_decompress(self, tar_url): - """"download tarball and decompress into downloaded_weights folder""" + shutil.unpack_archive( + filename=self.path_zipped_checkpoint, + extract_dir=self.extract_dir, + format=file_format, + ) + + def _download_tar_decompress(self, tar_url): + """ "download tarball and decompress into downloaded_weights folder""" if not os.path.exists(self._download_storage_path): os.mkdir(self._download_storage_path) - download_tar_name = str(self._download_storage_path) +"/downloaded_weights.tar.gz" - urllib.request.urlretrieve(tar_url,download_tar_name,DownloadProgressBar()) + download_tar_name = ( + str(self._download_storage_path) + "/downloaded_weights.tar.gz" + ) + urllib.request.urlretrieve(tar_url, download_tar_name, DownloadProgressBar()) with tarfile.open(download_tar_name) as pth_weights: - pth_weights.extractall(self._download_storage_path) - folder_name=pth_weights.getnames()[0] - download_path = str(self._download_storage_path)+'/'+str(folder_name) - new_download_path = str(self._download_storage_path)+'/downloaded_weights' + pth_weights.extractall(self._download_storage_path) + folder_name = pth_weights.getnames()[0] + download_path = str(self._download_storage_path) + "/" + str(folder_name) + new_download_path = str(self._download_storage_path) + "/downloaded_weights" if os.path.exists(new_download_path): shutil.rmtree(new_download_path) - os.rename(download_path,new_download_path) + os.rename(download_path, new_download_path) -class DownloadProgressBar(): +class DownloadProgressBar: """Downloading progress bar to show status of downloading""" + def __init__(self): self.dpb = None def __call__(self, b_num, b_size, size): widgets = [ - '\x1b[33mDownloading weights \x1b[39m', - progressbar.Percentage(), - progressbar.Bar(marker='\x1b[32m#\x1b[39m'), + "\x1b[33mDownloading weights \x1b[39m", + progressbar.Percentage(), + progressbar.Bar(marker="\x1b[32m#\x1b[39m"), ] if not self.dpb: - self.dpb=progressbar.ProgressBar(widgets=widgets, maxval=size,redirect_stdout=True) + self.dpb = progressbar.ProgressBar( + widgets=widgets, maxval=size, redirect_stdout=True + ) self.dpb.start() processed = b_num * b_size @@ -150,5 +212,3 @@ def __call__(self, b_num, b_size, size): self.dpb.update(processed) else: self.dpb.finish() - - diff --git a/aimet_zoo_torch/common/super_resolution/blocks.py b/aimet_zoo_torch/common/super_resolution/blocks.py index 78a5f3c..2fd091a 100644 --- a/aimet_zoo_torch/common/super_resolution/blocks.py +++ b/aimet_zoo_torch/common/super_resolution/blocks.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/common/super_resolution/imresize.py b/aimet_zoo_torch/common/super_resolution/imresize.py index 4ba1d9b..aff160c 100644 --- a/aimet_zoo_torch/common/super_resolution/imresize.py +++ b/aimet_zoo_torch/common/super_resolution/imresize.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/common/super_resolution/inference.py b/aimet_zoo_torch/common/super_resolution/inference.py index 363e1e5..e354913 100644 --- a/aimet_zoo_torch/common/super_resolution/inference.py +++ b/aimet_zoo_torch/common/super_resolution/inference.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/common/super_resolution/models.py b/aimet_zoo_torch/common/super_resolution/models.py index 4bdd72d..490d0d5 100644 --- a/aimet_zoo_torch/common/super_resolution/models.py +++ b/aimet_zoo_torch/common/super_resolution/models.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/common/super_resolution/psnr.py b/aimet_zoo_torch/common/super_resolution/psnr.py index 91e6f2a..77a4a8c 100644 --- a/aimet_zoo_torch/common/super_resolution/psnr.py +++ b/aimet_zoo_torch/common/super_resolution/psnr.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/common/super_resolution/utils.py b/aimet_zoo_torch/common/super_resolution/utils.py index f219e93..66233f7 100644 --- a/aimet_zoo_torch/common/super_resolution/utils.py +++ b/aimet_zoo_torch/common/super_resolution/utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/common/utils/image_net_data_loader.py b/aimet_zoo_torch/common/utils/image_net_data_loader.py index 6945269..c1f8f9f 100644 --- a/aimet_zoo_torch/common/utils/image_net_data_loader.py +++ b/aimet_zoo_torch/common/utils/image_net_data_loader.py @@ -20,12 +20,14 @@ import torch.utils.data as torch_data -logger = logging.getLogger('Dataloader') +logger = logging.getLogger("Dataloader") -IMG_EXTENSIONS = '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif' +IMG_EXTENSIONS = ".jpg", ".jpeg", ".png", ".ppm", ".bmp", ".pgm", ".tif" -def make_dataset(directory: str, class_to_idx: dict, extensions: tuple, num_samples_per_class: int) -> list: +def make_dataset( + directory: str, class_to_idx: dict, extensions: tuple, num_samples_per_class: int +) -> list: """ Creates a dataset of images with num_samples_per_class images in each class :param directory: The string path to the data directory. @@ -41,7 +43,9 @@ def make_dataset(directory: str, class_to_idx: dict, extensions: tuple, num_samp class_path = os.path.join(directory, class_name) if os.path.isdir(class_path): class_idx = class_to_idx[class_name] - class_images = add_images_for_class(class_path, extensions, num_samples_per_class, class_idx) + class_images = add_images_for_class( + class_path, extensions, num_samples_per_class, class_idx + ) images.extend(class_images) num_classes += 1 @@ -49,7 +53,9 @@ def make_dataset(directory: str, class_to_idx: dict, extensions: tuple, num_samp return images -def add_images_for_class(class_path: str, extensions: tuple, num_samples_per_class: int, class_idx: int) -> list: +def add_images_for_class( + class_path: str, extensions: tuple, num_samples_per_class: int, class_idx: int +) -> list: """ For a given class, adds num_samples_per_class images to a list. :param class_path: The string path to the class directory. @@ -78,9 +84,13 @@ class ImageFolder(Dataset): individual files grouped by category. """ - def __init__(self, root: str, transform=None, target_transform=None, - num_samples_per_class: int = None): - + def __init__( + self, + root: str, + transform=None, + target_transform=None, + num_samples_per_class: int = None, + ): """ :param root: The path to the data directory. :param transform: The required processing to be applied on the sample. @@ -89,11 +99,17 @@ def __init__(self, root: str, transform=None, target_transform=None, """ Dataset.__init__(self) classes, class_to_idx = self._find_classes(root) - self.samples = make_dataset(root, class_to_idx, IMG_EXTENSIONS, num_samples_per_class) + self.samples = make_dataset( + root, class_to_idx, IMG_EXTENSIONS, num_samples_per_class + ) if not self.samples: - raise (RuntimeError( - "Found 0 files in sub folders of: {}\nSupported extensions are: {}".format( - root, ",".join(IMG_EXTENSIONS)))) + raise ( + RuntimeError( + "Found 0 files in sub folders of: {}\nSupported extensions are: {}".format( + root, ",".join(IMG_EXTENSIONS) + ) + ) + ) self.root = root self.loader = default_loader @@ -110,8 +126,11 @@ def __init__(self, root: str, transform=None, target_transform=None, @staticmethod def _find_classes(directory: str): - classes = [d for d in os.listdir(directory) if - os.path.isdir(os.path.join(directory, d))] + classes = [ + d + for d in os.listdir(directory) + if os.path.isdir(os.path.join(directory, d)) + ] classes.sort() class_to_idx = {classes[i]: i for i in range(len(classes))} return classes, class_to_idx @@ -135,8 +154,15 @@ class ImageNetDataLoader: For loading Validation data from the ImageNet dataset. """ - def __init__(self, images_dir: str, image_size: int, batch_size: int = 128, - is_training: bool = False, num_workers: int = 8, num_samples_per_class: int = None): + def __init__( + self, + images_dir: str, + image_size: int, + batch_size: int = 128, + is_training: bool = False, + num_workers: int = 8, + num_samples_per_class: int = None, + ): """ :param images_dir: The path to the data directory :param image_size: The length of the image @@ -148,33 +174,48 @@ def __init__(self, images_dir: str, image_size: int, batch_size: int = 128, # For normalization, mean and std dev values are calculated per channel # and can be found on the web. - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) - - self.train_transforms = transforms.Compose([ - transforms.RandomResizedCrop(image_size), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - normalize]) - - self.val_transforms = transforms.Compose([ - transforms.Resize(image_size + 24), - transforms.CenterCrop(image_size), - transforms.ToTensor(), - normalize]) + normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + self.train_transforms = transforms.Compose( + [ + transforms.RandomResizedCrop(image_size), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ] + ) + + self.val_transforms = transforms.Compose( + [ + transforms.Resize(image_size + 24), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + normalize, + ] + ) if is_training: self.data_set = ImageFolder( - root=images_dir, transform=self.train_transforms, - num_samples_per_class=num_samples_per_class) + root=images_dir, + transform=self.train_transforms, + num_samples_per_class=num_samples_per_class, + ) else: self.data_set = ImageFolder( - root=images_dir, transform=self.val_transforms, - num_samples_per_class=num_samples_per_class) + root=images_dir, + transform=self.val_transforms, + num_samples_per_class=num_samples_per_class, + ) self._data_loader = torch_data.DataLoader( - self.data_set, batch_size=batch_size, shuffle=is_training, - num_workers=num_workers, pin_memory=True) + self.data_set, + batch_size=batch_size, + shuffle=is_training, + num_workers=num_workers, + pin_memory=True, + ) @property def data_loader(self) -> torch_data.DataLoader: @@ -182,4 +223,3 @@ def data_loader(self) -> torch_data.DataLoader: Returns the data-loader """ return self._data_loader - diff --git a/aimet_zoo_torch/common/utils/utils.py b/aimet_zoo_torch/common/utils/utils.py index 114c83c..e6252c4 100644 --- a/aimet_zoo_torch/common/utils/utils.py +++ b/aimet_zoo_torch/common/utils/utils.py @@ -7,13 +7,16 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - +""" util functions to get available device""" import torch -# Returns 'cuda' only when use_cuda is True and a GPU is available -# Throws exception when user enables use_cuda but no GPU is available + def get_device(args): + """ + Returns 'cuda' only when use_cuda is True and a GPU is available + Throws exception when user enables use_cuda but no GPU is available + """ + #pylint:disable = broad-exception-raised if args.use_cuda and not torch.cuda.is_available(): - raise Exception('use-cuda set to True, but cuda is not available') - return torch.device('cuda' if args.use_cuda else 'cpu') - + raise Exception("use-cuda set to True, but cuda is not available") + return torch.device("cuda" if args.use_cuda else "cpu") diff --git a/aimet_zoo_torch/deeplabv3/__init__.py b/aimet_zoo_torch/deeplabv3/__init__.py index 97d29ed..75d1579 100644 --- a/aimet_zoo_torch/deeplabv3/__init__.py +++ b/aimet_zoo_torch/deeplabv3/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import DeepLabV3_Plus \ No newline at end of file +""" loading deeplabv3 downloader class """ +from .model.model_definition import DeepLabV3_Plus diff --git a/aimet_zoo_torch/deeplabv3/dataloader/__init__.py b/aimet_zoo_torch/deeplabv3/dataloader/__init__.py index c086995..7661cb7 100644 --- a/aimet_zoo_torch/deeplabv3/dataloader/__init__.py +++ b/aimet_zoo_torch/deeplabv3/dataloader/__init__.py @@ -1 +1,2 @@ -from .dataloaders_and_eval_func import get_dataloaders_and_eval_func \ No newline at end of file +"""loading dataloader and evaluation function""" +from .dataloaders_and_eval_func import get_dataloaders_and_eval_func diff --git a/aimet_zoo_torch/deeplabv3/dataloader/ade20k_dataloader.py b/aimet_zoo_torch/deeplabv3/dataloader/ade20k_dataloader.py index 28799ec..639eef0 100644 --- a/aimet_zoo_torch/deeplabv3/dataloader/ade20k_dataloader.py +++ b/aimet_zoo_torch/deeplabv3/dataloader/ade20k_dataloader.py @@ -1,3 +1,4 @@ +# pylint: skip-file # --------------------------------------------- # Released under Apache 2.0 License # https://github.com/Tramac/awesome-semantic-segmentation-pytorch/tree/b8366310de50869f89e836ed24de24edd432ece5 diff --git a/aimet_zoo_torch/deeplabv3/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/deeplabv3/dataloader/dataloaders_and_eval_func.py index 84476a8..59b462b 100644 --- a/aimet_zoo_torch/deeplabv3/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/deeplabv3/dataloader/dataloaders_and_eval_func.py @@ -7,47 +7,60 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +""" module for getting dataloader and evaluation function for deeplabv3 """ -from aimet_zoo_torch.deeplabv3.model.dataloaders import make_data_loader -from aimet_zoo_torch.deeplabv3.model.utils.metrics import Evaluator -import torch import random import numpy as np from tqdm import tqdm +import torch +from aimet_zoo_torch.deeplabv3.model.dataloaders import make_data_loader +from aimet_zoo_torch.deeplabv3.model.utils.metrics import Evaluator -class DataloaderConfig(): +class DataloaderConfig: + """hardcode values for dataset config""" def __init__(self, dataset_path): self.use_sbd = False - self.dataset = 'pascal' + self.dataset = "pascal" self.num_classes = 21 self.batch_size = 8 self.crop_size = 513 self.base_size = 513 self.dataset_path = dataset_path + def work_init(work_id): """Define 'worker_init_fn' for data_loader""" seed = torch.initial_seed() % 2**32 random.seed(seed + work_id) np.random.seed(seed + work_id) + # Dataloader kwargs -kwargs = {'worker_init_fn': work_init, - 'num_workers': 0} +kwargs = {"worker_init_fn": work_init, "num_workers": 0} # '''Overwrite the Pascal dataloader with the ADE20K datalaoder''' # from aimet_zoo_torch.deeplabv3_plus.dataloader.ade20k_dataloader import ADE20KSegmentation # val_loader = ADE20KSegmentation(root=DataloaderConfig().path, split='val') + def get_dataloaders_and_eval_func(pascal_path): + """getting dataloader and evaluation function""" + #pylint:disable = unused-variable config = DataloaderConfig(dataset_path=pascal_path) - train_loader, val_loader, test_loader, num_class = make_data_loader(config, **kwargs) + train_loader, val_loader, test_loader, num_class = make_data_loader( + config, **kwargs + ) - def eval_func(model, args): + def eval_func(model, args): + """ + evaluation function for deeplabv3 + parameters: model, args + return: mIoU + """ iterations = args[0] device = args[1] - evaluator = Evaluator(21) # 21 for Pascal, 150 for ADE20k + evaluator = Evaluator(21) # 21 for Pascal, 150 for ADE20k evaluator.reset() model.eval() model.to(device) @@ -59,7 +72,12 @@ def eval_func(model, args): pred = torch.argmax(output, 1).data.cpu().numpy() evaluator.add_batch(label, pred) total_samples += images.size()[0] - if type(iterations)==int and iterations > 0 and total_samples >= iterations: + # pylint:disable = chained-comparison + if ( + isinstance(iterations, int) + and iterations > 0 + and total_samples >= iterations + ): break mIoU = evaluator.Mean_Intersection_over_Union() return mIoU @@ -67,10 +85,13 @@ def eval_func(model, args): return train_loader, val_loader, eval_func -def unlabeled_dataset(): - for X, Y in val_loader: +def unlabeled_dataset(val_loader): + """return unlabeled_dataset for dataloader""" + for X, _ in val_loader: yield X -def forward_pass(model, args): + +def forward_pass(model, val_loader): + """forward pass of dataloader""" for X in unlabeled_dataset(val_loader): - _ = model(X) \ No newline at end of file + _ = model(X) diff --git a/aimet_zoo_torch/deeplabv3/dataloader/segbase.py b/aimet_zoo_torch/deeplabv3/dataloader/segbase.py index 35e6a58..5a2beb2 100644 --- a/aimet_zoo_torch/deeplabv3/dataloader/segbase.py +++ b/aimet_zoo_torch/deeplabv3/dataloader/segbase.py @@ -1,3 +1,4 @@ +# pylint: skip-file # --------------------------------------------- # Released under Apache 2.0 License # https://github.com/Tramac/awesome-semantic-segmentation-pytorch/tree/b8366310de50869f89e836ed24de24edd432ece5 diff --git a/aimet_zoo_torch/deeplabv3/dataloader/utils.py b/aimet_zoo_torch/deeplabv3/dataloader/utils.py index 29e8b79..6c34c89 100644 --- a/aimet_zoo_torch/deeplabv3/dataloader/utils.py +++ b/aimet_zoo_torch/deeplabv3/dataloader/utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file # --------------------------------------------- # Released under Apache 2.0 License # https://github.com/Tramac/awesome-semantic-segmentation-pytorch/tree/b8366310de50869f89e836ed24de24edd432ece5 diff --git a/aimet_zoo_torch/deeplabv3/evaluators/deeplabv3_quanteval.py b/aimet_zoo_torch/deeplabv3/evaluators/deeplabv3_quanteval.py index 69e3f39..5f31356 100644 --- a/aimet_zoo_torch/deeplabv3/evaluators/deeplabv3_quanteval.py +++ b/aimet_zoo_torch/deeplabv3/evaluators/deeplabv3_quanteval.py @@ -8,37 +8,57 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantsim code for DeepLabV3+ ''' +""" AIMET Quantsim code for DeepLabV3+ """ # General Python related imports -import sys, os, tarfile -import urllib.request -from tqdm import tqdm import argparse +# Torch related imports +import torch from aimet_zoo_torch.deeplabv3 import DeepLabV3_Plus from aimet_zoo_torch.common.utils.utils import get_device from aimet_zoo_torch.deeplabv3.dataloader import get_dataloaders_and_eval_func -# Torch related imports -import torch def arguments(): - parser = argparse.ArgumentParser(description='Evaluation script for PyTorch ImageNet networks.') - parser.add_argument('--dataset-path', help='Fullpath to VOCdevkit/VOC2012/', type = str) - parser.add_argument('--model-config', help='Select the model configuration', type=str, default="dlv3_w4a8", choices=[ - "dlv3_w4a8", - "dlv3_w8a8"]) - parser.add_argument('--batch-size', help='Data batch size for a model', type = int, default=4) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type = int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type = int, default=8) - parser.add_argument('--use-cuda', help='Run evaluation on GPU.', type = bool, default=True) + """ argument parser""" + parser = argparse.ArgumentParser( + description="Evaluation script for PyTorch ImageNet networks." + ) + parser.add_argument( + "--dataset-path", help="Fullpath to VOCdevkit/VOC2012/", type=str + ) + parser.add_argument( + "--model-config", + help="Select the model configuration", + type=str, + default="dlv3_w4a8", + choices=["dlv3_w4a8", "dlv3_w8a8"], + ) + parser.add_argument( + "--batch-size", help="Data batch size for a model", type=int, default=4 + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True + ) args = parser.parse_args() return args -# Set seed for reproducibility def seed(seed_number): + """Set seed for reproducibility""" torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(seed_number) @@ -46,22 +66,30 @@ def seed(seed_number): torch.cuda.manual_seed_all(seed_number) +def get_num_classes(val_loader, device): + """get num of classes for model""" + uniques = torch.Tensor().to(device) + for _, Y in val_loader: + uniq = torch.unique(Y).to(device) + uniques = torch.cat([uniques, uniq]) + uniques = torch.unique(uniques) + num_classes = uniques.shape[0] - 1 + return num_classes + + def main(): + """ main evaluation function""" seed(0) args = arguments() device = get_device(args) iterations = -1 - print(f'device: {device}') + print(f"device: {device}") + # pylint: disable = unused-variable + train_loader, val_loader, eval_func = get_dataloaders_and_eval_func( + pascal_path=args.dataset_path + ) - train_loader, val_loader, eval_func = get_dataloaders_and_eval_func(pascal_path = args.dataset_path) - - uniques = torch.Tensor().to(device) - for X, Y in val_loader: - uniq = torch.unique(Y).to(device) - uniques = torch.cat([uniques, uniq]) - input_shape = X.shape - uniques = torch.unique(uniques) - num_classes = uniques.shape[0] - 1 + num_classes = get_num_classes(val_loader, device) # Load original model model_orig = DeepLabV3_Plus(model_config=args.model_config, num_classes=num_classes) @@ -70,14 +98,18 @@ def main(): model_orig.model.eval() # Load optimized model - model_optim = DeepLabV3_Plus(model_config=args.model_config, num_classes=num_classes) + model_optim = DeepLabV3_Plus( + model_config=args.model_config, num_classes=num_classes + ) model_optim.from_pretrained(quantized=True) model_optim.model.to(device) model_optim.model.eval() - print('Evaluating Original Model') + print("Evaluating Original Model") sim_orig = model_orig.get_quantsim(quantized=False) - sim_orig.compute_encodings(eval_func, [iterations, device]) # dont use AdaRound encodings for the original model + sim_orig.compute_encodings( + eval_func, [iterations, device] + ) # dont use AdaRound encodings for the original model mIoU_orig_fp32 = eval_func(model_orig.model, [iterations, device]) del model_orig torch.cuda.empty_cache() @@ -85,7 +117,7 @@ def main(): del sim_orig torch.cuda.empty_cache() - print('Evaluating Optimized Model') + print("Evaluating Optimized Model") sim_optim = model_optim.get_quantsim(quantized=True) sim_optim.compute_encodings(eval_func, [iterations, device]) mIoU_optim_fp32 = eval_func(model_optim.model, [iterations, device]) @@ -95,10 +127,15 @@ def main(): del sim_optim torch.cuda.empty_cache() - print(f'Original Model | 32-bit Environment | mIoU: {mIoU_orig_fp32:.4f}') - print(f'Original Model | {args.default_param_bw}-bit Environment | mIoU: {mIoU_orig_int8:.4f}') - print(f'Optimized Model | 32-bit Environment | mIoU: {mIoU_optim_fp32:.4f}') - print(f'Optimized Model | {args.default_param_bw}-bit Environment | mIoU: {mIoU_optim_int8:.4f}') + print(f"Original Model | 32-bit Environment | mIoU: {mIoU_orig_fp32:.4f}") + print( + f"Original Model | {args.default_param_bw}-bit Environment | mIoU: {mIoU_orig_int8:.4f}" + ) + print(f"Optimized Model | 32-bit Environment | mIoU: {mIoU_optim_fp32:.4f}") + print( + f"Optimized Model | {args.default_param_bw}-bit Environment | mIoU: {mIoU_optim_int8:.4f}" + ) + -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/__init__.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/__init__.py index cfb1fbe..88c1c4f 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/__init__.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/__init__.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # @@ -26,52 +27,69 @@ # @@-COPYRIGHT-START-@@ # # Copyright (c) 2022 of Qualcomm Innovation Center, Inc. All rights reserved. -# Changes from QuIC are licensed under the terms and conditions at +# Changes from QuIC are licensed under the terms and conditions at # https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf # # @@-COPYRIGHT-END-@@ # ============================================================================= -from aimet_zoo_torch.deeplabv3.model.dataloaders.datasets import pascal #cityscapes, coco, combine_dbs, pascal, sbd +from aimet_zoo_torch.deeplabv3.model.dataloaders.datasets import ( + pascal, +) # cityscapes, coco, combine_dbs, pascal, sbd from torch.utils.data import DataLoader -def make_data_loader(args, **kwargs): - if args.dataset == 'pascal': - train_set = pascal.VOCSegmentation(args, split='train') - val_set = pascal.VOCSegmentation(args, split='val') +def make_data_loader(args, **kwargs): + if args.dataset == "pascal": + train_set = pascal.VOCSegmentation(args, split="train") + val_set = pascal.VOCSegmentation(args, split="val") if args.use_sbd: - sbd_train = sbd.SBDSegmentation(args, split=['train', 'val']) - train_set = combine_dbs.CombineDBs([train_set, sbd_train], excluded=[val_set]) + sbd_train = sbd.SBDSegmentation(args, split=["train", "val"]) + train_set = combine_dbs.CombineDBs( + [train_set, sbd_train], excluded=[val_set] + ) num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False, **kwargs) + train_loader = DataLoader( + train_set, batch_size=args.batch_size, shuffle=True, **kwargs + ) + val_loader = DataLoader( + val_set, batch_size=args.batch_size, shuffle=False, **kwargs + ) test_loader = None return train_loader, val_loader, test_loader, num_class - elif args.dataset == 'cityscapes': - train_set = cityscapes.CityscapesSegmentation(args, split='train') - val_set = cityscapes.CityscapesSegmentation(args, split='val') - test_set = cityscapes.CityscapesSegmentation(args, split='test') + elif args.dataset == "cityscapes": + train_set = cityscapes.CityscapesSegmentation(args, split="train") + val_set = cityscapes.CityscapesSegmentation(args, split="val") + test_set = cityscapes.CityscapesSegmentation(args, split="test") num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False, **kwargs) - test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, **kwargs) + train_loader = DataLoader( + train_set, batch_size=args.batch_size, shuffle=True, **kwargs + ) + val_loader = DataLoader( + val_set, batch_size=args.batch_size, shuffle=False, **kwargs + ) + test_loader = DataLoader( + test_set, batch_size=args.batch_size, shuffle=False, **kwargs + ) return train_loader, val_loader, test_loader, num_class - elif args.dataset == 'coco': - train_set = coco.COCOSegmentation(args, split='train') - val_set = coco.COCOSegmentation(args, split='val') + elif args.dataset == "coco": + train_set = coco.COCOSegmentation(args, split="train") + val_set = coco.COCOSegmentation(args, split="val") num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False, **kwargs) + train_loader = DataLoader( + train_set, batch_size=args.batch_size, shuffle=True, **kwargs + ) + val_loader = DataLoader( + val_set, batch_size=args.batch_size, shuffle=False, **kwargs + ) test_loader = None return train_loader, val_loader, test_loader, num_class else: raise NotImplementedError - diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/custom_transforms.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/custom_transforms.py index 0020b6c..35abe59 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/custom_transforms.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/custom_transforms.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/cityscapes.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/cityscapes.py index 9b0ccd2..159ed12 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/cityscapes.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/cityscapes.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/coco.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/coco.py index 7955f32..2926032 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/coco.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/coco.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/combine_dbs.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/combine_dbs.py index 22d26bc..919591b 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/combine_dbs.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/combine_dbs.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/pascal.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/pascal.py index 030ee49..8fb2177 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/pascal.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/pascal.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/pascalbc.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/pascalbc.py index f6373a5..50ac917 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/pascalbc.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/pascalbc.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/sbd.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/sbd.py index c84033f..fe4a097 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/sbd.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/datasets/sbd.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/dataloaders/utils.py b/aimet_zoo_torch/deeplabv3/model/dataloaders/utils.py index 216157d..216134d 100644 --- a/aimet_zoo_torch/deeplabv3/model/dataloaders/utils.py +++ b/aimet_zoo_torch/deeplabv3/model/dataloaders/utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------- # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/model_cards/__init__.py b/aimet_zoo_torch/deeplabv3/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/deeplabv3/model/model_definition.py b/aimet_zoo_torch/deeplabv3/model/model_definition.py index a6c716b..e2cd914 100644 --- a/aimet_zoo_torch/deeplabv3/model/model_definition.py +++ b/aimet_zoo_torch/deeplabv3/model/model_definition.py @@ -7,54 +7,63 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - -import torch +"""Class for downloading and setting up of optmized and original deeplabv3plus model for AIMET model zoo""" +# pylint:disable = import-error import json import os -from aimet_zoo_torch.common.downloader import Downloader -from .modeling.deeplab import DeepLab +import torch from aimet_torch.cross_layer_equalization import equalize_model from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim +from aimet_zoo_torch.common.downloader import Downloader +from .modeling.deeplab import DeepLab class DeepLabV3_Plus(Downloader): + """Deeplabv3 objection detection parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" def __init__(self, model_config=None, num_classes=21): - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" with open(config_filepath) as f_in: self.cfg = json.load(f_in) if model_config: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.model = DeepLab(backbone = 'mobilenet', - sync_bn = False, - num_classes = self.cfg['model_args']['num_classes']) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.model = DeepLab( + backbone="mobilenet", + sync_bn=False, + num_classes=self.cfg["model_args"]["num_classes"], + ) else: - self.model = DeepLab(backbone = 'mobilenet', - sync_bn = False, - num_classes = num_classes) + self.model = DeepLab( + backbone="mobilenet", sync_bn=False, num_classes=num_classes + ) - self.input_shape = tuple(x if x != None else 1 for x in self.cfg['input_shape']) + self.input_shape = tuple(x if x is not None else 1 for x in self.cfg["input_shape"]) self.model_config = model_config def from_pretrained(self, quantized=False): + """get pretrained model from downloading or coping""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() self._download_aimet_encodings() self._download_adaround_encodings() if quantized: - if self.model_config == 'dlv3_w4a8': + if self.model_config == "dlv3_w4a8": self.model = torch.load(self.path_post_opt_weights) else: equalize_model(self.model, self.input_shape) @@ -63,23 +72,31 @@ def from_pretrained(self, quantized=False): del state_dict else: checkpoint = torch.load(self.path_pre_opt_weights) - self.model.load_state_dict(checkpoint['state_dict']) + self.model.load_state_dict(checkpoint["state_dict"]) del checkpoint self.model.cuda() def get_quantsim(self, quantized=False): + """" to get quantsim object for model from loading/computing proper encodings""" if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model.cuda(), **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/aspp.py b/aimet_zoo_torch/deeplabv3/model/modeling/aspp.py index fc79458..9395a6a 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/aspp.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/aspp.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- @@ -38,13 +39,23 @@ import torch import torch.nn as nn import torch.nn.functional as F -from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d +from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import ( + SynchronizedBatchNorm2d, +) + class _ASPPModule(nn.Module): def __init__(self, inplanes, planes, kernel_size, padding, dilation, BatchNorm): super(_ASPPModule, self).__init__() - self.atrous_conv = nn.Conv2d(inplanes, planes, kernel_size=kernel_size, - stride=1, padding=padding, dilation=dilation, bias=False) + self.atrous_conv = nn.Conv2d( + inplanes, + planes, + kernel_size=kernel_size, + stride=1, + padding=padding, + dilation=dilation, + bias=False, + ) self.bn = BatchNorm(planes) self.relu = nn.ReLU() @@ -67,12 +78,13 @@ def _init_weight(self): m.weight.data.fill_(1) m.bias.data.zero_() + class ASPP(nn.Module): def __init__(self, backbone, output_stride, BatchNorm): super(ASPP, self).__init__() - if backbone == 'drn': + if backbone == "drn": inplanes = 512 - elif backbone == 'mobilenet': + elif backbone == "mobilenet": inplanes = 320 else: inplanes = 2048 @@ -83,15 +95,40 @@ def __init__(self, backbone, output_stride, BatchNorm): else: raise NotImplementedError - self.aspp1 = _ASPPModule(inplanes, 256, 1, padding=0, dilation=dilations[0], BatchNorm=BatchNorm) - self.aspp2 = _ASPPModule(inplanes, 256, 3, padding=dilations[1], dilation=dilations[1], BatchNorm=BatchNorm) - self.aspp3 = _ASPPModule(inplanes, 256, 3, padding=dilations[2], dilation=dilations[2], BatchNorm=BatchNorm) - self.aspp4 = _ASPPModule(inplanes, 256, 3, padding=dilations[3], dilation=dilations[3], BatchNorm=BatchNorm) - - self.global_avg_pool = nn.Sequential(nn.AdaptiveAvgPool2d((1, 1)), - nn.Conv2d(inplanes, 256, 1, stride=1, bias=False), - BatchNorm(256), - nn.ReLU()) + self.aspp1 = _ASPPModule( + inplanes, 256, 1, padding=0, dilation=dilations[0], BatchNorm=BatchNorm + ) + self.aspp2 = _ASPPModule( + inplanes, + 256, + 3, + padding=dilations[1], + dilation=dilations[1], + BatchNorm=BatchNorm, + ) + self.aspp3 = _ASPPModule( + inplanes, + 256, + 3, + padding=dilations[2], + dilation=dilations[2], + BatchNorm=BatchNorm, + ) + self.aspp4 = _ASPPModule( + inplanes, + 256, + 3, + padding=dilations[3], + dilation=dilations[3], + BatchNorm=BatchNorm, + ) + + self.global_avg_pool = nn.Sequential( + nn.AdaptiveAvgPool2d((1, 1)), + nn.Conv2d(inplanes, 256, 1, stride=1, bias=False), + BatchNorm(256), + nn.ReLU(), + ) self.conv1 = nn.Conv2d(1280, 256, 1, bias=False) self.bn1 = BatchNorm(256) self.relu = nn.ReLU() @@ -104,7 +141,7 @@ def forward(self, x): x3 = self.aspp3(x) x4 = self.aspp4(x) x5 = self.global_avg_pool(x) - x5 = F.interpolate(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) + x5 = F.interpolate(x5, size=x4.size()[2:], mode="bilinear", align_corners=True) x = torch.cat((x1, x2, x3, x4, x5), dim=1) x = self.conv1(x) @@ -128,4 +165,4 @@ def _init_weight(self): def build_aspp(backbone, output_stride, BatchNorm): - return ASPP(backbone, output_stride, BatchNorm) \ No newline at end of file + return ASPP(backbone, output_stride, BatchNorm) diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/__init__.py b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/__init__.py index b2313a6..4c0df22 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/__init__.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/__init__.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- @@ -34,16 +35,22 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -from aimet_zoo_torch.deeplabv3.model.modeling.backbone import resnet, xception, drn, mobilenet +from aimet_zoo_torch.deeplabv3.model.modeling.backbone import ( + resnet, + xception, + drn, + mobilenet, +) + def build_backbone(backbone, output_stride, BatchNorm): - if backbone == 'resnet': + if backbone == "resnet": return resnet.ResNet101(output_stride, BatchNorm) - elif backbone == 'xception': + elif backbone == "xception": return xception.AlignedXception(output_stride, BatchNorm) - elif backbone == 'drn': + elif backbone == "drn": return drn.drn_d_54(BatchNorm) - elif backbone == 'mobilenet': + elif backbone == "mobilenet": return mobilenet.MobileNetV2(output_stride, BatchNorm) else: raise NotImplementedError diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/drn.py b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/drn.py index 73e6ad6..c7f37e6 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/drn.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/drn.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- @@ -37,39 +38,56 @@ import torch.nn as nn import math import torch.utils.model_zoo as model_zoo -from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d +from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import ( + SynchronizedBatchNorm2d, +) -webroot = 'http://dl.yf.io/drn/' +webroot = "http://dl.yf.io/drn/" model_urls = { - 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', - 'drn-c-26': webroot + 'drn_c_26-ddedf421.pth', - 'drn-c-42': webroot + 'drn_c_42-9d336e8c.pth', - 'drn-c-58': webroot + 'drn_c_58-0a53a92c.pth', - 'drn-d-22': webroot + 'drn_d_22-4bd2f8ea.pth', - 'drn-d-38': webroot + 'drn_d_38-eebb45f0.pth', - 'drn-d-54': webroot + 'drn_d_54-0e0534ff.pth', - 'drn-d-105': webroot + 'drn_d_105-12b40979.pth' + "resnet50": "https://download.pytorch.org/models/resnet50-19c8e357.pth", + "drn-c-26": webroot + "drn_c_26-ddedf421.pth", + "drn-c-42": webroot + "drn_c_42-9d336e8c.pth", + "drn-c-58": webroot + "drn_c_58-0a53a92c.pth", + "drn-d-22": webroot + "drn_d_22-4bd2f8ea.pth", + "drn-d-38": webroot + "drn_d_38-eebb45f0.pth", + "drn-d-54": webroot + "drn_d_54-0e0534ff.pth", + "drn-d-105": webroot + "drn_d_105-12b40979.pth", } def conv3x3(in_planes, out_planes, stride=1, padding=1, dilation=1): - return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, - padding=padding, bias=False, dilation=dilation) + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=padding, + bias=False, + dilation=dilation, + ) class BasicBlock(nn.Module): expansion = 1 - def __init__(self, inplanes, planes, stride=1, downsample=None, - dilation=(1, 1), residual=True, BatchNorm=None): + def __init__( + self, + inplanes, + planes, + stride=1, + downsample=None, + dilation=(1, 1), + residual=True, + BatchNorm=None, + ): super(BasicBlock, self).__init__() - self.conv1 = conv3x3(inplanes, planes, stride, - padding=dilation[0], dilation=dilation[0]) + self.conv1 = conv3x3( + inplanes, planes, stride, padding=dilation[0], dilation=dilation[0] + ) self.bn1 = BatchNorm(planes) self.relu = nn.ReLU(inplace=True) - self.conv2 = conv3x3(planes, planes, - padding=dilation[1], dilation=dilation[1]) + self.conv2 = conv3x3(planes, planes, padding=dilation[1], dilation=dilation[1]) self.bn2 = BatchNorm(planes) self.downsample = downsample self.stride = stride @@ -97,14 +115,28 @@ def forward(self, x): class Bottleneck(nn.Module): expansion = 4 - def __init__(self, inplanes, planes, stride=1, downsample=None, - dilation=(1, 1), residual=True, BatchNorm=None): + def __init__( + self, + inplanes, + planes, + stride=1, + downsample=None, + dilation=(1, 1), + residual=True, + BatchNorm=None, + ): super(Bottleneck, self).__init__() self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) self.bn1 = BatchNorm(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, - padding=dilation[1], bias=False, - dilation=dilation[1]) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=dilation[1], + bias=False, + dilation=dilation[1], + ) self.bn2 = BatchNorm(planes) self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) self.bn3 = BatchNorm(planes * 4) @@ -136,59 +168,118 @@ def forward(self, x): class DRN(nn.Module): - - def __init__(self, block, layers, arch='D', - channels=(16, 32, 64, 128, 256, 512, 512, 512), - BatchNorm=None): + def __init__( + self, + block, + layers, + arch="D", + channels=(16, 32, 64, 128, 256, 512, 512, 512), + BatchNorm=None, + ): super(DRN, self).__init__() self.inplanes = channels[0] self.out_dim = channels[-1] self.arch = arch - if arch == 'C': - self.conv1 = nn.Conv2d(3, channels[0], kernel_size=7, stride=1, - padding=3, bias=False) + if arch == "C": + self.conv1 = nn.Conv2d( + 3, channels[0], kernel_size=7, stride=1, padding=3, bias=False + ) self.bn1 = BatchNorm(channels[0]) self.relu = nn.ReLU(inplace=True) self.layer1 = self._make_layer( - BasicBlock, channels[0], layers[0], stride=1, BatchNorm=BatchNorm) + BasicBlock, channels[0], layers[0], stride=1, BatchNorm=BatchNorm + ) self.layer2 = self._make_layer( - BasicBlock, channels[1], layers[1], stride=2, BatchNorm=BatchNorm) + BasicBlock, channels[1], layers[1], stride=2, BatchNorm=BatchNorm + ) - elif arch == 'D': + elif arch == "D": self.layer0 = nn.Sequential( - nn.Conv2d(3, channels[0], kernel_size=7, stride=1, padding=3, - bias=False), + nn.Conv2d( + 3, channels[0], kernel_size=7, stride=1, padding=3, bias=False + ), BatchNorm(channels[0]), - nn.ReLU(inplace=True) + nn.ReLU(inplace=True), ) self.layer1 = self._make_conv_layers( - channels[0], layers[0], stride=1, BatchNorm=BatchNorm) + channels[0], layers[0], stride=1, BatchNorm=BatchNorm + ) self.layer2 = self._make_conv_layers( - channels[1], layers[1], stride=2, BatchNorm=BatchNorm) - - self.layer3 = self._make_layer(block, channels[2], layers[2], stride=2, BatchNorm=BatchNorm) - self.layer4 = self._make_layer(block, channels[3], layers[3], stride=2, BatchNorm=BatchNorm) - self.layer5 = self._make_layer(block, channels[4], layers[4], - dilation=2, new_level=False, BatchNorm=BatchNorm) - self.layer6 = None if layers[5] == 0 else \ - self._make_layer(block, channels[5], layers[5], dilation=4, - new_level=False, BatchNorm=BatchNorm) - - if arch == 'C': - self.layer7 = None if layers[6] == 0 else \ - self._make_layer(BasicBlock, channels[6], layers[6], dilation=2, - new_level=False, residual=False, BatchNorm=BatchNorm) - self.layer8 = None if layers[7] == 0 else \ - self._make_layer(BasicBlock, channels[7], layers[7], dilation=1, - new_level=False, residual=False, BatchNorm=BatchNorm) - elif arch == 'D': - self.layer7 = None if layers[6] == 0 else \ - self._make_conv_layers(channels[6], layers[6], dilation=2, BatchNorm=BatchNorm) - self.layer8 = None if layers[7] == 0 else \ - self._make_conv_layers(channels[7], layers[7], dilation=1, BatchNorm=BatchNorm) + channels[1], layers[1], stride=2, BatchNorm=BatchNorm + ) + + self.layer3 = self._make_layer( + block, channels[2], layers[2], stride=2, BatchNorm=BatchNorm + ) + self.layer4 = self._make_layer( + block, channels[3], layers[3], stride=2, BatchNorm=BatchNorm + ) + self.layer5 = self._make_layer( + block, + channels[4], + layers[4], + dilation=2, + new_level=False, + BatchNorm=BatchNorm, + ) + self.layer6 = ( + None + if layers[5] == 0 + else self._make_layer( + block, + channels[5], + layers[5], + dilation=4, + new_level=False, + BatchNorm=BatchNorm, + ) + ) + + if arch == "C": + self.layer7 = ( + None + if layers[6] == 0 + else self._make_layer( + BasicBlock, + channels[6], + layers[6], + dilation=2, + new_level=False, + residual=False, + BatchNorm=BatchNorm, + ) + ) + self.layer8 = ( + None + if layers[7] == 0 + else self._make_layer( + BasicBlock, + channels[7], + layers[7], + dilation=1, + new_level=False, + residual=False, + BatchNorm=BatchNorm, + ) + ) + elif arch == "D": + self.layer7 = ( + None + if layers[6] == 0 + else self._make_conv_layers( + channels[6], layers[6], dilation=2, BatchNorm=BatchNorm + ) + ) + self.layer8 = ( + None + if layers[7] == 0 + else self._make_conv_layers( + channels[7], layers[7], dilation=1, BatchNorm=BatchNorm + ) + ) self._init_weight() @@ -196,7 +287,7 @@ def _init_weight(self): for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2. / n)) + m.weight.data.normal_(0, math.sqrt(2.0 / n)) elif isinstance(m, SynchronizedBatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() @@ -204,49 +295,86 @@ def _init_weight(self): m.weight.data.fill_(1) m.bias.data.zero_() - - def _make_layer(self, block, planes, blocks, stride=1, dilation=1, - new_level=True, residual=True, BatchNorm=None): + def _make_layer( + self, + block, + planes, + blocks, + stride=1, + dilation=1, + new_level=True, + residual=True, + BatchNorm=None, + ): assert dilation == 1 or dilation % 2 == 0 downsample = None if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( - nn.Conv2d(self.inplanes, planes * block.expansion, - kernel_size=1, stride=stride, bias=False), + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), BatchNorm(planes * block.expansion), ) layers = list() - layers.append(block( - self.inplanes, planes, stride, downsample, - dilation=(1, 1) if dilation == 1 else ( - dilation // 2 if new_level else dilation, dilation), - residual=residual, BatchNorm=BatchNorm)) + layers.append( + block( + self.inplanes, + planes, + stride, + downsample, + dilation=(1, 1) + if dilation == 1 + else (dilation // 2 if new_level else dilation, dilation), + residual=residual, + BatchNorm=BatchNorm, + ) + ) self.inplanes = planes * block.expansion for i in range(1, blocks): - layers.append(block(self.inplanes, planes, residual=residual, - dilation=(dilation, dilation), BatchNorm=BatchNorm)) + layers.append( + block( + self.inplanes, + planes, + residual=residual, + dilation=(dilation, dilation), + BatchNorm=BatchNorm, + ) + ) return nn.Sequential(*layers) def _make_conv_layers(self, channels, convs, stride=1, dilation=1, BatchNorm=None): modules = [] for i in range(convs): - modules.extend([ - nn.Conv2d(self.inplanes, channels, kernel_size=3, - stride=stride if i == 0 else 1, - padding=dilation, bias=False, dilation=dilation), - BatchNorm(channels), - nn.ReLU(inplace=True)]) + modules.extend( + [ + nn.Conv2d( + self.inplanes, + channels, + kernel_size=3, + stride=stride if i == 0 else 1, + padding=dilation, + bias=False, + dilation=dilation, + ), + BatchNorm(channels), + nn.ReLU(inplace=True), + ] + ) self.inplanes = channels return nn.Sequential(*modules) def forward(self, x): - if self.arch == 'C': + if self.arch == "C": x = self.conv1(x) x = self.bn1(x) x = self.relu(x) - elif self.arch == 'D': + elif self.arch == "D": x = self.layer0(x) x = self.layer1(x) @@ -271,22 +399,24 @@ def forward(self, x): class DRN_A(nn.Module): - def __init__(self, block, layers, BatchNorm=None): self.inplanes = 64 super(DRN_A, self).__init__() self.out_dim = 512 * block.expansion - self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, - bias=False) + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) self.bn1 = BatchNorm(64) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0], BatchNorm=BatchNorm) - self.layer2 = self._make_layer(block, 128, layers[1], stride=2, BatchNorm=BatchNorm) - self.layer3 = self._make_layer(block, 256, layers[2], stride=1, - dilation=2, BatchNorm=BatchNorm) - self.layer4 = self._make_layer(block, 512, layers[3], stride=1, - dilation=4, BatchNorm=BatchNorm) + self.layer2 = self._make_layer( + block, 128, layers[1], stride=2, BatchNorm=BatchNorm + ) + self.layer3 = self._make_layer( + block, 256, layers[2], stride=1, dilation=2, BatchNorm=BatchNorm + ) + self.layer4 = self._make_layer( + block, 512, layers[3], stride=1, dilation=4, BatchNorm=BatchNorm + ) self._init_weight() @@ -294,7 +424,7 @@ def _init_weight(self): for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2. / n)) + m.weight.data.normal_(0, math.sqrt(2.0 / n)) elif isinstance(m, SynchronizedBatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() @@ -306,17 +436,33 @@ def _make_layer(self, block, planes, blocks, stride=1, dilation=1, BatchNorm=Non downsample = None if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( - nn.Conv2d(self.inplanes, planes * block.expansion, - kernel_size=1, stride=stride, bias=False), + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), BatchNorm(planes * block.expansion), ) layers = [] - layers.append(block(self.inplanes, planes, stride, downsample, BatchNorm=BatchNorm)) + layers.append( + block(self.inplanes, planes, stride, downsample, BatchNorm=BatchNorm) + ) self.inplanes = planes * block.expansion for i in range(1, blocks): - layers.append(block(self.inplanes, planes, - dilation=(dilation, dilation, ), BatchNorm=BatchNorm)) + layers.append( + block( + self.inplanes, + planes, + dilation=( + dilation, + dilation, + ), + BatchNorm=BatchNorm, + ) + ) return nn.Sequential(*layers) @@ -333,104 +479,107 @@ def forward(self, x): return x + def drn_a_50(BatchNorm, pretrained=True): model = DRN_A(Bottleneck, [3, 4, 6, 3], BatchNorm=BatchNorm) if pretrained: - model.load_state_dict(model_zoo.load_url(model_urls['resnet50'])) + model.load_state_dict(model_zoo.load_url(model_urls["resnet50"])) return model def drn_c_26(BatchNorm, pretrained=True): - model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 1, 1], arch='C', BatchNorm=BatchNorm) + model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 1, 1], arch="C", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-c-26']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-c-26"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model def drn_c_42(BatchNorm, pretrained=True): - model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 1, 1], arch='C', BatchNorm=BatchNorm) + model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 1, 1], arch="C", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-c-42']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-c-42"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model def drn_c_58(BatchNorm, pretrained=True): - model = DRN(Bottleneck, [1, 1, 3, 4, 6, 3, 1, 1], arch='C', BatchNorm=BatchNorm) + model = DRN(Bottleneck, [1, 1, 3, 4, 6, 3, 1, 1], arch="C", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-c-58']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-c-58"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model def drn_d_22(BatchNorm, pretrained=True): - model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 1, 1], arch='D', BatchNorm=BatchNorm) + model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 1, 1], arch="D", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-d-22']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-d-22"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model def drn_d_24(BatchNorm, pretrained=True): - model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 2, 2], arch='D', BatchNorm=BatchNorm) + model = DRN(BasicBlock, [1, 1, 2, 2, 2, 2, 2, 2], arch="D", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-d-24']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-d-24"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model def drn_d_38(BatchNorm, pretrained=True): - model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 1, 1], arch='D', BatchNorm=BatchNorm) + model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 1, 1], arch="D", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-d-38']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-d-38"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model def drn_d_40(BatchNorm, pretrained=True): - model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 2, 2], arch='D', BatchNorm=BatchNorm) + model = DRN(BasicBlock, [1, 1, 3, 4, 6, 3, 2, 2], arch="D", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-d-40']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-d-40"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model def drn_d_54(BatchNorm, pretrained=True): - model = DRN(Bottleneck, [1, 1, 3, 4, 6, 3, 1, 1], arch='D', BatchNorm=BatchNorm) + model = DRN(Bottleneck, [1, 1, 3, 4, 6, 3, 1, 1], arch="D", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-d-54']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-d-54"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model def drn_d_105(BatchNorm, pretrained=True): - model = DRN(Bottleneck, [1, 1, 3, 4, 23, 3, 1, 1], arch='D', BatchNorm=BatchNorm) + model = DRN(Bottleneck, [1, 1, 3, 4, 23, 3, 1, 1], arch="D", BatchNorm=BatchNorm) if pretrained: - pretrained = model_zoo.load_url(model_urls['drn-d-105']) - del pretrained['fc.weight'] - del pretrained['fc.bias'] + pretrained = model_zoo.load_url(model_urls["drn-d-105"]) + del pretrained["fc.weight"] + del pretrained["fc.bias"] model.load_state_dict(pretrained) return model + if __name__ == "__main__": import torch + model = drn_a_50(BatchNorm=nn.BatchNorm2d, pretrained=True) input = torch.rand(1, 3, 512, 512) output, low_level_feat = model(input) diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/mobilenet.py b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/mobilenet.py index e9afdc1..a9a6af5 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/mobilenet.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/mobilenet.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- @@ -38,14 +39,17 @@ import torch.nn.functional as F import torch.nn as nn import math -from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d +from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import ( + SynchronizedBatchNorm2d, +) import torch.utils.model_zoo as model_zoo + def conv_bn(inp, oup, stride, BatchNorm): return nn.Sequential( nn.Conv2d(inp, oup, 3, stride, 1, bias=False), BatchNorm(oup), - nn.ReLU6(inplace=True) + nn.ReLU6(inplace=True), ) @@ -72,7 +76,16 @@ def __init__(self, inp, oup, stride, dilation, expand_ratio, BatchNorm): if expand_ratio == 1: self.conv = nn.Sequential( # dw - nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 0, dilation, groups=hidden_dim, bias=False), + nn.Conv2d( + hidden_dim, + hidden_dim, + 3, + stride, + 0, + dilation, + groups=hidden_dim, + bias=False, + ), BatchNorm(hidden_dim), nn.ReLU6(inplace=True), # pw-linear @@ -86,7 +99,16 @@ def __init__(self, inp, oup, stride, dilation, expand_ratio, BatchNorm): BatchNorm(hidden_dim), nn.ReLU6(inplace=True), # dw - nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 0, dilation, groups=hidden_dim, bias=False), + nn.Conv2d( + hidden_dim, + hidden_dim, + 3, + stride, + 0, + dilation, + groups=hidden_dim, + bias=False, + ), BatchNorm(hidden_dim), nn.ReLU6(inplace=True), # pw-linear @@ -104,7 +126,9 @@ def forward(self, x): class MobileNetV2(nn.Module): - def __init__(self, output_stride=8, BatchNorm=None, width_mult=1., pretrained=True): + def __init__( + self, output_stride=8, BatchNorm=None, width_mult=1.0, pretrained=True + ): super(MobileNetV2, self).__init__() block = InvertedResidual input_channel = 32 @@ -138,9 +162,20 @@ def __init__(self, output_stride=8, BatchNorm=None, width_mult=1., pretrained=Tr output_channel = int(c * width_mult) for i in range(n): if i == 0: - self.features.append(block(input_channel, output_channel, stride, dilation, t, BatchNorm)) + self.features.append( + block( + input_channel, + output_channel, + stride, + dilation, + t, + BatchNorm, + ) + ) else: - self.features.append(block(input_channel, output_channel, 1, dilation, t, BatchNorm)) + self.features.append( + block(input_channel, output_channel, 1, dilation, t, BatchNorm) + ) input_channel = output_channel self.features = nn.Sequential(*self.features) self._initialize_weights() @@ -157,7 +192,9 @@ def forward(self, x): return x, low_level_feat def _load_pretrained_model(self): - pretrain_dict = model_zoo.load_url('http://jeff95.me/models/mobilenet_v2-6a65762b.pth') + pretrain_dict = model_zoo.load_url( + "http://jeff95.me/models/mobilenet_v2-6a65762b.pth" + ) model_dict = {} state_dict = self.state_dict() for k, v in pretrain_dict.items(): @@ -179,6 +216,7 @@ def _initialize_weights(self): m.weight.data.fill_(1) m.bias.data.zero_() + if __name__ == "__main__": input = torch.rand(1, 3, 512, 512) model = MobileNetV2(output_stride=16, BatchNorm=nn.BatchNorm2d) diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/resnet.py b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/resnet.py index 773cb81..814ae85 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/resnet.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/resnet.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- @@ -37,17 +38,29 @@ import math import torch.nn as nn import torch.utils.model_zoo as model_zoo -from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d +from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import ( + SynchronizedBatchNorm2d, +) + class Bottleneck(nn.Module): expansion = 4 - def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None, BatchNorm=None): + def __init__( + self, inplanes, planes, stride=1, dilation=1, downsample=None, BatchNorm=None + ): super(Bottleneck, self).__init__() self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) self.bn1 = BatchNorm(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, - dilation=dilation, padding=dilation, bias=False) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + dilation=dilation, + padding=dilation, + bias=False, + ) self.bn2 = BatchNorm(planes) self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) self.bn3 = BatchNorm(planes * 4) @@ -78,8 +91,8 @@ def forward(self, x): return out -class ResNet(nn.Module): +class ResNet(nn.Module): def __init__(self, block, layers, output_stride, BatchNorm, pretrained=True): self.inplanes = 64 super(ResNet, self).__init__() @@ -94,16 +107,43 @@ def __init__(self, block, layers, output_stride, BatchNorm, pretrained=True): raise NotImplementedError # Modules - self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, - bias=False) + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) self.bn1 = BatchNorm(64) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) - self.layer1 = self._make_layer(block, 64, layers[0], stride=strides[0], dilation=dilations[0], BatchNorm=BatchNorm) - self.layer2 = self._make_layer(block, 128, layers[1], stride=strides[1], dilation=dilations[1], BatchNorm=BatchNorm) - self.layer3 = self._make_layer(block, 256, layers[2], stride=strides[2], dilation=dilations[2], BatchNorm=BatchNorm) - self.layer4 = self._make_MG_unit(block, 512, blocks=blocks, stride=strides[3], dilation=dilations[3], BatchNorm=BatchNorm) + self.layer1 = self._make_layer( + block, + 64, + layers[0], + stride=strides[0], + dilation=dilations[0], + BatchNorm=BatchNorm, + ) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=strides[1], + dilation=dilations[1], + BatchNorm=BatchNorm, + ) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=strides[2], + dilation=dilations[2], + BatchNorm=BatchNorm, + ) + self.layer4 = self._make_MG_unit( + block, + 512, + blocks=blocks, + stride=strides[3], + dilation=dilations[3], + BatchNorm=BatchNorm, + ) # self.layer4 = self._make_layer(block, 512, layers[3], stride=strides[3], dilation=dilations[3], BatchNorm=BatchNorm) self._init_weight() @@ -114,35 +154,66 @@ def _make_layer(self, block, planes, blocks, stride=1, dilation=1, BatchNorm=Non downsample = None if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( - nn.Conv2d(self.inplanes, planes * block.expansion, - kernel_size=1, stride=stride, bias=False), + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), BatchNorm(planes * block.expansion), ) layers = [] - layers.append(block(self.inplanes, planes, stride, dilation, downsample, BatchNorm)) + layers.append( + block(self.inplanes, planes, stride, dilation, downsample, BatchNorm) + ) self.inplanes = planes * block.expansion for i in range(1, blocks): - layers.append(block(self.inplanes, planes, dilation=dilation, BatchNorm=BatchNorm)) + layers.append( + block(self.inplanes, planes, dilation=dilation, BatchNorm=BatchNorm) + ) return nn.Sequential(*layers) - def _make_MG_unit(self, block, planes, blocks, stride=1, dilation=1, BatchNorm=None): + def _make_MG_unit( + self, block, planes, blocks, stride=1, dilation=1, BatchNorm=None + ): downsample = None if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( - nn.Conv2d(self.inplanes, planes * block.expansion, - kernel_size=1, stride=stride, bias=False), + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), BatchNorm(planes * block.expansion), ) layers = [] - layers.append(block(self.inplanes, planes, stride, dilation=blocks[0]*dilation, - downsample=downsample, BatchNorm=BatchNorm)) + layers.append( + block( + self.inplanes, + planes, + stride, + dilation=blocks[0] * dilation, + downsample=downsample, + BatchNorm=BatchNorm, + ) + ) self.inplanes = planes * block.expansion for i in range(1, len(blocks)): - layers.append(block(self.inplanes, planes, stride=1, - dilation=blocks[i]*dilation, BatchNorm=BatchNorm)) + layers.append( + block( + self.inplanes, + planes, + stride=1, + dilation=blocks[i] * dilation, + BatchNorm=BatchNorm, + ) + ) return nn.Sequential(*layers) @@ -163,7 +234,7 @@ def _init_weight(self): for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2. / n)) + m.weight.data.normal_(0, math.sqrt(2.0 / n)) elif isinstance(m, SynchronizedBatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() @@ -172,7 +243,9 @@ def _init_weight(self): m.bias.data.zero_() def _load_pretrained_model(self): - pretrain_dict = model_zoo.load_url('https://download.pytorch.org/models/resnet101-5d3b4d8f.pth') + pretrain_dict = model_zoo.load_url( + "https://download.pytorch.org/models/resnet101-5d3b4d8f.pth" + ) model_dict = {} state_dict = self.state_dict() for k, v in pretrain_dict.items(): @@ -181,18 +254,23 @@ def _load_pretrained_model(self): state_dict.update(model_dict) self.load_state_dict(state_dict) + def ResNet101(output_stride, BatchNorm, pretrained=True): """Constructs a ResNet-101 model. Args: pretrained (bool): If True, returns a model pre-trained on ImageNet """ - model = ResNet(Bottleneck, [3, 4, 23, 3], output_stride, BatchNorm, pretrained=pretrained) + model = ResNet( + Bottleneck, [3, 4, 23, 3], output_stride, BatchNorm, pretrained=pretrained + ) return model + if __name__ == "__main__": import torch + model = ResNet101(BatchNorm=nn.BatchNorm2d, pretrained=True, output_stride=8) input = torch.rand(1, 3, 512, 512) output, low_level_feat = model(input) print(output.size()) - print(low_level_feat.size()) \ No newline at end of file + print(low_level_feat.size()) diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/xception.py b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/xception.py index 4c32126..df6bdc0 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/backbone/xception.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/backbone/xception.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- @@ -39,7 +40,10 @@ import torch.nn as nn import torch.nn.functional as F import torch.utils.model_zoo as model_zoo -from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d +from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import ( + SynchronizedBatchNorm2d, +) + def fixed_padding(inputs, kernel_size, dilation): kernel_size_effective = kernel_size + (kernel_size - 1) * (dilation - 1) @@ -51,11 +55,28 @@ def fixed_padding(inputs, kernel_size, dilation): class SeparableConv2d(nn.Module): - def __init__(self, inplanes, planes, kernel_size=3, stride=1, dilation=1, bias=False, BatchNorm=None): + def __init__( + self, + inplanes, + planes, + kernel_size=3, + stride=1, + dilation=1, + bias=False, + BatchNorm=None, + ): super(SeparableConv2d, self).__init__() - self.conv1 = nn.Conv2d(inplanes, inplanes, kernel_size, stride, 0, dilation, - groups=inplanes, bias=bias) + self.conv1 = nn.Conv2d( + inplanes, + inplanes, + kernel_size, + stride, + 0, + dilation, + groups=inplanes, + bias=bias, + ) self.bn = BatchNorm(inplanes) self.pointwise = nn.Conv2d(inplanes, planes, 1, 1, 0, 1, 1, bias=bias) @@ -68,8 +89,18 @@ def forward(self, x): class Block(nn.Module): - def __init__(self, inplanes, planes, reps, stride=1, dilation=1, BatchNorm=None, - start_with_relu=True, grow_first=True, is_last=False): + def __init__( + self, + inplanes, + planes, + reps, + stride=1, + dilation=1, + BatchNorm=None, + start_with_relu=True, + grow_first=True, + is_last=False, + ): super(Block, self).__init__() if planes != inplanes or stride != 1: @@ -84,18 +115,24 @@ def __init__(self, inplanes, planes, reps, stride=1, dilation=1, BatchNorm=None, filters = inplanes if grow_first: rep.append(self.relu) - rep.append(SeparableConv2d(inplanes, planes, 3, 1, dilation, BatchNorm=BatchNorm)) + rep.append( + SeparableConv2d(inplanes, planes, 3, 1, dilation, BatchNorm=BatchNorm) + ) rep.append(BatchNorm(planes)) filters = planes for i in range(reps - 1): rep.append(self.relu) - rep.append(SeparableConv2d(filters, filters, 3, 1, dilation, BatchNorm=BatchNorm)) + rep.append( + SeparableConv2d(filters, filters, 3, 1, dilation, BatchNorm=BatchNorm) + ) rep.append(BatchNorm(filters)) if not grow_first: rep.append(self.relu) - rep.append(SeparableConv2d(inplanes, planes, 3, 1, dilation, BatchNorm=BatchNorm)) + rep.append( + SeparableConv2d(inplanes, planes, 3, 1, dilation, BatchNorm=BatchNorm) + ) rep.append(BatchNorm(planes)) if stride != 1: @@ -131,8 +168,8 @@ class AlignedXception(nn.Module): """ Modified Alighed Xception """ - def __init__(self, output_stride, BatchNorm, - pretrained=True): + + def __init__(self, output_stride, BatchNorm, pretrained=True): super(AlignedXception, self).__init__() if output_stride == 16: @@ -146,7 +183,6 @@ def __init__(self, output_stride, BatchNorm, else: raise NotImplementedError - # Entry flow self.conv1 = nn.Conv2d(3, 32, 3, stride=2, padding=1, bias=False) self.bn1 = BatchNorm(32) @@ -155,57 +191,232 @@ def __init__(self, output_stride, BatchNorm, self.conv2 = nn.Conv2d(32, 64, 3, stride=1, padding=1, bias=False) self.bn2 = BatchNorm(64) - self.block1 = Block(64, 128, reps=2, stride=2, BatchNorm=BatchNorm, start_with_relu=False) - self.block2 = Block(128, 256, reps=2, stride=2, BatchNorm=BatchNorm, start_with_relu=False, - grow_first=True) - self.block3 = Block(256, 728, reps=2, stride=entry_block3_stride, BatchNorm=BatchNorm, - start_with_relu=True, grow_first=True, is_last=True) + self.block1 = Block( + 64, 128, reps=2, stride=2, BatchNorm=BatchNorm, start_with_relu=False + ) + self.block2 = Block( + 128, + 256, + reps=2, + stride=2, + BatchNorm=BatchNorm, + start_with_relu=False, + grow_first=True, + ) + self.block3 = Block( + 256, + 728, + reps=2, + stride=entry_block3_stride, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + is_last=True, + ) # Middle flow - self.block4 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block5 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block6 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block7 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block8 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block9 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block10 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block11 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block12 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block13 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block14 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block15 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block16 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block17 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block18 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) - self.block19 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, - BatchNorm=BatchNorm, start_with_relu=True, grow_first=True) + self.block4 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block5 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block6 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block7 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block8 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block9 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block10 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block11 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block12 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block13 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block14 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block15 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block16 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block17 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block18 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) + self.block19 = Block( + 728, + 728, + reps=3, + stride=1, + dilation=middle_block_dilation, + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=True, + ) # Exit flow - self.block20 = Block(728, 1024, reps=2, stride=1, dilation=exit_block_dilations[0], - BatchNorm=BatchNorm, start_with_relu=True, grow_first=False, is_last=True) - - self.conv3 = SeparableConv2d(1024, 1536, 3, stride=1, dilation=exit_block_dilations[1], BatchNorm=BatchNorm) + self.block20 = Block( + 728, + 1024, + reps=2, + stride=1, + dilation=exit_block_dilations[0], + BatchNorm=BatchNorm, + start_with_relu=True, + grow_first=False, + is_last=True, + ) + + self.conv3 = SeparableConv2d( + 1024, + 1536, + 3, + stride=1, + dilation=exit_block_dilations[1], + BatchNorm=BatchNorm, + ) self.bn3 = BatchNorm(1536) - self.conv4 = SeparableConv2d(1536, 1536, 3, stride=1, dilation=exit_block_dilations[1], BatchNorm=BatchNorm) + self.conv4 = SeparableConv2d( + 1536, + 1536, + 3, + stride=1, + dilation=exit_block_dilations[1], + BatchNorm=BatchNorm, + ) self.bn4 = BatchNorm(1536) - self.conv5 = SeparableConv2d(1536, 2048, 3, stride=1, dilation=exit_block_dilations[1], BatchNorm=BatchNorm) + self.conv5 = SeparableConv2d( + 1536, + 2048, + 3, + stride=1, + dilation=exit_block_dilations[1], + BatchNorm=BatchNorm, + ) self.bn5 = BatchNorm(2048) # Init weights @@ -271,7 +482,7 @@ def _init_weight(self): for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2. / n)) + m.weight.data.normal_(0, math.sqrt(2.0 / n)) elif isinstance(m, SynchronizedBatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() @@ -279,44 +490,45 @@ def _init_weight(self): m.weight.data.fill_(1) m.bias.data.zero_() - def _load_pretrained_model(self): - pretrain_dict = model_zoo.load_url('http://data.lip6.fr/cadene/pretrainedmodels/xception-b5690688.pth') + pretrain_dict = model_zoo.load_url( + "http://data.lip6.fr/cadene/pretrainedmodels/xception-b5690688.pth" + ) model_dict = {} state_dict = self.state_dict() for k, v in pretrain_dict.items(): if k in state_dict: - if 'pointwise' in k: + if "pointwise" in k: v = v.unsqueeze(-1).unsqueeze(-1) - if k.startswith('block11'): + if k.startswith("block11"): model_dict[k] = v - model_dict[k.replace('block11', 'block12')] = v - model_dict[k.replace('block11', 'block13')] = v - model_dict[k.replace('block11', 'block14')] = v - model_dict[k.replace('block11', 'block15')] = v - model_dict[k.replace('block11', 'block16')] = v - model_dict[k.replace('block11', 'block17')] = v - model_dict[k.replace('block11', 'block18')] = v - model_dict[k.replace('block11', 'block19')] = v - elif k.startswith('block12'): - model_dict[k.replace('block12', 'block20')] = v - elif k.startswith('bn3'): + model_dict[k.replace("block11", "block12")] = v + model_dict[k.replace("block11", "block13")] = v + model_dict[k.replace("block11", "block14")] = v + model_dict[k.replace("block11", "block15")] = v + model_dict[k.replace("block11", "block16")] = v + model_dict[k.replace("block11", "block17")] = v + model_dict[k.replace("block11", "block18")] = v + model_dict[k.replace("block11", "block19")] = v + elif k.startswith("block12"): + model_dict[k.replace("block12", "block20")] = v + elif k.startswith("bn3"): model_dict[k] = v - model_dict[k.replace('bn3', 'bn4')] = v - elif k.startswith('conv4'): - model_dict[k.replace('conv4', 'conv5')] = v - elif k.startswith('bn4'): - model_dict[k.replace('bn4', 'bn5')] = v + model_dict[k.replace("bn3", "bn4")] = v + elif k.startswith("conv4"): + model_dict[k.replace("conv4", "conv5")] = v + elif k.startswith("bn4"): + model_dict[k.replace("bn4", "bn5")] = v else: model_dict[k] = v state_dict.update(model_dict) self.load_state_dict(state_dict) - if __name__ == "__main__": import torch + model = AlignedXception(BatchNorm=nn.BatchNorm2d, pretrained=True, output_stride=16) input = torch.rand(1, 3, 512, 512) output, low_level_feat = model(input) diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/decoder.py b/aimet_zoo_torch/deeplabv3/model/modeling/decoder.py index d2d5896..208661a 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/decoder.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/decoder.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- @@ -38,16 +39,19 @@ import torch import torch.nn as nn import torch.nn.functional as F -from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d +from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import ( + SynchronizedBatchNorm2d, +) + class Decoder(nn.Module): def __init__(self, num_classes, backbone, BatchNorm): super(Decoder, self).__init__() - if backbone == 'resnet' or backbone == 'drn': + if backbone == "resnet" or backbone == "drn": low_level_inplanes = 256 - elif backbone == 'xception': + elif backbone == "xception": low_level_inplanes = 128 - elif backbone == 'mobilenet': + elif backbone == "mobilenet": low_level_inplanes = 24 else: raise NotImplementedError @@ -55,24 +59,27 @@ def __init__(self, num_classes, backbone, BatchNorm): self.conv1 = nn.Conv2d(low_level_inplanes, 48, 1, bias=False) self.bn1 = BatchNorm(48) self.relu = nn.ReLU() - self.last_conv = nn.Sequential(nn.Conv2d(304, 256, kernel_size=3, stride=1, padding=1, bias=False), - BatchNorm(256), - nn.ReLU(), - nn.Dropout(0.5), - nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=False), - BatchNorm(256), - nn.ReLU(), - nn.Dropout(0.1), - nn.Conv2d(256, num_classes, kernel_size=1, stride=1)) + self.last_conv = nn.Sequential( + nn.Conv2d(304, 256, kernel_size=3, stride=1, padding=1, bias=False), + BatchNorm(256), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=False), + BatchNorm(256), + nn.ReLU(), + nn.Dropout(0.1), + nn.Conv2d(256, num_classes, kernel_size=1, stride=1), + ) self._init_weight() - def forward(self, x, low_level_feat): low_level_feat = self.conv1(low_level_feat) low_level_feat = self.bn1(low_level_feat) low_level_feat = self.relu(low_level_feat) - x = F.interpolate(x, size=low_level_feat.size()[2:], mode='bilinear', align_corners=True) + x = F.interpolate( + x, size=low_level_feat.size()[2:], mode="bilinear", align_corners=True + ) x = torch.cat((x, low_level_feat), dim=1) x = self.last_conv(x) @@ -89,5 +96,6 @@ def _init_weight(self): m.weight.data.fill_(1) m.bias.data.zero_() + def build_decoder(num_classes, backbone, BatchNorm): - return Decoder(num_classes, backbone, BatchNorm) \ No newline at end of file + return Decoder(num_classes, backbone, BatchNorm) diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/deeplab.py b/aimet_zoo_torch/deeplabv3/model/modeling/deeplab.py index bc0038c..bd8f8d9 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/deeplab.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/deeplab.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- @@ -38,16 +39,25 @@ import torch import torch.nn as nn import torch.nn.functional as F -from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d +from aimet_zoo_torch.deeplabv3.model.modeling.sync_batchnorm.batchnorm import ( + SynchronizedBatchNorm2d, +) from aimet_zoo_torch.deeplabv3.model.modeling.aspp import build_aspp from aimet_zoo_torch.deeplabv3.model.modeling.decoder import build_decoder from aimet_zoo_torch.deeplabv3.model.modeling.backbone import build_backbone + class DeepLab(nn.Module): - def __init__(self, backbone='mobilenet', output_stride=16, num_classes=21, - sync_bn=True, freeze_bn=False): + def __init__( + self, + backbone="mobilenet", + output_stride=16, + num_classes=21, + sync_bn=True, + freeze_bn=False, + ): super(DeepLab, self).__init__() - if backbone == 'drn': + if backbone == "drn": output_stride = 8 if sync_bn == True: @@ -65,7 +75,7 @@ def forward(self, input): x, low_level_feat = self.backbone(input) x = self.aspp(x) x = self.decoder(x, low_level_feat) - x = F.interpolate(x, size=input.size()[2:], mode='bilinear', align_corners=True) + x = F.interpolate(x, size=input.size()[2:], mode="bilinear", align_corners=True) return x @@ -86,8 +96,11 @@ def get_1x_lr_params(self): if p.requires_grad: yield p else: - if isinstance(m[1], nn.Conv2d) or isinstance(m[1], SynchronizedBatchNorm2d) \ - or isinstance(m[1], nn.BatchNorm2d): + if ( + isinstance(m[1], nn.Conv2d) + or isinstance(m[1], SynchronizedBatchNorm2d) + or isinstance(m[1], nn.BatchNorm2d) + ): for p in m[1].parameters(): if p.requires_grad: yield p @@ -102,17 +115,19 @@ def get_10x_lr_params(self): if p.requires_grad: yield p else: - if isinstance(m[1], nn.Conv2d) or isinstance(m[1], SynchronizedBatchNorm2d) \ - or isinstance(m[1], nn.BatchNorm2d): + if ( + isinstance(m[1], nn.Conv2d) + or isinstance(m[1], SynchronizedBatchNorm2d) + or isinstance(m[1], nn.BatchNorm2d) + ): for p in m[1].parameters(): if p.requires_grad: yield p + if __name__ == "__main__": - model = DeepLab(backbone='mobilenet', output_stride=16) + model = DeepLab(backbone="mobilenet", output_stride=16) model.eval() input = torch.rand(1, 3, 513, 513) output = model(input) print(output.size()) - - diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/__init__.py b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/__init__.py index 540118f..a607675 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/__init__.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/__init__.py @@ -1,3 +1,4 @@ +#pylint: skip-file # -*- coding: utf-8 -*- # File : __init__.py # Author : Jiayuan Mao @@ -8,5 +9,9 @@ # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch # Distributed under MIT License. -from .batchnorm import SynchronizedBatchNorm1d, SynchronizedBatchNorm2d, SynchronizedBatchNorm3d -from .replicate import DataParallelWithCallback, patch_replication_callback \ No newline at end of file +from .batchnorm import ( + SynchronizedBatchNorm1d, + SynchronizedBatchNorm2d, + SynchronizedBatchNorm3d, +) +from .replicate import DataParallelWithCallback, patch_replication_callback diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/batchnorm.py b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/batchnorm.py index aa9dd37..6f3d811 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/batchnorm.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/batchnorm.py @@ -1,3 +1,4 @@ +# pylint: skip-file # -*- coding: utf-8 -*- # File : batchnorm.py # Author : Jiayuan Mao diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/comm.py b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/comm.py index 8f2f701..837e328 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/comm.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/comm.py @@ -1,3 +1,4 @@ +# pylint: skip-file # -*- coding: utf-8 -*- # File : comm.py # Author : Jiayuan Mao diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/replicate.py b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/replicate.py index 3734266..183cdcc 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/replicate.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/replicate.py @@ -1,3 +1,4 @@ +# pylint: skip-file # -*- coding: utf-8 -*- # File : replicate.py # Author : Jiayuan Mao diff --git a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/unittest.py b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/unittest.py index f826560..ee153c2 100644 --- a/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/unittest.py +++ b/aimet_zoo_torch/deeplabv3/model/modeling/sync_batchnorm/unittest.py @@ -1,3 +1,4 @@ +# pylint: skip-file # -*- coding: utf-8 -*- # File : unittest.py # Author : Jiayuan Mao diff --git a/aimet_zoo_torch/deeplabv3/model/train.py b/aimet_zoo_torch/deeplabv3/model/train.py index 48a1031..0c5c8aa 100644 --- a/aimet_zoo_torch/deeplabv3/model/train.py +++ b/aimet_zoo_torch/deeplabv3/model/train.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------ # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/utils/__init__.py b/aimet_zoo_torch/deeplabv3/model/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/deeplabv3/model/utils/calculate_weights.py b/aimet_zoo_torch/deeplabv3/model/utils/calculate_weights.py index fdb0637..5a8b7c4 100644 --- a/aimet_zoo_torch/deeplabv3/model/utils/calculate_weights.py +++ b/aimet_zoo_torch/deeplabv3/model/utils/calculate_weights.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------ # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/utils/loss.py b/aimet_zoo_torch/deeplabv3/model/utils/loss.py index 43fa81e..9294b45 100644 --- a/aimet_zoo_torch/deeplabv3/model/utils/loss.py +++ b/aimet_zoo_torch/deeplabv3/model/utils/loss.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------ # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/utils/lr_scheduler.py b/aimet_zoo_torch/deeplabv3/model/utils/lr_scheduler.py index 0f790e6..e02996c 100644 --- a/aimet_zoo_torch/deeplabv3/model/utils/lr_scheduler.py +++ b/aimet_zoo_torch/deeplabv3/model/utils/lr_scheduler.py @@ -1,3 +1,4 @@ +# pylint: skip-file ##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ## Created by: Hang Zhang ## ECE Department, Rutgers University diff --git a/aimet_zoo_torch/deeplabv3/model/utils/metrics.py b/aimet_zoo_torch/deeplabv3/model/utils/metrics.py index d7be4aa..010acbd 100644 --- a/aimet_zoo_torch/deeplabv3/model/utils/metrics.py +++ b/aimet_zoo_torch/deeplabv3/model/utils/metrics.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------ # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/utils/saver.py b/aimet_zoo_torch/deeplabv3/model/utils/saver.py index f5095a1..225589e 100644 --- a/aimet_zoo_torch/deeplabv3/model/utils/saver.py +++ b/aimet_zoo_torch/deeplabv3/model/utils/saver.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------ # MIT License # diff --git a/aimet_zoo_torch/deeplabv3/model/utils/summaries.py b/aimet_zoo_torch/deeplabv3/model/utils/summaries.py index b4c37f3..1d0b97b 100644 --- a/aimet_zoo_torch/deeplabv3/model/utils/summaries.py +++ b/aimet_zoo_torch/deeplabv3/model/utils/summaries.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ------------------------------------------------------------------------------ # MIT License # diff --git a/aimet_zoo_torch/deepspeech2/evaluators/deepspeech2_quanteval.py b/aimet_zoo_torch/deepspeech2/evaluators/deepspeech2_quanteval.py index df2c841..b6322ff 100644 --- a/aimet_zoo_torch/deepspeech2/evaluators/deepspeech2_quanteval.py +++ b/aimet_zoo_torch/deepspeech2/evaluators/deepspeech2_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -32,9 +32,9 @@ from deepspeech_pytorch.utils import load_model, load_decoder from deepspeech_pytorch.testing import run_evaluation + def run_quantsim_evaluation(args): - """ function to run quantization evaluation - """ + """function to run quantization evaluation""" device = get_device(args) def wrapped_forward_function(self, x, lengths=None): @@ -47,19 +47,14 @@ def wrapped_forward_function(self, x, lengths=None): ) deepspeech_pytorch.model.DeepSpeech.forward = wrapped_forward_function - model = load_model( - device=device, - model_path=args.model_path, - use_half=False) + model = load_model(device=device, model_path=args.model_path, use_half=False) decoder = load_decoder(labels=model.labels, cfg=LMConfig) - target_decoder = GreedyDecoder( - model.labels, blank_index=model.labels.index("_")) + target_decoder = GreedyDecoder(model.labels, blank_index=model.labels.index("_")) def eval_func(model, iterations=None, device=device): - """ evaluation function - """ + """evaluation function""" model = model.to(device) test_dataset = SpectrogramDataset( audio_conf=model.audio_conf, @@ -72,9 +67,8 @@ def eval_func(model, iterations=None, device=device): test_dataset.size = iterations test_loader = AudioDataLoader( - test_dataset, - batch_size=args.batch_size, - num_workers=args.num_workers) + test_dataset, batch_size=args.batch_size, num_workers=args.num_workers + ) wer, cer, output_data = run_evaluation( test_loader=test_loader, @@ -91,7 +85,7 @@ def eval_func(model, iterations=None, device=device): quant_scheme = QuantScheme.post_training_tf_enhanced # Test original model on GPU - #pylint: disable=W0612 + # pylint: disable=W0612 wer, cer, output_data = eval_func(model, None) print(f"Original Model | 32-bit Environment | Average WER {wer:.4f}") @@ -292,7 +286,7 @@ def manually_configure_quant_ops(sim): "output_quantizer": True, }, } - #pylint: disable=W0212 + # pylint: disable=W0212 quant_ops = QuantizationSimModel._get_qc_quantized_layers(sim.model) for name, op in quant_ops: mc = manual_config[name] @@ -311,8 +305,7 @@ def manually_configure_quant_ops(sim): def arguments(): - """argument parser - """ + """argument parser""" parser = argparse.ArgumentParser( description="Evaluation script for an DeepSpeech2 network." ) @@ -322,16 +315,8 @@ def arguments(): type=str, default="libri_test_clean_manifest.csv", ) - parser.add_argument( - "--batch-size", - help="Batch size.", - type=int, - default=20) - parser.add_argument( - "--num-workers", - help="Number of workers.", - type=int, - default=1) + parser.add_argument("--batch-size", help="Batch size.", type=int, default=20) + parser.add_argument("--num-workers", help="Number of workers.", type=int, default=1) parser.add_argument( "--default-output-bw", help="Default output bitwidth for quantization.", @@ -350,8 +335,7 @@ def arguments(): def download_weights(): - """Downloading model weights - """ + """Downloading model weights""" # Download original model if not os.path.exists("./librispeech_pretrained_v2.pth"): urllib.request.urlretrieve( @@ -368,8 +352,8 @@ def download_weights(): class ModelConfig: - """Hardcoded model configuration - """ + """Hardcoded model configuration""" + def __init__(self, args): self.model_path = "librispeech_pretrained_v2.pth" self.quantsim_config_file = "default_config.json" @@ -380,8 +364,7 @@ def __init__(self, args): def main(): - """Main function - """ + """Main function""" args = arguments() config = ModelConfig(args) run_quantsim_evaluation(config) diff --git a/aimet_zoo_torch/distilbert/dataloader/utils/__init__.py b/aimet_zoo_torch/distilbert/dataloader/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/distilbert/evaluators/__init__.py b/aimet_zoo_torch/distilbert/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/distilbert/evaluators/distilbert_quanteval.py b/aimet_zoo_torch/distilbert/evaluators/distilbert_quanteval.py index 46f5168..a84343c 100644 --- a/aimet_zoo_torch/distilbert/evaluators/distilbert_quanteval.py +++ b/aimet_zoo_torch/distilbert/evaluators/distilbert_quanteval.py @@ -38,11 +38,11 @@ def parse_args(): parser.add_argument( "--model_config", default="distilbert_w8a8_mnli", - help="choice [ mobilebert_w8a8_cola,"\ - "mobilebert_w8a8_mnli,mobilebert_w8a8_mrpc,"\ - "mobilebert_w8a8_qnli,mobilebert_w8a8_qqp,"\ - "mobilebert_w8a8_rte,mobilebert_w8a8_squad,"\ - "mobilebert_w8a8_sst2,mobilebert_w8a8_stsb]", + help="choice [ mobilebert_w8a8_cola," + "mobilebert_w8a8_mnli,mobilebert_w8a8_mrpc," + "mobilebert_w8a8_qnli,mobilebert_w8a8_qqp," + "mobilebert_w8a8_rte,mobilebert_w8a8_squad," + "mobilebert_w8a8_sst2,mobilebert_w8a8_stsb]", ) parser.add_argument( "--per_device_eval_batch_size", diff --git a/aimet_zoo_torch/distilbert/model/__init__.py b/aimet_zoo_torch/distilbert/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/distilbert/model/baseline_models/__init__.py b/aimet_zoo_torch/distilbert/model/baseline_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/distilbert/model/model_cards/__init__.py b/aimet_zoo_torch/distilbert/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_cola.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_cola.json index b775b5f..6cc9b73 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_cola.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_cola.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/cola_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/cola_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mnli.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mnli.json index 17ba0d1..361ec31 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mnli.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mnli.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/mnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/mnli_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mrpc.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mrpc.json index ad6b623..f84fd96 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mrpc.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mrpc.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/mrpc_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/mrpc_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qnli.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qnli.json index 7a50e25..5225dcf 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qnli.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qnli.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/qnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/qnli_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qqp.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qqp.json index dc6a17a..d4c8d24 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qqp.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qqp.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/qqp_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/qqp_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_rte.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_rte.json index 2a4dfa1..5e3c98d 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_rte.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_rte.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/rte_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/rte_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_squad.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_squad.json index b326f33..e9cee9e 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_squad.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_squad.json @@ -46,7 +46,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/squad_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/squad_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_sst2.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_sst2.json index ee2a368..340a6fb 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_sst2.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_sst2.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/sst2_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/sst2_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_stsb.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_stsb.json index 1dc92e2..f96e3a6 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_stsb.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_stsb.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/stsb_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_distilbert/stsb_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/distilbert/model/model_definition.py b/aimet_zoo_torch/distilbert/model/model_definition.py index b4d4706..59d55aa 100644 --- a/aimet_zoo_torch/distilbert/model/model_definition.py +++ b/aimet_zoo_torch/distilbert/model/model_definition.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- mode: python -*- -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 # ============================================================================= # @@-COPYRIGHT-START-@@ # @@ -9,47 +9,43 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """Class for downloading and setting up of optimized and original distilbert model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet import json import os -import csv import sys +import pathlib from collections import defaultdict import torch -import datasets -from transformers import AutoConfig as Config -from transformers import AutoFeatureExtractor as FeatureExtractor -from aimet_common.defs import QuantScheme + # AIMET imports from aimet_torch.quantsim import load_checkpoint -from aimet_torch.quantsim import QuantizationSimModel -from aimet_torch.qc_quantize_op import QcQuantizeWrapper + +from transformers import HfArgumentParser +from transformers import AutoConfig, AutoTokenizer, TrainingArguments + # transformers import from aimet_zoo_torch.distilbert.model.baseline_models import distilbert from aimet_zoo_torch.common.downloader import Downloader -sys.modules['baseline_models.distilbert'] = distilbert -from transformers import HfArgumentParser -from transformers import ( - AutoConfig, - AutoTokenizer, - TrainingArguments -) +sys.modules["baseline_models.distilbert"] = distilbert class DistilBert(Downloader): - """ model distilbert configuration class """ + """model distilbert configuration class""" + #pylint:disable = import-outside-toplevel def __init__(self, model_config=None): - if model_config=="distilbert_w8a8_squad": + if model_config == "distilbert_w8a8_squad": from aimet_zoo_torch.distilbert.model.utils.utils_qa_dataclass import ( ModelArguments, DataTrainingArguments, AuxArguments, - ) + ) else: from aimet_zoo_torch.distilbert.model.utils.utils_nlclassifier_dataclass import ( ModelArguments, DataTrainingArguments, AuxArguments, - ) + ) parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: @@ -65,47 +61,43 @@ def __init__(self, model_config=None): ) # Parse arguments parser = HfArgumentParser( - (ModelArguments, - DataTrainingArguments, - TrainingArguments, - AuxArguments)) + (ModelArguments, DataTrainingArguments, TrainingArguments, AuxArguments) + ) ( - model_args, - data_args, - training_args, - aux_args, + model_args, + data_args, + training_args, + aux_args, ) = parser.parse_args_into_dataclasses() - self.model = None - self.model_args=model_args - self.data_args =data_args + self.model_args = model_args + self.data_args = data_args self.training_args = training_args self.aux_args = aux_args - #additional setup of the argsumetns from model_config - if model_config=="distilbert_w8a8_squad": - self.data_args.dataset_name=self.cfg["data_training_args"]["dataset_name"] + # additional setup of the argsumetns from model_config + if model_config == "distilbert_w8a8_squad": + self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] else: - self.data_args.task_name=self.cfg["data_training_args"]["task_name"] + self.data_args.task_name = self.cfg["data_training_args"]["task_name"] - self.aux_args.model_config=model_config - self.training_args.do_eval=True + self.aux_args.model_config = model_config + self.training_args.do_eval = True # setup the download path from arguments - self.path_pre_opt_weights = self.aux_args.fmodel_path + self.path_pre_opt_weights = self.aux_args.fmodel_path self.path_post_opt_weights = self.aux_args.qmodel_path - def get_model_from_pretrained(self): """get original or optimized model Parameters: dataset: Return: - model : pretrained/optimized model + model : pretrained/optimized model """ - #case1. model for squad dataset - if hasattr(self.data_args,'dataset_name'): + # case1. model for squad dataset + if hasattr(self.data_args, "dataset_name"): self._download_pre_opt_weights(show_progress=True) - self._download_aimet_config() + self._download_aimet_config() config = AutoConfig.from_pretrained( self.model_args.config_name if self.model_args.config_name @@ -128,10 +120,10 @@ def get_model_from_pretrained(self): use_auth_token=True if self.model_args.use_auth_token else None, ) - self.model = torch.load(self.aux_args.fmodel_path) - return self.model,tokenizer - #case2. model for glue dataset - num_labels=2 + self.model = torch.load(self.aux_args.fmodel_path) + return self.model, tokenizer + # case2. model for glue dataset + num_labels = 2 self._download_pre_opt_weights(show_progress=True) self._download_aimet_config() # Load pretrained model and tokenizer @@ -148,7 +140,9 @@ def get_model_from_pretrained(self): # ++++ config.return_dict = False config.classifier_dropout = None - config.attention_probs_dropout_prob = self.model_args.attention_probs_dropout_prob + config.attention_probs_dropout_prob = ( + self.model_args.attention_probs_dropout_prob + ) # ++++ tokenizer = AutoTokenizer.from_pretrained( @@ -162,12 +156,12 @@ def get_model_from_pretrained(self): ) self.model = torch.load(self.aux_args.fmodel_path) - return self.model,tokenizer + return self.model, tokenizer def get_quantsim(self): - """get quantsim object """ + """get quantsim object""" self._download_post_opt_weights(show_progress=True) # Load the Quantsim_model object quantsim_model = load_checkpoint(self.aux_args.qmodel_path) - return quantsim_model + return quantsim_model diff --git a/aimet_zoo_torch/distilbert/model/utils/__init__.py b/aimet_zoo_torch/distilbert/model/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/efficientnetlite0/__init__.py b/aimet_zoo_torch/efficientnetlite0/__init__.py index 6098f33..0c8e9cc 100644 --- a/aimet_zoo_torch/efficientnetlite0/__init__.py +++ b/aimet_zoo_torch/efficientnetlite0/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import EfficientNetLite0 \ No newline at end of file +"""loading efficientnetlite0 downloader class""" +from .model.model_definition import EfficientNetLite0 diff --git a/aimet_zoo_torch/efficientnetlite0/dataloader/__init__.py b/aimet_zoo_torch/efficientnetlite0/dataloader/__init__.py index 773a063..8c2686e 100644 --- a/aimet_zoo_torch/efficientnetlite0/dataloader/__init__.py +++ b/aimet_zoo_torch/efficientnetlite0/dataloader/__init__.py @@ -1 +1,2 @@ -from .dataloaders_and_eval_func import eval_func, forward_pass \ No newline at end of file +""" datasets and eval function are defined and loaded""" +from .dataloaders_and_eval_func import eval_func, forward_pass diff --git a/aimet_zoo_torch/efficientnetlite0/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/efficientnetlite0/dataloader/dataloaders_and_eval_func.py index 4c4462b..ab926dc 100644 --- a/aimet_zoo_torch/efficientnetlite0/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/efficientnetlite0/dataloader/dataloaders_and_eval_func.py @@ -7,15 +7,17 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +""" module for getting imagenet dataloader, evaluation function and forward pass""" -import torch import torchvision from torchvision import transforms as T -from torch.utils.data import Dataset, DataLoader +import torch +from torch.utils.data import DataLoader from tqdm import tqdm -# ImageNet data loader + def get_imagenet_dataloader(image_dir, BATCH_SIZE=64): + """get imagenet dataloder""" def generate_dataloader(data, transform, batch_size=BATCH_SIZE): if data is None: return None @@ -25,22 +27,26 @@ def generate_dataloader(data, transform, batch_size=BATCH_SIZE): else: dataset = torchvision.datasets.ImageFolder(data, transform=transform) - dataloader = DataLoader(dataset, batch_size=batch_size, - shuffle=False, num_workers=4) + dataloader = DataLoader( + dataset, batch_size=batch_size, shuffle=False, num_workers=4 + ) return dataloader # Define transformation - preprocess_transform_pretrain = T.Compose([ - T.Resize(256), # Resize images to 256 x 256 - T.CenterCrop(224), # Center crop image - T.RandomHorizontalFlip(), - T.ToTensor(), # Converting cropped images to tensors - T.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) - ]) - - dataloader = generate_dataloader(image_dir, transform=preprocess_transform_pretrain, batch_size=BATCH_SIZE) + preprocess_transform_pretrain = T.Compose( + [ + T.Resize(256), # Resize images to 256 x 256 + T.CenterCrop(224), # Center crop image + T.RandomHorizontalFlip(), + T.ToTensor(), # Converting cropped images to tensors + T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + ) + + dataloader = generate_dataloader( + image_dir, transform=preprocess_transform_pretrain, batch_size=BATCH_SIZE + ) return dataloader @@ -69,11 +75,12 @@ def pass_calibration_data(model, args): """Forward pass for encoding calculations""" # Get Dataloader - dataloader_encoding = get_imagenet_dataloader(args['evaluation_dataset']) + dataloader_encoding = get_imagenet_dataloader(args["evaluation_dataset"]) on_cuda = next(model.parameters()).is_cuda model.eval() batch_counter = 0 samples = 100 # number of samples for validation + #pylint: disable = unused-variable with torch.no_grad(): for input_data, target_data in dataloader_encoding: if on_cuda: @@ -81,13 +88,14 @@ def pass_calibration_data(model, args): output_data = model(input_data) batch_counter += 1 - if (batch_counter * args['batch_size']) > samples: + if (batch_counter * args["batch_size"]) > samples: break + def forward_pass(model, dataloader): """forward pass through the calibration dataset""" + #pylint:disable = unused-variable model.eval() - on_cuda = next(model.parameters()).is_cuda with torch.no_grad(): for data, label in tqdm(dataloader): @@ -95,5 +103,4 @@ def forward_pass(model, dataloader): data, label = data.cuda(), label.cuda() output = model(data) - del dataloader diff --git a/aimet_zoo_torch/efficientnetlite0/evaluators/efficientnetlite0_quanteval.py b/aimet_zoo_torch/efficientnetlite0/evaluators/efficientnetlite0_quanteval.py index e514f05..aff0d45 100755 --- a/aimet_zoo_torch/efficientnetlite0/evaluators/efficientnetlite0_quanteval.py +++ b/aimet_zoo_torch/efficientnetlite0/evaluators/efficientnetlite0_quanteval.py @@ -8,7 +8,7 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET evaluation code for Efficientnet Lite0 ''' +""" AIMET evaluation code for Efficientnet Lite0 """ # general python imports import argparse @@ -20,21 +20,41 @@ from aimet_zoo_torch.efficientnetlite0 import EfficientNetLite0 -# add arguments def arguments(): - parser = argparse.ArgumentParser(description='0725 changed script for efficientnet_lite0 quantization') - parser.add_argument('--dataset-path', help='path to image evaluation dataset', type=str) - parser.add_argument('--model-config', help='model configuration to be tested', type=str) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) - parser.add_argument('--batch-size',help='batch_size for loading data',type=int,default=16) - parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) + """add arguments""" + parser = argparse.ArgumentParser( + description="0725 changed script for efficientnet_lite0 quantization" + ) + parser.add_argument( + "--dataset-path", help="path to image evaluation dataset", type=str + ) + parser.add_argument( + "--model-config", help="model configuration to be tested", type=str + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--batch-size", help="batch_size for loading data", type=int, default=16 + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU", type=bool, default=True + ) args = parser.parse_args() return args -# set seed for reproducibility def seed(seednum, use_cuda): + """set seed for reproducibility""" torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(seednum) @@ -42,38 +62,58 @@ def seed(seednum, use_cuda): torch.cuda.manual_seed(seednum) torch.cuda.manual_seed_all(seednum) -# adding hardcoded values into args from parseargs() and return config object -class ModelConfig(): + +class ModelConfig: + """adding hardcoded values into args from parseargs() and return config object""" def __init__(self, args): - self.seed=23 - self.input_shape=(1,3,224,224) - self.checkpoint="model_efficientnetlite0_w" + str(args.default_param_bw) + "a" + str(args.default_output_bw) + "_pc_checkpoint.pth" - self.encoding="efficientnetlite0_w" + str(args.default_param_bw) + "a" + str(args.default_output_bw) + "_pc.encodings" - self.quant_scheme='tf_enhanced' - self.config_file='default_config_per_channel.json' + self.seed = 23 + self.input_shape = (1, 3, 224, 224) + self.checkpoint = ( + "model_efficientnetlite0_w" + + str(args.default_param_bw) + + "a" + + str(args.default_output_bw) + + "_pc_checkpoint.pth" + ) + self.encoding = ( + "efficientnetlite0_w" + + str(args.default_param_bw) + + "a" + + str(args.default_output_bw) + + "_pc.encodings" + ) + self.quant_scheme = "tf_enhanced" + self.config_file = "default_config_per_channel.json" for arg in vars(args): setattr(self, arg, getattr(args, arg)) -def main(): +def main(): + """ main evaluation function""" + #pylint:disable = no-member args = arguments() config = ModelConfig(args) seed(seednum=23, use_cuda=args.use_cuda) # ===================================fp32 model ================================== - fp32_model = EfficientNetLite0(model_config = config.model_config) + fp32_model = EfficientNetLite0(model_config=config.model_config) fp32_model.from_pretrained(quantized=False) fp32_model.model.eval() fp32_acc = eval_func(fp32_model.model, config.dataset_path, config.batch_size) - print(f'=========FP32 Model Accuracy : {fp32_acc:0.2f}% ') + print(f"=========FP32 Model Accuracy : {fp32_acc:0.2f}% ") # ===================================Quantized model ================================== - model_int8 = EfficientNetLite0(model_config = config.model_config) + model_int8 = EfficientNetLite0(model_config=config.model_config) sim = model_int8.get_quantsim(quantized=True) - encoding_dataloader = ImageNetDataLoader(config.dataset_path, image_size=224, num_samples_per_class=2) - sim.compute_encodings(forward_pass, forward_pass_callback_args=encoding_dataloader.data_loader) + encoding_dataloader = ImageNetDataLoader( + config.dataset_path, image_size=224, num_samples_per_class=2 + ) + sim.compute_encodings( + forward_pass, forward_pass_callback_args=encoding_dataloader.data_loader + ) quant_acc = eval_func(sim.model, config.dataset_path, config.batch_size) - print(f'=========Quantized model Accuracy: {quant_acc:0.2f}% ') + print(f"=========Quantized model Accuracy: {quant_acc:0.2f}% ") + -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/efficientnetlite0/model/model_cards/__init__.py b/aimet_zoo_torch/efficientnetlite0/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/efficientnetlite0/model/model_definition.py b/aimet_zoo_torch/efficientnetlite0/model/model_definition.py index 019595d..ee8e126 100644 --- a/aimet_zoo_torch/efficientnetlite0/model/model_definition.py +++ b/aimet_zoo_torch/efficientnetlite0/model/model_definition.py @@ -7,40 +7,50 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - -import torch +"""Class for downloading and setting up of optmized and original effiecientlite0 model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet import json import os +import torch from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim from aimet_zoo_torch.common.downloader import Downloader -import geffnet +import geffnet # pylint:disable = import-error -class EfficientNetLite0(Downloader): +class EfficientNetLite0(Downloader): + """efficientnetlite0 parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" def __init__(self, model_config: str = None): - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) - self.model = getattr(geffnet, 'efficientnet_lite0')(pretrained=True) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + self.model = getattr(geffnet, "efficientnet_lite0")(pretrained=True) self.model.eval() - + def from_pretrained(self, quantized=False): + """get pretrained model from downloading or coping""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() @@ -49,34 +59,44 @@ def from_pretrained(self, quantized=False): if quantized: self.model = torch.load(self.path_post_opt_weights) else: - self.model = getattr(geffnet, 'efficientnet_lite0')(pretrained=True) + self.model = getattr(geffnet, "efficientnet_lite0")(pretrained=True) self.model.cuda() self.model.eval() def get_quantsim(self, quantized=False): + """" to get quantsim object for model from loading/computing proper encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model.cuda(), **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') + print("set_and_freeze_param_encodings finished!") sim.model.eval() return sim def __call__(self, x): - return self.model(x) \ No newline at end of file + return self.model(x) diff --git a/aimet_zoo_torch/ffnet/__init__.py b/aimet_zoo_torch/ffnet/__init__.py index 5a3ee0a..4146f53 100644 --- a/aimet_zoo_torch/ffnet/__init__.py +++ b/aimet_zoo_torch/ffnet/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import FFNet \ No newline at end of file +""" loading ffnet model downloader class""" +from .model.model_definition import FFNet diff --git a/aimet_zoo_torch/ffnet/dataloader/__init__.py b/aimet_zoo_torch/ffnet/dataloader/__init__.py index c086995..a454cde 100644 --- a/aimet_zoo_torch/ffnet/dataloader/__init__.py +++ b/aimet_zoo_torch/ffnet/dataloader/__init__.py @@ -1 +1,2 @@ -from .dataloaders_and_eval_func import get_dataloaders_and_eval_func \ No newline at end of file +""" datasets and eval function are defined and loaded""" +from .dataloaders_and_eval_func import get_dataloaders_and_eval_func diff --git a/aimet_zoo_torch/ffnet/dataloader/base_loader.py b/aimet_zoo_torch/ffnet/dataloader/base_loader.py index 3d72c1b..6e085e2 100644 --- a/aimet_zoo_torch/ffnet/dataloader/base_loader.py +++ b/aimet_zoo_torch/ffnet/dataloader/base_loader.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Copyright 2020 Nvidia Corporation Redistribution and use in source and binary forms, with or without diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/cityscapes.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/cityscapes.py index 274bf45..e8cea36 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/cityscapes.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/cityscapes.py @@ -1,3 +1,4 @@ +# pylint: skip-file import os import os.path as path from . import cityscapes_labels diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/cityscapes_labels.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/cityscapes_labels.py index f3906a6..da71f9d 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/cityscapes_labels.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/cityscapes_labels.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # File taken from https://github.com/mcordts/cityscapesScripts/ # License File Available at: diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/base_loader.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/base_loader.py index 39e9f83..5ba49e3 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/base_loader.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/base_loader.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Copyright 2020 Nvidia Corporation Redistribution and use in source and binary forms, with or without diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/get_dataloaders.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/get_dataloaders.py index 59029bc..b4ae413 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/get_dataloaders.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/get_dataloaders.py @@ -1,3 +1,4 @@ +# pylint: skip-file # import datasets.cityscapes.dataloader.joint_transforms as joint_transforms from . import transforms as extended_transforms from torch.utils.data import DataLoader diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/sampler.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/sampler.py index 5be17c4..0cc0282 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/sampler.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/sampler.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code adapted from: # https://github.com/pytorch/pytorch/blob/master/torch/utils/data/distributed.py diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/transforms.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/transforms.py index 59ad387..ecc5915 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/transforms.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/dataloader/transforms.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code borrowded from: # https://github.com/zijundeng/pytorch-semantic-segmentation/blob/master/utils/transforms.py diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/attr_dict.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/attr_dict.py index dee785a..314a16e 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/attr_dict.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/attr_dict.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code adapted from: # https://github.com/facebookresearch/Detectron/blob/master/detectron/utils/collections.py diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/misc.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/misc.py index 626a41b..891f10f 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/misc.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/misc.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Miscellanous Functions : From HRNet semantic segmentation : https://github.com/HRNet/HRNet-Semantic-Segmentation """ diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/my_data_parallel.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/my_data_parallel.py index b649d72..0797025 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/my_data_parallel.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/my_data_parallel.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code adapted from: # https://github.com/pytorch/pytorch/blob/master/torch/nn/parallel/data_parallel.py diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/progress_bar.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/progress_bar.py index a93dafa..68ca753 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/progress_bar.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/progress_bar.py @@ -1,11 +1,13 @@ # Copyright (c) 2021 Qualcomm Technologies, Inc. # All Rights Reserved. - +""" download progress bar class""" import sys def printProgressBar(i, max, postText): + """function for printing progress of downloading""" + #pylint:disable = redefined-builtin n_bar = 10 j = i / max sys.stdout.write("\r") diff --git a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/trnval_utils.py b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/trnval_utils.py index 4b3ec2b..85fbbc8 100644 --- a/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/trnval_utils.py +++ b/aimet_zoo_torch/ffnet/dataloader/cityscapes/utils/trnval_utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Copyright 2020 Nvidia Corporation diff --git a/aimet_zoo_torch/ffnet/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/ffnet/dataloader/dataloaders_and_eval_func.py index 8af59fa..38cb923 100644 --- a/aimet_zoo_torch/ffnet/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/ffnet/dataloader/dataloaders_and_eval_func.py @@ -5,29 +5,39 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +""" module for getting dataloaders and eval function for cityscapes dataset""" - +from tqdm import tqdm from .cityscapes.utils.misc import eval_metrics from .cityscapes.utils.trnval_utils import eval_minibatch from .cityscapes.dataloader.get_dataloaders import return_dataloader -from tqdm import tqdm - def get_dataloaders_and_eval_func(dataset_path, batch_size, num_workers=4): - val_loader = return_dataloader(num_workers, batch_size, cityscapes_base_path=dataset_path) + """ + Summary: function to get cityscape dataset dataloader + Parameters: + dataset_path(str): + batch_size(int): + num_workers(int): + Returns: + dataloader + """ + val_loader = return_dataloader( + num_workers, batch_size, cityscapes_base_path=dataset_path + ) # Define evaluation func to evaluate model with data_loader def eval_func(model, args=None): - iterations = args[0] if type(args)==list and len(args)>0 else float('inf') + #pylint:disable = unused-argument model.eval() iou_acc = 0 - for data in tqdm(val_loader, desc='evaluate'): + for data in tqdm(val_loader, desc="evaluate"): _iou_acc = eval_minibatch(data, model, True, 0, False, False) iou_acc += _iou_acc mean_iou = eval_metrics(iou_acc, model) return mean_iou - return None, val_loader, eval_func \ No newline at end of file + return None, val_loader, eval_func diff --git a/aimet_zoo_torch/ffnet/evaluators/ffnet_quanteval.py b/aimet_zoo_torch/ffnet/evaluators/ffnet_quanteval.py index 5e6c711..4dc2e87 100755 --- a/aimet_zoo_torch/ffnet/evaluators/ffnet_quanteval.py +++ b/aimet_zoo_torch/ffnet/evaluators/ffnet_quanteval.py @@ -8,34 +8,39 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantsim code for FFNet ''' +""" AIMET Quantsim code for FFNet """ +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet # General Python related imports from __future__ import absolute_import from __future__ import division import os import sys -sys.path.append(os.path.dirname(sys.path[0])) + import argparse -from tqdm import tqdm from functools import partial -from aimet_zoo_torch.common.utils.utils import get_device -from aimet_zoo_torch.ffnet.dataloader import get_dataloaders_and_eval_func -from aimet_zoo_torch.ffnet import FFNet +from tqdm import tqdm # Torch related imports import torch +# AIMET related imports +from aimet_torch.model_validator.model_validator import ModelValidator # Dataloader and Model Evaluation imports +from aimet_zoo_torch.common.utils.utils import get_device from aimet_zoo_torch.ffnet.dataloader.cityscapes.utils.misc import eval_metrics -from aimet_zoo_torch.ffnet.dataloader.cityscapes.utils.trnval_utils import eval_minibatch +from aimet_zoo_torch.ffnet.dataloader.cityscapes.utils.trnval_utils import ( + eval_minibatch, +) +from aimet_zoo_torch.ffnet.dataloader import get_dataloaders_and_eval_func +from aimet_zoo_torch.ffnet import FFNet +sys.path.append(os.path.dirname(sys.path[0])) -# AIMET related imports -from aimet_torch.model_validator.model_validator import ModelValidator -# Set seed for reproducibility def seed(seed_number): + """Set seed for reproducibility""" torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True torch.manual_seed(seed_number) @@ -43,12 +48,12 @@ def seed(seed_number): torch.cuda.manual_seed_all(seed_number) -# Define evaluation func to evaluate model with data_loader def eval_func(model, dataloader): + """Define evaluation func to evaluate model with data_loader""" model.eval() iou_acc = 0 - for data in tqdm(dataloader, desc='evaluate'): + for data in tqdm(dataloader, desc="evaluate"): _iou_acc = eval_minibatch(data, model, True, 0, False, False) iou_acc += _iou_acc mean_iou = eval_metrics(iou_acc, model) @@ -56,43 +61,73 @@ def eval_func(model, dataloader): return mean_iou -# Forward pass for compute encodings def forward_pass(device, model, data_loader): + """Forward pass for compute encodings""" model = model.to(device) model.eval() for data in tqdm(data_loader): - images, gt_image, edge, img_names, scale_float = data + images, gt_image, edge, img_names, scale_float = data # pylint: disable = unused-variable assert isinstance(images, torch.Tensor) assert len(images.size()) == 4 and len(gt_image.size()) == 3 assert images.size()[2:] == gt_image.size()[1:] - batch_pixel_size = images.size(0) * images.size(2) * images.size(3) - input_size = images.size(2), images.size(3) with torch.no_grad(): inputs = images _pred = model(inputs.to(device)) - + def arguments(): - parser = argparse.ArgumentParser(description='Evaluation script for PyTorch FFNet models.') - parser.add_argument('--model-config', help='Select the model configuration', type=str, default="segmentation_ffnet78S_dBBB_mobile", choices=[ - "segmentation_ffnet78S_dBBB_mobile", - "segmentation_ffnet54S_dBBB_mobile", - "segmentation_ffnet40S_dBBB_mobile", - "segmentation_ffnet78S_BCC_mobile_pre_down", - "segmentation_ffnet122NS_CCC_mobile_pre_down"]) - parser.add_argument('--dataset-path', help='Path to cityscapes parent folder containing leftImg8bit', type=str, default='') - parser.add_argument('--batch-size', help='Data batch size for a model', type=int, default=8) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) - parser.add_argument('--use-cuda', help='Run evaluation on GPU.', type=bool, default=True) + """ argument parser""" + #pylint: disable=redefined-outer-name + parser = argparse.ArgumentParser( + description="Evaluation script for PyTorch FFNet models." + ) + parser.add_argument( + "--model-config", + help="Select the model configuration", + type=str, + default="segmentation_ffnet78S_dBBB_mobile", + choices=[ + "segmentation_ffnet78S_dBBB_mobile", + "segmentation_ffnet54S_dBBB_mobile", + "segmentation_ffnet40S_dBBB_mobile", + "segmentation_ffnet78S_BCC_mobile_pre_down", + "segmentation_ffnet122NS_CCC_mobile_pre_down", + ], + ) + parser.add_argument( + "--dataset-path", + help="Path to cityscapes parent folder containing leftImg8bit", + type=str, + default="", + ) + parser.add_argument( + "--batch-size", help="Data batch size for a model", type=int, default=8 + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True + ) args = parser.parse_args() return args -class ModelConfig(): +class ModelConfig: + """hardcoded values for parsed arguments""" def __init__(self, args): + #pylint: disable=redefined-outer-name self.input_shape = (1, 3, 1024, 2048) self.prepared_checkpoint_path = f"prepared_{args.model_config}.pth" self.optimized_checkpoint_path = f"{args.model_config}_W{args.default_param_bw}A{args.default_output_bw}_CLE_tfe_perchannel.pth" @@ -103,27 +138,32 @@ def __init__(self, args): def main(args): + """ main evaluation function""" + # pylint: disable=redefined-outer-name, too-many-locals, no-member seed(1234) config = ModelConfig(args) device = get_device(args) - print(f'device: {device}') + print(f"device: {device}") # Load original model - model_orig = FFNet(model_config = config.model_config) + model_orig = FFNet(model_config=config.model_config) model_orig.from_pretrained(quantized=False) # model_orig = torch.load(config.prepared_checkpoint_path) model_orig.model = model_orig.model.to(device) model_orig.model.eval() # Load optimized model - model_optim = FFNet(model_config = config.model_config) + model_optim = FFNet(model_config=config.model_config) model_optim.from_pretrained(quantized=True) - #model_optim = torch.load(config.optimized_checkpoint_path) + # model_optim = torch.load(config.optimized_checkpoint_path) model_optim.model = model_optim.model.to(device) model_optim.model.eval() # Get Dataloader - train_loader, val_loader, eval_func = get_dataloaders_and_eval_func(dataset_path = config.dataset_path, batch_size = config.batch_size, num_workers = 4) + # pylint: disable = unused-variable + train_loader, val_loader, eval_func = get_dataloaders_and_eval_func( + dataset_path=config.dataset_path, batch_size=config.batch_size, num_workers=4 + ) # Initialize Quantized model dummy_input = torch.rand(config.input_shape, device=device) @@ -132,15 +172,15 @@ def main(args): ModelValidator.validate_model(model_orig.model, dummy_input) ModelValidator.validate_model(model_optim.model, dummy_input) - print('Evaluating Original Model') + print("Evaluating Original Model") sim_orig = model_orig.get_quantsim(quantized=False) - #sim_orig = QuantizationSimModel(model_orig, **kwargs) + # sim_orig = QuantizationSimModel(model_orig, **kwargs) if "pre_down" in config.model_config: sim_orig.model.smoothing.output_quantizer.enabled = False - sim_orig.model.smoothing.param_quantizers['weight'].enabled = False + sim_orig.model.smoothing.param_quantizers["weight"].enabled = False # forward_func = partial(forward_pass, device) # sim_orig.compute_encodings(forward_func, forward_pass_callback_args=val_loader) - + mIoU_orig_fp32 = eval_func(model_orig.model, None) del model_orig torch.cuda.empty_cache() @@ -148,12 +188,12 @@ def main(args): del sim_orig torch.cuda.empty_cache() - print('Evaluating Optimized Model') + print("Evaluating Optimized Model") sim_optim = model_optim.get_quantsim(quantized=True) - #sim_optim = QuantizationSimModel(model_optim, **kwargs) + # sim_optim = QuantizationSimModel(model_optim, **kwargs) if "pre_down" in config.model_config: sim_orig.model.smoothing.output_quantizer.enabled = False - sim_orig.model.smoothing.param_quantizers['weight'].enabled = False + sim_orig.model.smoothing.param_quantizers["weight"].enabled = False forward_func = partial(forward_pass, device) sim_optim.compute_encodings(forward_func, forward_pass_callback_args=val_loader) @@ -164,12 +204,16 @@ def main(args): del sim_optim torch.cuda.empty_cache() - print(f'Original Model | 32-bit Environment | mIoU: {mIoU_orig_fp32:.4f}') - print(f'Original Model | {config.default_param_bw}-bit Environment | mIoU: {mIoU_orig_int8:.4f}') - print(f'Optimized Model | 32-bit Environment | mIoU: {mIoU_optim_fp32:.4f}') - print(f'Optimized Model | {config.default_param_bw}-bit Environment | mIoU: {mIoU_optim_int8:.4f}') + print(f"Original Model | 32-bit Environment | mIoU: {mIoU_orig_fp32:.4f}") + print( + f"Original Model | {config.default_param_bw}-bit Environment | mIoU: {mIoU_orig_int8:.4f}" + ) + print(f"Optimized Model | 32-bit Environment | mIoU: {mIoU_optim_fp32:.4f}") + print( + f"Optimized Model | {config.default_param_bw}-bit Environment | mIoU: {mIoU_optim_int8:.4f}" + ) -if __name__ == '__main__': +if __name__ == "__main__": args = arguments() - main(args) \ No newline at end of file + main(args) diff --git a/aimet_zoo_torch/ffnet/model/__init__.py b/aimet_zoo_torch/ffnet/model/__init__.py index 2bd8d14..c6ebb7c 100644 --- a/aimet_zoo_torch/ffnet/model/__init__.py +++ b/aimet_zoo_torch/ffnet/model/__init__.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/config.py b/aimet_zoo_torch/ffnet/model/config.py index 11a6bb3..c30a336 100644 --- a/aimet_zoo_torch/ffnet/model/config.py +++ b/aimet_zoo_torch/ffnet/model/config.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/ffnet/model/ffnet_NS_mobile.py b/aimet_zoo_torch/ffnet/model/ffnet_NS_mobile.py index 364b429..61d3143 100644 --- a/aimet_zoo_torch/ffnet/model/ffnet_NS_mobile.py +++ b/aimet_zoo_torch/ffnet/model/ffnet_NS_mobile.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/ffnet_N_gpu_large.py b/aimet_zoo_torch/ffnet/model/ffnet_N_gpu_large.py index 84a29b1..dfe136e 100644 --- a/aimet_zoo_torch/ffnet/model/ffnet_N_gpu_large.py +++ b/aimet_zoo_torch/ffnet/model/ffnet_N_gpu_large.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/ffnet_S_gpu_large.py b/aimet_zoo_torch/ffnet/model/ffnet_S_gpu_large.py index a21645e..6078bb4 100644 --- a/aimet_zoo_torch/ffnet/model/ffnet_S_gpu_large.py +++ b/aimet_zoo_torch/ffnet/model/ffnet_S_gpu_large.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/ffnet_S_gpu_small.py b/aimet_zoo_torch/ffnet/model/ffnet_S_gpu_small.py index 8ac763b..53e4e9f 100644 --- a/aimet_zoo_torch/ffnet/model/ffnet_S_gpu_small.py +++ b/aimet_zoo_torch/ffnet/model/ffnet_S_gpu_small.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/ffnet_S_mobile.py b/aimet_zoo_torch/ffnet/model/ffnet_S_mobile.py index 7fe79ce..8935a5d 100644 --- a/aimet_zoo_torch/ffnet/model/ffnet_S_mobile.py +++ b/aimet_zoo_torch/ffnet/model/ffnet_S_mobile.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/ffnet_blocks.py b/aimet_zoo_torch/ffnet/model/ffnet_blocks.py index 2d07c21..c8ca1e9 100644 --- a/aimet_zoo_torch/ffnet/model/ffnet_blocks.py +++ b/aimet_zoo_torch/ffnet/model/ffnet_blocks.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/ffnet_gpu_large.py b/aimet_zoo_torch/ffnet/model/ffnet_gpu_large.py index d3f16c8..3720cc6 100644 --- a/aimet_zoo_torch/ffnet/model/ffnet_gpu_large.py +++ b/aimet_zoo_torch/ffnet/model/ffnet_gpu_large.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/ffnet_gpu_small.py b/aimet_zoo_torch/ffnet/model/ffnet_gpu_small.py index 24f8161..7296bd0 100644 --- a/aimet_zoo_torch/ffnet/model/ffnet_gpu_small.py +++ b/aimet_zoo_torch/ffnet/model/ffnet_gpu_small.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/model_cards/__init__.py b/aimet_zoo_torch/ffnet/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/ffnet/model/model_definition.py b/aimet_zoo_torch/ffnet/model/model_definition.py index bdce657..84293e6 100644 --- a/aimet_zoo_torch/ffnet/model/model_definition.py +++ b/aimet_zoo_torch/ffnet/model/model_definition.py @@ -7,40 +7,53 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - -import torch +"""Class for downloading and setting up of optmized and original ffnet model for AIMET model zoo""" import json import os +import torch +from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim #pylint:disable = import-error from aimet_zoo_torch.common.downloader import Downloader from aimet_zoo_torch.ffnet.model.model_registry import model_entrypoint -from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim class FFNet(Downloader): + """model FFNet configuration class""" def __init__(self, model_config: str = None): - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x != None else 1 for x in self.cfg['input_shape']) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) if model_config is not None: self.model = model_entrypoint(model_config) def from_pretrained(self, quantized=False): + """get original or optimized model + Parameters: + dataset: + Return: + model : pretrained/optimized model + """ if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() @@ -52,25 +65,35 @@ def from_pretrained(self, quantized=False): self.model = torch.load(self.path_pre_opt_weights) def get_quantsim(self, quantized=False): + """get quantsim object""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model.cuda(), **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') - return sim \ No newline at end of file + print("set_and_freeze_param_encodings finished!") + return sim diff --git a/aimet_zoo_torch/ffnet/model/model_registry.py b/aimet_zoo_torch/ffnet/model/model_registry.py index 15b1c57..172342b 100644 --- a/aimet_zoo_torch/ffnet/model/model_registry.py +++ b/aimet_zoo_torch/ffnet/model/model_registry.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/resnet.py b/aimet_zoo_torch/ffnet/model/resnet.py index 340496a..c3d81b1 100644 --- a/aimet_zoo_torch/ffnet/model/resnet.py +++ b/aimet_zoo_torch/ffnet/model/resnet.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/ffnet/model/utils.py b/aimet_zoo_torch/ffnet/model/utils.py index 5ef1e71..cc76f0d 100644 --- a/aimet_zoo_torch/ffnet/model/utils.py +++ b/aimet_zoo_torch/ffnet/model/utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2022 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/gpt2/__init__.py b/aimet_zoo_torch/gpt2/__init__.py index aec1843..ef53649 100644 --- a/aimet_zoo_torch/gpt2/__init__.py +++ b/aimet_zoo_torch/gpt2/__init__.py @@ -1 +1,2 @@ +""" package for getting gpt2 original model and quantized model""" from .model.model_definition import gpt2 diff --git a/aimet_zoo_torch/gpt2/dataloader/__init__.py b/aimet_zoo_torch/gpt2/dataloader/__init__.py index 4d76f6a..34ce1ac 100644 --- a/aimet_zoo_torch/gpt2/dataloader/__init__.py +++ b/aimet_zoo_torch/gpt2/dataloader/__init__.py @@ -1 +1,2 @@ +""" datasets and eval function are defined and loaded""" from .dataloaders import get_dataloaders diff --git a/aimet_zoo_torch/gpt2/evaluators/__init__.py b/aimet_zoo_torch/gpt2/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpt2/evaluators/gpt2_quanteval.py b/aimet_zoo_torch/gpt2/evaluators/gpt2_quanteval.py index 31bb723..98f5341 100644 --- a/aimet_zoo_torch/gpt2/evaluators/gpt2_quanteval.py +++ b/aimet_zoo_torch/gpt2/evaluators/gpt2_quanteval.py @@ -7,7 +7,7 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0902,R0903,C0103 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0902,R0903,C0103 """ Quatization evaluation script for GPT-2 model""" @@ -20,8 +20,9 @@ logger = get_logger(__name__) -def parse_args(): - """ argument parser""" + +def parse_args(raw_args): + """argument parser""" parser = argparse.ArgumentParser( description="Finetune a transformers model on a causal language modeling task" ) @@ -38,13 +39,13 @@ def parse_args(): default=8, help="Batch size (per device) for the evaluation dataloader.", ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(): - """ main evaluation script""" - args = parse_args() +def main(raw_args=None): + """main evaluation script""" + args = parse_args(raw_args) logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", diff --git a/aimet_zoo_torch/gpt2/model/__init__.py b/aimet_zoo_torch/gpt2/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpt2/model/model_cards/__init__.py b/aimet_zoo_torch/gpt2/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpt2/model/model_cards/gpt2_w8a8.json b/aimet_zoo_torch/gpt2/model/model_cards/gpt2_w8a8.json index a9da394..da61c5d 100644 --- a/aimet_zoo_torch/gpt2/model/model_cards/gpt2_w8a8.json +++ b/aimet_zoo_torch/gpt2/model/model_cards/gpt2_w8a8.json @@ -29,6 +29,6 @@ "tar_url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_gpt2/gpt2_wikitext_finetune.tar.gz", "tar_url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_gpt2/gpt2_wikitext_5e-5_1e-3_150_8.tar.gz", "url_aimet_encodings": null, - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_gpt2/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.24/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/gpt2/model/model_definition.py b/aimet_zoo_torch/gpt2/model/model_definition.py index e9a78c1..f830209 100644 --- a/aimet_zoo_torch/gpt2/model/model_definition.py +++ b/aimet_zoo_torch/gpt2/model/model_definition.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- mode: python -*- -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201 # ============================================================================= # @@-COPYRIGHT-START-@@ # @@ -10,6 +10,8 @@ # ============================================================================= """ model gpt2 configuration class """ +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet import json import os @@ -31,7 +33,8 @@ class gpt2(Downloader): - """ model gpt2 configuration class""" + """model gpt2 configuration class""" + def __init__(self, model_config=None, quantized=False): """ dataloader @@ -42,7 +45,9 @@ def __init__(self, model_config=None, quantized=False): self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: - config_filepath = os.path.join( self.parent_dir, "/model_cards/", model_config , ".json") + config_filepath = os.path.join( + self.parent_dir, "model_cards", model_config + ".json" + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( @@ -55,7 +60,9 @@ def __init__(self, model_config=None, quantized=False): ) self.model = None self.quantized = quantized - self.model_name_or_path = os.path.join(self.parent_dir,self.cfg["model_args"]["model_name_or_path"]) + self.model_name_or_path = os.path.join( + self.parent_dir, self.cfg["model_args"]["model_name_or_path"] + ) def get_model_from_pretrained(self): """downloading model from github and return model object""" @@ -69,7 +76,7 @@ def get_model_from_pretrained(self): if self.cfg["model_args"]["model_type"]: config = AutoConfig.from_pretrained(self.cfg["model_args"]["model_type"]) else: - raise ValueError('model type in model cards is not valid') + raise ValueError("model type in model cards is not valid") tokenizer = AutoTokenizer.from_pretrained( self.model_name_or_path, @@ -98,24 +105,21 @@ def get_quantsim(self, dataloader, eval_function): """ dummy_input = self._get_dummy_input(dataloader) - quant_scheme_card = self.cfg["optimization_config"]["quantization_configuration"]["quant_scheme"] - if ( - quant_scheme_card == "tf" - ): + quant_scheme_card = self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"] + if quant_scheme_card == "tf": quant_scheme = QuantScheme.post_training_tf - elif ( - quant_scheme_card == "tf_enhanced" - ): + elif quant_scheme_card == "tf_enhanced": quant_scheme = QuantScheme.post_training_tf_enhanced - elif ( - - quant_scheme_card == "tf_range_learning" - ): + elif quant_scheme_card == "tf_range_learning": quant_scheme = QuantScheme.training_range_learning_with_tf_init else: - raise ValueError('quant_scheme not valid in model cards') - - config_file = os.path.join(self.parent_dir,self.cfg["model_args"]["config_file"]) + raise ValueError("quant_scheme not valid in model cards") + + config_file = os.path.join( + self.parent_dir, self.cfg["model_args"]["config_file"] + ) quant_sim = QuantizationSimModel( model=self.model.cuda(), quant_scheme=quant_scheme, @@ -132,8 +136,11 @@ def get_quantsim(self, dataloader, eval_function): ) # remove dropout quantizers disable_list = [] - for name, module in quant_sim.model.named_modules(): - if isinstance(module, QcQuantizeWrapper) and isinstance(module._module_to_wrap, torch.nn.Dropout): + # pylint: disable = protected-access + for _, module in quant_sim.model.named_modules(): + if isinstance(module, QcQuantizeWrapper) and isinstance( + module._module_to_wrap, torch.nn.Dropout + ): disable_list.append(module) for module in disable_list: module.output_quantizers[0].enabled = False @@ -142,13 +149,12 @@ def get_quantsim(self, dataloader, eval_function): quant_sim.compute_encodings(eval_function, (10, dataloader, metric)) # load encodings if there is encodings.csv - self._load_encoding_data( - quant_sim, self.model_name_or_path - ) + self._load_encoding_data(quant_sim, self.model_name_or_path) return quant_sim - def _get_dummy_input(self, dataloader): - """getting dummy input from dataloader """ + @staticmethod + def _get_dummy_input(dataloader): + """getting dummy input from dataloader""" for batch in dataloader: output = [] input_args = ["input_ids"] @@ -160,7 +166,8 @@ def _get_dummy_input(self, dataloader): raise ValueError("dummy data error") return tuple(output) - def _load_encoding_data(self, quant_sim, save_dir): + @staticmethod + def _load_encoding_data(quant_sim, save_dir): """loading encoding data from previously saved encoding.csv file zipped in tar file""" fname = os.path.join(save_dir, "encodings.csv") if not os.path.exists(fname): diff --git a/aimet_zoo_torch/gpunet0/evaluator/__init__.py b/aimet_zoo_torch/gpunet0/evaluator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpunet0/evaluator/gpunet0_quanteval.py b/aimet_zoo_torch/gpunet0/evaluator/gpunet0_quanteval.py index f047867..6c2b5c9 100755 --- a/aimet_zoo_torch/gpunet0/evaluator/gpunet0_quanteval.py +++ b/aimet_zoo_torch/gpunet0/evaluator/gpunet0_quanteval.py @@ -8,13 +8,10 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantsim code for GPUNet-0 ''' +""" AIMET Quantsim code for GPUNet-0 """ # General Python related imports -import os -import json import argparse -import pathlib # Torch related imports import torch @@ -23,8 +20,9 @@ from aimet_zoo_torch.gpunet0 import GPUNet0 from aimet_zoo_torch.gpunet0.model.src.evaluate_model import evaluate + def seed(seed_number): - """" Set seed for reproducibility """ + """ " Set seed for reproducibility""" torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True torch.manual_seed(seed_number) @@ -33,59 +31,85 @@ def seed(seed_number): def arguments(): - """ argument parser """ - parser = argparse.ArgumentParser(description='Evaluation script for PyTorch GPUNet-0 models.') - parser.add_argument('--dataset-path', help='The path to load your dataset', type=str) - parser.add_argument('--model-config', help='Model Configuration to test', type=str, default='gpunet0_w8a8', choices=['gpunet0_w8a8']) - parser.add_argument('--batch-size', help='Data batch size to evaluate your model', type=int, default=200) - parser.add_argument('--use-cuda', help='Run evaluation on GPU.', type=bool, default=True) + """argument parser""" + # pylint: disable = redefined-outer-name + parser = argparse.ArgumentParser( + description="Evaluation script for PyTorch GPUNet-0 models." + ) + parser.add_argument( + "--dataset-path", help="The path to load your dataset", type=str + ) + parser.add_argument( + "--model-config", + help="Model Configuration to test", + type=str, + default="gpunet0_w8a8", + choices=["gpunet0_w8a8"], + ) + parser.add_argument( + "--batch-size", + help="Data batch size to evaluate your model", + type=int, + default=200, + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True + ) args = parser.parse_args() return args def main(args): - """" main function for quantization evaluation """ + """ " main function for quantization evaluation""" + # pylint: disable = redefined-outer-name seed(1234) - evaluator = evaluate(testBatch=args.batch_size, imgRes=(3,320,320), crop_pct=1, dtype="fp32", val_path = args.dataset_path+"/val") - - #load original model + evaluator = evaluate( + testBatch=args.batch_size, + imgRes=(3, 320, 320), + crop_pct=1, + dtype="fp32", + val_path=args.dataset_path + "/val", + ) + + # load original model model = GPUNet0(model_config=args.model_config) model.from_pretrained(quantized=False) model_orig = model.model - print('Evaluating original Model') + print("Evaluating original Model") acc_top1_orig_fp32, acc_top5_orig_fp32 = evaluator.test_model(model_orig) torch.cuda.empty_cache() - #get quantsimmodel of original model + # get quantsimmodel of original model sim_orig = model.get_quantsim(quantized=False) sim_orig.compute_encodings(evaluator.test_model, forward_pass_callback_args=2000) - print('Evaluating original Model After Quansim') - acc_top1_orig_quantsim, acc_top5_orig_quantsim = evaluator.test_model(sim_orig.model) + print("Evaluating original Model After Quansim") + acc_top1_orig_quantsim, acc_top5_orig_quantsim = evaluator.test_model( + sim_orig.model + ) del sim_orig torch.cuda.empty_cache() - #load quantsim w8a8 model + # load quantsim w8a8 model sim_w8a8 = model.get_quantsim(quantized=True) - print('Evaluating Optimized Model') + print("Evaluating Optimized Model") acc_top1_optim_w8a8, acc_top5_optim_w8a8 = evaluator.test_model(sim_w8a8.model) del sim_w8a8 torch.cuda.empty_cache() - parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent.parent) - config_filepath = os.path.join(parent_dir, 'model/model_cards', args.model_config + '.json') - if not os.path.exists(config_filepath): - raise NotImplementedError('Model_config file doesn\'t exist') - with open(config_filepath) as f_in: - cfg = json.load(f_in) - default_param_bw = cfg['optimization_config']['quantization_configuration']['param_bw'] - print(f'Original Model | 32-bit Environment | acc top1: {acc_top1_orig_fp32:.4f} | acc top5: {acc_top5_orig_fp32:.4f}') - print(f'Original Model | {default_param_bw}-bit Environment | acc top1: {acc_top1_orig_quantsim:.4f} | acc top5: {acc_top5_orig_quantsim:.4f}') - print(f'Optimized Model | 8-bit Environment | acc top1: {acc_top1_optim_w8a8:.4f} | acc top5: {acc_top5_optim_w8a8:.4f}') + print( + f"Original Model | 32-bit Environment | acc top1: {acc_top1_orig_fp32:.4f} | acc top5: {acc_top5_orig_fp32:.4f}" + ) + print( + f"Original Model | 8-bit Environment | acc top1: {acc_top1_orig_quantsim:.4f} | acc top5: {acc_top5_orig_quantsim:.4f}" + ) + print( + f"Optimized Model | 8-bit Environment | acc top1: {acc_top1_optim_w8a8:.4f} | acc top5: {acc_top5_optim_w8a8:.4f}" + ) -if __name__ == '__main__': +if __name__ == "__main__": args = arguments() main(args) diff --git a/aimet_zoo_torch/gpunet0/model/__init__.py b/aimet_zoo_torch/gpunet0/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpunet0/model/model_cards/__init__.py b/aimet_zoo_torch/gpunet0/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpunet0/model/model_definition.py b/aimet_zoo_torch/gpunet0/model/model_definition.py index 506fb67..e35ef70 100755 --- a/aimet_zoo_torch/gpunet0/model/model_definition.py +++ b/aimet_zoo_torch/gpunet0/model/model_definition.py @@ -7,6 +7,9 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +"""Class for downloading and setting up of optmized and original gpunet0 model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet import json import os @@ -24,7 +27,8 @@ class GPUNet0(Downloader): """GPUNet-0 Image Classification parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" - def __init__(self, model_config = 'gpunet0_w8a8'): + + def __init__(self, model_config="gpunet0_w8a8"): """ :param model_config: named model config from which to obtain model artifacts and arguments. If provided, overwrites the other arguments passed to this object @@ -32,38 +36,48 @@ def __init__(self, model_config = 'gpunet0_w8a8'): self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = False if model_config: - config_filepath = self.parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = self.parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = self.parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=self.parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) self.dummy_input = torch.rand(self.input_shape, device=torch.device("cuda")) self.model = None def from_pretrained(self, quantized=False): """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: - if self.url_post_opt_weights and not os.path.exists(self.path_post_opt_weights): + if self.url_post_opt_weights and not os.path.exists( + self.path_post_opt_weights + ): self._download_post_opt_weights() self.from_pretrained(quantized=False) state_dict = torch.load(self.path_post_opt_weights) self.model.load_state_dict(state_dict) else: - if self.url_pre_opt_weights and not os.path.exists(self.path_pre_opt_weights): + if self.url_pre_opt_weights and not os.path.exists( + self.path_pre_opt_weights + ): self._download_pre_opt_weights() cpkPath = self.path_pre_opt_weights - configPath = self.parent_dir + '/src/configs/batch1/GV100/0.65ms.json' + configPath = self.parent_dir + "/src/configs/batch1/GV100/0.65ms.json" with open(configPath) as configFile: modelJSON = json.load(configFile) configFile.close() @@ -78,30 +92,43 @@ def from_pretrained(self, quantized=False): def get_quantsim(self, quantized=False): """get quantsim object with pre-loaded encodings or pretrained model""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if self.url_aimet_config and not os.path.exists(self.path_aimet_config): self._download_aimet_config() if quantized: - if self.url_aimet_encodings and not os.path.exists(self.path_aimet_encodings): + if self.url_aimet_encodings and not os.path.exists( + self.path_aimet_encodings + ): self._download_aimet_encodings() - if self.url_adaround_encodings and not os.path.exists(self.path_adaround_encodings): + if self.url_adaround_encodings and not os.path.exists( + self.path_adaround_encodings + ): self._download_adaround_encodings() self.from_pretrained(quantized) - dummy_input = torch.rand(self.input_shape, device = torch.device('cuda')) + dummy_input = torch.rand(self.input_shape, device=torch.device("cuda")) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model, **kwargs) sim.set_percentile_value(99.999) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') + print("set_and_freeze_param_encodings finished!") sim.model.cuda() sim.model.eval() return sim diff --git a/aimet_zoo_torch/gpunet0/model/src/__init__.py b/aimet_zoo_torch/gpunet0/model/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpunet0/model/src/configs/__init__.py b/aimet_zoo_torch/gpunet0/model/src/configs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpunet0/model/src/configs/batch1/GV100/__init__.py b/aimet_zoo_torch/gpunet0/model/src/configs/batch1/GV100/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpunet0/model/src/configs/batch1/__init__.py b/aimet_zoo_torch/gpunet0/model/src/configs/batch1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpunet0/model/src/evaluate_model.py b/aimet_zoo_torch/gpunet0/model/src/evaluate_model.py index a5be66d..ca97b75 100755 --- a/aimet_zoo_torch/gpunet0/model/src/evaluate_model.py +++ b/aimet_zoo_torch/gpunet0/model/src/evaluate_model.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/gpunet0/model/src/models/__init__.py b/aimet_zoo_torch/gpunet0/model/src/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/gpunet0/model/src/models/gpunet_builder.py b/aimet_zoo_torch/gpunet0/model/src/models/gpunet_builder.py index a93fc28..78d6047 100644 --- a/aimet_zoo_torch/gpunet0/model/src/models/gpunet_builder.py +++ b/aimet_zoo_torch/gpunet0/model/src/models/gpunet_builder.py @@ -1,4 +1,5 @@ # pylint: skip-file +# pylint: skip-file # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/aimet_zoo_torch/gpunet0/model/src/models/gpunet_modules.py b/aimet_zoo_torch/gpunet0/model/src/models/gpunet_modules.py index c1aba54..13612be 100644 --- a/aimet_zoo_torch/gpunet0/model/src/models/gpunet_modules.py +++ b/aimet_zoo_torch/gpunet0/model/src/models/gpunet_modules.py @@ -1,4 +1,5 @@ # pylint: skip-file +# pylint: skip-file # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/aimet_zoo_torch/hrnet_image_classification/__init__.py b/aimet_zoo_torch/hrnet_image_classification/__init__.py index 4cddc71..858c0d0 100644 --- a/aimet_zoo_torch/hrnet_image_classification/__init__.py +++ b/aimet_zoo_torch/hrnet_image_classification/__init__.py @@ -1 +1,2 @@ +""" package for getting bert original model and quantized model""" from .model.model_definition import HRNetImageClassification diff --git a/aimet_zoo_torch/hrnet_image_classification/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/hrnet_image_classification/dataloader/dataloaders_and_eval_func.py index fe2cce2..62f1b97 100644 --- a/aimet_zoo_torch/hrnet_image_classification/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/hrnet_image_classification/dataloader/dataloaders_and_eval_func.py @@ -8,13 +8,14 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= +""" module for getting evaluation function """ import torch from tqdm import tqdm -def eval_func(model, dataloader, device = torch.device('cuda')): - ''' Evaluates the model on validation dataset and returns the classification accuracy ''' - #Get Dataloader +def eval_func(model, dataloader, device=torch.device("cuda")): + """Evaluates the model on validation dataset and returns the classification accuracy""" + # Get Dataloader model.eval() correct = 0 total_samples = 0 @@ -26,11 +27,12 @@ def eval_func(model, dataloader, device = torch.device('cuda')): correct += (prediction == label).sum() total_samples += len(output) del dataloader - return float(100* correct / total_samples) + return float(100 * correct / total_samples) -def forward_pass(model, dataloader, device = torch.device('cuda')): - ''' forward pass through the calibration dataset ''' +def forward_pass(model, dataloader, device=torch.device("cuda")): + """forward pass through the calibration dataset""" + #pylint:disable = unused-variable model.eval() on_cuda = next(model.parameters()).is_cuda with torch.no_grad(): diff --git a/aimet_zoo_torch/hrnet_image_classification/dataloader/list/__init__.py b/aimet_zoo_torch/hrnet_image_classification/dataloader/list/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_image_classification/dataloader/list/cityscapes/__init__.py b/aimet_zoo_torch/hrnet_image_classification/dataloader/list/cityscapes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_image_classification/dataloader/list/lip/__init__.py b/aimet_zoo_torch/hrnet_image_classification/dataloader/list/lip/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_image_classification/evaluators/hrnet_image_classification_quanteval.py b/aimet_zoo_torch/hrnet_image_classification/evaluators/hrnet_image_classification_quanteval.py index c0c1aef..bd808f0 100644 --- a/aimet_zoo_torch/hrnet_image_classification/evaluators/hrnet_image_classification_quanteval.py +++ b/aimet_zoo_torch/hrnet_image_classification/evaluators/hrnet_image_classification_quanteval.py @@ -8,51 +8,65 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantsim code of HRNet for image classification ''' +""" AIMET Quantsim code of HRNet for image classification """ import argparse import logging -import torch -from aimet_zoo_torch.hrnet_image_classification.dataloader.dataloaders_and_eval_func import eval_func +from aimet_zoo_torch.hrnet_image_classification.dataloader.dataloaders_and_eval_func import ( + eval_func, +) from aimet_zoo_torch.hrnet_image_classification import HRNetImageClassification -from aimet_zoo_torch.common.utils.utils import get_device +from aimet_zoo_torch.common.utils.utils import get_device from aimet_zoo_torch.common.utils.image_net_data_loader import ImageNetDataLoader + def arguments(): - """ parse command line arguments """ - parser = argparse.ArgumentParser(description='Evaluation script for HRNet') - parser.add_argument('--model-config', help='model configuration to use', required=True, type=str, default='hrnet_w32_w8a8', choices=['hrnet_w32_w8a8']) - parser.add_argument('--dataset-path', help='Use GPU for evaluation', type=str) - parser.add_argument('--batch-size',help='batch_size for loading data',type=int,default=16) - parser.add_argument('--use-cuda', help='Use GPU for evaluation', default=True, type=bool) + """parse command line arguments""" + parser = argparse.ArgumentParser(description="Evaluation script for HRNet") + parser.add_argument( + "--model-config", + help="model configuration to use", + required=True, + type=str, + default="hrnet_w32_w8a8", + choices=["hrnet_w32_w8a8"], + ) + parser.add_argument("--dataset-path", help="Use GPU for evaluation", type=str) + parser.add_argument( + "--batch-size", help="batch_size for loading data", type=int, default=16 + ) + parser.add_argument( + "--use-cuda", help="Use GPU for evaluation", default=True, type=bool + ) args = parser.parse_args() return args - def main(): - """ run evaluator """ + """run evaluator""" + #pylint:disable = undefined-variable, logging-fstring-interpolation args = arguments() + logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger(_name_) device = get_device(args) - # get imagenet validation dataloader - eval_dataloader = ImageNetDataLoader(args.dataset_path,image_size=224, batch_size=args.batch_size).data_loader + # get imagenet validation dataloader + eval_dataloader = ImageNetDataLoader( + args.dataset_path, image_size=224, batch_size=args.batch_size + ).data_loader # Load quantized model, compute encodings and evaluate - fp32_model = HRNetImageClassification(model_config = args.model_config, device = device) + fp32_model = HRNetImageClassification(model_config=args.model_config, device=device) fp32_model.from_pretrained(quantized=False) - fp32_acc = eval_func(fp32_model.model, eval_dataloader, device = device) - + fp32_acc = eval_func(fp32_model.model, eval_dataloader, device=device) - int8_model= HRNetImageClassification(model_config = args.model_config, device = device) + int8_model = HRNetImageClassification(model_config=args.model_config, device=device) sim = int8_model.get_quantsim(quantized=True) - int8_acc = eval_func(sim.model, eval_dataloader, device = device) - - - logging.info(f'=========FP32 Model Accuracy : {fp32_acc:0.2f}% ') - logging.info(f'=========W8A8 Model | Accuracy: {int8_acc:0.2f}%') + int8_acc = eval_func(sim.model, eval_dataloader, device=device) + logger.info(f"=========FP32 Model Accuracy : {fp32_acc:0.2f}% ") + logger.info(f"=========W8A8 Model | Accuracy: {int8_acc:0.2f}%") -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/hrnet_image_classification/model/experiments/__init__.py b/aimet_zoo_torch/hrnet_image_classification/model/experiments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_image_classification/model/lib/__init__.py b/aimet_zoo_torch/hrnet_image_classification/model/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_image_classification/model/lib/config/__init__.py b/aimet_zoo_torch/hrnet_image_classification/model/lib/config/__init__.py index 59be749..c36aeb1 100644 --- a/aimet_zoo_torch/hrnet_image_classification/model/lib/config/__init__.py +++ b/aimet_zoo_torch/hrnet_image_classification/model/lib/config/__init__.py @@ -1,3 +1,4 @@ +#pylint: skip-file # ------------------------------------------------------------------------------ # Copyright (c) Microsoft # Licensed under the MIT License. diff --git a/aimet_zoo_torch/hrnet_image_classification/model/lib/core/__init__.py b/aimet_zoo_torch/hrnet_image_classification/model/lib/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_image_classification/model/lib/utils/__init__.py b/aimet_zoo_torch/hrnet_image_classification/model/lib/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_image_classification/model/model_cards/__init__.py b/aimet_zoo_torch/hrnet_image_classification/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_image_classification/model/model_definition.py b/aimet_zoo_torch/hrnet_image_classification/model/model_definition.py index e549332..20eed7e 100644 --- a/aimet_zoo_torch/hrnet_image_classification/model/model_definition.py +++ b/aimet_zoo_torch/hrnet_image_classification/model/model_definition.py @@ -21,7 +21,7 @@ from aimet_torch.cross_layer_equalization import equalize_model from aimet_torch.model_preparer import prepare_model from aimet_zoo_torch.common.downloader import Downloader -from aimet_zoo_torch.hrnet_image_classification.model.lib import models +from aimet_zoo_torch.hrnet_image_classification.model.lib import models #pylint:disable = unused-import from aimet_zoo_torch.hrnet_image_classification.model.lib.config import config diff --git a/aimet_zoo_torch/hrnet_posenet/dataloader/dataloader_and_eval_func.py b/aimet_zoo_torch/hrnet_posenet/dataloader/dataloader_and_eval_func.py index 3217395..14078a0 100644 --- a/aimet_zoo_torch/hrnet_posenet/dataloader/dataloader_and_eval_func.py +++ b/aimet_zoo_torch/hrnet_posenet/dataloader/dataloader_and_eval_func.py @@ -11,9 +11,9 @@ import torch import torch.utils.data -import torchvision.transforms as transforms +from torchvision import transforms from aimet_zoo_torch.hrnet_posenet.models.core.function import validate -import aimet_zoo_torch.hrnet_posenet.models.dataset as dataset +from aimet_zoo_torch.hrnet_posenet.models import dataset from aimet_zoo_torch.hrnet_posenet.models.core.loss import JointsMSELoss diff --git a/aimet_zoo_torch/hrnet_posenet/evaluators/hrnet_posenet_quanteval.py b/aimet_zoo_torch/hrnet_posenet/evaluators/hrnet_posenet_quanteval.py index 5391370..124f555 100644 --- a/aimet_zoo_torch/hrnet_posenet/evaluators/hrnet_posenet_quanteval.py +++ b/aimet_zoo_torch/hrnet_posenet/evaluators/hrnet_posenet_quanteval.py @@ -27,7 +27,7 @@ from aimet_zoo_torch.hrnet_posenet.models.core.loss import JointsMSELoss -def parse_args(): +def parse_args(raw_args): """parse user arguments""" parser = argparse.ArgumentParser(description="Evaluate keypoints network") parser.add_argument( @@ -63,14 +63,14 @@ def parse_args(): nargs=argparse.REMAINDER, ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(): +def main(raw_args=None): """execute evaluation""" # Load parameters from arguments - args = parse_args() + args = parse_args(raw_args) # Set dir args to default args.modelDir = "./" diff --git a/aimet_zoo_torch/hrnet_posenet/models/model_cards/__init__.py b/aimet_zoo_torch/hrnet_posenet/models/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/cityscapes/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/cityscapes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/lip/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/dataloader/list/lip/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/cityscapes/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/cityscapes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/lip/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/lip/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/pascal_ctx/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/experiments/pascal_ctx/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/hrnet_sem_seg_quanteval.py b/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/hrnet_sem_seg_quanteval.py index c0bec89..71c3669 100644 --- a/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/hrnet_sem_seg_quanteval.py +++ b/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/hrnet_sem_seg_quanteval.py @@ -21,7 +21,7 @@ from aimet_zoo_torch.hrnet_semantic_segmentation import HRNetSemSeg -def arguments(): +def arguments(raw_args): """parse command line arguments""" parser = argparse.ArgumentParser(description="Evaluation script for HRNet") parser.add_argument( @@ -36,7 +36,7 @@ def arguments(): "--use-cuda", help="Use GPU for evaluation", default=True, type=bool ) parser.add_argument("--dataset-path", help="Use GPU for evaluation", type=str) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -67,9 +67,9 @@ def __init__(self, args): setattr(self, arg, getattr(args, arg)) -def main(): +def main(raw_args=None): """run evaluator""" - args = arguments() + args = arguments(raw_args) config = ModelConfig(args) seed(config.seed, config.use_cuda) diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/model/core/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/model/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/model/datasets/cityscapes.py b/aimet_zoo_torch/hrnet_semantic_segmentation/model/datasets/cityscapes.py index 594d617..2eefab2 100644 --- a/aimet_zoo_torch/hrnet_semantic_segmentation/model/datasets/cityscapes.py +++ b/aimet_zoo_torch/hrnet_semantic_segmentation/model/datasets/cityscapes.py @@ -21,7 +21,7 @@ import cv2 import numpy as np from PIL import Image - +import pathlib import torch from torch.nn import functional as F diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/model/model_cards/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/model/models/sync_bn/inplace_abn/src/__init__.py b/aimet_zoo_torch/hrnet_semantic_segmentation/model/models/sync_bn/inplace_abn/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/inverseform/__init__.py b/aimet_zoo_torch/inverseform/__init__.py index 223b700..9dbef70 100644 --- a/aimet_zoo_torch/inverseform/__init__.py +++ b/aimet_zoo_torch/inverseform/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import HRNetInverseForm \ No newline at end of file +""" package for getting hrnetinverseform original model and quantized model""" +from .model.model_definition import HRNetInverseForm diff --git a/aimet_zoo_torch/inverseform/dataloader/__init__.py b/aimet_zoo_torch/inverseform/dataloader/__init__.py index c086995..a454cde 100644 --- a/aimet_zoo_torch/inverseform/dataloader/__init__.py +++ b/aimet_zoo_torch/inverseform/dataloader/__init__.py @@ -1 +1,2 @@ -from .dataloaders_and_eval_func import get_dataloaders_and_eval_func \ No newline at end of file +""" datasets and eval function are defined and loaded""" +from .dataloaders_and_eval_func import get_dataloaders_and_eval_func diff --git a/aimet_zoo_torch/inverseform/dataloader/data/cityscapes.py b/aimet_zoo_torch/inverseform/dataloader/data/cityscapes.py index 61b145a..f912c23 100644 --- a/aimet_zoo_torch/inverseform/dataloader/data/cityscapes.py +++ b/aimet_zoo_torch/inverseform/dataloader/data/cityscapes.py @@ -1,7 +1,8 @@ +# pylint: skip-file import os import os.path as path from aimet_zoo_torch.inverseform.model.utils.config import cfg -import aimet_model_zoo.aimet_zoo_torch.inverseform.dataloader.data.cityscapes_labels as cityscapes_labels +import aimet_zoo_torch.inverseform.dataloader.data.cityscapes_labels as cityscapes_labels def find_directories(root): diff --git a/aimet_zoo_torch/inverseform/dataloader/data/cityscapes_labels.py b/aimet_zoo_torch/inverseform/dataloader/data/cityscapes_labels.py index 7622fc6..dbafd10 100644 --- a/aimet_zoo_torch/inverseform/dataloader/data/cityscapes_labels.py +++ b/aimet_zoo_torch/inverseform/dataloader/data/cityscapes_labels.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # File taken from https://github.com/mcordts/cityscapesScripts/ # License File Available at: diff --git a/aimet_zoo_torch/inverseform/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/inverseform/dataloader/dataloaders_and_eval_func.py index 60e33cc..99cd781 100644 --- a/aimet_zoo_torch/inverseform/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/inverseform/dataloader/dataloaders_and_eval_func.py @@ -1,3 +1,4 @@ +# pylint: skip-file # from ..transforms.joint_transforms import joint_transforms from .transforms import transforms as extended_transforms from torch.utils.data import DataLoader @@ -56,7 +57,7 @@ def eval_func(model, N = -1): S = 0 with torch.no_grad(): for i, batch in enumerate(tqdm(dataloader)): - if i >= N and N >= 0: + if type(N) is int and N >= 0 and i >= N: break images, gt_image, edge, _, _ = batch inputs = torch.cat((images, gt_image.unsqueeze(dim=1), edge), dim=1) diff --git a/aimet_zoo_torch/inverseform/dataloader/datasets/base_loader.py b/aimet_zoo_torch/inverseform/dataloader/datasets/base_loader.py index 743a1a8..cc50233 100644 --- a/aimet_zoo_torch/inverseform/dataloader/datasets/base_loader.py +++ b/aimet_zoo_torch/inverseform/dataloader/datasets/base_loader.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Copyright 2020 Nvidia Corporation Redistribution and use in source and binary forms, with or without @@ -31,10 +32,10 @@ from PIL import Image from torch.utils import data -from aimet_model_zoo.zoo_torch.inverseform.model.utils.config import cfg -from aimet_model_zoo.zoo_torch.inverseform.model.utils.misc import tensor_to_pil -from aimet_model_zoo.zoo_torch.inverseform.dataloader.data.cityscapes import find_directories -import aimet_model_zoo.zoo_torch.inverseform.dataloader.data.cityscapes_labels as cityscapes_labels +from aimet_zoo_torch.inverseform.model.utils.config import cfg +from aimet_zoo_torch.inverseform.model.utils.misc import tensor_to_pil +from aimet_zoo_torch.inverseform.dataloader.data.cityscapes import find_directories +import aimet_zoo_torch.inverseform.dataloader.data.cityscapes_labels as cityscapes_labels from scipy.ndimage.morphology import distance_transform_edt diff --git a/aimet_zoo_torch/inverseform/dataloader/datasets/get_dataloaders.py b/aimet_zoo_torch/inverseform/dataloader/datasets/get_dataloaders.py index b10b838..b8cb2f3 100644 --- a/aimet_zoo_torch/inverseform/dataloader/datasets/get_dataloaders.py +++ b/aimet_zoo_torch/inverseform/dataloader/datasets/get_dataloaders.py @@ -1,3 +1,4 @@ +# pylint: skip-file from aimet_zoo_torch.inverseform.dataloader.transforms import transforms as extended_transforms from torch.utils.data import DataLoader diff --git a/aimet_zoo_torch/inverseform/dataloader/datasets/sampler.py b/aimet_zoo_torch/inverseform/dataloader/datasets/sampler.py index 5bd3251..c508e13 100644 --- a/aimet_zoo_torch/inverseform/dataloader/datasets/sampler.py +++ b/aimet_zoo_torch/inverseform/dataloader/datasets/sampler.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code adapted from: # https://github.com/pytorch/pytorch/blob/master/torch/utils/data/distributed.py diff --git a/aimet_zoo_torch/inverseform/dataloader/helper.py b/aimet_zoo_torch/inverseform/dataloader/helper.py index 60e33cc..82822e1 100644 --- a/aimet_zoo_torch/inverseform/dataloader/helper.py +++ b/aimet_zoo_torch/inverseform/dataloader/helper.py @@ -1,3 +1,4 @@ +# pylint: skip-file # from ..transforms.joint_transforms import joint_transforms from .transforms import transforms as extended_transforms from torch.utils.data import DataLoader diff --git a/aimet_zoo_torch/inverseform/dataloader/transforms/joint_transforms.py b/aimet_zoo_torch/inverseform/dataloader/transforms/joint_transforms.py index a22f61b..885a448 100644 --- a/aimet_zoo_torch/inverseform/dataloader/transforms/joint_transforms.py +++ b/aimet_zoo_torch/inverseform/dataloader/transforms/joint_transforms.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code borrowded from: # https://github.com/zijundeng/pytorch-semantic-segmentation/blob/master/utils/joint_transforms.py diff --git a/aimet_zoo_torch/inverseform/dataloader/transforms/transforms.py b/aimet_zoo_torch/inverseform/dataloader/transforms/transforms.py index 3c8b083..216074c 100644 --- a/aimet_zoo_torch/inverseform/dataloader/transforms/transforms.py +++ b/aimet_zoo_torch/inverseform/dataloader/transforms/transforms.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code borrowded from: # https://github.com/zijundeng/pytorch-semantic-segmentation/blob/master/utils/transforms.py diff --git a/aimet_zoo_torch/inverseform/evaluators/inverseform_quanteval.py b/aimet_zoo_torch/inverseform/evaluators/inverseform_quanteval.py index bfde079..f8156ab 100644 --- a/aimet_zoo_torch/inverseform/evaluators/inverseform_quanteval.py +++ b/aimet_zoo_torch/inverseform/evaluators/inverseform_quanteval.py @@ -8,16 +8,14 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantization code for InverseForm ''' +""" AIMET Quantization code for InverseForm """ # General imports -#import fire import argparse # PyTorch imports import torch -from torch.nn import functional as F # InverseForm imports from aimet_zoo_torch.inverseform import HRNetInverseForm @@ -25,46 +23,78 @@ def arguments(): - parser = argparse.ArgumentParser(description='Evaluation script for PyTorch InverseForm models.') - parser.add_argument('--model-config', help='Select the model configuration', type=str, default="hrnet_16_slim_if", choices=[ - "hrnet_16_slim_if", - "ocrnet_48_if"]) - parser.add_argument('--dataset-path', help='Path to cityscapes parent folder containing leftImg8bit', type=str, default='') - parser.add_argument('--batch-size', help='Data batch size for a model', type=int, default=8) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) - parser.add_argument('--use-cuda', help='Run evaluation on GPU.', type=bool, default=True) + """argument parser""" + parser = argparse.ArgumentParser( + description="Evaluation script for PyTorch InverseForm models." + ) + parser.add_argument( + "--model-config", + help="Select the model configuration", + type=str, + default="hrnet_16_slim_if", + choices=["hrnet_16_slim_if", "ocrnet_48_if"], + ) + parser.add_argument( + "--dataset-path", + help="Path to cityscapes parent folder containing leftImg8bit", + type=str, + default="", + ) + parser.add_argument( + "--batch-size", help="Data batch size for a model", type=int, default=8 + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True + ) args = parser.parse_args() return args def seed(seednum, use_cuda): - torch.manual_seed(seednum) - if use_cuda: - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - torch.cuda.manual_seed(seednum) - torch.cuda.manual_seed_all(seednum) + """random seed generator""" + torch.manual_seed(seednum) + if use_cuda: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.cuda.manual_seed(seednum) + torch.cuda.manual_seed_all(seednum) def main(): - args = arguments() - seed(0, args.use_cuda) - - # Load model - model = HRNetInverseForm(model_config=args.model_config) - sim = model.get_quantsim(quantized=True) - sim.model.cuda() - - _, val_loader, eval_func = get_dataloaders_and_eval_func(dataset_path=args.dataset_path) - - sim.compute_encodings(forward_pass_callback = eval_func, - forward_pass_callback_args = -1) - - # Evaluate quantized model - mIoU = eval_func(sim.model) - print("Quantized mIoU : {:0.4f}".format(mIoU)) - -if __name__ == '__main__': - # fire.Fire(main) - main() + """main evaluation function""" + args = arguments() + seed(0, args.use_cuda) + + # Load model + model = HRNetInverseForm(model_config=args.model_config) + sim = model.get_quantsim(quantized=True) + sim.model.cuda() + # pylint:disable = unused-variable + _, val_loader, eval_func = get_dataloaders_and_eval_func( + dataset_path=args.dataset_path + ) + + sim.compute_encodings( + forward_pass_callback=eval_func, forward_pass_callback_args=-1 + ) + + # Evaluate quantized model + mIoU = eval_func(sim.model) + print("Quantized mIoU : {:0.4f}".format(mIoU)) + + +if __name__ == "__main__": + # fire.Fire(main) + main() diff --git a/aimet_zoo_torch/inverseform/model/model_cards/__init__.py b/aimet_zoo_torch/inverseform/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/inverseform/model/model_definition.py b/aimet_zoo_torch/inverseform/model/model_definition.py index 070722e..af6614f 100644 --- a/aimet_zoo_torch/inverseform/model/model_definition.py +++ b/aimet_zoo_torch/inverseform/model/model_definition.py @@ -7,55 +7,70 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +"""Class for downloading and setting up of optmized and original inverseform model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet -import torch import json import os +import torch +from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim +from aimet_torch.cross_layer_equalization import equalize_model from aimet_zoo_torch.common.downloader import Downloader from aimet_zoo_torch.inverseform.model.utils.config import assert_and_infer_cfg, cfg from aimet_zoo_torch.inverseform.model.models.lighthrnet import HRNet16 from aimet_zoo_torch.inverseform.model.models.ocrnet import OCRNet -from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim -from aimet_torch.cross_layer_equalization import equalize_model -class HRNetInverseForm(Downloader): - def __init__(self, model_config: str = None, num_classes = 19): - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) +class HRNetInverseForm(Downloader): + """HRNetInverseForm parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + def __init__(self, model_config: str = None, num_classes=19): + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) - if '16' in model_config: - assert_and_infer_cfg(result_dir='', global_rank=0, syncbn=False, hrnet_base=16) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + if "16" in model_config: + assert_and_infer_cfg( + result_dir="", global_rank=0, syncbn=False, hrnet_base=16 + ) C = cfg C.MODEL.HR18 = False C.MODEL.HR16 = True C.MODEL.DOWN_CONV = True - self.model = HRNet16(num_classes = num_classes, criterion=None) - elif '48' in model_config: - assert_and_infer_cfg(result_dir='', global_rank=0, syncbn=False, hrnet_base=48) + self.model = HRNet16(num_classes=num_classes, criterion=None) + elif "48" in model_config: + assert_and_infer_cfg( + result_dir="", global_rank=0, syncbn=False, hrnet_base=48 + ) C = cfg C.MODEL.HR18 = False C.MODEL.HR16 = False C.MODEL.DOWN_CONV = False - self.model = OCRNet(num_classes = num_classes, criterion=None) - + self.model = OCRNet(num_classes=num_classes, criterion=None) + def from_pretrained(self, quantized=False): + """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() @@ -69,25 +84,35 @@ def from_pretrained(self, quantized=False): self.model = torch.load(self.path_pre_opt_weights) def get_quantsim(self, quantized=False): + """get quantsim object with pre-loaded encodings or pretrained model""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model.cuda(), **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') - return sim \ No newline at end of file + print("set_and_freeze_param_encodings finished!") + return sim diff --git a/aimet_zoo_torch/inverseform/model/models/InverseForm.py b/aimet_zoo_torch/inverseform/model/models/InverseForm.py index 98cd115..5463fa9 100644 --- a/aimet_zoo_torch/inverseform/model/models/InverseForm.py +++ b/aimet_zoo_torch/inverseform/model/models/InverseForm.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2021 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/inverseform/model/models/__init__.py b/aimet_zoo_torch/inverseform/model/models/__init__.py index 02304a0..1351cef 100644 --- a/aimet_zoo_torch/inverseform/model/models/__init__.py +++ b/aimet_zoo_torch/inverseform/model/models/__init__.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Network Initializations """ @@ -27,6 +28,7 @@ def wrap_network_in_dataparallel(net, use_apex_data_parallel=False): """ if use_apex_data_parallel: import apex + net = apex.parallel.DistributedDataParallel(net) else: net = torch.nn.DataParallel(net) @@ -37,13 +39,15 @@ def get_model(network, num_classes, criterion, has_edge=False): """ Fetch Network Function Pointer """ - module = network[:network.rfind('.')] - model = network[network.rfind('.') + 1:] + module = network[: network.rfind(".")] + model = network[network.rfind(".") + 1 :] mod = importlib.import_module(module) net_func = getattr(mod, model) if not has_edge: net = net_func(num_classes=num_classes, criterion=criterion) else: - net = net_func(num_classes=num_classes, criterion=criterion, has_edge_head=has_edge) + net = net_func( + num_classes=num_classes, criterion=criterion, has_edge_head=has_edge + ) return net diff --git a/aimet_zoo_torch/inverseform/model/models/attnscale.py b/aimet_zoo_torch/inverseform/model/models/attnscale.py index 6c46358..586bded 100644 --- a/aimet_zoo_torch/inverseform/model/models/attnscale.py +++ b/aimet_zoo_torch/inverseform/model/models/attnscale.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file """ Copyright 2020 Nvidia Corporation diff --git a/aimet_zoo_torch/inverseform/model/models/basic.py b/aimet_zoo_torch/inverseform/model/models/basic.py index a606da0..e9c1dc5 100644 --- a/aimet_zoo_torch/inverseform/model/models/basic.py +++ b/aimet_zoo_torch/inverseform/model/models/basic.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file """ Copyright 2020 Nvidia Corporation diff --git a/aimet_zoo_torch/inverseform/model/models/bn_helper.py b/aimet_zoo_torch/inverseform/model/models/bn_helper.py index 5e3cefa..599fbde 100644 --- a/aimet_zoo_torch/inverseform/model/models/bn_helper.py +++ b/aimet_zoo_torch/inverseform/model/models/bn_helper.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file import torch import functools diff --git a/aimet_zoo_torch/inverseform/model/models/hrnetv2.py b/aimet_zoo_torch/inverseform/model/models/hrnetv2.py index 3bfc359..17386d6 100644 --- a/aimet_zoo_torch/inverseform/model/models/hrnetv2.py +++ b/aimet_zoo_torch/inverseform/model/models/hrnetv2.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file # Most of the code below is from the following repo: # https://github.com/HRNet/HRNet-Semantic-Segmentation/tree/HRNet-OCR # diff --git a/aimet_zoo_torch/inverseform/model/models/lighthrnet.py b/aimet_zoo_torch/inverseform/model/models/lighthrnet.py index a3bec7e..3ca688e 100644 --- a/aimet_zoo_torch/inverseform/model/models/lighthrnet.py +++ b/aimet_zoo_torch/inverseform/model/models/lighthrnet.py @@ -1,3 +1,4 @@ +# pylint: skip-file # Copyright (c) 2021 Qualcomm Technologies, Inc. # All Rights Reserved. diff --git a/aimet_zoo_torch/inverseform/model/models/loss/utils.py b/aimet_zoo_torch/inverseform/model/models/loss/utils.py index eddd65f..d802a83 100644 --- a/aimet_zoo_torch/inverseform/model/models/loss/utils.py +++ b/aimet_zoo_torch/inverseform/model/models/loss/utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Copyright (c) 2021 Qualcomm Technologies, Inc. diff --git a/aimet_zoo_torch/inverseform/model/models/model_loader.py b/aimet_zoo_torch/inverseform/model/models/model_loader.py index e88af23..de6d674 100644 --- a/aimet_zoo_torch/inverseform/model/models/model_loader.py +++ b/aimet_zoo_torch/inverseform/model/models/model_loader.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file def load_model(net, pretrained): pretrained_dict = pretrained['state_dict'] model_dict = net.state_dict() diff --git a/aimet_zoo_torch/inverseform/model/models/mscale.py b/aimet_zoo_torch/inverseform/model/models/mscale.py index 05cf9ba..1f9109e 100644 --- a/aimet_zoo_torch/inverseform/model/models/mscale.py +++ b/aimet_zoo_torch/inverseform/model/models/mscale.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file """ Copyright 2020 Nvidia Corporation diff --git a/aimet_zoo_torch/inverseform/model/models/mscale2.py b/aimet_zoo_torch/inverseform/model/models/mscale2.py index ace1aac..b6452d7 100644 --- a/aimet_zoo_torch/inverseform/model/models/mscale2.py +++ b/aimet_zoo_torch/inverseform/model/models/mscale2.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file """ Copyright 2020 Nvidia Corporation diff --git a/aimet_zoo_torch/inverseform/model/models/mynn.py b/aimet_zoo_torch/inverseform/model/models/mynn.py index 6ca92bf..cf26b94 100644 --- a/aimet_zoo_torch/inverseform/model/models/mynn.py +++ b/aimet_zoo_torch/inverseform/model/models/mynn.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file """ Code adapted from: https://github.com/NVIDIA/semantic-segmentation diff --git a/aimet_zoo_torch/inverseform/model/models/ocr_utils.py b/aimet_zoo_torch/inverseform/model/models/ocr_utils.py index 4629369..df1ab04 100644 --- a/aimet_zoo_torch/inverseform/model/models/ocr_utils.py +++ b/aimet_zoo_torch/inverseform/model/models/ocr_utils.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file # ------------------------------------------------------------------------------ # Copyright (c) Microsoft # Licensed under the MIT License. diff --git a/aimet_zoo_torch/inverseform/model/models/ocrnet.py b/aimet_zoo_torch/inverseform/model/models/ocrnet.py index 6b3534f..2f4101b 100644 --- a/aimet_zoo_torch/inverseform/model/models/ocrnet.py +++ b/aimet_zoo_torch/inverseform/model/models/ocrnet.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file """ Copyright 2020 Nvidia Corporation diff --git a/aimet_zoo_torch/inverseform/model/models/utils.py b/aimet_zoo_torch/inverseform/model/models/utils.py index b36aed0..2395161 100644 --- a/aimet_zoo_torch/inverseform/model/models/utils.py +++ b/aimet_zoo_torch/inverseform/model/models/utils.py @@ -1,3 +1,5 @@ +# pylint: skip-file +# pylint: skip-file """ Copyright 2020 Nvidia Corporation diff --git a/aimet_zoo_torch/inverseform/model/utils/attr_dict.py b/aimet_zoo_torch/inverseform/model/utils/attr_dict.py index b97d95d..951cc51 100644 --- a/aimet_zoo_torch/inverseform/model/utils/attr_dict.py +++ b/aimet_zoo_torch/inverseform/model/utils/attr_dict.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code adapted from: # https://github.com/facebookresearch/Detectron/blob/master/detectron/utils/collections.py diff --git a/aimet_zoo_torch/inverseform/model/utils/cityscapes_labels.py b/aimet_zoo_torch/inverseform/model/utils/cityscapes_labels.py index 7622fc6..dbafd10 100644 --- a/aimet_zoo_torch/inverseform/model/utils/cityscapes_labels.py +++ b/aimet_zoo_torch/inverseform/model/utils/cityscapes_labels.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # File taken from https://github.com/mcordts/cityscapesScripts/ # License File Available at: diff --git a/aimet_zoo_torch/inverseform/model/utils/config.py b/aimet_zoo_torch/inverseform/model/utils/config.py index d90129f..69d33b5 100644 --- a/aimet_zoo_torch/inverseform/model/utils/config.py +++ b/aimet_zoo_torch/inverseform/model/utils/config.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code adapted from: # https://github.com/facebookresearch/Detectron/blob/master/detectron/core/config.py diff --git a/aimet_zoo_torch/inverseform/model/utils/misc.py b/aimet_zoo_torch/inverseform/model/utils/misc.py index 3aeffd3..f7d2fda 100644 --- a/aimet_zoo_torch/inverseform/model/utils/misc.py +++ b/aimet_zoo_torch/inverseform/model/utils/misc.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Miscellanous Functions : From HRNet semantic segmentation : https://github.com/HRNet/HRNet-Semantic-Segmentation """ diff --git a/aimet_zoo_torch/inverseform/model/utils/my_data_parallel.py b/aimet_zoo_torch/inverseform/model/utils/my_data_parallel.py index 1da210a..696da05 100644 --- a/aimet_zoo_torch/inverseform/model/utils/my_data_parallel.py +++ b/aimet_zoo_torch/inverseform/model/utils/my_data_parallel.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ # Code adapted from: # https://github.com/pytorch/pytorch/blob/master/torch/nn/parallel/data_parallel.py diff --git a/aimet_zoo_torch/inverseform/model/utils/progress_bar.py b/aimet_zoo_torch/inverseform/model/utils/progress_bar.py index 31cfb06..f1f127e 100644 --- a/aimet_zoo_torch/inverseform/model/utils/progress_bar.py +++ b/aimet_zoo_torch/inverseform/model/utils/progress_bar.py @@ -2,11 +2,15 @@ # All Rights Reserved. +"""module for download progress bar""" import sys -def printProgressBar(i,max,postText): + +def printProgressBar(i, max, postText): + """ print download progress bar""" + #pylint:disable = redefined-builtin n_bar = 10 - j= i/max - sys.stdout.write('\r') + j = i / max + sys.stdout.write("\r") sys.stdout.write(f"[{'=' * int(n_bar * j):{n_bar}s}] {postText}") - sys.stdout.flush() \ No newline at end of file + sys.stdout.flush() diff --git a/aimet_zoo_torch/inverseform/model/utils/trnval_utils.py b/aimet_zoo_torch/inverseform/model/utils/trnval_utils.py index 19c2032..7ab4ac0 100644 --- a/aimet_zoo_torch/inverseform/model/utils/trnval_utils.py +++ b/aimet_zoo_torch/inverseform/model/utils/trnval_utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Copyright 2020 Nvidia Corporation diff --git a/aimet_zoo_torch/minilm/__init__.py b/aimet_zoo_torch/minilm/__init__.py index a3ce928..8df698f 100644 --- a/aimet_zoo_torch/minilm/__init__.py +++ b/aimet_zoo_torch/minilm/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import Minilm +""" package for getting minilm original model and quantized model""" +from .model.model_definition import Minilm diff --git a/aimet_zoo_torch/minilm/dataloader/__init__.py b/aimet_zoo_torch/minilm/dataloader/__init__.py index a82b7e0..01744ee 100644 --- a/aimet_zoo_torch/minilm/dataloader/__init__.py +++ b/aimet_zoo_torch/minilm/dataloader/__init__.py @@ -1 +1,2 @@ -from .dataloaders import get_datasets,get_num_labels,eval_function +""" package for getting minilm original model and quantized model""" +from .dataloaders import get_datasets, get_num_labels, eval_function diff --git a/aimet_zoo_torch/minilm/dataloader/utils/__init__.py b/aimet_zoo_torch/minilm/dataloader/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/minilm/evaluators/__init__.py b/aimet_zoo_torch/minilm/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/minilm/evaluators/minilm_quanteval.py b/aimet_zoo_torch/minilm/evaluators/minilm_quanteval.py index fcaa522..0374ff6 100644 --- a/aimet_zoo_torch/minilm/evaluators/minilm_quanteval.py +++ b/aimet_zoo_torch/minilm/evaluators/minilm_quanteval.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201 # ============================================================================= # @@-COPYRIGHT-START-@@ # # Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. -# Changes from QuIC are licensed under the terms and conditions at +# Changes from QuIC are licensed under the terms and conditions at # https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf" # # @@-COPYRIGHT-END-@@ @@ -27,21 +27,20 @@ import argparse import logging -from transformers.utils.versions import require_version -from datasets import load_dataset, load_metric from aimet_zoo_torch.minilm import Minilm -from aimet_zoo_torch.minilm.dataloader import get_datasets, eval_function +from aimet_zoo_torch.minilm.dataloader import get_datasets, eval_function def parse_args(): """argument parser""" parser = argparse.ArgumentParser( - description="Evaluating MiniLM model on GLUE datasets") + description="Evaluating MiniLM model on GLUE datasets" + ) parser.add_argument( "--model_config", default="minilm_w8a8_mnli", - help="choice [mobilebert_w8a8_cola, mobilebert_w8a8_mnli, mobilebert_w8a8_mrpc, mobilebert_w8a8_qnli, mobilebert_w8a8_qqp, mobilebert_w8a8_rte, mobilebert_w8a8_squad, mobilebert_w8a8_sst2, mobilebert_w8a8_stsb]", + help="choice [mobilebert_w8a8_cola, mobilebert_w8a8_mnli, mobilebert_w8a8_mrpc, mobilebert_w8a8_qnli, mobilebert_w8a8_qqp, mobilebert_w8a8_rte, mobilebert_w8a8_squad, mobilebert_w8a8_sst2, mobilebert_w8a8_stsb]", ) parser.add_argument( "--per_device_eval_batch_size", @@ -54,7 +53,7 @@ def parse_args(): type=str, default=None, help="Output directory", - ) + ) args = parser.parse_args() for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) @@ -63,7 +62,7 @@ def parse_args(): def main(): - """main function for quantization evaluation""" + """main function for quantization evaluation""" args = parse_args() logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", @@ -71,21 +70,31 @@ def main(): level=logging.INFO, ) model = Minilm(model_config=args.model_config) - - # get original model and tokenizer - model_orig,tokenizer = model.get_model_from_pretrained() - # get datasets - datasets=get_datasets(data_args=model.data_args,training_args=model.training_args,model_args=model.model_args,model=model_orig,tokenizer=tokenizer) + # get original model and tokenizer + model_orig, tokenizer = model.get_model_from_pretrained() - # evaluation of original model - original_eval_results=eval_function(model_orig,tokenizer,datasets,model.data_args,model.training_args) + # get datasets + datasets = get_datasets( + data_args=model.data_args, + training_args=model.training_args, + model_args=model.model_args, + model=model_orig, + tokenizer=tokenizer, + ) - # get quantsim object + # evaluation of original model + original_eval_results = eval_function( + model_orig, tokenizer, datasets, model.data_args, model.training_args + ) + + # get quantsim object quantsim_model = model.get_quantsim() - # evaluation of quantsim model - optimized_eval_results=eval_function(quantsim_model.model,tokenizer,datasets,model.data_args,model.training_args) + # evaluation of quantsim model + optimized_eval_results = eval_function( + quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args + ) logging.info(f"***** Original Eval results *****") for key, value in sorted(original_eval_results.items()): @@ -93,8 +102,7 @@ def main(): logging.info(f"***** Optimized Quantized Model Eval results *****") for key, value in sorted(optimized_eval_results.items()): - logging.info(f" {key} = {value}") - + logging.info(f" {key} = {value}") if __name__ == "__main__": diff --git a/aimet_zoo_torch/minilm/model/__init__.py b/aimet_zoo_torch/minilm/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/minilm/model/model_cards/__init__.py b/aimet_zoo_torch/minilm/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_cola.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_cola.json index 37b349d..8a3ed5a 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_cola.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_cola.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/cola_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/cola_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mnli.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mnli.json index 540fb08..8d04fe4 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mnli.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mnli.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/mnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/mnli_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mrpc.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mrpc.json index 7513075..699fd5f 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mrpc.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mrpc.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/mrpc_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/mrpc_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qnli.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qnli.json index 6af10b1..7451b01 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qnli.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qnli.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/qnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/qnli_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qqp.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qqp.json index ca2b7e7..6c26996 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qqp.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qqp.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/qqp_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/qqp_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_rte.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_rte.json index 4a4284f..1e58e59 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_rte.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_rte.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/rte_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/rte_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_squad.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_squad.json index d18a644..6a70ee6 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_squad.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_squad.json @@ -46,7 +46,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/squad_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/squad_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_sst2.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_sst2.json index 4a73c68..650786f 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_sst2.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_sst2.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/sst2_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/sst2_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_stsb.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_stsb.json index 33e9380..6ef4354 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_stsb.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_stsb.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/stsb_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_minilm/stsb_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/minilm/model/model_definition.py b/aimet_zoo_torch/minilm/model/model_definition.py index 472932f..cb881de 100644 --- a/aimet_zoo_torch/minilm/model/model_definition.py +++ b/aimet_zoo_torch/minilm/model/model_definition.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- mode: python -*- -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 # ============================================================================= # @@-COPYRIGHT-START-@@ # @@ -9,47 +9,44 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """Class for downloading and setting up of optmized and original vit model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet + import json import os -import csv import sys +import pathlib from collections import defaultdict import torch -import datasets -from transformers import AutoConfig as Config -from transformers import AutoFeatureExtractor as FeatureExtractor -from aimet_common.defs import QuantScheme + # AIMET imports from aimet_torch.quantsim import load_checkpoint -from aimet_torch.quantsim import QuantizationSimModel -from aimet_torch.qc_quantize_op import QcQuantizeWrapper + +from transformers import HfArgumentParser +from transformers import AutoConfig, AutoTokenizer, TrainingArguments + # transformers import -from aimet_zoo_torch.minilm.model.baseline_models import bert +from aimet_zoo_torch.minilm.model import baseline_models from aimet_zoo_torch.common.downloader import Downloader -sys.modules['baseline_models.bert'] = bert +sys.modules["baseline_models"] = baseline_models -from transformers import HfArgumentParser -from transformers import ( - AutoConfig, - AutoTokenizer, - TrainingArguments -) class Minilm(Downloader): - """ model minilm configuration class """ + """model minilm configuration class""" + #pylint:disable = import-outside-toplevel def __init__(self, model_config=None): - if model_config=="minilm_w8a8_squad": + if model_config == "minilm_w8a8_squad": from aimet_zoo_torch.minilm.model.utils.utils_qa_dataclass import ( ModelArguments, DataTrainingArguments, AuxArguments, - ) + ) else: from aimet_zoo_torch.minilm.model.utils.utils_nlclassifier_dataclass import ( ModelArguments, DataTrainingArguments, AuxArguments, - ) + ) parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: @@ -65,47 +62,43 @@ def __init__(self, model_config=None): ) # Parse arguments parser = HfArgumentParser( - (ModelArguments, - DataTrainingArguments, - TrainingArguments, - AuxArguments)) + (ModelArguments, DataTrainingArguments, TrainingArguments, AuxArguments) + ) ( - model_args, - data_args, - training_args, - aux_args, + model_args, + data_args, + training_args, + aux_args, ) = parser.parse_args_into_dataclasses() - self.model = None - self.model_args=model_args - self.data_args =data_args + self.model_args = model_args + self.data_args = data_args self.training_args = training_args self.aux_args = aux_args - #additional setup of the argsumetns from model_config - if model_config=="minilm_w8a8_squad": - self.data_args.dataset_name=self.cfg["data_training_args"]["dataset_name"] + # additional setup of the argsumetns from model_config + if model_config == "minilm_w8a8_squad": + self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] else: - self.data_args.task_name=self.cfg["data_training_args"]["task_name"] + self.data_args.task_name = self.cfg["data_training_args"]["task_name"] - self.aux_args.model_config=model_config - self.training_args.do_eval=True + self.aux_args.model_config = model_config + self.training_args.do_eval = True # setup the download path from arguments - self.path_pre_opt_weights = self.aux_args.fmodel_path + self.path_pre_opt_weights = self.aux_args.fmodel_path self.path_post_opt_weights = self.aux_args.qmodel_path - def get_model_from_pretrained(self): """get original or optmized model Parameters: dataset: Return: - model : pretrained/optmized model + model : pretrained/optmized model """ - #case1. model for squad dataset - if hasattr(self.data_args,'dataset_name'): + # case1. model for squad dataset + if hasattr(self.data_args, "dataset_name"): self._download_pre_opt_weights(show_progress=True) - self._download_aimet_config() + self._download_aimet_config() config = AutoConfig.from_pretrained( self.model_args.config_name if self.model_args.config_name @@ -128,10 +121,10 @@ def get_model_from_pretrained(self): use_auth_token=True if self.model_args.use_auth_token else None, ) - self.model = torch.load(self.aux_args.fmodel_path) - return self.model,tokenizer - #case2. model for glue dataset - num_labels=2 + self.model = torch.load(self.aux_args.fmodel_path) + return self.model, tokenizer + # case2. model for glue dataset + num_labels = 2 self._download_pre_opt_weights(show_progress=True) self._download_aimet_config() # Load pretrained model and tokenizer @@ -148,7 +141,9 @@ def get_model_from_pretrained(self): # ++++ config.return_dict = False config.classifier_dropout = None - config.attention_probs_dropout_prob = self.model_args.attention_probs_dropout_prob + config.attention_probs_dropout_prob = ( + self.model_args.attention_probs_dropout_prob + ) # ++++ tokenizer = AutoTokenizer.from_pretrained( @@ -162,12 +157,12 @@ def get_model_from_pretrained(self): ) self.model = torch.load(self.aux_args.fmodel_path) - return self.model,tokenizer + return self.model, tokenizer def get_quantsim(self): - """get quantsim object """ + """get quantsim object""" self._download_post_opt_weights(show_progress=True) # Load the Quantsim_model object quantsim_model = load_checkpoint(self.aux_args.qmodel_path) - return quantsim_model + return quantsim_model diff --git a/aimet_zoo_torch/minilm/model/utils/__init__.py b/aimet_zoo_torch/minilm/model/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilebert/__init__.py b/aimet_zoo_torch/mobilebert/__init__.py index a44a2f4..0754246 100644 --- a/aimet_zoo_torch/mobilebert/__init__.py +++ b/aimet_zoo_torch/mobilebert/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import MobileBert +""" package for getting distilbert original model and quantized model""" +from .model.model_definition import MobileBert diff --git a/aimet_zoo_torch/mobilebert/dataloader/__init__.py b/aimet_zoo_torch/mobilebert/dataloader/__init__.py index a82b7e0..229cc2a 100644 --- a/aimet_zoo_torch/mobilebert/dataloader/__init__.py +++ b/aimet_zoo_torch/mobilebert/dataloader/__init__.py @@ -1 +1,2 @@ -from .dataloaders import get_datasets,get_num_labels,eval_function +""" datasets and eval function are defined and loaded""" +from .dataloaders import get_datasets, get_num_labels, eval_function diff --git a/aimet_zoo_torch/mobilebert/dataloader/utils/__init__.py b/aimet_zoo_torch/mobilebert/dataloader/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilebert/evaluators/__init__.py b/aimet_zoo_torch/mobilebert/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilebert/evaluators/mobilebert_quanteval.py b/aimet_zoo_torch/mobilebert/evaluators/mobilebert_quanteval.py index 0377fd4..109d930 100644 --- a/aimet_zoo_torch/mobilebert/evaluators/mobilebert_quanteval.py +++ b/aimet_zoo_torch/mobilebert/evaluators/mobilebert_quanteval.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201 # ============================================================================= # @@-COPYRIGHT-START-@@ # # Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. -# Changes from QuIC are licensed under the terms and conditions at +# Changes from QuIC are licensed under the terms and conditions at # https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf" # # @@-COPYRIGHT-END-@@ @@ -27,17 +27,16 @@ import argparse import logging -from transformers.utils.versions import require_version -from datasets import load_dataset, load_metric from aimet_zoo_torch.mobilebert import MobileBert -from aimet_zoo_torch.mobilebert.dataloader import get_datasets, eval_function +from aimet_zoo_torch.mobilebert.dataloader import get_datasets, eval_function def parse_args(): """argument parser""" parser = argparse.ArgumentParser( - description="Evaluating MobileBert model on GLUE datasets") + description="Evaluating MobileBert model on GLUE datasets" + ) parser.add_argument( "--model_config", default="mobilebert_w8a8_mnli", @@ -54,7 +53,7 @@ def parse_args(): type=str, default=None, help="Output directory", - ) + ) args = parser.parse_args() for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) @@ -63,7 +62,7 @@ def parse_args(): def main(): - """main function for quantization evaluation""" + """main function for quantization evaluation""" args = parse_args() logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", @@ -71,21 +70,31 @@ def main(): level=logging.INFO, ) model = MobileBert(model_config=args.model_config) - - # get original model and tokenizer - model_orig,tokenizer = model.get_model_from_pretrained() - # get datasets - datasets=get_datasets(data_args=model.data_args,training_args=model.training_args,model_args=model.model_args,model=model_orig,tokenizer=tokenizer) + # get original model and tokenizer + model_orig, tokenizer = model.get_model_from_pretrained() - # evaluation of original model - original_eval_results=eval_function(model_orig,tokenizer,datasets,model.data_args,model.training_args) + # get datasets + datasets = get_datasets( + data_args=model.data_args, + training_args=model.training_args, + model_args=model.model_args, + model=model_orig, + tokenizer=tokenizer, + ) - # get quantsim object + # evaluation of original model + original_eval_results = eval_function( + model_orig, tokenizer, datasets, model.data_args, model.training_args + ) + + # get quantsim object quantsim_model = model.get_quantsim() - # evaluation of quantsim model - optimized_eval_results=eval_function(quantsim_model.model,tokenizer,datasets,model.data_args,model.training_args) + # evaluation of quantsim model + optimized_eval_results = eval_function( + quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args + ) logging.info(f"***** Original Eval results *****") for key, value in sorted(original_eval_results.items()): @@ -93,8 +102,7 @@ def main(): logging.info(f"***** Optimized Quantized Model Eval results *****") for key, value in sorted(optimized_eval_results.items()): - logging.info(f" {key} = {value}") - + logging.info(f" {key} = {value}") if __name__ == "__main__": diff --git a/aimet_zoo_torch/mobilebert/model/__init__.py b/aimet_zoo_torch/mobilebert/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilebert/model/baseline_models/__init__.py b/aimet_zoo_torch/mobilebert/model/baseline_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/__init__.py b/aimet_zoo_torch/mobilebert/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json index 1d9c7c6..ae8c9e7 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/cola_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/cola_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json index 1f0645a..08525c7 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/mnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/mnli_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json index 0fd7118..b9bec7e 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/mrpc_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/mrpc_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json index dfed240..e86236c 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/qnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/qnli_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json index 587a0eb..a2c58d2 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/qqp_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/qqp_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json index 3868a4f..f869f4e 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/rte_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/rte_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json index 2b688e6..11e100e 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json @@ -46,7 +46,7 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/squad_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/squad_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json index 2f65f91..774d496 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/sst2_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/sst2_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json index 87b7a0d..3ac4f31 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/stsb_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/stsb_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_definition.py b/aimet_zoo_torch/mobilebert/model/model_definition.py index b531c22..d68f130 100644 --- a/aimet_zoo_torch/mobilebert/model/model_definition.py +++ b/aimet_zoo_torch/mobilebert/model/model_definition.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- mode: python -*- -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 # ============================================================================= # @@-COPYRIGHT-START-@@ # @@ -8,48 +8,42 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -"""Class for downloading and setting up of optmized and original vit model for AIMET model zoo""" +"""Class for downloading and setting up of optmized and original mobilebert model for AIMET model zoo""" import json import os -import csv import sys +import pathlib from collections import defaultdict import torch -import datasets -from transformers import AutoConfig as Config -from transformers import AutoFeatureExtractor as FeatureExtractor -from aimet_common.defs import QuantScheme + + +from transformers import HfArgumentParser +from transformers import AutoConfig, AutoTokenizer, TrainingArguments # AIMET imports from aimet_torch.quantsim import load_checkpoint -from aimet_torch.quantsim import QuantizationSimModel -from aimet_torch.qc_quantize_op import QcQuantizeWrapper # transformers import from aimet_zoo_torch.mobilebert.model import baseline_models from aimet_zoo_torch.common.downloader import Downloader -sys.modules['baseline_models'] = baseline_models -from transformers import HfArgumentParser -from transformers import ( - AutoConfig, - AutoTokenizer, - TrainingArguments -) +sys.modules["baseline_models"] = baseline_models + class MobileBert(Downloader): - """ model mobilebert configuration class """ + """model mobilebert configuration class""" + #pylint:disable = import-outside-toplevel def __init__(self, model_config=None): - if model_config=="mobilebert_w8a8_squad": + if model_config == "mobilebert_w8a8_squad": from aimet_zoo_torch.mobilebert.model.utils.utils_qa_dataclass import ( ModelArguments, DataTrainingArguments, AuxArguments, - ) + ) else: from aimet_zoo_torch.mobilebert.model.utils.utils_nlclassifier_dataclass import ( ModelArguments, DataTrainingArguments, AuxArguments, - ) + ) parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: @@ -65,47 +59,43 @@ def __init__(self, model_config=None): ) # Parse arguments parser = HfArgumentParser( - (ModelArguments, - DataTrainingArguments, - TrainingArguments, - AuxArguments)) + (ModelArguments, DataTrainingArguments, TrainingArguments, AuxArguments) + ) ( - model_args, - data_args, - training_args, - aux_args, + model_args, + data_args, + training_args, + aux_args, ) = parser.parse_args_into_dataclasses() - self.model = None - self.model_args=model_args - self.data_args =data_args + self.model_args = model_args + self.data_args = data_args self.training_args = training_args self.aux_args = aux_args - #additional setup of the argsumetns from model_config - if model_config=="mobilebert_w8a8_squad": - self.data_args.dataset_name=self.cfg["data_training_args"]["dataset_name"] + # additional setup of the argsumetns from model_config + if model_config == "mobilebert_w8a8_squad": + self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] else: - self.data_args.task_name=self.cfg["data_training_args"]["task_name"] + self.data_args.task_name = self.cfg["data_training_args"]["task_name"] - self.aux_args.model_config=model_config - self.training_args.do_eval=True + self.aux_args.model_config = model_config + self.training_args.do_eval = True # setup the download path from arguments - self.path_pre_opt_weights = self.aux_args.fmodel_path + self.path_pre_opt_weights = self.aux_args.fmodel_path self.path_post_opt_weights = self.aux_args.qmodel_path - def get_model_from_pretrained(self): """get original or optmized model Parameters: dataset: Return: - model : pretrained/optmized model + model : pretrained/optmized model """ - #case1. model for squad dataset - if hasattr(self.data_args,'dataset_name'): + # case1. model for squad dataset + if hasattr(self.data_args, "dataset_name"): self._download_pre_opt_weights(show_progress=True) - self._download_aimet_config() + self._download_aimet_config() config = AutoConfig.from_pretrained( self.model_args.config_name if self.model_args.config_name @@ -128,10 +118,10 @@ def get_model_from_pretrained(self): use_auth_token=True if self.model_args.use_auth_token else None, ) - self.model = torch.load(self.aux_args.fmodel_path) - return self.model,tokenizer - #case2. model for glue dataset - num_labels=2 + self.model = torch.load(self.aux_args.fmodel_path) + return self.model, tokenizer + # case2. model for glue dataset + num_labels = 2 self._download_pre_opt_weights(show_progress=True) self._download_aimet_config() # Load pretrained model and tokenizer @@ -148,7 +138,9 @@ def get_model_from_pretrained(self): # ++++ config.return_dict = False config.classifier_dropout = None - config.attention_probs_dropout_prob = self.model_args.attention_probs_dropout_prob + config.attention_probs_dropout_prob = ( + self.model_args.attention_probs_dropout_prob + ) # ++++ tokenizer = AutoTokenizer.from_pretrained( @@ -162,12 +154,12 @@ def get_model_from_pretrained(self): ) self.model = torch.load(self.aux_args.fmodel_path) - return self.model,tokenizer + return self.model, tokenizer def get_quantsim(self): - """get quantsim object """ + """get quantsim object""" self._download_post_opt_weights(show_progress=True) # Load the Quantsim_model object quantsim_model = load_checkpoint(self.aux_args.qmodel_path) - return quantsim_model + return quantsim_model diff --git a/aimet_zoo_torch/mobilebert/model/utils/__init__.py b/aimet_zoo_torch/mobilebert/model/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilenetv2/__init__.py b/aimet_zoo_torch/mobilenetv2/__init__.py index ac67a86..e684808 100644 --- a/aimet_zoo_torch/mobilenetv2/__init__.py +++ b/aimet_zoo_torch/mobilenetv2/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import MobileNetV2 \ No newline at end of file +""" package for getting mobilenetv2 original model and quantized model""" +from .model.model_definition import MobileNetV2 diff --git a/aimet_zoo_torch/mobilenetv2/dataloader/__init__.py b/aimet_zoo_torch/mobilenetv2/dataloader/__init__.py index c086995..a454cde 100644 --- a/aimet_zoo_torch/mobilenetv2/dataloader/__init__.py +++ b/aimet_zoo_torch/mobilenetv2/dataloader/__init__.py @@ -1 +1,2 @@ -from .dataloaders_and_eval_func import get_dataloaders_and_eval_func \ No newline at end of file +""" datasets and eval function are defined and loaded""" +from .dataloaders_and_eval_func import get_dataloaders_and_eval_func diff --git a/aimet_zoo_torch/mobilenetv2/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/mobilenetv2/dataloader/dataloaders_and_eval_func.py index 37a74d3..899dbea 100644 --- a/aimet_zoo_torch/mobilenetv2/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/mobilenetv2/dataloader/dataloaders_and_eval_func.py @@ -7,49 +7,65 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +""" modules for getting dataloader and eval functions""" - -import torch -from torch.utils.data import DataLoader -from torchvision import transforms, datasets import random import numpy as np from tqdm import tqdm +import torch +from torch.utils.data import DataLoader +from torchvision import transforms, datasets def work_init(work_id): - init_seed = torch.initial_seed() % 2 ** 32 + """random seed generator""" + init_seed = torch.initial_seed() % 2**32 random.seed(init_seed + work_id) np.random.seed(init_seed + work_id) def make_dataloader(dataset_path, image_size, batch_size=16, num_workers=8): - data_loader_kwargs = {'worker_init_fn': work_init, 'num_workers': min(num_workers, batch_size//2)} - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) - val_transforms = transforms.Compose([ - transforms.Resize(image_size + 24), - transforms.CenterCrop(image_size), - transforms.ToTensor(), - normalize]) + """ + getting dataloder from dataset path + """ + data_loader_kwargs = { + "worker_init_fn": work_init, + "num_workers": min(num_workers, batch_size // 2), + } + normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + val_transforms = transforms.Compose( + [ + transforms.Resize(image_size + 24), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + normalize, + ] + ) data_folder = datasets.ImageFolder(dataset_path, val_transforms) - dataloader = DataLoader(data_folder, batch_size=batch_size, shuffle=True, pin_memory=True, **data_loader_kwargs) + dataloader = DataLoader( + data_folder, + batch_size=batch_size, + shuffle=True, + pin_memory=True, + **data_loader_kwargs + ) return dataloader def get_dataloaders_and_eval_func(imagenet_path, image_size=224): - train_loader = make_dataloader(dataset_path = imagenet_path, - image_size = image_size) - val_loader = make_dataloader(dataset_path = imagenet_path, - image_size = image_size) + """getting imagnet dataloader and evaluation function""" + train_loader = make_dataloader(dataset_path=imagenet_path, image_size=image_size) + val_loader = make_dataloader(dataset_path=imagenet_path, image_size=image_size) def eval_func(model, args): - num_samples = args[0] if args[0] > 0 else float('inf') + num_samples = args[0] if args[0] > 0 else float("inf") device = args[1] top1_acc = 0.0 total_num = 0 model.to(device) - for idx, (sample, label) in enumerate(tqdm(val_loader)): + for _, (sample, label) in enumerate(tqdm(val_loader)): total_num += sample.size()[0] sample = sample.to(device) label = label.to(device) @@ -59,15 +75,19 @@ def eval_func(model, args): top1_acc += correct if total_num >= num_samples: break - avg_acc = top1_acc * 100. / total_num + avg_acc = top1_acc * 100.0 / total_num return avg_acc return train_loader, val_loader, eval_func -def unlabeled_dataset(): +#pylint:disable = unused-variable +def unlabeled_dataset(val_loader): + """generating unlabelled dataset""" for X, Y in val_loader: yield X -def forward_pass(model, args): + +def forward_pass(model, val_loader): + """forward pass of dataloader""" for X in unlabeled_dataset(val_loader): - _ = model(X) \ No newline at end of file + _ = model(X) diff --git a/aimet_zoo_torch/mobilenetv2/evaluators/mobilenetv2_quanteval.py b/aimet_zoo_torch/mobilenetv2/evaluators/mobilenetv2_quanteval.py index d219359..2d48056 100644 --- a/aimet_zoo_torch/mobilenetv2/evaluators/mobilenetv2_quanteval.py +++ b/aimet_zoo_torch/mobilenetv2/evaluators/mobilenetv2_quanteval.py @@ -8,7 +8,7 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantsim code for MobileNetV2 ''' +""" AIMET Quantsim code for MobileNetV2 """ import argparse import torch @@ -18,18 +18,44 @@ def arguments(): - parser = argparse.ArgumentParser(description='Evaluation script for PyTorch ImageNet networks.') - parser.add_argument('--model-config', help='Select the model configuration', type=str, default="mobilenetv2_w8a8", choices=["mobilenetv2_w8a8"]) - parser.add_argument('--dataset-path', help='Imagenet eval image', default='./ILSVRC2012/', type=str) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) - parser.add_argument('--batch-size', help='Data batch size for a model', type=int, default=16) - parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) + """argument parser""" + parser = argparse.ArgumentParser( + description="Evaluation script for PyTorch ImageNet networks." + ) + parser.add_argument( + "--model-config", + help="Select the model configuration", + type=str, + default="mobilenetv2_w8a8", + choices=["mobilenetv2_w8a8"], + ) + parser.add_argument( + "--dataset-path", help="Imagenet eval image", default="./ILSVRC2012/", type=str + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--batch-size", help="Data batch size for a model", type=int, default=16 + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU", type=bool, default=True + ) args = parser.parse_args() return args def seed(seed_num): + """random seed generator""" torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(seed_num) @@ -38,14 +64,18 @@ def seed(seed_num): def main(): + """main evaluation function""" + # pylint:disable = too-many-locals, unused-variable seed(0) args = arguments() device = get_device(args) eval_samples = -1 encoding_samples = 2000 - train_loader, val_loader, eval_func = get_dataloaders_and_eval_func(imagenet_path = args.dataset_path) - + train_loader, val_loader, eval_func = get_dataloaders_and_eval_func( + imagenet_path=args.dataset_path + ) + X, Y = next(iter(val_loader)) input_shape = X.shape @@ -57,21 +87,21 @@ def main(): num_classes = uniques.shape[0] dummy_input = torch.randn(input_shape) - print('### Simulating original model performance ###') - model_fp32 = MobileNetV2(model_config = args.model_config) + print("### Simulating original model performance ###") + model_fp32 = MobileNetV2(model_config=args.model_config) model_fp32.from_pretrained(quantized=False) model_fp32.model.eval() - #sim = QuantizationSimModel(model_fp32, dummy_input=dummy_input, **kwargs) + # sim = QuantizationSimModel(model_fp32, dummy_input=dummy_input, **kwargs) sim = model_fp32.get_quantsim(quantized=False) sim.compute_encodings(eval_func, [encoding_samples, device]) orig_acc_fp32 = eval_func(model_fp32.model.to(device), [eval_samples, device]) orig_acc_int8 = eval_func(sim.model.to(device), [eval_samples, device]) - print('### Simulating quantized model performance ###') - model_int8 = MobileNetV2(model_config = args.model_config) + print("### Simulating quantized model performance ###") + model_int8 = MobileNetV2(model_config=args.model_config) model_int8.from_pretrained(quantized=True) model_int8.model.eval() - #sim = QuantizationSimModel(model_int8, dummy_input=dummy_input, **kwargs) + # sim = QuantizationSimModel(model_int8, dummy_input=dummy_input, **kwargs) sim = model_fp32.get_quantsim(quantized=True) sim.compute_encodings(eval_func, [encoding_samples, device]) optim_acc_fp32 = eval_func(model_int8.model.to(device), [eval_samples, device]) @@ -80,10 +110,14 @@ def main(): print() print("Evaluation Summary:") print(f"Original Model | Accuracy on 32-bit device: {orig_acc_fp32:.4f}") - print(f"Original Model | Accuracy on {args.default_param_bw}-bit device: {orig_acc_int8:.4f}") + print( + f"Original Model | Accuracy on {args.default_param_bw}-bit device: {orig_acc_int8:.4f}" + ) print(f"Optimized Model | Accuracy on 32-bit device: {optim_acc_fp32:.4f}") - print(f"Optimized Model | Accuracy on {args.default_param_bw}-bit device: {optim_acc_int8:.4f}") + print( + f"Optimized Model | Accuracy on {args.default_param_bw}-bit device: {optim_acc_int8:.4f}" + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/aimet_zoo_torch/mobilenetv2/model/MobileNetV2.py b/aimet_zoo_torch/mobilenetv2/model/MobileNetV2.py index 3e342d9..2b034c6 100644 --- a/aimet_zoo_torch/mobilenetv2/model/MobileNetV2.py +++ b/aimet_zoo_torch/mobilenetv2/model/MobileNetV2.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ------------------------------------------------------------------------------- diff --git a/aimet_zoo_torch/mobilenetv2/model/model_cards/__init__.py b/aimet_zoo_torch/mobilenetv2/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilenetv2/model/model_definition.py b/aimet_zoo_torch/mobilenetv2/model/model_definition.py index 3c9f87c..8bd458a 100644 --- a/aimet_zoo_torch/mobilenetv2/model/model_definition.py +++ b/aimet_zoo_torch/mobilenetv2/model/model_definition.py @@ -7,82 +7,109 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +"""Class for downloading and setting up of optimized and original mobilenetv2 model for AIMET model zoo""" + +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet -from .MobileNetV2 import MobileNetV2 as Mobile_Net_V2 -from aimet_zoo_torch.common.downloader import Downloader -import torch import os import json +import torch from aimet_torch.cross_layer_equalization import equalize_model from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim +from aimet_zoo_torch.common.downloader import Downloader +from .MobileNetV2 import MobileNetV2 as Mobile_Net_V2 class MobileNetV2(Downloader): """MobileNetV2 parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" - def __init__(self, model_config= None, num_classes=1000, input_size=224, width_mult=1.): - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + + def __init__( + self, model_config=None, num_classes=1000, input_size=224, width_mult=1.0 + ): + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" with open(config_filepath) as f_in: self.cfg = json.load(f_in) if model_config: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.model = Mobile_Net_V2(n_class = self.cfg['model_args']['num_classes'], - input_size = self.cfg['model_args']['input_size'], - width_mult = self.cfg['model_args']['width_mult']) - self.input_shape = tuple(x if x != None else 1 for x in self.cfg['input_shape']) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.model = Mobile_Net_V2( + n_class=self.cfg["model_args"]["num_classes"], + input_size=self.cfg["model_args"]["input_size"], + width_mult=self.cfg["model_args"]["width_mult"], + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) else: - self.model = Mobile_Net_V2(n_class = num_classes, - input_size = input_size, - width_mult = width_mult) + self.model = Mobile_Net_V2( + n_class=num_classes, input_size=input_size, width_mult=width_mult + ) def from_pretrained(self, quantized=False): """load pretrained weights""" self._download_aimet_config() self._download_post_opt_weights() if quantized: - equalize_model(self.model, (1, 3, 224, 224)) + equalize_model(self.model, (1, 3, 224, 224)) quantized_state_dict = torch.load(self.path_post_opt_weights) # need to rename some state dict keys due to differences in aimet naming between when the state dict was generated and now - quantized_state_dict['state_dict']['classifier.weight'] = quantized_state_dict['state_dict']['classifier.1.weight'] - del quantized_state_dict['state_dict']['classifier.1.weight'] - quantized_state_dict['state_dict']['classifier.bias'] = quantized_state_dict['state_dict']['classifier.1.bias'] - del quantized_state_dict['state_dict']['classifier.1.bias'] - self.model.load_state_dict(quantized_state_dict['state_dict']) + quantized_state_dict["state_dict"][ + "classifier.weight" + ] = quantized_state_dict["state_dict"]["classifier.1.weight"] + del quantized_state_dict["state_dict"]["classifier.1.weight"] + quantized_state_dict["state_dict"][ + "classifier.bias" + ] = quantized_state_dict["state_dict"]["classifier.1.bias"] + del quantized_state_dict["state_dict"]["classifier.1.bias"] + self.model.load_state_dict(quantized_state_dict["state_dict"]) del quantized_state_dict else: + #pylint:disable=import-outside-toplevel try: from torch.hub import load_state_dict_from_url except ImportError: from torch.utils.model_zoo import load_url as load_state_dict_from_url state_dict = load_state_dict_from_url( - 'https://www.dropbox.com/s/47tyzpofuuyyv1b/mobilenetv2_1.0-f2a8633.pth.tar?dl=1', progress=True) + "https://www.dropbox.com/s/47tyzpofuuyyv1b/mobilenetv2_1.0-f2a8633.pth.tar?dl=1", + progress=True, + ) self.model.load_state_dict(state_dict) - def get_quantsim(self, quantized=False): + """get quantsim object with pre-loaded encodings or pretrained model""" if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model.cuda(), **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - return sim \ No newline at end of file + return sim diff --git a/aimet_zoo_torch/mobilevit/__init__.py b/aimet_zoo_torch/mobilevit/__init__.py index 8f6d546..a0d75e9 100644 --- a/aimet_zoo_torch/mobilevit/__init__.py +++ b/aimet_zoo_torch/mobilevit/__init__.py @@ -1 +1,2 @@ +""" package for getting mobilevit original model and quantized model""" from .model.model_definition import mobilevit diff --git a/aimet_zoo_torch/mobilevit/dataloader/__init__.py b/aimet_zoo_torch/mobilevit/dataloader/__init__.py index 03d7e3f..9d2befd 100644 --- a/aimet_zoo_torch/mobilevit/dataloader/__init__.py +++ b/aimet_zoo_torch/mobilevit/dataloader/__init__.py @@ -1,2 +1,3 @@ +"""modules for getting dataloaders and dataset""" from .dataloaders import get_dataloaders from .dataloaders import get_dataset diff --git a/aimet_zoo_torch/mobilevit/dataloader/utils/__init__.py b/aimet_zoo_torch/mobilevit/dataloader/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilevit/evaluators/__init__.py b/aimet_zoo_torch/mobilevit/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py b/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py index c7c43a4..132293a 100644 --- a/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py +++ b/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py @@ -18,7 +18,7 @@ from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.utils import set_seed -from aimet_zoo_torch.mobilevit.dataloader import get_dataloaders, get_dataset +from aimet_zoo_torch.mobilevit.dataloader import get_dataloaders from aimet_zoo_torch.mobilevit import mobilevit logger = get_logger(__name__) @@ -109,12 +109,10 @@ def main(): set_seed(config.seed) accelerator.wait_for_everyone() - # get dataset for gathering information to load model - dataset = get_dataset(config) # loading finetuned original model model = mobilevit(model_config=args.model_config, quantized=False) - model_orig = model.get_model_from_pretrained(dataset) + model_orig = model.get_model_from_pretrained() feature_extractor = model.get_feature_extractor_from_pretrained() @@ -146,7 +144,7 @@ def main(): # loading optimized model model = mobilevit(model_config=args.model_config, quantized=True) - model_w8a8 = model.get_model_from_pretrained(dataset) + model_w8a8 = model.get_model_from_pretrained() # Prepare everything with our `accelerator`. model_w8a8, train_dataloader, eval_dataloader = accelerator.prepare( diff --git a/aimet_zoo_torch/mobilevit/model/__init__.py b/aimet_zoo_torch/mobilevit/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilevit/model/huggingface/__init__.py b/aimet_zoo_torch/mobilevit/model/huggingface/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilevit/model/huggingface/baseline_models/__init__.py b/aimet_zoo_torch/mobilevit/model/huggingface/baseline_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilevit/model/huggingface/baseline_models/mobilevit/__init__.py b/aimet_zoo_torch/mobilevit/model/huggingface/baseline_models/mobilevit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilevit/model/model_cards/__init__.py b/aimet_zoo_torch/mobilevit/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/mobilevit/model/model_cards/mobilevit_w8a8.json b/aimet_zoo_torch/mobilevit/model/model_cards/mobilevit_w8a8.json index ec3a162..66f00a6 100644 --- a/aimet_zoo_torch/mobilevit/model/model_cards/mobilevit_w8a8.json +++ b/aimet_zoo_torch/mobilevit/model/model_cards/mobilevit_w8a8.json @@ -28,6 +28,6 @@ "tar_url_pre_opt_weights":null, "tar_url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilevit/imgnet_mobilevit_5e4_clamp_rl.tar.gz", "url_aimet_encodings": null, - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilevit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.24/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/mobilevit/model/model_definition.py b/aimet_zoo_torch/mobilevit/model/model_definition.py index eb700f2..f0d6152 100644 --- a/aimet_zoo_torch/mobilevit/model/model_definition.py +++ b/aimet_zoo_torch/mobilevit/model/model_definition.py @@ -12,22 +12,22 @@ import json import os import csv -import datasets -import torch from collections import defaultdict +import torch from transformers import AutoConfig as Config from transformers import AutoFeatureExtractor as FeatureExtractor -from aimet_common.defs import QuantScheme -from aimet_torch.quantsim import QuantizationSimModel +from aimet_common.defs import QuantScheme #pylint:disable = import-error +from aimet_torch.quantsim import QuantizationSimModel #pylint:disable = import-error from aimet_zoo_torch.common.downloader import Downloader from aimet_zoo_torch.mobilevit.model.huggingface.baseline_models.mobilevit.modeling_mobilevit import ( MobileViTForImageClassification as MobileVitModel, ) - +import datasets # pylint: disable = import-error class mobilevit(Downloader): - """class for mobilevit configuration """ + """class for mobilevit configuration""" + def __init__(self, model_config=None, quantized=False): """ dataloader @@ -50,7 +50,7 @@ def __init__(self, model_config=None, quantized=False): self.model = None self.quantized = quantized - def get_model_from_pretrained(self, dataset): + def get_model_from_pretrained(self): """get original or optmized model Parameters: dataset: @@ -156,7 +156,8 @@ def get_quantsim(self, train_dataloader, eval_dataloader, eval_function): self._load_encoding_data(quant_sim, model_name_or_path) return quant_sim - def _get_dummy_input(self, dataloader): + @staticmethod + def _get_dummy_input(dataloader): """get dummy input of dataloader for vit model""" for batch in dataloader: output = [] @@ -167,7 +168,8 @@ def _get_dummy_input(self, dataloader): output.append(batch[k].to("cuda")) return tuple(output) - def _load_encoding_data(self, quant_sim, save_dir): + @staticmethod + def _load_encoding_data(quant_sim, save_dir): """loading saved encodings.csv file""" fname = os.path.join(save_dir, "encodings.csv") if not os.path.exists(fname): diff --git a/aimet_zoo_torch/poseestimation/evaluators/pose_estimation_quanteval.py b/aimet_zoo_torch/poseestimation/evaluators/pose_estimation_quanteval.py index f0d1bb1..57b60aa 100644 --- a/aimet_zoo_torch/poseestimation/evaluators/pose_estimation_quanteval.py +++ b/aimet_zoo_torch/poseestimation/evaluators/pose_estimation_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3.6 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,C0111 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,C0111 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -31,7 +31,7 @@ from scipy.ndimage.filters import gaussian_filter import torch -import torch.nn as nn +from torch import nn # aimet import from aimet_torch import quantsim @@ -190,8 +190,9 @@ def define_base_layers(block, layer_size): one_ = block[i] for k, v in zip(one_.keys(), one_.values()): if "pool" in k: - layers += [nn.MaxPool2d(kernel_size=v[0], - stride=v[1], padding=v[2])] + layers += [ + nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2]) + ] elif "sequential" in k: conv2d_1 = nn.Conv2d( in_channels=v[0][0], @@ -279,6 +280,7 @@ class PoseModel(nn.Module): """ def __init__(self, model_dict, upsample=False): + #pylint:disable = super-with-arguments super(PoseModel, self).__init__() self.upsample = upsample self.basemodel = model_dict["block0"] @@ -379,7 +381,8 @@ def pad_image(img, stride, padding): return img_padded, pad -#pylint: disable=I1101 + +# pylint: disable=I1101 def encode_input(image, scale, stride, padding): image_scaled = cv2.resize( image, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC @@ -394,8 +397,7 @@ def decode_output(data, stride, padding, input_shape, image_shape): output = cv2.resize( output, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC ) - output = output[: input_shape[0] - padding[2], - : input_shape[1] - padding[3], :] + output = output[: input_shape[0] - padding[2], : input_shape[1] - padding[3], :] output = cv2.resize( output, (image_shape[1], image_shape[0]), interpolation=cv2.INTER_CUBIC ) @@ -452,13 +454,10 @@ def run_model(model, image, fast=False): ) else: image_encoded, pad = encode_input(image, scale, stride, padValue) - image_encoded_ = preprocess( - image_encoded, [ - "addchannel", "normalize", "bgr"]) + image_encoded_ = preprocess(image_encoded, ["addchannel", "normalize", "bgr"]) image_encoded_ = np.transpose(image_encoded_, (0, 3, 1, 2)) with torch.no_grad(): - input_image = torch.FloatTensor( - torch.from_numpy(image_encoded_).float()) + input_image = torch.FloatTensor(torch.from_numpy(image_encoded_).float()) if next(model.parameters()).is_cuda: input_image = input_image.to(device="cuda") output = model(input_image) @@ -466,18 +465,11 @@ def run_model(model, image, fast=False): heatmap = output[3].cpu().data.numpy().transpose((0, 2, 3, 1)) if fast: paf = cv2.resize(paf[0], (image.shape[1], image.shape[0])) - heatmap = cv2.resize( - heatmap[0], dsize=( - image.shape[1], image.shape[0])) + heatmap = cv2.resize(heatmap[0], dsize=(image.shape[1], image.shape[0])) else: # paf = paf.transpose((0, 3, 1, 2)) # heatmap = heatmap.transpose((0, 3, 1, 2)) - paf = decode_output( - paf, - stride, - pad, - image_encoded.shape, - image.shape) + paf = decode_output(paf, stride, pad, image_encoded.shape, image.shape) heatmap = decode_output( heatmap, stride, pad, image_encoded.shape, image.shape ) @@ -501,12 +493,7 @@ def get_keypoints(heatmap): return keypoints_all -def get_limb_consistency( - paf, - start_keypoint, - end_keypoint, - image_h, - div_num=10): +def get_limb_consistency(paf, start_keypoint, end_keypoint, image_h, div_num=10): vec_key = np.subtract(end_keypoint[:2], start_keypoint[:2]) vec_key_norm = math.sqrt(vec_key[0] * vec_key[0] + vec_key[1] * vec_key[1]) if vec_key_norm == 0: @@ -520,17 +507,14 @@ def get_limb_consistency( ) ) - vec_paf_x = np.array([paf[vec_paf[k][1], vec_paf[k][0], 0] - for k in range(div_num)]) - vec_paf_y = np.array([paf[vec_paf[k][1], vec_paf[k][0], 1] - for k in range(div_num)]) + vec_paf_x = np.array([paf[vec_paf[k][1], vec_paf[k][0], 0] for k in range(div_num)]) + vec_paf_y = np.array([paf[vec_paf[k][1], vec_paf[k][0], 1] for k in range(div_num)]) # To see how well the direction of the prediction over the line connecting the limbs aligns # with the vec_key we compute the integral of the dot product of the "affinity vector at point # 'u' on the line" and the "vec_key". # In discrete form, this integral is done as below: - vec_sims = np.multiply( - vec_paf_x, vec_key[0]) + np.multiply(vec_paf_y, vec_key[1]) + vec_sims = np.multiply(vec_paf_x, vec_key[0]) + np.multiply(vec_paf_y, vec_key[1]) # this is just a heuristic approach to punish very long predicted limbs vec_sims_prior = vec_sims.mean() + min(0.5 * image_h / vec_key_norm - 1, 0) @@ -542,6 +526,7 @@ def connect_keypoints(image_shape, keypoints, paf, limbs, limbsInds): thre2 = 0.05 connections = [] small_limb_list = [1, 15, 16, 17, 18] + #pylint:disable = consider-using-enumerate for k in range(len(limbsInds)): paf_limb = paf[:, :, limbsInds[k]] limb_strs = keypoints[limbs[k][0]] @@ -555,10 +540,12 @@ def connect_keypoints(image_shape, keypoints, paf, limbs, limbsInds): # measure a score using the get_limb_consistency function if limbs[k][0] in small_limb_list or limbs[k][1] in small_limb_list: sims, sims_p = get_limb_consistency( - paf_limb, limb_str, limb_end, image_shape[0], div_num=10) + paf_limb, limb_str, limb_end, image_shape[0], div_num=10 + ) else: sims, sims_p = get_limb_consistency( - paf_limb, limb_str, limb_end, image_shape[0], div_num=10) + paf_limb, limb_str, limb_end, image_shape[0], div_num=10 + ) if ( len(np.where(sims > thre2)[0]) > int(0.80 * len(sims)) and sims_p > 0 @@ -589,15 +576,16 @@ def create_skeletons(keypoints, connections, limbs): # the second last number in each row is the score of the overall # configuration skeletons = -1 * np.ones((0, 20)) - keypoints_flatten = np.array( - [item for sublist in keypoints for item in sublist]) + keypoints_flatten = np.array([item for sublist in keypoints for item in sublist]) + #pylint:disable = consider-using-enumerate for k in range(len(limbs)): if connections[k]: detected_str = connections[k][:, 0] detected_end = connections[k][:, 1] limb_str, limb_end = np.array(limbs[k]) + #pylint:disable = consider-using-enumerate for i in range(len(connections[k])): found = 0 subset_idx = [-1, -1] @@ -625,7 +613,7 @@ def create_skeletons(keypoints, connections, limbs): (skeletons[j1] >= 0).astype(int) + (skeletons[j2] >= 0).astype(int) )[:-2] - #pylint: disable=C1801 + # pylint: disable=C1801 if len(np.nonzero(membership == 2)[0]) == 0: # merge skeletons[j1][:-2] += skeletons[j2][:-2] + 1 skeletons[j1][-2:] += skeletons[j2][-2:] @@ -645,12 +633,15 @@ def create_skeletons(keypoints, connections, limbs): row[limb_str] = detected_str[i] row[limb_end] = detected_end[i] row[-1] = 2 - row[-2] = (sum(keypoints_flatten[connections[k] - [i, :2].astype(int), 2]) + connections[k][i][2]) + row[-2] = ( + sum(keypoints_flatten[connections[k][i, :2].astype(int), 2]) + + connections[k][i][2] + ) skeletons = np.vstack([skeletons, row]) # delete some rows of subset which has few parts occur deleteIdx = [] + #pylint:disable = consider-using-enumerate for i in range(len(skeletons)): if skeletons[i][-1] < 4 or skeletons[i][-2] / skeletons[i][-1] < 0.4: deleteIdx.append(i) @@ -710,22 +701,20 @@ def estimate_pose(image_shape, heatmap, paf): keypoints = get_keypoints(heatmap) # Computing which pairs of joints should be connected based on the paf. - connections = connect_keypoints( - image_shape, keypoints, paf, limbs, limbsInd) + connections = connect_keypoints(image_shape, keypoints, paf, limbs, limbsInd) skeletons = create_skeletons(keypoints, connections, limbs) - return skeletons, np.array( - [item for sublist in keypoints for item in sublist]) + return skeletons, np.array([item for sublist in keypoints for item in sublist]) def parse_results(skeletons, points): - coco_indices = [0, -1, 6, 8, 10, 5, 7, 9, - 12, 14, 16, 11, 13, 15, 2, 1, 4, 3] + coco_indices = [0, -1, 6, 8, 10, 5, 7, 9, 12, 14, 16, 11, 13, 15, 2, 1, 4, 3] skeletons_out, scores = [], [] for score, keypoints in zip(skeletons["scores"], skeletons["keypoints"]): skeleton = [] + #pylint:disable = consider-using-enumerate for p in range(len(keypoints)): if p == 1: continue @@ -776,12 +765,11 @@ def evaluate_json(self, obj): cocoEval.summarize() return cocoEval.stats[0::5] - #pylint: disable=R0201 + # pylint: disable=R0201 def get_results_json(self, results, imgs): results_obj = [] for img, result in list(zip(imgs, results)): - for score, skeleton in list( - zip(result["scores"], result["skeletons"])): + for score, skeleton in list(zip(result["scores"], result["skeletons"])): obj = { "image_id": img["id"], "category_id": 1, @@ -815,8 +803,7 @@ def cocoGT(self): cocoGT = COCO(annFile) if not cocoGT: - raise AttributeError( - "COCO ground truth demo failed to initialize!") + raise AttributeError("COCO ground truth demo failed to initialize!") return cocoGT @@ -890,13 +877,12 @@ def download_weights(): def pose_estimation_quanteval(args): - download_weights() # load the model checkpoint from meta model_builder = ModelBuilder() model_builder.create_model() model = model_builder.model - #pylint: disable = no-value-for-parameter + # pylint: disable = no-value-for-parameter state_dict = torch.load("pe_weights.pth") state = model.state_dict() state.update(state_dict) diff --git a/aimet_zoo_torch/quicksrnet/__init__.py b/aimet_zoo_torch/quicksrnet/__init__.py index 3b2e423..91a2b0f 100644 --- a/aimet_zoo_torch/quicksrnet/__init__.py +++ b/aimet_zoo_torch/quicksrnet/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import QuickSRNet \ No newline at end of file +""" package for getting quicksrnet original model and quantized model""" +from .model.model_definition import QuickSRNet diff --git a/aimet_zoo_torch/quicksrnet/dataloader/imresize.py b/aimet_zoo_torch/quicksrnet/dataloader/imresize.py index 4100b59..4a7afb4 100644 --- a/aimet_zoo_torch/quicksrnet/dataloader/imresize.py +++ b/aimet_zoo_torch/quicksrnet/dataloader/imresize.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/quicksrnet/dataloader/utils.py b/aimet_zoo_torch/quicksrnet/dataloader/utils.py index e4c1f6f..e2fbff7 100644 --- a/aimet_zoo_torch/quicksrnet/dataloader/utils.py +++ b/aimet_zoo_torch/quicksrnet/dataloader/utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= @@ -35,7 +36,7 @@ def load_dataset(test_images_dir, scaling_factor=2): IMAGES_HR = [] # Load the test images - for img_path in glob.glob(os.path.join(test_images_dir, '*')): + for img_path in glob.glob(os.path.join(test_images_dir, "*")): img = cv2.imread(img_path) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) @@ -61,8 +62,12 @@ def create_hr_lr_pair(img, scaling_factor=2): height, width = img.shape[0:2] # Take the largest possible center-crop of it such that its dimensions are perfectly divisible by the scaling factor - x_remainder = width % (2 * scaling_factor if scaling_factor == 1.5 else scaling_factor) - y_remainder = height % (2 * scaling_factor if scaling_factor == 1.5 else scaling_factor) + x_remainder = width % ( + 2 * scaling_factor if scaling_factor == 1.5 else scaling_factor + ) + y_remainder = height % ( + 2 * scaling_factor if scaling_factor == 1.5 else scaling_factor + ) left = int(x_remainder // 2) top = int(y_remainder // 2) right = int(left + (width - x_remainder)) @@ -71,15 +76,20 @@ def create_hr_lr_pair(img, scaling_factor=2): hr_height, hr_width = hr_img.shape[0:2] - hr_img = np.array(hr_img, dtype='float64') - lr_img = imresize(hr_img, 1. / scaling_factor) # equivalent to matlab's imresize - lr_img = np.uint8(np.clip(lr_img, 0., 255.)) # this is to simulate matlab's imwrite operation + hr_img = np.array(hr_img, dtype="float64") + lr_img = imresize(hr_img, 1.0 / scaling_factor) # equivalent to matlab's imresize + lr_img = np.uint8( + np.clip(lr_img, 0.0, 255.0) + ) # this is to simulate matlab's imwrite operation hr_img = np.uint8(hr_img) lr_height, lr_width = lr_img.shape[0:2] # Sanity check - assert hr_width == lr_width * scaling_factor and hr_height == lr_height * scaling_factor + assert ( + hr_width == lr_width * scaling_factor + and hr_height == lr_height * scaling_factor + ) lr_img = torch.from_numpy(lr_img.transpose((2, 0, 1))).contiguous() lr_img = lr_img.to(dtype=torch.float32).div(255) @@ -100,8 +110,8 @@ def post_process(img): The image after reverting the changes done in the pre-processing steps """ img = img.permute((1, 2, 0)) # CHW > HWC - img = img.cpu().numpy() # torch > numpy - img = np.clip(255. * img, 0., 255.) # float [0, 1] to [0, 255] + img = img.cpu().numpy() # torch > numpy + img = np.clip(255.0 * img, 0.0, 255.0) # float [0, 1] to [0, 255] img = np.uint8(img) return img @@ -113,11 +123,8 @@ def imshow(image): :param image: Image to plot """ - plt.imshow(image, interpolation='nearest') - plt.tick_params(left=False, - bottom=False, - labelleft=False, - labelbottom=False) + plt.imshow(image, interpolation="nearest") + plt.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False) def pass_calibration_data(model, calibration_data): @@ -132,15 +139,17 @@ def pass_calibration_data(model, calibration_data): images, use_cuda = calibration_data if use_cuda: - device = torch.device('cuda') + device = torch.device("cuda") else: - device = torch.device('cpu') + device = torch.device("cpu") model.eval() with torch.no_grad(): for itx, img in enumerate(images): - print(f'\rCalibrate activation encodings: {itx + 1} / {len(images)}', end='') + print( + f"\rCalibrate activation encodings: {itx + 1} / {len(images)}", end="" + ) input_img = img.unsqueeze(0).to(device) _ = model(input_img) - print('\n') \ No newline at end of file + print("\n") diff --git a/aimet_zoo_torch/quicksrnet/evaluators/quicksrnet_quanteval.py b/aimet_zoo_torch/quicksrnet/evaluators/quicksrnet_quanteval.py index 741aace..5e66da7 100644 --- a/aimet_zoo_torch/quicksrnet/evaluators/quicksrnet_quanteval.py +++ b/aimet_zoo_torch/quicksrnet/evaluators/quicksrnet_quanteval.py @@ -8,47 +8,72 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET evaluation code for QuickSRNet ''' +""" AIMET evaluation code for QuickSRNet """ import argparse from aimet_zoo_torch.quicksrnet import QuickSRNet from aimet_zoo_torch.quicksrnet.model.helpers import evaluate_average_psnr -from aimet_zoo_torch.quicksrnet.dataloader.utils import load_dataset, pass_calibration_data +from aimet_zoo_torch.quicksrnet.dataloader.utils import ( + load_dataset, + pass_calibration_data, +) from aimet_zoo_torch.quicksrnet.model.inference import run_model # add arguments -def arguments(): +def arguments(raw_args): """parses command line arguments""" - parser = argparse.ArgumentParser(description='Arguments for evaluating model') - parser.add_argument('--dataset-path', help='path to image evaluation dataset', type=str) - parser.add_argument('--model-config', help='model configuration to be tested', type=str) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) - parser.add_argument('--batch-size',help='batch_size for loading data',type=int,default=16) - parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) - args = parser.parse_args() + parser = argparse.ArgumentParser(description="Arguments for evaluating model") + parser.add_argument( + "--dataset-path", help="path to image evaluation dataset", type=str + ) + parser.add_argument( + "--model-config", help="model configuration to be tested", type=str + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--batch-size", help="batch_size for loading data", type=int, default=16 + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU", type=bool, default=True + ) + args = parser.parse_args(raw_args) return args -def main(): +def main(raw_args=None): """exeutes evaluation""" - args = arguments() + args = arguments(raw_args) - model_fp32 = QuickSRNet(model_config = args.model_config) + model_fp32 = QuickSRNet(model_config=args.model_config) model_fp32.from_pretrained(quantized=False) sim_fp32 = model_fp32.get_quantsim(quantized=False) - model_int8 = QuickSRNet(model_config = args.model_config) + model_int8 = QuickSRNet(model_config=args.model_config) model_int8.from_pretrained(quantized=True) sim_int8 = model_int8.get_quantsim(quantized=True) IMAGES_LR, IMAGES_HR = load_dataset(args.dataset_path, model_fp32.scaling_factor) - sim_fp32.compute_encodings(forward_pass_callback=pass_calibration_data, - forward_pass_callback_args=(IMAGES_LR, args.use_cuda)) - sim_int8.compute_encodings(forward_pass_callback=pass_calibration_data, - forward_pass_callback_args=(IMAGES_LR, args.use_cuda)) + sim_fp32.compute_encodings( + forward_pass_callback=pass_calibration_data, + forward_pass_callback_args=(IMAGES_LR, args.use_cuda), + ) + sim_int8.compute_encodings( + forward_pass_callback=pass_calibration_data, + forward_pass_callback_args=(IMAGES_LR, args.use_cuda), + ) # Run model inference on test images and get super-resolved images IMAGES_SR_original_fp32 = run_model(model_fp32, IMAGES_LR, args.use_cuda) @@ -58,13 +83,14 @@ def main(): # Get the average PSNR for all test-images avg_psnr = evaluate_average_psnr(IMAGES_SR_original_fp32, IMAGES_HR) - print(f'Original Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Original Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_original_int8, IMAGES_HR) - print(f'Original Model | Quantized Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Original Model | Quantized Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_optimized_fp32, IMAGES_HR) - print(f'Optimized Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Optimized Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_optimized_int8, IMAGES_HR) - print(f'Optimized Model | Quantized Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Optimized Model | Quantized Environment | Avg. PSNR: {avg_psnr:.3f}") -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/aimet_zoo_torch/quicksrnet/model/blocks.py b/aimet_zoo_torch/quicksrnet/model/blocks.py index e0f2d9c..c11d673 100644 --- a/aimet_zoo_torch/quicksrnet/model/blocks.py +++ b/aimet_zoo_torch/quicksrnet/model/blocks.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/quicksrnet/model/downloader.py b/aimet_zoo_torch/quicksrnet/model/downloader.py index 5a64d73..aaea33d 100644 --- a/aimet_zoo_torch/quicksrnet/model/downloader.py +++ b/aimet_zoo_torch/quicksrnet/model/downloader.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/quicksrnet/model/helpers.py b/aimet_zoo_torch/quicksrnet/model/helpers.py index 5a65e6d..d5855aa 100644 --- a/aimet_zoo_torch/quicksrnet/model/helpers.py +++ b/aimet_zoo_torch/quicksrnet/model/helpers.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/quicksrnet/model/imresize.py b/aimet_zoo_torch/quicksrnet/model/imresize.py index 4100b59..4a7afb4 100644 --- a/aimet_zoo_torch/quicksrnet/model/imresize.py +++ b/aimet_zoo_torch/quicksrnet/model/imresize.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/quicksrnet/model/inference.py b/aimet_zoo_torch/quicksrnet/model/inference.py index f68bea8..cca43d0 100644 --- a/aimet_zoo_torch/quicksrnet/model/inference.py +++ b/aimet_zoo_torch/quicksrnet/model/inference.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/quicksrnet/model/model_cards/__init__.py b/aimet_zoo_torch/quicksrnet/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/quicksrnet/model/model_definition.py b/aimet_zoo_torch/quicksrnet/model/model_definition.py index ab6c7d6..172a493 100644 --- a/aimet_zoo_torch/quicksrnet/model/model_definition.py +++ b/aimet_zoo_torch/quicksrnet/model/model_definition.py @@ -8,9 +8,13 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -import torch +"""Class for downloading and setting up of optmized and original QuickSRNet model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet + import json import os +import torch from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim from aimet_zoo_torch.common.downloader import Downloader from aimet_zoo_torch.quicksrnet.model.models import QuickSRNetBase @@ -18,54 +22,80 @@ class QuickSRNet(QuickSRNetBase, Downloader): """QuickSRNet parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" - def __init__(self, model_config = None, num_channels=16, num_intermediate_layers=3, scaling_factor=2, use_ito_connection=False, **kwargs): + + def __init__( + self, + model_config=None, + num_channels=16, + num_intermediate_layers=3, + scaling_factor=2, + use_ito_connection=False, + **kwargs + ): """ :param model_config: named model config from which to obtain model artifacts and arguments. - If provided, overwrites the other arguments passed to this object + If provided, overwrites the other arguments passed to this object :param scaling_factor: scaling factor for LR-to-HR upscaling (2x, 3x, 4x... or 1.5x) :param num_channels: number of feature channels for convolutional layers :param num_intermediate_layers: number of intermediate conv layers :param use_ito_connection: whether to use an input-to-output residual connection or not - (using one facilitates quantization) + (using one facilitates quantization) """ - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) - QuickSRNetBase.__init__(self, scaling_factor = self.cfg['model_args']['scaling_factor'] if self.cfg else scaling_factor, - num_channels = self.cfg['model_args']['num_channels'] if self.cfg else num_channels, - num_intermediate_layers = self.cfg['model_args']['num_intermediate_layers'] if self.cfg else num_intermediate_layers, - use_ito_connection = self.cfg['model_args']['use_ito_connection'] if self.cfg else use_ito_connection, - **kwargs) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + QuickSRNetBase.__init__( + self, + scaling_factor=self.cfg["model_args"]["scaling_factor"] + if self.cfg + else scaling_factor, + num_channels=self.cfg["model_args"]["num_channels"] + if self.cfg + else num_channels, + num_intermediate_layers=self.cfg["model_args"]["num_intermediate_layers"] + if self.cfg + else num_intermediate_layers, + use_ito_connection=self.cfg["model_args"]["use_ito_connection"] + if self.cfg + else use_ito_connection, + **kwargs + ) def from_pretrained(self, quantized=False): """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() self._download_aimet_encodings() self._download_adaround_encodings() if quantized: - state_dict = torch.load(self.path_post_opt_weights)['state_dict'] + state_dict = torch.load(self.path_post_opt_weights)["state_dict"] self.load_state_dict(state_dict) self.cuda() else: - state_dict = torch.load(self.path_pre_opt_weights)['state_dict'] + state_dict = torch.load(self.path_pre_opt_weights)["state_dict"] self.load_state_dict(state_dict) self.cuda() self.eval() @@ -73,25 +103,34 @@ def from_pretrained(self, quantized=False): def get_quantsim(self, quantized=False): """get quantsim object with pre-loaded encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self, **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') + print("set_and_freeze_param_encodings finished!") sim.model.eval() - return sim \ No newline at end of file + return sim diff --git a/aimet_zoo_torch/quicksrnet/model/models.py b/aimet_zoo_torch/quicksrnet/model/models.py index b01e50c..2e461f3 100644 --- a/aimet_zoo_torch/quicksrnet/model/models.py +++ b/aimet_zoo_torch/quicksrnet/model/models.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/rangenet/evaluators/__init__.py b/aimet_zoo_torch/rangenet/evaluators/__init__.py index a6cde28..090ecc8 100644 --- a/aimet_zoo_torch/rangenet/evaluators/__init__.py +++ b/aimet_zoo_torch/rangenet/evaluators/__init__.py @@ -1,4 +1,5 @@ """ init file """ import sys + TRAIN_PATH = "../models/train" sys.path.insert(0, TRAIN_PATH) diff --git a/aimet_zoo_torch/rangenet/evaluators/rangenet_quanteval.py b/aimet_zoo_torch/rangenet/evaluators/rangenet_quanteval.py index f1a625b..20732fb 100644 --- a/aimet_zoo_torch/rangenet/evaluators/rangenet_quanteval.py +++ b/aimet_zoo_torch/rangenet/evaluators/rangenet_quanteval.py @@ -34,7 +34,7 @@ def seed(seed_number): torch.cuda.manual_seed_all(seed_number) -def arguments(): +def arguments(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluation script for PyTorch RangeNet++ models." @@ -63,17 +63,19 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(args): +def main(raw_args=None): """execute evaluation""" + args = arguments(raw_args) seed(1234) models_dir = os.path.join( str(pathlib.Path(os.path.abspath(__file__)).parent.parent), "models" ) + #pylint:disable = consider-using-with DATA = yaml.safe_load(open(os.path.join(models_dir, "semantic-kitti.yaml"), "r")) ARCH = yaml.safe_load(open(os.path.join(models_dir, "darknet21.yaml"), "r")) @@ -136,5 +138,4 @@ def main(args): if __name__ == "__main__": - args = arguments() - main(args) + main() diff --git a/aimet_zoo_torch/rangenet/models/model_cards/__init__.py b/aimet_zoo_torch/rangenet/models/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/rangenet/models/model_definition.py b/aimet_zoo_torch/rangenet/models/model_definition.py index 89dd7f6..3e301f3 100644 --- a/aimet_zoo_torch/rangenet/models/model_definition.py +++ b/aimet_zoo_torch/rangenet/models/model_definition.py @@ -80,6 +80,7 @@ def from_pretrained(self, quantized=False): original_checkpoint_path = os.path.join( self.parent_dir, self.extract_dir, "darknet21" ) + #pylint:disable = consider-using-with model_orig_ARCH = yaml.safe_load( open(os.path.join(self.extract_dir, "darknet21", "arch_cfg.yaml"), "r") ) @@ -87,9 +88,9 @@ def from_pretrained(self, quantized=False): ARCH=model_orig_ARCH, nclasses=20, path=original_checkpoint_path ) self.model = prepare_model(self.model) - ModelValidator.validate_model(self.model, self.dummy_input) self.model.cuda() self.model.eval() + ModelValidator.validate_model(self.model, self.dummy_input) def get_quantsim(self, quantized=False): """get quantsim object with pre-loaded encodings""" diff --git a/aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/__init__.py b/aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/arch/__init__.py b/aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/arch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/labels/__init__.py b/aimet_zoo_torch/rangenet/models/train/tasks/semantic/config/labels/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/rangenet/models/train/tasks/semantic/dataset/__init__.py b/aimet_zoo_torch/rangenet/models/train/tasks/semantic/dataset/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/regnet/__init__.py b/aimet_zoo_torch/regnet/__init__.py index 3c5bf54..a32ee3a 100644 --- a/aimet_zoo_torch/regnet/__init__.py +++ b/aimet_zoo_torch/regnet/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import RegNet \ No newline at end of file +""" package for getting regnet original model and quantized model""" +from .model.model_definition import RegNet diff --git a/aimet_zoo_torch/regnet/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/regnet/dataloader/dataloaders_and_eval_func.py index c572d93..807a215 100644 --- a/aimet_zoo_torch/regnet/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/regnet/dataloader/dataloaders_and_eval_func.py @@ -8,15 +8,15 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -import torch -from tqdm import tqdm -from ...common.utils.image_net_data_loader import ImageNetDataLoader +""" module for getting evalution function and forward pass of dataloader""" +import torch def eval_func(model, dataloader, BATCH_SIZE=128): - ''' Evaluates the model on validation dataset and returns the classification accuracy ''' - #Get Dataloader + #pylint:disable = unused-argument + """Evaluates the model on validation dataset and returns the classification accuracy""" + # Get Dataloader model.eval() correct = 0 total_samples = 0 @@ -30,11 +30,12 @@ def eval_func(model, dataloader, BATCH_SIZE=128): correct += (prediction == label).sum() total_samples += len(output) del dataloader - return float(100* correct / total_samples) + return float(100 * correct / total_samples) def forward_pass(model, dataloader): - ''' forward pass through the calibration dataset ''' + """forward pass through the calibration dataset""" + #pylint:disable = unused-variable model.eval() on_cuda = next(model.parameters()).is_cuda with torch.no_grad(): diff --git a/aimet_zoo_torch/regnet/evaluator/regnet_quanteval.py b/aimet_zoo_torch/regnet/evaluator/regnet_quanteval.py index 2211756..d45631d 100644 --- a/aimet_zoo_torch/regnet/evaluator/regnet_quanteval.py +++ b/aimet_zoo_torch/regnet/evaluator/regnet_quanteval.py @@ -7,50 +7,62 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantsim evaluation code for Regnet_x_3_2gf ''' +""" AIMET Quantsim evaluation code for Regnet_x_3_2gf """ import argparse from aimet_zoo_torch.common.utils.image_net_data_loader import ImageNetDataLoader -from aimet_zoo_torch.regnet.dataloader.dataloaders_and_eval_func import eval_func, forward_pass +from aimet_zoo_torch.regnet.dataloader.dataloaders_and_eval_func import ( + eval_func, + forward_pass, +) from aimet_zoo_torch.regnet import RegNet def arguments(): - """ Parse input arguments """ - parser = argparse.ArgumentParser(description='script for classification model quantization') - parser.add_argument('--model-config', help='model configuration to use', default="regnet_x_3_2gf_w8a8", - choices = ['regnet_x_3_2gf_w4a8', 'regnet_x_3_2gf_w8a8'], - type=str, required=True) - parser.add_argument('--dataset-path', help='path to evaluation dataset',type=str, required=True) - parser.add_argument('--use-cuda', help='Use cuda', default=True, type=bool) + """Parse input arguments""" + parser = argparse.ArgumentParser( + description="script for classification model quantization" + ) + parser.add_argument( + "--model-config", + help="model configuration to use", + default="regnet_x_3_2gf_w8a8", + choices=["regnet_x_3_2gf_w4a8", "regnet_x_3_2gf_w8a8"], + type=str, + required=True, + ) + parser.add_argument( + "--dataset-path", help="path to evaluation dataset", type=str, required=True + ) + parser.add_argument("--use-cuda", help="Use cuda", default=True, type=bool) args = parser.parse_args() return args def main(): - """ Run evaluations """ + """Run evaluations""" args = arguments() - + # Dataloaders - encoding_dataloader = ImageNetDataLoader(args.dataset_path,image_size=224,num_samples_per_class=2).data_loader - eval_dataloader = ImageNetDataLoader(args.dataset_path,image_size=224).data_loader + encoding_dataloader = ImageNetDataLoader( + args.dataset_path, image_size=224, num_samples_per_class=2 + ).data_loader + eval_dataloader = ImageNetDataLoader(args.dataset_path, image_size=224).data_loader # Models - model = RegNet(model_config = args.model_config) + model = RegNet(model_config=args.model_config) model.from_pretrained(quantized=False) sim = model.get_quantsim(quantized=True) # Evaluate original - fp32_acc = eval_func(model = model.model, dataloader = eval_dataloader) - print(f'FP32 accuracy: {fp32_acc:0.3f}%') + fp32_acc = eval_func(model=model.model, dataloader=eval_dataloader) + print(f"FP32 accuracy: {fp32_acc:0.3f}%") # Evaluate optimized sim.compute_encodings(forward_pass, forward_pass_callback_args=encoding_dataloader) - quant_acc = eval_func(model = sim.model.cuda(), dataloader = eval_dataloader) - print(f'Quantized quantized accuracy: {quant_acc:0.3f}%') - -if __name__ == '__main__': - main() - + quant_acc = eval_func(model=sim.model.cuda(), dataloader=eval_dataloader) + print(f"Quantized quantized accuracy: {quant_acc:0.3f}%") +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/regnet/model/model_cards/__init__.py b/aimet_zoo_torch/regnet/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/regnet/model/model_definition.py b/aimet_zoo_torch/regnet/model/model_definition.py index f77325b..bc8f6a1 100644 --- a/aimet_zoo_torch/regnet/model/model_definition.py +++ b/aimet_zoo_torch/regnet/model/model_definition.py @@ -8,6 +8,10 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= +"""Class for downloading and setting up of optmized and original regnet model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet + import os import json import torch @@ -19,28 +23,33 @@ class RegNet(Downloader): """RegNet parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" - def __init__(self, model_config = None, **kwargs): + #pylint:disable = unused-argument + def __init__(self, model_config=None, **kwargs): """ :param model_config: named model config from which to obtain model artifacts and arguments. - If provided, overwrites the other arguments passed to this object + If provided, overwrites the other arguments passed to this object """ - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) self.model = getattr(torchvision.models, "regnet_x_3_2gf")(pretrained=True) self.model.cuda() self.model.eval() @@ -48,7 +57,9 @@ def __init__(self, model_config = None, **kwargs): def from_pretrained(self, quantized=False): """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() @@ -67,25 +78,34 @@ def from_pretrained(self, quantized=False): def get_quantsim(self, quantized=False): """get quantsim object with pre-loaded encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model, **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') + print("set_and_freeze_param_encodings finished!") sim.model.eval() return sim diff --git a/aimet_zoo_torch/resnet/__init__.py b/aimet_zoo_torch/resnet/__init__.py index e0db2f2..adc83d1 100644 --- a/aimet_zoo_torch/resnet/__init__.py +++ b/aimet_zoo_torch/resnet/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import ResNet \ No newline at end of file +""" package for getting resnet original model and quantized model""" +from .model.model_definition import ResNet diff --git a/aimet_zoo_torch/resnet/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/resnet/dataloader/dataloaders_and_eval_func.py index 6206594..2eba920 100644 --- a/aimet_zoo_torch/resnet/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/resnet/dataloader/dataloaders_and_eval_func.py @@ -7,14 +7,14 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - +""" module for evaluation function and forward pass of dataloader""" import torch from tqdm import tqdm def eval_func(model, dataloader): - ''' Evaluates the model on validation dataset and returns the classification accuracy ''' - #Get Dataloader + """Evaluates the model on validation dataset and returns the classification accuracy""" + # Get Dataloader model.eval() correct = 0 total_samples = 0 @@ -28,11 +28,12 @@ def eval_func(model, dataloader): correct += (prediction == label).sum() total_samples += len(output) del dataloader - return float(100* correct / total_samples) + return float(100 * correct / total_samples) def forward_pass(model, dataloader): - ''' forward pass through the calibration dataset ''' + """forward pass through the calibration dataset""" + #pylint:disable = unused-variable model.eval() on_cuda = next(model.parameters()).is_cuda with torch.no_grad(): diff --git a/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py b/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py index 8f1df74..57a08f4 100644 --- a/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py +++ b/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py @@ -15,22 +15,22 @@ from aimet_zoo_torch.resnet import ResNet -def arguments(): +def arguments(raw_args): """ Parse input arguments """ parser = argparse.ArgumentParser(description='script for classification model quantization') - parser.add_argument('--model-config', help='model configuration to use', default="resnet50_w8a8", - choices = ['resnet18_w4a8', 'resnet18_w8a8', 'resnet50_w4a8', 'resnet50_w8a8', 'resnet101_w8a8'], + parser.add_argument('--model-config', help='model configuration to use', default="resnet50_w8a8", + choices = ['resnet18_w4a8', 'resnet18_w8a8', 'resnet50_w4a8', 'resnet50_w8a8', 'resnet101_w8a8'], type=str, required=True) parser.add_argument('--dataset-path', help='path to evaluation dataset',type=str, required=True) parser.add_argument('--use-cuda', help='Use cuda', default=True, type=bool) - args = parser.parse_args() + args = parser.parse_args(raw_args) + print(vars(args)) return args -def main(): +def main(raw_args=None): """ Run evaluations """ - args = arguments() - + args = arguments(raw_args) # Dataloaders encoding_dataloader = ImageNetDataLoader(args.dataset_path,image_size=224,num_samples_per_class=2).data_loader eval_dataloader = ImageNetDataLoader(args.dataset_path,image_size=224).data_loader @@ -49,8 +49,7 @@ def main(): quant_acc = eval_func(model = sim.model.cuda(), dataloader = eval_dataloader) print(f'Quantized quantized accuracy: {quant_acc:0.3f}%') + return quant_acc + if __name__ == '__main__': main() - - - diff --git a/aimet_zoo_torch/resnet/model/model_cards/__init__.py b/aimet_zoo_torch/resnet/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/resnet/model/model_definition.py b/aimet_zoo_torch/resnet/model/model_definition.py index 9499850..435572e 100644 --- a/aimet_zoo_torch/resnet/model/model_definition.py +++ b/aimet_zoo_torch/resnet/model/model_definition.py @@ -8,6 +8,10 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= +"""Class for downloading and setting up of optmized and original resnet model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet + import os import json import torch @@ -17,35 +21,41 @@ from aimet_zoo_torch.common.downloader import Downloader - class ResNet(Downloader): """ResNet parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" - def __init__(self, model_config = None, **kwargs): + #pylint:disable = unused-argument + def __init__(self, model_config=None, **kwargs): """ :param model_config: named model config from which to obtain model artifacts and arguments. - If provided, overwrites the other arguments passed to this object + If provided, overwrites the other arguments passed to this object """ - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) - self.resnet_variant=model_config.split('_')[0] - supported_resnet_variants = {'resnet18', 'resnet50', 'resnet101'} + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + self.resnet_variant = model_config.split("_")[0] + supported_resnet_variants = {"resnet18", "resnet50", "resnet101"} if self.resnet_variant not in supported_resnet_variants: - raise NotImplementedError(f'Only support variants in {supported_resnet_variants}') + raise NotImplementedError( + f"Only support variants in {supported_resnet_variants}" + ) self.model = getattr(torchvision.models, self.resnet_variant)(pretrained=True) self.model.cuda() self.model.eval() @@ -53,7 +63,9 @@ def __init__(self, model_config = None, **kwargs): def from_pretrained(self, quantized=False): """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() @@ -65,32 +77,43 @@ def from_pretrained(self, quantized=False): self.model.load_state_dict(state_dict) self.model.cuda() else: - self.model = getattr(torchvision.models, self.resnet_variant)(pretrained=True) + self.model = getattr(torchvision.models, self.resnet_variant)( + pretrained=True + ) self.model.cuda() self.model.eval() def get_quantsim(self, quantized=False): """get quantsim object with pre-loaded encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model, **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') + print("set_and_freeze_param_encodings finished!") sim.model.eval() return sim diff --git a/aimet_zoo_torch/resnext/__init__.py b/aimet_zoo_torch/resnext/__init__.py index 1f5aa0f..0414736 100644 --- a/aimet_zoo_torch/resnext/__init__.py +++ b/aimet_zoo_torch/resnext/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import ResNext \ No newline at end of file +""" package for getting resnext original model and quantized model""" +from .model.model_definition import ResNext diff --git a/aimet_zoo_torch/resnext/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/resnext/dataloader/dataloaders_and_eval_func.py index dc6b281..98e25e1 100644 --- a/aimet_zoo_torch/resnext/dataloader/dataloaders_and_eval_func.py +++ b/aimet_zoo_torch/resnext/dataloader/dataloaders_and_eval_func.py @@ -7,16 +7,14 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - +"""module for getting evaluation function of dataloader""" import torch -from tqdm import tqdm -from ...common.utils.image_net_data_loader import ImageNetDataLoader - -def eval_func(model, dataloader, BATCH_SIZE=128, device = torch.device('cuda')): - ''' Evaluates the model on validation dataset and returns the classification accuracy ''' - #Get Dataloader +def eval_func(model, dataloader, BATCH_SIZE=128, device=torch.device("cuda")): + """Evaluates the model on validation dataset and returns the classification accuracy""" + #pylint:disable = unused-argument + # Get Dataloader model.eval() correct = 0 total_samples = 0 @@ -28,14 +26,4 @@ def eval_func(model, dataloader, BATCH_SIZE=128, device = torch.device('cuda')): correct += (prediction == label).sum() total_samples += len(output) del dataloader - return float(100* correct / total_samples) - - -def forward_pass(model, dataloader): - ''' forward pass through the calibration dataset ''' - model.eval() - with torch.no_grad(): - for data, label in dataloader: - data, label = data.to(device), label.to(device) - output = model(data) - del dataloader + return float(100 * correct / total_samples) diff --git a/aimet_zoo_torch/resnext/evaluator/resnext_quanteval.py b/aimet_zoo_torch/resnext/evaluator/resnext_quanteval.py index a7f388e..c424b17 100644 --- a/aimet_zoo_torch/resnext/evaluator/resnext_quanteval.py +++ b/aimet_zoo_torch/resnext/evaluator/resnext_quanteval.py @@ -7,54 +7,66 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantsim evaluation code for ResNeXt ''' +""" AIMET Quantsim evaluation code for ResNeXt """ import argparse from aimet_zoo_torch.common.utils.image_net_data_loader import ImageNetDataLoader -from aimet_zoo_torch.resnext.dataloader.dataloaders_and_eval_func import eval_func, forward_pass +from aimet_zoo_torch.resnext.dataloader.dataloaders_and_eval_func import ( + eval_func, +) from aimet_zoo_torch.resnext import ResNext from aimet_zoo_torch.common.utils.utils import get_device + def arguments(): - """ Parse input arguments """ - parser = argparse.ArgumentParser(description='script for classification model quantization') - parser.add_argument('--model-config', help='model configuration to use', default="resnext101_w8a8", - choices = ['resnext101_w8a8'], - type=str, required=True) - parser.add_argument('--dataset-path', help='path to evaluation dataset',type=str, required=True) - parser.add_argument('--use-cuda', help='Use cuda', default=True, type=bool) + """Parse input arguments""" + parser = argparse.ArgumentParser( + description="script for classification model quantization" + ) + parser.add_argument( + "--model-config", + help="model configuration to use", + default="resnext101_w8a8", + choices=["resnext101_w8a8"], + type=str, + required=True, + ) + parser.add_argument( + "--dataset-path", help="path to evaluation dataset", type=str, required=True + ) + parser.add_argument("--use-cuda", help="Use cuda", default=True, type=bool) args = parser.parse_args() return args def main(): - """ Run evaluations """ + """Run evaluations""" args = arguments() device = get_device(args) # Dataloaders - encoding_dataloader = ImageNetDataLoader(args.dataset_path,image_size=224,num_samples_per_class=2).data_loader - eval_dataloader = ImageNetDataLoader(args.dataset_path,image_size=224).data_loader + eval_dataloader = ImageNetDataLoader(args.dataset_path, image_size=224).data_loader # Original Model - model = ResNext(model_config = args.model_config, device = device , quantized = False ) + model = ResNext(model_config=args.model_config, device=device, quantized=False) model.from_pretrained() - fp32_acc = eval_func(model = model.model.to(device), dataloader = eval_dataloader, device = device) + fp32_acc = eval_func( + model=model.model.to(device), dataloader=eval_dataloader, device=device + ) del model - - # Quantized Model - model = ResNext(model_config = args.model_config, device = device , quantized = True ) + + # Quantized Model + model = ResNext(model_config=args.model_config, device=device, quantized=True) model.from_pretrained() sim = model.get_quantsim() - quant_acc = eval_func(model = sim.model.to(device), dataloader = eval_dataloader, device = device) + quant_acc = eval_func( + model=sim.model.to(device), dataloader=eval_dataloader, device=device + ) del model - print(f'FP32 accuracy: {fp32_acc:0.3f}%') - print(f'Quantized quantized accuracy: {quant_acc:0.3f}%') + print(f"FP32 accuracy: {fp32_acc:0.3f}%") + print(f"Quantized quantized accuracy: {quant_acc:0.3f}%") -if __name__ == '__main__': +if __name__ == "__main__": main() - - - diff --git a/aimet_zoo_torch/resnext/model/model_cards/__init__.py b/aimet_zoo_torch/resnext/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/resnext/model/model_cards/resnext101_w8a8.json b/aimet_zoo_torch/resnext/model/model_cards/resnext101_w8a8.json index 990b90d..3c917bd 100644 --- a/aimet_zoo_torch/resnext/model/model_cards/resnext101_w8a8.json +++ b/aimet_zoo_torch/resnext/model/model_cards/resnext101_w8a8.json @@ -19,9 +19,9 @@ }, "artifacts": { "url_pre_opt_weights": null, - "url_post_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnext101_w8a8/resnext101_w8a8_state_dict.pth", + "url_post_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnext101/resnext101_w8a8_state_dict.pth", "url_adaround_encodings": null, - "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnext101_w8a8/resnext101_w8a8_state_dict.encodings", + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnext101/resnext101_w8a8_state_dict.encodings", "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.24/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/resnext/model/model_definition.py b/aimet_zoo_torch/resnext/model/model_definition.py index 2a7f602..dd43dd1 100644 --- a/aimet_zoo_torch/resnext/model/model_definition.py +++ b/aimet_zoo_torch/resnext/model/model_definition.py @@ -8,83 +8,102 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= +"""Class for downloading and setting up of optmized and original resnext model for AIMET model zoo""" import os import json -import pathlib +import pathlib import torch -import torchvision -from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim -from aimet_torch.cross_layer_equalization import equalize_model +from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim # pylint:disable = import-error +from aimet_torch.cross_layer_equalization import equalize_model # pylint:disable = import-error from aimet_zoo_torch.common.downloader import Downloader class ResNext(Downloader): """ResNext parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" - def __init__(self, model_config = None, device = torch.device('cuda'), quantized = False): + + def __init__(self, model_config=None, device=torch.device("cuda"), quantized=False): """ :param model_config: named model config from which to obtain model artifacts and arguments. - If provided, overwrites the other arguments passed to this object + If provided, overwrites the other arguments passed to this object """ parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) - self.device = device + self.device = device self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) - self.model = None + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + self.model = None self.quantized = quantized def from_pretrained(self): """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() self._download_aimet_encodings() self._download_adaround_encodings() if self.quantized: - self.model = torch.hub.load('pytorch/vision:v0.10.0', 'resnext101_32x8d', pretrained=True) + self.model = torch.hub.load( + "pytorch/vision:v0.10.0", "resnext101_32x8d", pretrained=True + ) equalize_model(self.model, self.input_shape) state_dict = torch.load(self.path_post_opt_weights) self.model.load_state_dict(state_dict) self.model.to(self.device) else: - self.model = torch.hub.load('pytorch/vision:v0.10.0', 'resnext101_32x8d', pretrained=True) + self.model = torch.hub.load( + "pytorch/vision:v0.10.0", "resnext101_32x8d", pretrained=True + ) self.model.to(self.device) self.model.eval() def get_quantsim(self): """get quantsim object with pre-loaded encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') - dummy_input = torch.rand(self.input_shape, device = self.device) + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) + dummy_input = torch.rand(self.input_shape, device=self.device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self.model, **kwargs) if self.path_aimet_encodings and self.quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and self.quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') + print("set_and_freeze_param_encodings finished!") sim.model.to(self.device) sim.model.eval() return sim - diff --git a/aimet_zoo_torch/roberta/__init__.py b/aimet_zoo_torch/roberta/__init__.py index 84e1ae7..fe578ca 100644 --- a/aimet_zoo_torch/roberta/__init__.py +++ b/aimet_zoo_torch/roberta/__init__.py @@ -1 +1,2 @@ +""" package for getting roberta original model and quantized model""" from .model.model_definition import Roberta diff --git a/aimet_zoo_torch/roberta/dataloader/__init__.py b/aimet_zoo_torch/roberta/dataloader/__init__.py index 87b3f3f..b03e63e 100644 --- a/aimet_zoo_torch/roberta/dataloader/__init__.py +++ b/aimet_zoo_torch/roberta/dataloader/__init__.py @@ -1 +1,2 @@ -from .dataloaders import get_datasets,eval_function +""" datasets and eval function are defined and loaded""" +from .dataloaders import get_datasets, eval_function diff --git a/aimet_zoo_torch/roberta/dataloader/utils/__init__.py b/aimet_zoo_torch/roberta/dataloader/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/roberta/evaluators/__init__.py b/aimet_zoo_torch/roberta/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/roberta/evaluators/roberta_quanteval.py b/aimet_zoo_torch/roberta/evaluators/roberta_quanteval.py index 496d381..faf7585 100644 --- a/aimet_zoo_torch/roberta/evaluators/roberta_quanteval.py +++ b/aimet_zoo_torch/roberta/evaluators/roberta_quanteval.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201 # ============================================================================= # @@-COPYRIGHT-START-@@ # # Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. -# Changes from QuIC are licensed under the terms and conditions at +# Changes from QuIC are licensed under the terms and conditions at # https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf" # # @@-COPYRIGHT-END-@@ @@ -27,21 +27,20 @@ import argparse import logging -from transformers.utils.versions import require_version -from datasets import load_dataset, load_metric from aimet_zoo_torch.roberta import Roberta -from aimet_zoo_torch.roberta.dataloader import get_datasets, eval_function +from aimet_zoo_torch.roberta.dataloader import get_datasets, eval_function def parse_args(): """argument parser""" parser = argparse.ArgumentParser( - description="Evaluating Bert model on GLUE datasets") + description="Evaluating Bert model on GLUE datasets" + ) parser.add_argument( "--model_config", default="roberta_w8a8_mnli", - help="choice [mobilebert_w8a8_cola, mobilebert_w8a8_mnli, mobilebert_w8a8_mrpc, mobilebert_w8a8_qnli, mobilebert_w8a8_qqp, mobilebert_w8a8_rte, mobilebert_w8a8_squad, mobilebert_w8a8_sst2, mobilebert_w8a8_stsb]", + help="choice [mobilebert_w8a8_cola, mobilebert_w8a8_mnli, mobilebert_w8a8_mrpc, mobilebert_w8a8_qnli, mobilebert_w8a8_qqp, mobilebert_w8a8_rte, mobilebert_w8a8_squad, mobilebert_w8a8_sst2, mobilebert_w8a8_stsb]", ) parser.add_argument( "--per_device_eval_batch_size", @@ -54,7 +53,7 @@ def parse_args(): type=str, default=None, help="Output directory", - ) + ) args = parser.parse_args() for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) @@ -63,7 +62,7 @@ def parse_args(): def main(): - """main function for quantization evaluation""" + """main function for quantization evaluation""" args = parse_args() logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", @@ -71,21 +70,31 @@ def main(): level=logging.INFO, ) model = Roberta(model_config=args.model_config) - - # get original model and tokenizer - model_orig,tokenizer = model.get_model_from_pretrained() - # get datasets - datasets=get_datasets(data_args=model.data_args,training_args=model.training_args,model_args=model.model_args,model=model_orig,tokenizer=tokenizer) + # get original model and tokenizer + model_orig, tokenizer = model.get_model_from_pretrained() - # evaluation of original model - original_eval_results=eval_function(model_orig,tokenizer,datasets,model.data_args,model.training_args) + # get datasets + datasets = get_datasets( + data_args=model.data_args, + training_args=model.training_args, + model_args=model.model_args, + model=model_orig, + tokenizer=tokenizer, + ) - # get quantsim object + # evaluation of original model + original_eval_results = eval_function( + model_orig, tokenizer, datasets, model.data_args, model.training_args + ) + + # get quantsim object quantsim_model = model.get_quantsim() - # evaluation of quantsim model - optimized_eval_results=eval_function(quantsim_model.model,tokenizer,datasets,model.data_args,model.training_args) + # evaluation of quantsim model + optimized_eval_results = eval_function( + quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args + ) logging.info(f"***** Original Eval results *****") for key, value in sorted(original_eval_results.items()): @@ -93,8 +102,7 @@ def main(): logging.info(f"***** Optimized Quantized Model Eval results *****") for key, value in sorted(optimized_eval_results.items()): - logging.info(f" {key} = {value}") - + logging.info(f" {key} = {value}") if __name__ == "__main__": diff --git a/aimet_zoo_torch/roberta/model/baseline_models/__init__.py b/aimet_zoo_torch/roberta/model/baseline_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/roberta/model/model_cards/__init__.py b/aimet_zoo_torch/roberta/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_cola.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_cola.json index f39dda0..c6ca00d 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_cola.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_cola.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/cola_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/cola_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mnli.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mnli.json index 5c23fb1..8239d90 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mnli.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mnli.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/mnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/mnli_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mrpc.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mrpc.json index 795ecb1..8fc09a2 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mrpc.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mrpc.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/mrpc_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/mrpc_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qnli.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qnli.json index c67ed98..07cebac 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qnli.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qnli.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/qnli_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/qnli_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qqp.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qqp.json index d81a79f..961d966 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qqp.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qqp.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/qqp_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/qqp_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_rte.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_rte.json index 8cb80a9..bb2a3df 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_rte.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_rte.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/rte_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/rte_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_sst2.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_sst2.json index 0914be1..724169e 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_sst2.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_sst2.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/sst2_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/sst2_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_stsb.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_stsb.json index 10a3cfa..00a7b6e 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_stsb.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_stsb.json @@ -39,6 +39,6 @@ "artifacts": { "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/stsb_fp.pth", "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_roberta/stsb_qat.ckpt", - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/model/model_definition.py b/aimet_zoo_torch/roberta/model/model_definition.py index 7e7554a..2a0c8fd 100644 --- a/aimet_zoo_torch/roberta/model/model_definition.py +++ b/aimet_zoo_torch/roberta/model/model_definition.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- mode: python -*- -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 # ============================================================================= # @@-COPYRIGHT-START-@@ # @@ -9,47 +9,44 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= """Class for downloading and setting up of optmized and original roberta model for AIMET model zoo""" +# pylint:disable = wrong-import-order + import json import os -import csv import sys from collections import defaultdict +import pathlib import torch -import datasets -from transformers import AutoConfig as Config -from transformers import AutoFeatureExtractor as FeatureExtractor -from aimet_common.defs import QuantScheme + # AIMET imports from aimet_torch.quantsim import load_checkpoint -from aimet_torch.quantsim import QuantizationSimModel -from aimet_torch.qc_quantize_op import QcQuantizeWrapper + +from transformers import HfArgumentParser +from transformers import AutoConfig, AutoTokenizer, TrainingArguments + # transformers import from aimet_zoo_torch.roberta.model import baseline_models from aimet_zoo_torch.common.downloader import Downloader -sys.modules['baseline_models'] = baseline_models -from transformers import HfArgumentParser -from transformers import ( - AutoConfig, - AutoTokenizer, - TrainingArguments -) +sys.modules["baseline_models"] = baseline_models + class Roberta(Downloader): - """ model roberta configuration class """ + """model roberta configuration class""" + #pylint:disable = import-outside-toplevel def __init__(self, model_config=None): - if model_config=="roberta_w8a8_squad": + if model_config == "roberta_w8a8_squad": from aimet_zoo_torch.roberta.model.utils.utils_qa_dataclass import ( ModelArguments, DataTrainingArguments, AuxArguments, - ) + ) else: from aimet_zoo_torch.roberta.model.utils.utils_nlclassifier_dataclass import ( ModelArguments, DataTrainingArguments, AuxArguments, - ) + ) parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: @@ -65,47 +62,43 @@ def __init__(self, model_config=None): ) # Parse arguments parser = HfArgumentParser( - (ModelArguments, - DataTrainingArguments, - TrainingArguments, - AuxArguments)) + (ModelArguments, DataTrainingArguments, TrainingArguments, AuxArguments) + ) ( - model_args, - data_args, - training_args, - aux_args, + model_args, + data_args, + training_args, + aux_args, ) = parser.parse_args_into_dataclasses() - self.model = None - self.model_args=model_args - self.data_args =data_args + self.model_args = model_args + self.data_args = data_args self.training_args = training_args self.aux_args = aux_args - #additional setup of the argsumetns from model_config - if model_config=="roberta_w8a8_squad": - self.data_args.dataset_name=self.cfg["data_training_args"]["dataset_name"] + # additional setup of the argsumetns from model_config + if model_config == "roberta_w8a8_squad": + self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] else: - self.data_args.task_name=self.cfg["data_training_args"]["task_name"] + self.data_args.task_name = self.cfg["data_training_args"]["task_name"] - self.aux_args.model_config=model_config - self.training_args.do_eval=True + self.aux_args.model_config = model_config + self.training_args.do_eval = True # setup the download path from arguments - self.path_pre_opt_weights = self.aux_args.fmodel_path + self.path_pre_opt_weights = self.aux_args.fmodel_path self.path_post_opt_weights = self.aux_args.qmodel_path - def get_model_from_pretrained(self): """get original or optmized model Parameters: dataset: Return: - model : pretrained/optmized model + model : pretrained/optmized model """ - #case1. model for squad dataset - if hasattr(self.data_args,'dataset_name'): + # case1. model for squad dataset + if hasattr(self.data_args, "dataset_name"): self._download_pre_opt_weights(show_progress=True) - self._download_aimet_config() + self._download_aimet_config() config = AutoConfig.from_pretrained( self.model_args.config_name if self.model_args.config_name @@ -128,10 +121,10 @@ def get_model_from_pretrained(self): use_auth_token=True if self.model_args.use_auth_token else None, ) - self.model = torch.load(self.aux_args.fmodel_path) - return self.model,tokenizer - #case2. model for glue dataset - num_labels=2 + self.model = torch.load(self.aux_args.fmodel_path) + return self.model, tokenizer + # case2. model for glue dataset + num_labels = 2 self._download_pre_opt_weights(show_progress=True) self._download_aimet_config() # Load pretrained model and tokenizer @@ -148,7 +141,9 @@ def get_model_from_pretrained(self): # ++++ config.return_dict = False config.classifier_dropout = None - config.attention_probs_dropout_prob = self.model_args.attention_probs_dropout_prob + config.attention_probs_dropout_prob = ( + self.model_args.attention_probs_dropout_prob + ) # ++++ tokenizer = AutoTokenizer.from_pretrained( @@ -162,12 +157,12 @@ def get_model_from_pretrained(self): ) self.model = torch.load(self.aux_args.fmodel_path) - return self.model,tokenizer + return self.model, tokenizer def get_quantsim(self): - """get quantsim object """ + """get quantsim object""" self._download_post_opt_weights(show_progress=True) # Load the Quantsim_model object quantsim_model = load_checkpoint(self.aux_args.qmodel_path) - return quantsim_model + return quantsim_model diff --git a/aimet_zoo_torch/roberta/model/utils/__init__.py b/aimet_zoo_torch/roberta/model/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/salsaNext/evaluators/evaluation_func.py b/aimet_zoo_torch/salsaNext/evaluators/evaluation_func.py deleted file mode 100755 index fe943e1..0000000 --- a/aimet_zoo_torch/salsaNext/evaluators/evaluation_func.py +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env python3 -# -*- mode: python -*- -# This file is covered by the MIT LICENSE. - -""" -This script applies and evaluates a pre-trained salsaNext model taken from -https://github.com/TiagoCortinhal/SalsaNext. -Such model is for the semantic segmentation task with the metric (mIoU) and semantic-kitti dataset. -For quantization instructions, please refer to zoo_torch/salsaNext/salsaNext.md -""" - -import torch -import torch.nn as nn -import torch.optim as optim -import torch.backends.cudnn as cudnn -import imp -import yaml -import time -from PIL import Image -import __init__ as booger -import collections -import copy -import cv2 -import os -import numpy as np -import argparse -from tasks.semantic.modules.SalsaNext import * -from tasks.semantic.modules.SalsaNextAdf import * -from tasks.semantic.postproc.KNN import KNN -from tasks.semantic.modules.ioueval import * -from common.avgmeter import * - -# AIMET IMPORTS - -from aimet_torch.quantsim import QuantizationSimModel -from aimet_torch.cross_layer_equalization import equalize_model -from aimet_torch import bias_correction -from aimet_torch.quantsim import QuantParams -from aimet_torch.utils import create_fake_data_loader -from aimet_torch.model_validator.model_validator import ModelValidator -from aimet_torch.quantsim import QuantizationDataType -from aimet_torch.quantsim import save_checkpoint -from aimet_torch.quantsim import load_checkpoint -from aimet_torch.quantsim import QuantizationDataType -from aimet_common.defs import QuantScheme - -def str2bool(v): - if isinstance(v, bool): - return v - if v.lower() in ('yes', 'true', 't', 'y'): - return True - elif v.lower() in ('no', 'false', 'f', 'n'): - return False - else: - raise argparse.ArgumentTypeError('Boolean expected') - -def save_to_log(logdir,logfile,message): - f = open(logdir+'/'+logfile, "a") - f.write(message+'\n') - f.close() - return - -class User(): - def __init__(self, ARCH, DATA, datadir, logdir, modeldir,split,uncertainty,mc=30,model_given=None): - # parameters - self.ARCH = ARCH - self.DATA = DATA - self.datadir = datadir - self.logdir = logdir - self.modeldir = modeldir - self.uncertainty = uncertainty - self.split = split - self.mc = mc - - # get the data - parserModule = imp.load_source("parserModule", - os.getcwd()+ '/train' + '/tasks/semantic/dataset/' + - self.DATA["name"] + '/parser.py') - self.parser = parserModule.Parser(root=self.datadir, - train_sequences=self.DATA["split"]["train"], - valid_sequences=self.DATA["split"]["valid"], - test_sequences=self.DATA["split"]["test"], - labels=self.DATA["labels"], - color_map=self.DATA["color_map"], - learning_map=self.DATA["learning_map"], - learning_map_inv=self.DATA["learning_map_inv"], - sensor=self.ARCH["dataset"]["sensor"], - max_points=self.ARCH["dataset"]["max_points"], - batch_size=1, - workers=self.ARCH["train"]["workers"], - gt=True, - shuffle_train=False) - - # concatenate the encoder and the head - with torch.no_grad(): - torch.nn.Module.dump_patches = True - self.model = model_given - - - # use knn post processing? - self.post = None - if self.ARCH["post"]["KNN"]["use"]: - self.post = KNN(self.ARCH["post"]["KNN"]["params"], - self.parser.get_n_classes()) - - # GPU? - self.gpu = False - self.model_single = self.model - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - #self.device = torch.device("cpu") - print("Infering in device: ", self.device) - if torch.cuda.is_available() and torch.cuda.device_count() > 0: - cudnn.benchmark = True - cudnn.fastest = True - self.gpu = True - self.model.cuda() - - def infer(self): - cnn = [] - knn = [] - if self.split == None: - - self.infer_subset(loader=self.parser.get_train_set(), - to_orig_fn=self.parser.to_original, cnn=cnn, knn=knn) - - # do valid set - self.infer_subset(loader=self.parser.get_valid_set(), - to_orig_fn=self.parser.to_original, cnn=cnn, knn=knn) - # do test set - self.infer_subset(loader=self.parser.get_test_set(), - to_orig_fn=self.parser.to_original, cnn=cnn, knn=knn) - - - elif self.split == 'valid': - self.infer_subset(loader=self.parser.get_valid_set(), - to_orig_fn=self.parser.to_original, cnn=cnn, knn=knn) - elif self.split == 'train': - self.infer_subset(loader=self.parser.get_train_set(), - to_orig_fn=self.parser.to_original, cnn=cnn, knn=knn) - else: - self.infer_subset(loader=self.parser.get_test_set(), - to_orig_fn=self.parser.to_original, cnn=cnn, knn=knn) - print("Mean CNN inference time:{}\t std:{}".format(np.mean(cnn), np.std(cnn))) - print("Mean KNN inference time:{}\t std:{}".format(np.mean(knn), np.std(knn))) - print("Total Frames:{}".format(len(cnn))) - print("Finished Infering") - - return - - def infer_subset(self, loader, to_orig_fn,cnn,knn): - parser = argparse.ArgumentParser("./user.py") - FLAGS, unparsed = parser.parse_known_args() - # switch to evaluate mode - if not self.uncertainty: - self.model.eval() - # empty the cache to infer in high res - if self.gpu: - torch.cuda.empty_cache() - - total_time=0 - total_frames=0 - - with torch.no_grad(): - end = time.time() - - for i, (proj_in, proj_mask, _, _, path_seq, path_name, p_x, p_y, proj_range, unproj_range, _, _, _, _, npoints) in enumerate(loader): - # first cut to rela size (batch size one allows it) - p_x = p_x[0, :npoints] - p_y = p_y[0, :npoints] - proj_range = proj_range[0, :npoints] - unproj_range = unproj_range[0, :npoints] - path_seq = path_seq[0] - path_name = path_name[0] - - if self.gpu: - proj_in = proj_in.cuda() - p_x = p_x.cuda() - p_y = p_y.cuda() - if self.post: - proj_range = proj_range.cuda() - unproj_range = unproj_range.cuda() - - #compute output - if self.uncertainty: - proj_output_r,log_var_r = self.model(proj_in) - for i in range(self.mc): - log_var, proj_output = self.model(proj_in) - log_var_r = torch.cat((log_var, log_var_r)) - proj_output_r = torch.cat((proj_output, proj_output_r)) - - proj_output2,log_var2 = self.model(proj_in) - proj_output = proj_output_r.var(dim=0, keepdim=True).mean(dim=1) - log_var2 = log_var_r.mean(dim=0, keepdim=True).mean(dim=1) - if self.post: - # knn postproc - unproj_argmax = self.post(proj_range, - unproj_range, - proj_argmax, - p_x, - p_y) - else: - # put in original pointcloud using indexes - unproj_argmax = proj_argmax[p_y, p_x] - - # measure elapsed time - if torch.cuda.is_available(): - torch.cuda.synchronize() - frame_time = time.time() - end - print("Infered seq", path_seq, "scan", path_name, - "in", frame_time, "sec") - total_time += frame_time - total_frames += 1 - end = time.time() - - # save scan - # get the first scan in batch and project scan - pred_np = unproj_argmax.cpu().numpy() - pred_np = pred_np.reshape((-1)).astype(np.int32) - - # log_var2 = log_var2[0][p_y, p_x] - # log_var2 = log_var2.cpu().numpy() - # log_var2 = log_var2.reshape((-1)).astype(np.float32) - - log_var2 = log_var2[0][p_y, p_x] - log_var2 = log_var2.cpu().numpy() - log_var2 = log_var2.reshape((-1)).astype(np.float32) - # assert proj_output.reshape((-1)).shape == log_var2.reshape((-1)).shape == pred_np.reshape((-1)).shape - - # map to original label - pred_np = to_orig_fn(pred_np) - - # save scan - path = os.path.join(self.logdir, "sequences", - path_seq, "predictions", path_name) - pred_np.tofile(path) - - path = os.path.join(self.logdir, "sequences", - path_seq, "log_var", path_name) - if not os.path.exists(os.path.join(self.logdir, "sequences", - path_seq, "log_var")): - os.makedirs(os.path.join(self.logdir, "sequences", - path_seq, "log_var")) - log_var2.tofile(path) - - proj_output = proj_output[0][p_y, p_x] - proj_output = proj_output.cpu().numpy() - proj_output = proj_output.reshape((-1)).astype(np.float32) - - path = os.path.join(self.logdir, "sequences", - path_seq, "uncert", path_name) - if not os.path.exists(os.path.join(self.logdir, "sequences", - path_seq, "uncert")): - os.makedirs(os.path.join(self.logdir, "sequences", - path_seq, "uncert")) - proj_output.tofile(path) - - print(total_time / total_frames) - else: - proj_output = self.model(proj_in) - - proj_argmax = proj_output[0].argmax(dim=0) - if torch.cuda.is_available(): - torch.cuda.synchronize() - res = time.time() - end - print("Network seq", path_seq, "scan", path_name, - "in", res, "sec") - end = time.time() - cnn.append(res) - - if torch.cuda.is_available(): - torch.cuda.synchronize() - res = time.time() - end - print("Network seq", path_seq, "scan", path_name, - "in", res, "sec") - end = time.time() - cnn.append(res) - - if self.post: - # knn postproc - unproj_argmax = self.post(proj_range, - unproj_range, - proj_argmax, - p_x, - p_y) - else: - # put in original pointcloud using indexes - unproj_argmax = proj_argmax[p_y, p_x] - - # measure elapsed time - if torch.cuda.is_available(): - torch.cuda.synchronize() - res = time.time() - end - print("KNN Infered seq", path_seq, "scan", path_name, - "in", res, "sec") - knn.append(res) - end = time.time() - - # save scan - # get the first scan in batch and project scan - pred_np = unproj_argmax.cpu().numpy() - pred_np = pred_np.reshape((-1)).astype(np.int32) - - # map to original label - pred_np = to_orig_fn(pred_np) - - # save scan - path = os.path.join(self.logdir, "sequences", - path_seq, "predictions", path_name) - pred_np.tofile(path) diff --git a/aimet_zoo_torch/salsaNext/evaluators/salsaNext_quanteval.py b/aimet_zoo_torch/salsaNext/evaluators/salsaNext_quanteval.py deleted file mode 100755 index 20fde6c..0000000 --- a/aimet_zoo_torch/salsaNext/evaluators/salsaNext_quanteval.py +++ /dev/null @@ -1,636 +0,0 @@ -#!/usr/bin/env python3 -# -*- mode: python -*- -# ============================================================================= -# @@-COPYRIGHT-START-@@ -# -# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. -# Changes from QuIC are licensed under the terms and conditions at -# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf -# -# @@-COPYRIGHT-END-@@ -# ============================================================================= - -""" -This script applies and evaluates a pre-trained salsaNext model taken from -https://github.com/TiagoCortinhal/SalsaNext. -Such model is for the semantic segmentation task with the metric (mIoU) and semantic-kitti dataset. -For quantization instructions, please refer to zoo_torch/salsaNext/salsaNext.md -""" - -import datetime -import os -import sys -import argparse -import urllib -import shutil -import yaml -import numpy as np -import imp -import os - -# torch imports -import torch -import torch.nn as nn -import pathlib - -parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) -sys.path.append(os.path.join(parent_dir, 'train')) - -# source code imports -from tasks.semantic.modules.ioueval import iouEval -from common.laserscan import SemLaserScan -from tasks.semantic.modules.SalsaNext import * -from tasks.semantic.modules.SalsaNextAdf import * -from tasks.semantic.modules.ioueval import * -from common.avgmeter import * - -# AIMET IMPORTS -from aimet_torch.quantsim import QuantizationSimModel -from aimet_torch.model_validator.model_validator import ModelValidator -from aimet_torch.model_preparer import prepare_model -from aimet_torch.quantsim import load_checkpoint -from aimet_common.defs import QuantScheme -from aimet_torch import batch_norm_fold -from aimet_torch.quantsim import load_encodings_to_sim - -import evaluation_func -from evaluation_func import * - -# possible splits -splits = ['train','valid','test'] - - -# Set seed for reproducibility -def seed(seed_number): - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = True - torch.manual_seed(seed_number) - torch.cuda.manual_seed(seed_number) - torch.cuda.manual_seed_all(seed_number) - -""" -Arguments to run the script, at least including: ---dataset, dataset folder ---model, pretrained model folder ---log, output log folder - -One example: -python salsaNext_quanteval.py --dataset /tf/semntic_kitti/datasets_segmentation/dataset --log ./logs --model ./pretrained -""" -def arguments(): - - parser = argparse.ArgumentParser("./salsaNext_quanteval.py") - - parser.add_argument( - '--dataset', '-d', - type=str, - required=True, - help='Dataset to train with. No Default', - ) - parser.add_argument( - '--log', '-l', - type=str, - required=True, - help='Directory to put the predictions.' - ) - parser.add_argument( - '--model', '-m', - type=str, - required=True, - default=None, - help='Directory to get the trained model.' - ) - - parser.add_argument( - '--uncertainty', '-u', - type=str2bool, nargs='?', - const=True, default=False, - help='Set this if you want to use the Uncertainty Version' - ) - - parser.add_argument( - '--monte-carlo', '-c', - type=int, default=30, - help='Number of samplings per scan' - ) - - - parser.add_argument( - '--split', '-s', - type=str, - required=False, - default='valid', - help='Split to evaluate on. One of ' + - str(splits) + '. Defaults to %(default)s', - ) - - parser.add_argument( - '--predictions', '-p', - type=str, - required=False, - default=None, - help='Prediction dir. Same organization as dataset, but predictions in' - 'each sequences "prediction" directory. No Default. If no option is set' - ' we look for the labels in the same directory as dataset' - ) - - parser.add_argument( - '--data_cfg', '-dc', - type=str, - required=False, - default="config/labels/semantic-kitti.yaml", - help='Dataset config file. Defaults to %(default)s', - ) - - parser.add_argument( - '--limit', '-li', - type=int, - required=False, - default=None, - help='Limit to the first "--limit" points of each scan. Useful for' - ' evaluating single scan from aggregated pointcloud.' - ' Defaults to %(default)s', - ) - - - FLAGS, unparsed = parser.parse_known_args() - - if FLAGS.predictions == None: - FLAGS.predictions = FLAGS.log - - print("input configuration") - print(FLAGS) - - return FLAGS - -""" -Download the related files and checkpoints -""" -def download_weights(FLAGS): - """ Download weights to cache directory """ - # Download original model - FILE_NAME = os.path.join(FLAGS.model, "SalsaNext") - ORIGINAL_MODEL_URL = "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/SalsaNext" - if not os.path.exists(FILE_NAME): - urllib.request.urlretrieve(ORIGINAL_MODEL_URL, FILE_NAME) - - # Download optimized w8a8 weights - FILE_NAME = os.path.join(FLAGS.model, "SalsaNext_optimized_model.pth") - OPTIMIZED_CHECKPOINT_URL = "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/SalsaNext_optimized_model.pth" - if not os.path.exists(FILE_NAME): - urllib.request.urlretrieve(OPTIMIZED_CHECKPOINT_URL, FILE_NAME) - - # Download optimized w8a8 encodings - FILE_NAME = os.path.join(FLAGS.model, "SalsaNext_optimized_encoding.encodings") - OPTIMIZED_ENCODINGS_URL = "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/SalsaNext_optimized_encoding.encodings" - if not os.path.exists(FILE_NAME): - urllib.request.urlretrieve(OPTIMIZED_ENCODINGS_URL, FILE_NAME) - - # Download config file - FILE_NAME = os.path.join(FLAGS.model, "htp_quantsim_config_pt_pertensor.json") - QUANTSIM_CONFIG_URL = "https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" - if not os.path.exists(FILE_NAME): - urllib.request.urlretrieve(QUANTSIM_CONFIG_URL, FILE_NAME) - - # Downlod model config files - FILE_NAME = os.path.join(FLAGS.model, "arch_cfg.yaml") - QUANTSIM_CONFIG_URL = "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/arch_cfg.yaml" - if not os.path.exists(FILE_NAME): - urllib.request.urlretrieve(QUANTSIM_CONFIG_URL, FILE_NAME) - - FILE_NAME = os.path.join(FLAGS.model, "data_cfg.yaml") - QUANTSIM_CONFIG_URL = "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/data_cfg.yaml" - if not os.path.exists(FILE_NAME): - urllib.request.urlretrieve(QUANTSIM_CONFIG_URL, FILE_NAME) - -""" -First step in eval_func() -Make the inference. Save the inference output. -""" -def infer_main(FLAGS, model_given): - # print summary of what we will do - print("----------") - print("INTERFACE:") - print("dataset", FLAGS.dataset) - print("log", FLAGS.log) - print("model", FLAGS.model) - print("Uncertainty", FLAGS.uncertainty) - print("Monte Carlo Sampling", FLAGS.monte_carlo) - print("infering", FLAGS.split) - print("----------\n") - #print("Commit hash (training version): ", str( - # subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip())) - print("----------\n") - - # open arch config file - try: - print("Opening arch config file from %s" % FLAGS.model) - ARCH = yaml.safe_load(open(os.path.join(FLAGS.model, "arch_cfg.yaml"), 'r')) - except Exception as e: - print(e) - print("Error opening arch yaml file.") - quit() - - # open data config file - try: - print("Opening data config file from %s" % FLAGS.model) - DATA = yaml.safe_load(open(os.path.join(FLAGS.model, "data_cfg.yaml"), 'r')) - except Exception as e: - print(e) - print("Error opening data yaml file.") - quit() - - # create log folder - try: - if os.path.isdir(FLAGS.log): - shutil.rmtree(FLAGS.log) - os.makedirs(FLAGS.log) - os.makedirs(os.path.join(FLAGS.log, "sequences")) - for seq in DATA["split"]["train"]: - seq = '{0:02d}'.format(int(seq)) - print("train", seq) - os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) - os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) - for seq in DATA["split"]["valid"]: - seq = '{0:02d}'.format(int(seq)) - print("valid", seq) - os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) - os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) - for seq in DATA["split"]["test"]: - seq = '{0:02d}'.format(int(seq)) - print("test", seq) - os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) - os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) - except Exception as e: - print(e) - print("Error creating log directory. Check permissions!") - raise - - except Exception as e: - print(e) - print("Error creating log directory. Check permissions!") - quit() - - # does model folder exist? - if os.path.isdir(FLAGS.model): - print("model folder exists! Using model from %s" % (FLAGS.model)) - else: - print("model folder doesnt exist! Can't infer...") - quit() - - # create user and infer dataset - user = User(ARCH, DATA, FLAGS.dataset, FLAGS.log, FLAGS.model,FLAGS.split,FLAGS.uncertainty,FLAGS.monte_carlo, model_given) - user.infer() - -def eval(test_sequences,splits,pred,remap_lut,evaluator, class_strings, class_inv_remap, ignore): - # get scan paths - scan_names = [] - for sequence in test_sequences: - sequence = '{0:02d}'.format(int(sequence)) - scan_paths = os.path.join(FLAGS.dataset, "sequences", - str(sequence), "velodyne") - # populate the scan names - seq_scan_names = [os.path.join(dp, f) for dp, dn, fn in os.walk( - os.path.expanduser(scan_paths)) for f in fn if ".bin" in f] - seq_scan_names.sort() - scan_names.extend(seq_scan_names) - # print(scan_names) - - # get label paths - label_names = [] - for sequence in test_sequences: - sequence = '{0:02d}'.format(int(sequence)) - label_paths = os.path.join(FLAGS.dataset, "sequences", - str(sequence), "labels") - # populate the label names - seq_label_names = [os.path.join(dp, f) for dp, dn, fn in os.walk( - os.path.expanduser(label_paths)) for f in fn if ".label" in f] - seq_label_names.sort() - label_names.extend(seq_label_names) - # print(label_names) - - # get predictions paths - pred_names = [] - for sequence in test_sequences: - sequence = '{0:02d}'.format(int(sequence)) - pred_paths = os.path.join(FLAGS.predictions, "sequences", - sequence, "predictions") - # populate the label names - seq_pred_names = [os.path.join(dp, f) for dp, dn, fn in os.walk( - os.path.expanduser(pred_paths)) for f in fn if ".label" in f] - seq_pred_names.sort() - pred_names.extend(seq_pred_names) - # print(pred_names) - - # check that I have the same number of files - # print("labels: ", len(label_names)) - # print("predictions: ", len(pred_names)) - assert (len(label_names) == len(scan_names) and - len(label_names) == len(pred_names)) - - print("Evaluating sequences: ") - # open each file, get the tensor, and make the iou comparison - for scan_file, label_file, pred_file in zip(scan_names, label_names, pred_names): - print("evaluating label ", label_file, "with", pred_file) - # open label - label = SemLaserScan(project=False) - label.open_scan(scan_file) - label.open_label(label_file) - u_label_sem = remap_lut[label.sem_label] # remap to xentropy format - if FLAGS.limit is not None: - u_label_sem = u_label_sem[:FLAGS.limit] - - # open prediction - pred = SemLaserScan(project=False) - pred.open_scan(scan_file) - pred.open_label(pred_file) - u_pred_sem = remap_lut[pred.sem_label] # remap to xentropy format - if FLAGS.limit is not None: - u_pred_sem = u_pred_sem[:FLAGS.limit] - - # add single scan to evaluation - evaluator.addBatch(u_pred_sem, u_label_sem) - - # when I am done, print the evaluation - m_accuracy = evaluator.getacc() - m_jaccard, class_jaccard = evaluator.getIoU() - - print('{split} set:\n' - 'Acc avg {m_accuracy:.3f}\n' - 'IoU avg {m_jaccard:.3f}'.format(split=splits, - m_accuracy=m_accuracy, - m_jaccard=m_jaccard)) - - save_to_log(FLAGS.predictions,'pred.txt','{split} set:\n' - 'Acc avg {m_accuracy:.3f}\n' - 'IoU avg {m_jaccard:.3f}'.format(split=splits, - m_accuracy=m_accuracy, - m_jaccard=m_jaccard)) - # print also classwise - for i, jacc in enumerate(class_jaccard): - if i not in ignore: - print('IoU class {i:} [{class_str:}] = {jacc:.3f}'.format( - i=i, class_str=class_strings[class_inv_remap[i]], jacc=jacc)) - save_to_log(FLAGS.predictions, 'pred.txt', 'IoU class {i:} [{class_str:}] = {jacc:.3f}'.format( - i=i, class_str=class_strings[class_inv_remap[i]], jacc=jacc)) - - # print for spreadsheet - print("*" * 80) - print("below is the final result") - for i, jacc in enumerate(class_jaccard): - if i not in ignore: - sys.stdout.write('{jacc:.3f}'.format(jacc=jacc.item())) - sys.stdout.write(",") - sys.stdout.write('{jacc:.3f}'.format(jacc=m_jaccard.item())) - sys.stdout.write(",") - sys.stdout.write('{acc:.3f}'.format(acc=m_accuracy.item())) - sys.stdout.write('\n') - sys.stdout.flush() - - return m_jaccard.item() - -""" -second step in eval_func() -Function to evaluate the model, and output the results. -""" -def evaluate_main(FLAGS): - splits = ['train','valid','test'] - # fill in real predictions dir - if FLAGS.predictions is None: - FLAGS.predictions = FLAGS.dataset - - # print summary of what we will do - print("*" * 80) - print("INTERFACE:") - print("Data: ", FLAGS.dataset) - print("Predictions: ", FLAGS.predictions) - print("Split: ", FLAGS.split) - print("Config: ", FLAGS.data_cfg) - print("Limit: ", FLAGS.limit) - print("*" * 80) - - # assert split - assert (FLAGS.split in splits) - - # open data config file - try: - FLAGS.data_cfg_1 = os.path.join(parent_dir, 'train/tasks/semantic/', FLAGS.data_cfg) - print("Opening data config file %s" % FLAGS.data_cfg_1) - DATA = yaml.safe_load(open(FLAGS.data_cfg_1, 'r')) - except Exception as e: - print(e) - print("Error opening data yaml file.") - quit() - - # get number of interest classes, and the label mappings - class_strings = DATA["labels"] - class_remap = DATA["learning_map"] - class_inv_remap = DATA["learning_map_inv"] - class_ignore = DATA["learning_ignore"] - nr_classes = len(class_inv_remap) - - # make lookup table for mapping - maxkey = 0 - for key, data in class_remap.items(): - if key > maxkey: - maxkey = key - # +100 hack making lut bigger just in case there are unknown labels - remap_lut = np.zeros((maxkey + 100), dtype=np.int32) - for key, data in class_remap.items(): - try: - remap_lut[key] = data - except IndexError: - print("Wrong key ", key) - # print(remap_lut) - - # create evaluator - ignore = [] - for cl, ign in class_ignore.items(): - if ign: - x_cl = int(cl) - ignore.append(x_cl) - print("Ignoring xentropy class ", x_cl, " in IoU evaluation") - - # create evaluator - device = torch.device("cpu") - evaluator = iouEval(nr_classes, device, ignore) - evaluator.reset() - - # get test set - if FLAGS.split is None: - for splits in ('train','valid'): - mIoU = eval((DATA["split"][splits]),splits,FLAGS.predictions,remap_lut,evaluator, class_strings, class_inv_remap,ignore) - else: - mIoU = eval(DATA["split"][FLAGS.split],splits,FLAGS.predictions,remap_lut,evaluator, class_strings, class_inv_remap,ignore) - - return mIoU - -""" -Main function to evaluate the model, including two steps: -1st: make the inferene, and save the prediction. -2nd: load prediction, and further make the final evaluation. -""" -def eval_func(temp_model, FLAGS): - temp_model.eval() - infer_main(FLAGS, temp_model) - mIoU = evaluate_main(FLAGS) - return mIoU - - -""" -The function to output the salsaNext FP32 model, and the related configuration of the dataset. -""" -def build_FP32_model(FLAGS): - try: - print("Opening arch config file from %s" % FLAGS.model) - ARCH = yaml.safe_load(open(os.path.join(FLAGS.model, "arch_cfg.yaml"), 'r')) - except Exception as e: - print(e) - print("Error opening arch yaml file.") - quit() - - # open data config file - try: - print("Opening data config file from %s" % FLAGS.model) - DATA = yaml.safe_load(open(os.path.join(FLAGS.model, "data_cfg.yaml"), 'r')) - except Exception as e: - print(e) - print("Error opening data yaml file.") - quit() - - parserModule = imp.load_source("parserModule", - parent_dir+ '/train' + '/tasks/semantic/dataset/' + - DATA["name"] + '/parser.py') - parser = parserModule.Parser(root=FLAGS.dataset, - train_sequences=DATA["split"]["train"], - valid_sequences=DATA["split"]["valid"], - test_sequences=DATA["split"]["test"], - labels=DATA["labels"], - color_map=DATA["color_map"], - learning_map=DATA["learning_map"], - learning_map_inv=DATA["learning_map_inv"], - sensor=ARCH["dataset"]["sensor"], - max_points=ARCH["dataset"]["max_points"], - batch_size=1, - workers=ARCH["train"]["workers"], - gt=True, - shuffle_train=False) - - - temp_model = SalsaNext(parser.get_n_classes()) - w_dict = torch.load(os.path.join(FLAGS.model, "SalsaNext"), - map_location=lambda storage, loc: storage) - s_dict=w_dict['state_dict'] - from collections import OrderedDict - new_dict=OrderedDict() - for key,values in s_dict.items(): - key=key[7:] - # print(key) - new_dict[key]=values - temp_model.load_state_dict(new_dict, strict=True) - - return temp_model, parser - -""" -parameters configuration for AIMET. -""" -class ModelConfig(): - def __init__(self, FLAGS): - self.input_shape = (1, 5, 64, 2048) - self.config_file = 'htp_quantsim_config_pt_pertensor.json' - self.param_bw = 8 - self.output_bw = 8 - for arg in vars(FLAGS): - setattr(self, arg, getattr(FLAGS, arg)) - -""" -The simplified forward function for model compute_encoding in AIMET. -""" -def forward_func(model,cal_dataloader): - iterations = 0 - with torch.no_grad(): - idx = 0 - for i, (proj_in, proj_mask, _, _, path_seq, path_name, p_x, p_y, proj_range, unproj_range, _, _, _, _, npoints) in enumerate(cal_dataloader): - - if i > 20: - print(i) - proj_output = model(proj_in.cuda()) - idx += 1 - if idx > iterations: - break - else: - continue - - return 0.5 - - -""" -The main function. -""" -def main(FLAGS): - seed(1234) - - """ - create the FP32 model, and futher verify the baseline FP32 performance. - """ - - # build the original FP32 model - print("build the salsaNext model baseline, FP32") - temp_model_FP32, parser = build_FP32_model(FLAGS) - print("evaluate the FP32 performance") - mIoU_FP32 = eval_func(temp_model_FP32, FLAGS) - - """ - Make the basic W8A8 PTQ, - including the model validation/pre and folding. - """ - - # Quant configuration - config = ModelConfig(FLAGS) - size_data = torch.rand(config.input_shape) - kwargs = { - 'quant_scheme': QuantScheme.post_training_percentile, - 'default_param_bw': config.param_bw, - 'default_output_bw': config.output_bw, - 'config_file': os.path.join(FLAGS.model, config.config_file), - 'dummy_input': size_data.cuda() - } - - print("make the validation and model-preparing") - ModelValidator.validate_model(temp_model_FP32.cuda(), model_input=size_data.cuda()) - temp_model_FP32 = prepare_model(temp_model_FP32.eval()) - ModelValidator.validate_model(temp_model_FP32.cuda(), model_input=size_data.cuda()) - - print("make the norm folding") - batch_norm_fold.fold_all_batch_norms(temp_model_FP32, config.input_shape) - - print("W8A8 PTQ quantization") - sim = QuantizationSimModel(temp_model_FP32.cuda(), **kwargs) - cal_dataloader = parser.get_train_set() - sim.set_percentile_value(99.9) - sim.compute_encodings(forward_pass_callback=forward_func, forward_pass_callback_args=cal_dataloader) - temp_model = sim.model - mIoU_INT8 = eval_func(temp_model, FLAGS) - - """ - Load the encoding file of the optimized W8A8 model. - """ - model_reload = torch.load(os.path.join(FLAGS.model, 'SalsaNext_optimized_model.pth')) - sim_reload = QuantizationSimModel(model_reload.cuda(), **kwargs) - load_encodings_to_sim(sim_reload, os.path.join(FLAGS.model, 'SalsaNext_optimized_encoding.encodings')) - mIoU_INT8_pre_encoding = eval_func(sim_reload.model.eval(), FLAGS) - - """ - Print the evaluation results, - including: baseline FP32, W8A8 (PTQ), the optimized W8A8 checkpoint. - """ - print(f'Original Model | 32-bit Environment | mIoU: {mIoU_FP32:.3f}') - print(f'Original Model | 8-bit Environment | mIoU: {mIoU_INT8:.3f}') - print(f'Optimized Model, load encoding | 8-bit Environment | mIoU: {mIoU_INT8_pre_encoding:.3f}') - -if __name__ == '__main__': - - FLAGS = arguments() - download_weights(FLAGS) - main(FLAGS) - diff --git a/aimet_zoo_torch/salsaNext/salsaNext.md b/aimet_zoo_torch/salsaNext/salsaNext.md deleted file mode 100644 index 0282763..0000000 --- a/aimet_zoo_torch/salsaNext/salsaNext.md +++ /dev/null @@ -1,171 +0,0 @@ -# Pytorch SalsaNext for LiDAR semantic segmentation - -## Setup AI Model Efficiency Toolkit (AIMET) -Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.24/packaging/install.md) before proceeding further. -This model was tested with the `torch_gpu` variant of AIMET 1.25.0 - -## Experiment setup -- If `aimet-model-zoo` source code is not ready, clone the [aimet-model-zoo](https://github.com/quic/aimet-model-zoo.git) repo -```bash - git clone https://github.com/quic/aimet-model-zoo.git - cd ./aimet-model-zoo/aimet_zoo_torch/salsaNext/evaluators -``` - -- Clone the [salsaNext](https://github.com/TiagoCortinhal/SalsaNext.git) repo -```bash - git clone --recursive https://github.com/TiagoCortinhal/SalsaNext.git - cd SalsaNext -``` - -- Create folders for log/output and pretrained model -```bash - mkdir -p logs/sequences/08 - mkdir pretrained -``` - -- Prepare the AIMET code in salsaNext -```bash - cp ../evaluation_func.py ./ - cp ../salsaNext_quanteval.py ./ -``` - -- Update the salsaNext source code to adapt the AIMET quantization -```bash - cp ../../models/SalsaNext.py ./train/tasks/semantic/modules/ -``` - -## Model checkpoints and configuration -- Downloading checkpoints is handled through evaluation script. Configuration is set to default by evaluation script. -- The model checkpoints can be downloaded from - - FP32 checkpoint `SalsaNext` [FP32](https://drive.google.com/file/d/10fxIwPK10UVVB9jsgXDZSDwj4vy9MyTl/view). - - The corresponding `arch_cfg.yaml` and `data_cfg.yaml` are also downloaded from the above link. - - Quantized files W8A8 `SalsaNext_optimized_model.pth` and `SalsaNext_optimized_encoding.encodings` [INT8](https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/tag/torch_salsanext_models). -- The Quantization Simulation (*Quantsim*) Configuration file `htp_quantsim_config_pt_pertensor.json` can be downloaded from [JSON](https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/download/torch_salsanext_models/htp_quantsim_config_pt_pertensor.json). (Please see [this page](https://quic.github.io/aimet-pages/releases/1.21.0/user_guide/quantization_configuration.html) for more information on this file). -- These files should be arranged in the `./pretrained` folder with the following way -``` - /SalsaNext/pretrained/ - ├── SalsaNext - ├── SalsaNext_optimized_encoding.encodings - ├── SalsaNext_optimized_model.pth - ├── arch_cfg.yaml - ├── data_cfg.yaml - ├── htp_quantsim_config_pt_pertensor.json -``` - -## Dataset -- Semantic-kitti dataset can be downloaded from here: - - (http://semantic-kitti.org/tasks.html#semseg) - -- Downloaded datasets should be arranged in one directory - - The should be arranged in the following way -``` - /sequences/ - ├── 00 - │ ├── labels/ - │ ├── velodyne/ - │ ├── calib.txt - │ ├── poses.txt - │ ├── times.txt - ├── 01 - │ ├── labels/ - │ ├── velodyne/ - │ ├── calib.txt - │ ├── poses.txt - │ ├── times.txt -``` - -## Usage -- To run evaluation with QuantSim in AIMET, use the following -```bash -python salsaNext_quanteval.py \ - --dataset \ - --log \ - --model -``` -- One example -```bash -python salsaNext_quanteval.py --dataset --log ./logs/ --model ./pretrained/ -``` -## Quantization configuration -- Weight quantization: 8 bits -- Activation quantization: 8 bits -- PTQ techniques: - - Firstly, apply batch_norm_fold API to make the folding, by `batch_norm_fold.fold_all_batch_norms` - - Secondly, apply the Adaround API to optimize the weight, by `AdaroundParameters(*)` and `Adaround.apply_adaround(*)` - - Finally, set the percentile (99.9%) as the quant scheme, by `sim.set_percentile_value(99.9)` -- The checkpoint is with 3 activation output with 16 bitwidth with QAT. - - `sim.model.downCntx.conv1.input_quantizers[0].bitwidth = 16` - - `sim.model.resBlock1.pool.output_quantizers[0].bitwidth = 16` - - `sim.model.module_softmax.output_quantizers[0].bitwidth = 16` - -## Results - - - - - - - - - - - - - - - -
ModelIoU avgAcc avg
FP32 - INT8 - FP32 - INT8 -
SalsaNext0.5580.5490.8790.874
- -## FP32 results -``` -Acc avg 0.879 -IoU avg 0.558 -IoU class 1 [car] = 0.862 -IoU class 2 [bicycle] = 0.394 -IoU class 3 [motorcycle] = 0.420 -IoU class 4 [truck] = 0.777 -IoU class 5 [other-vehicle] = 0.420 -IoU class 6 [person] = 0.621 -IoU class 7 [bicyclist] = 0.683 -IoU class 8 [motorcyclist] = 0.000 -IoU class 9 [road] = 0.943 -IoU class 10 [parking] = 0.422 -IoU class 11 [sidewalk] = 0.800 -IoU class 12 [other-ground] = 0.041 -IoU class 13 [building] = 0.800 -IoU class 14 [fence] = 0.484 -IoU class 15 [vegetation] = 0.803 -IoU class 16 [trunk] = 0.579 -IoU class 17 [terrain] = 0.642 -IoU class 18 [pole] = 0.466 -IoU class 19 [traffic-sign] = 0.445 -``` - -## W8A8 results -``` -Acc avg 0.874 -IoU avg 0.549 -IoU class 1 [car] = 0.863 -IoU class 2 [bicycle] = 0.372 -IoU class 3 [motorcycle] = 0.426 -IoU class 4 [truck] = 0.780 -IoU class 5 [other-vehicle] = 0.447 -IoU class 6 [person] = 0.600 -IoU class 7 [bicyclist] = 0.663 -IoU class 8 [motorcyclist] = 0.000 -IoU class 9 [road] = 0.936 -IoU class 10 [parking] = 0.395 -IoU class 11 [sidewalk] = 0.789 -IoU class 12 [other-ground] = 0.035 -IoU class 13 [building] = 0.791 -IoU class 14 [fence] = 0.444 -IoU class 15 [vegetation] = 0.796 -IoU class 16 [trunk] = 0.562 -IoU class 17 [terrain] = 0.647 -IoU class 18 [pole] = 0.457 -IoU class 19 [traffic-sign] = 0.420 -``` diff --git a/aimet_zoo_torch/salsanext/SalsaNext.md b/aimet_zoo_torch/salsanext/SalsaNext.md new file mode 100644 index 0000000..cc3c37f --- /dev/null +++ b/aimet_zoo_torch/salsanext/SalsaNext.md @@ -0,0 +1,206 @@ +# Pytorch SalsaNext for LiDAR semantic segmentation + +## Environment Setup +### Setup AI Model Efficiency Toolkit (AIMET) +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.25/packaging/install.md) before proceeding further. +This model was tested with the `torch_gpu` variant of AIMET 1.25.0 + +### Add AIMET Model Zoo to Pythonpath +- If `aimet-model-zoo` source code is not ready, clone the [aimet-model-zoo](https://github.com/quic/aimet-model-zoo.git) repo +```bash + git clone https://github.com/quic/aimet-model-zoo.git + export PYTHONPATH=$PYTHONPATH: +``` + +### Dataset +- Semantic-kitti dataset can be downloaded from here: + - (http://semantic-kitti.org/tasks.html#semseg) + +- Downloaded datasets should be arranged in one directory + - The should be arranged in the following way +``` + /sequences/ + ├── 00 + │ ├── labels/ + │ ├── velodyne/ + │ ├── calib.txt + │ ├── poses.txt + │ ├── times.txt + ├── 01 + │ ├── labels/ + │ ├── velodyne/ + │ ├── calib.txt + │ ├── poses.txt + │ ├── times.txt +``` + +--- + +## Usage +- To run evaluation with QuantSim in AIMET, use the following +```bash +python salsanext_quanteval.py \ + --model-config + --dataset-path \ +``` +- Example +```bash +python salsaNext_quanteval.py --model-config salsanext_w8a8 --dataset-path +``` +Or +```bash +python salsaNext_quanteval.py --model-config salsanext_w4a8 --dataset-path +``` + +Available model configurations are: +- salsanext_w8a8 +- salsanext_w4a8 + +--- + +## Model checkpoints and configuration +- Downloading checkpoints is handled through evaluation script. Configuration is set to default by evaluation script. +- The model checkpoints can be downloaded from + - FP32 checkpoint `SalsaNext` [FP32](https://drive.google.com/file/d/10fxIwPK10UVVB9jsgXDZSDwj4vy9MyTl/view). + - Quantized files W8A8 `SalsaNext_optimized_model.pth` and `SalsaNext_optimized_encoding.encodings` [W8A8](https://github.com/quic/aimet-model-zoo/releases/tag/torch_salsanext_models). + - Quantized files W4A8 `SalsaNext_optimized_w4A8_model.pth` and `SalsaNext_optimized_w4A8_encoding.encodings` [W4A8](https://github.com/quic/aimet-model-zoo/releases/tag/torch_salsanext_models). +- The Quantization Simulation (*Quantsim*) Configuration file `default_config.json` can be downloaded from [W4A8_JSON](https://raw.githubusercontent.com/quic/aimet/release-aimet-1.25/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json) and [W8A8_JSON](https://raw.githubusercontent.com/quic/aimet/develop/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json). (Please see [this page](https://quic.github.io/aimet-pages/releases/1.21.0/user_guide/quantization_configuration.html) for more information on this file). + +--- + +## Quantization configuration for W8A8 +- Weight quantization: 8 bits +- Activation quantization: 8 bits +- PTQ techniques: + - Firstly, apply batch_norm_fold API to make the folding, by `batch_norm_fold.fold_all_batch_norms` + - Secondly, apply the Adaround API to optimize the weight, by `AdaroundParameters(*)` and `Adaround.apply_adaround(*)` + - Finally, set the percentile (99.9%) as the quant scheme, by `sim.set_percentile_value(99.9)` +- The checkpoint is with 3 activation output with 16 bitwidth with QAT. + - `sim.model.downCntx.conv1.input_quantizers[0].bitwidth = 16` + - `sim.model.resBlock1.pool.output_quantizers[0].bitwidth = 16` + - `sim.model.module_softmax.output_quantizers[0].bitwidth = 16` + +## Quantization configuration for W4A8 +- Weight quantization: 4 bits +- Activation quantization: 8 bits +- PTQ techniques: + - Set the QuantScheme with `QuantScheme.training_range_learning_with_tf_init` + - Apply batch_norm_fold API to make the folding, by `batch_norm_fold.fold_all_batch_norms` +- The checkpoint is with 9 activation output with 16 bitwidth with QAT. + - `sim.model.downCntx.conv3.output_quantizers[0].bitwidth = 16` + - `sim.model.downCntx.conv2.output_quantizers[0].bitwidth = 16` + - `sim.model.downCntx.conv1.output_quantizers[0].bitwidth = 16` + - `sim.model.downCntx.conv1.input_quantizers[0].bitwidth = 16` + - `sim.model.downCntx.act2.output_quantizers[0].bitwidth = 16` + - `sim.model.downCntx2.act2.output_quantizers[0].bitwidth = 16` + - `sim.model.downCntx3.act2.output_quantizers[0].bitwidth = 16` + - `sim.model.downCntx.act1.output_quantizers[0].bitwidth = 16` + - `sim.model.module_softmax.output_quantizers[0].bitwidth = 16` +- The checkpoint is with 6 weight with 8 bitwidth with QAT. + - `sim.model.downCntx.conv1.param_quantizers['weight'].bitwidth = 8` + - `sim.model.downCntx.conv3.param_quantizers['weight'].bitwidth = 8` + - `sim.model.downCntx2.conv3.param_quantizers['weight'].bitwidth = 8` + - `sim.model.downCntx3.conv3.param_quantizers['weight'].bitwidth = 8` + - `sim.model.downCntx.conv2.param_quantizers['weight'].bitwidth = 8` + - `sim.model.downCntx.bn2.param_quantizers['weight'].bitwidth = 8` +--- + +## Results + + + + + + + + + + + + + + + + + +
ModelIoU avgAcc avg
FP32 + W8A8 + W4A8 + FP32 + W8A8 + W4A8 +
SalsaNext0.5580.5490.5510.8790.8740.876
+ +## FP32 results +``` +Acc avg 0.879 +IoU avg 0.558 +IoU class 1 [car] = 0.862 +IoU class 2 [bicycle] = 0.394 +IoU class 3 [motorcycle] = 0.420 +IoU class 4 [truck] = 0.777 +IoU class 5 [other-vehicle] = 0.420 +IoU class 6 [person] = 0.621 +IoU class 7 [bicyclist] = 0.683 +IoU class 8 [motorcyclist] = 0.000 +IoU class 9 [road] = 0.943 +IoU class 10 [parking] = 0.422 +IoU class 11 [sidewalk] = 0.800 +IoU class 12 [other-ground] = 0.041 +IoU class 13 [building] = 0.800 +IoU class 14 [fence] = 0.484 +IoU class 15 [vegetation] = 0.803 +IoU class 16 [trunk] = 0.579 +IoU class 17 [terrain] = 0.642 +IoU class 18 [pole] = 0.466 +IoU class 19 [traffic-sign] = 0.445 +``` + +## W8A8 results +``` +Acc avg 0.874 +IoU avg 0.549 +IoU class 1 [car] = 0.863 +IoU class 2 [bicycle] = 0.372 +IoU class 3 [motorcycle] = 0.426 +IoU class 4 [truck] = 0.780 +IoU class 5 [other-vehicle] = 0.447 +IoU class 6 [person] = 0.600 +IoU class 7 [bicyclist] = 0.663 +IoU class 8 [motorcyclist] = 0.000 +IoU class 9 [road] = 0.936 +IoU class 10 [parking] = 0.395 +IoU class 11 [sidewalk] = 0.789 +IoU class 12 [other-ground] = 0.035 +IoU class 13 [building] = 0.791 +IoU class 14 [fence] = 0.444 +IoU class 15 [vegetation] = 0.796 +IoU class 16 [trunk] = 0.562 +IoU class 17 [terrain] = 0.647 +IoU class 18 [pole] = 0.457 +IoU class 19 [traffic-sign] = 0.420 +``` + +## W4A8 results +``` +Acc avg 0.876 +IoU avg 0.551 +IoU class 1 [car] = 0.863 +IoU class 2 [bicycle] = 0.381 +IoU class 3 [motorcycle] = 0.437 +IoU class 4 [truck] = 0.770 +IoU class 5 [other-vehicle] = 0.428 +IoU class 6 [person] = 0.606 +IoU class 7 [bicyclist] = 0.670 +IoU class 8 [motorcyclist] = 0.000 +IoU class 9 [road] = 0.937 +IoU class 10 [parking] = 0.378 +IoU class 11 [sidewalk] = 0.788 +IoU class 12 [other-ground] = 0.053 +IoU class 13 [building] = 0.798 +IoU class 14 [fence] = 0.467 +IoU class 15 [vegetation] = 0.800 +IoU class 16 [trunk] = 0.567 +IoU class 17 [terrain] = 0.646 +IoU class 18 [pole] = 0.456 +IoU class 19 [traffic-sign] = 0.428 diff --git a/aimet_zoo_torch/salsanext/__init__.py b/aimet_zoo_torch/salsanext/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/salsanext/dataloader/sematickitti_parser.py b/aimet_zoo_torch/salsanext/dataloader/sematickitti_parser.py new file mode 100644 index 0000000..e756c3a --- /dev/null +++ b/aimet_zoo_torch/salsanext/dataloader/sematickitti_parser.py @@ -0,0 +1,452 @@ +# pylint: skip-file + + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import os +import numpy as np +import torch +from torch.utils.data import Dataset +from aimet_zoo_torch.salsanext.models.common.laserscan import LaserScan, SemLaserScan +import torchvision + +import torch +import math +import random +from PIL import Image +try: + import accimage +except ImportError: + accimage = None +import numpy as np +import numbers +import types +from collections.abc import Sequence, Iterable +import warnings + + +EXTENSIONS_SCAN = ['.bin'] +EXTENSIONS_LABEL = ['.label'] + + +def is_scan(filename): + return any(filename.endswith(ext) for ext in EXTENSIONS_SCAN) + + +def is_label(filename): + return any(filename.endswith(ext) for ext in EXTENSIONS_LABEL) + + +def my_collate(batch): + data = [item[0] for item in batch] + project_mask = [item[1] for item in batch] + proj_labels = [item[2] for item in batch] + data = torch.stack(data,dim=0) + project_mask = torch.stack(project_mask,dim=0) + proj_labels = torch.stack(proj_labels, dim=0) + + to_augment =(proj_labels == 12).nonzero() + to_augment_unique_12 = torch.unique(to_augment[:, 0]) + + to_augment = (proj_labels == 5).nonzero() + to_augment_unique_5 = torch.unique(to_augment[:, 0]) + + to_augment = (proj_labels == 8).nonzero() + to_augment_unique_8 = torch.unique(to_augment[:, 0]) + + to_augment_unique = torch.cat((to_augment_unique_5,to_augment_unique_8,to_augment_unique_12),dim=0) + to_augment_unique = torch.unique(to_augment_unique) + + for k in to_augment_unique: + data = torch.cat((data,torch.flip(data[k.item()], [2]).unsqueeze(0)),dim=0) + proj_labels = torch.cat((proj_labels,torch.flip(proj_labels[k.item()], [1]).unsqueeze(0)),dim=0) + project_mask = torch.cat((project_mask,torch.flip(project_mask[k.item()], [1]).unsqueeze(0)),dim=0) + + return data, project_mask,proj_labels + +class SemanticKitti(Dataset): + + def __init__(self, root, # directory where data is + sequences, # sequences for this data (e.g. [1,3,4,6]) + labels, # label dict: (e.g 10: "car") + color_map, # colors dict bgr (e.g 10: [255, 0, 0]) + learning_map, # classes to learn (0 to N-1 for xentropy) + learning_map_inv, # inverse of previous (recover labels) + sensor, # sensor to parse scans from + max_points=150000, # max number of points present in dataset + gt=True, + transform=False): # send ground truth? + # save deats + self.root = os.path.join(root, "sequences") + self.sequences = sequences + self.labels = labels + self.color_map = color_map + self.learning_map = learning_map + self.learning_map_inv = learning_map_inv + self.sensor = sensor + self.sensor_img_H = sensor["img_prop"]["height"] + self.sensor_img_W = sensor["img_prop"]["width"] + self.sensor_img_means = torch.tensor(sensor["img_means"], + dtype=torch.float) + self.sensor_img_stds = torch.tensor(sensor["img_stds"], + dtype=torch.float) + self.sensor_fov_up = sensor["fov_up"] + self.sensor_fov_down = sensor["fov_down"] + self.max_points = max_points + self.gt = gt + self.transform = transform + + # get number of classes (can't be len(self.learning_map) because there + # are multiple repeated entries, so the number that matters is how many + # there are for the xentropy) + self.nclasses = len(self.learning_map_inv) + + # sanity checks + + # make sure directory exists + if os.path.isdir(self.root): + print("Sequences folder exists! Using sequences from %s" % self.root) + else: + raise ValueError("Sequences folder doesn't exist! Exiting...") + + # make sure labels is a dict + assert(isinstance(self.labels, dict)) + + # make sure color_map is a dict + assert(isinstance(self.color_map, dict)) + + # make sure learning_map is a dict + assert(isinstance(self.learning_map, dict)) + + # make sure sequences is a list + assert(isinstance(self.sequences, list)) + + # placeholder for filenames + self.scan_files = [] + self.label_files = [] + + # fill in with names, checking that all sequences are complete + for seq in self.sequences: + # to string + seq = '{0:02d}'.format(int(seq)) + + print("parsing seq {}".format(seq)) + + # get paths for each + scan_path = os.path.join(self.root, seq, "velodyne") + label_path = os.path.join(self.root, seq, "labels") + + # get files + scan_files = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(scan_path)) for f in fn if is_scan(f)] + label_files = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(label_path)) for f in fn if is_label(f)] + + # check all scans have labels + if self.gt: + assert(len(scan_files) == len(label_files)) + + # extend list + self.scan_files.extend(scan_files) + self.label_files.extend(label_files) + + # sort for correspondance + self.scan_files.sort() + self.label_files.sort() + + print("Using {} scans from sequences {}".format(len(self.scan_files), + self.sequences)) + + def __getitem__(self, index): + # get item in tensor shape + scan_file = self.scan_files[index] + if self.gt: + label_file = self.label_files[index] + + # open a semantic laserscan + DA = False + flip_sign = False + rot = False + drop_points = False + if self.transform: + if random.random() > 0.5: + if random.random() > 0.5: + DA = True + if random.random() > 0.5: + flip_sign = True + if random.random() > 0.5: + rot = True + drop_points = random.uniform(0, 0.5) + + if self.gt: + scan = SemLaserScan(self.color_map, + project=True, + H=self.sensor_img_H, + W=self.sensor_img_W, + fov_up=self.sensor_fov_up, + fov_down=self.sensor_fov_down, + DA=DA, + flip_sign=flip_sign, + drop_points=drop_points) + else: + scan = LaserScan(project=True, + H=self.sensor_img_H, + W=self.sensor_img_W, + fov_up=self.sensor_fov_up, + fov_down=self.sensor_fov_down, + DA=DA, + rot=rot, + flip_sign=flip_sign, + drop_points=drop_points) + + # open and obtain scan + scan.open_scan(scan_file) + if self.gt: + scan.open_label(label_file) + # map unused classes to used classes (also for projection) + scan.sem_label = self.map(scan.sem_label, self.learning_map) + scan.proj_sem_label = self.map(scan.proj_sem_label, self.learning_map) + + # make a tensor of the uncompressed data (with the max num points) + unproj_n_points = scan.points.shape[0] + unproj_xyz = torch.full((self.max_points, 3), -1.0, dtype=torch.float) + unproj_xyz[:unproj_n_points] = torch.from_numpy(scan.points) + unproj_range = torch.full([self.max_points], -1.0, dtype=torch.float) + unproj_range[:unproj_n_points] = torch.from_numpy(scan.unproj_range) + unproj_remissions = torch.full([self.max_points], -1.0, dtype=torch.float) + unproj_remissions[:unproj_n_points] = torch.from_numpy(scan.remissions) + if self.gt: + unproj_labels = torch.full([self.max_points], -1.0, dtype=torch.int32) + unproj_labels[:unproj_n_points] = torch.from_numpy(scan.sem_label) + else: + unproj_labels = [] + + # get points and labels + proj_range = torch.from_numpy(scan.proj_range).clone() + proj_xyz = torch.from_numpy(scan.proj_xyz).clone() + proj_remission = torch.from_numpy(scan.proj_remission).clone() + proj_mask = torch.from_numpy(scan.proj_mask) + if self.gt: + proj_labels = torch.from_numpy(scan.proj_sem_label).clone() + proj_labels = proj_labels * proj_mask + else: + proj_labels = [] + proj_x = torch.full([self.max_points], -1, dtype=torch.long) + proj_x[:unproj_n_points] = torch.from_numpy(scan.proj_x) + proj_y = torch.full([self.max_points], -1, dtype=torch.long) + proj_y[:unproj_n_points] = torch.from_numpy(scan.proj_y) + proj = torch.cat([proj_range.unsqueeze(0).clone(), + proj_xyz.clone().permute(2, 0, 1), + proj_remission.unsqueeze(0).clone()]) + proj = (proj - self.sensor_img_means[:, None, None] + ) / self.sensor_img_stds[:, None, None] + proj = proj * proj_mask.float() + + # get name and sequence + path_norm = os.path.normpath(scan_file) + path_split = path_norm.split(os.sep) + path_seq = path_split[-3] + path_name = path_split[-1].replace(".bin", ".label") + + # return + return proj, proj_mask, proj_labels, unproj_labels, path_seq, path_name, proj_x, proj_y, proj_range, unproj_range, proj_xyz, unproj_xyz, proj_remission, unproj_remissions, unproj_n_points + + def __len__(self): + return len(self.scan_files) + + @staticmethod + def map(label, mapdict): + # put label from original values to xentropy + # or vice-versa, depending on dictionary values + # make learning map a lookup table + maxkey = 0 + for key, data in mapdict.items(): + if isinstance(data, list): + nel = len(data) + else: + nel = 1 + if key > maxkey: + maxkey = key + # +100 hack making lut bigger just in case there are unknown labels + if nel > 1: + lut = np.zeros((maxkey + 100, nel), dtype=np.int32) + else: + lut = np.zeros((maxkey + 100), dtype=np.int32) + for key, data in mapdict.items(): + try: + lut[key] = data + except IndexError: + print("Wrong key ", key) + # do the mapping + return lut[label] + + +class Parser(): + # standard conv, BN, relu + def __init__(self, + root, # directory for data + train_sequences, # sequences to train + valid_sequences, # sequences to validate. + test_sequences, # sequences to test (if none, don't get) + labels, # labels in data + color_map, # color for each label + learning_map, # mapping for training labels + learning_map_inv, # recover labels from xentropy + sensor, # sensor to use + max_points, # max points in each scan in entire dataset + batch_size, # batch size for train and val + workers, # threads to load data + gt=True, # get gt? + shuffle_train=True): # shuffle training set? + super(Parser, self).__init__() + + # if I am training, get the dataset + self.root = root + self.train_sequences = train_sequences + self.valid_sequences = valid_sequences + self.test_sequences = test_sequences + self.labels = labels + self.color_map = color_map + self.learning_map = learning_map + self.learning_map_inv = learning_map_inv + self.sensor = sensor + self.max_points = max_points + self.batch_size = batch_size + self.workers = workers + self.gt = gt + self.shuffle_train = shuffle_train + + # number of classes that matters is the one for xentropy + self.nclasses = len(self.learning_map_inv) + + # Data loading code + self.train_dataset = SemanticKitti(root=self.root, + sequences=self.train_sequences, + labels=self.labels, + color_map=self.color_map, + learning_map=self.learning_map, + learning_map_inv=self.learning_map_inv, + sensor=self.sensor, + max_points=max_points, + transform=True, + gt=self.gt) + + self.trainloader = torch.utils.data.DataLoader(self.train_dataset, + batch_size=self.batch_size, + shuffle=self.shuffle_train, + num_workers=self.workers, + drop_last=True) + assert len(self.trainloader) > 0 + self.trainiter = iter(self.trainloader) + + self.valid_dataset = SemanticKitti(root=self.root, + sequences=self.valid_sequences, + labels=self.labels, + color_map=self.color_map, + learning_map=self.learning_map, + learning_map_inv=self.learning_map_inv, + sensor=self.sensor, + max_points=max_points, + gt=self.gt) + + self.validloader = torch.utils.data.DataLoader(self.valid_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.workers, + drop_last=True) + assert len(self.validloader) > 0 + self.validiter = iter(self.validloader) + + if self.test_sequences: + self.test_dataset = SemanticKitti(root=self.root, + sequences=self.test_sequences, + labels=self.labels, + color_map=self.color_map, + learning_map=self.learning_map, + learning_map_inv=self.learning_map_inv, + sensor=self.sensor, + max_points=max_points, + gt=False) + + self.testloader = torch.utils.data.DataLoader(self.test_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.workers, + drop_last=True) + assert len(self.testloader) > 0 + self.testiter = iter(self.testloader) + + def get_train_batch(self): + scans = self.trainiter.next() + return scans + + def get_train_set(self): + return self.trainloader + + def get_valid_batch(self): + scans = self.validiter.next() + return scans + + def get_valid_set(self): + return self.validloader + + def get_test_batch(self): + scans = self.testiter.next() + return scans + + def get_test_set(self): + return self.testloader + + def get_train_size(self): + return len(self.trainloader) + + def get_valid_size(self): + return len(self.validloader) + + def get_test_size(self): + return len(self.testloader) + + def get_n_classes(self): + return self.nclasses + + def get_original_class_string(self, idx): + return self.labels[idx] + + def get_xentropy_class_string(self, idx): + return self.labels[self.learning_map_inv[idx]] + + def to_original(self, label): + # put label in original values + return SemanticKitti.map(label, self.learning_map_inv) + + def to_xentropy(self, label): + # put label in xentropy values + return SemanticKitti.map(label, self.learning_map) + + def to_color(self, label): + # put label in original values + label = SemanticKitti.map(label, self.learning_map_inv) + # put label in color + return SemanticKitti.map(label, self.color_map) \ No newline at end of file diff --git a/aimet_zoo_torch/salsanext/evaluators/__init__.py b/aimet_zoo_torch/salsanext/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/salsanext/evaluators/evaluation_func.py b/aimet_zoo_torch/salsanext/evaluators/evaluation_func.py new file mode 100755 index 0000000..511d11f --- /dev/null +++ b/aimet_zoo_torch/salsanext/evaluators/evaluation_func.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R1732,R0902,R0913,C0303,C0330 + + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + + + +""" +This script applies and evaluates a pre-trained salsaNext model taken from +https://github.com/TiagoCortinhal/SalsaNext. +Such model is for the semantic segmentation task with the metric (mIoU) and semantic-kitti dataset. +For quantization instructions, please refer to zoo_torch/salsanext/SalsaNext.md +""" + +import argparse +import os +import time +import torch +from torch.backends import cudnn +import numpy as np +from tqdm import tqdm +from aimet_zoo_torch.salsanext.models.tasks.semantic.dataset.kitti import ( + parser as parserModule, +) +from aimet_zoo_torch.salsanext.models.tasks.semantic.postproc.KNN import KNN + + +def str2bool(v): + """ convert string to boolean """ + if isinstance(v, bool): + return v + if v.lower() in ("yes", "true", "t", "y"): + return True + if v.lower() in ("no", "false", "f", "n"): + return False + raise argparse.ArgumentTypeError("Boolean expected") + + +def save_to_log(logdir, logfile, message): + """ save input message to logs """ + f = open(logdir + "/" + logfile, "a", encoding="utf8") + f.write(message + "\n") + f.close() + + +class User: + """ User class for running evaluation """ + def __init__( + self, + ARCH, + DATA, + datadir, + logdir, + modeldir, + split, + uncertainty, + mc=30, + model_given=None, + ): + # parameters + self.ARCH = ARCH + self.DATA = DATA + self.datadir = datadir + self.logdir = logdir + self.modeldir = modeldir + self.uncertainty = uncertainty + self.split = split + self.mc = mc + + # get the data + # parserModule = imp.load_source("parserModule", + # os.getcwd()+ '/train' + '/tasks/semantic/dataset/' + + # self.DATA["name"] + '/parser.py') + self.parser = parserModule.Parser( + root=self.datadir, + train_sequences=self.DATA["split"]["train"], + valid_sequences=self.DATA["split"]["valid"], + test_sequences=self.DATA["split"]["test"], + labels=self.DATA["labels"], + color_map=self.DATA["color_map"], + learning_map=self.DATA["learning_map"], + learning_map_inv=self.DATA["learning_map_inv"], + sensor=self.ARCH["dataset"]["sensor"], + max_points=self.ARCH["dataset"]["max_points"], + batch_size=1, + workers=self.ARCH["train"]["workers"], + gt=True, + shuffle_train=False, + ) + + # concatenate the encoder and the head + with torch.no_grad(): + torch.nn.Module.dump_patches = True + self.model = model_given + + # use knn post processing? + self.post = None + if self.ARCH["post"]["KNN"]["use"]: + self.post = KNN( + self.ARCH["post"]["KNN"]["params"], self.parser.get_n_classes() + ) + + # GPU? + self.gpu = False + self.model_single = self.model + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + # self.device = torch.device("cpu") + print("Infering on device: ", self.device) + if torch.cuda.is_available() and torch.cuda.device_count() > 0: + cudnn.benchmark = True + cudnn.fastest = True + self.gpu = True + self.model.cuda() + + def infer(self): + """ run inference """ + cnn = [] + knn = [] + if self.split is None: + self.infer_subset( + loader=self.parser.get_train_set(), + to_orig_fn=self.parser.to_original, + cnn=cnn, + knn=knn, + ) + + # do valid set + self.infer_subset( + loader=self.parser.get_valid_set(), + to_orig_fn=self.parser.to_original, + cnn=cnn, + knn=knn, + ) + # do test set + self.infer_subset( + loader=self.parser.get_test_set(), + to_orig_fn=self.parser.to_original, + cnn=cnn, + knn=knn, + ) + + elif self.split == "valid": + self.infer_subset( + loader=self.parser.get_valid_set(), + to_orig_fn=self.parser.to_original, + cnn=cnn, + knn=knn, + ) + elif self.split == "train": + self.infer_subset( + loader=self.parser.get_train_set(), + to_orig_fn=self.parser.to_original, + cnn=cnn, + knn=knn, + ) + else: + self.infer_subset( + loader=self.parser.get_test_set(), + to_orig_fn=self.parser.to_original, + cnn=cnn, + knn=knn, + ) + print(f"Mean CNN inference time:{np.mean(cnn)}\t std:{np.std(cnn)}") + print(f"Mean KNN inference time:{np.mean(knn)}\t std:{np.std(knn)}") + print(f"Total Frames:{len(cnn)}") + print("Finished Infering") + + + def infer_subset(self, loader, to_orig_fn, cnn, knn): + """ infer on a subset of the data """ + # parser = argparse.ArgumentParser("./user.py") + # FLAGS, unparsed = parser.parse_known_args() + # switch to evaluate mode + if not self.uncertainty: + self.model.eval() + # empty the cache to infer in high res + if self.gpu: + torch.cuda.empty_cache() + + total_time = 0 + total_frames = 0 + + with torch.no_grad(): + end = time.time() + + for _, ( + proj_in, + _, + _, + _, + path_seq, + path_name, + p_x, + p_y, + proj_range, + unproj_range, + _, + _, + _, + _, + npoints, + ) in tqdm(enumerate(loader), total=len(loader)): + # first cut to rela size (batch size one allows it) + p_x = p_x[0, :npoints] + p_y = p_y[0, :npoints] + proj_range = proj_range[0, :npoints] + unproj_range = unproj_range[0, :npoints] + path_seq = path_seq[0] + path_name = path_name[0] + + if self.gpu: + proj_in = proj_in.cuda() + p_x = p_x.cuda() + p_y = p_y.cuda() + if self.post: + proj_range = proj_range.cuda() + unproj_range = unproj_range.cuda() + + # compute output + if self.uncertainty: + proj_output_r, log_var_r = self.model(proj_in) + for _ in range(self.mc): + log_var, proj_output = self.model(proj_in) + log_var_r = torch.cat((log_var, log_var_r)) + proj_output_r = torch.cat((proj_output, proj_output_r)) + + _, log_var2 = self.model(proj_in) + proj_output = proj_output_r.var(dim=0, keepdim=True).mean(dim=1) + log_var2 = log_var_r.mean(dim=0, keepdim=True).mean(dim=1) + if self.post: + # knn postproc + unproj_argmax = self.post( + proj_range, unproj_range, proj_argmax, p_x, p_y + ) + else: + # put in original pointcloud using indexes + unproj_argmax = proj_argmax[p_y, p_x] + + # measure elapsed time + if torch.cuda.is_available(): + torch.cuda.synchronize() + frame_time = time.time() - end + # print("Infered seq", path_seq, "scan", path_name, + # "in", frame_time, "sec") + total_time += frame_time + total_frames += 1 + end = time.time() + + # save scan + # get the first scan in batch and project scan + pred_np = unproj_argmax.cpu().numpy() + pred_np = pred_np.reshape((-1)).astype(np.int32) + + # log_var2 = log_var2[0][p_y, p_x] + # log_var2 = log_var2.cpu().numpy() + # log_var2 = log_var2.reshape((-1)).astype(np.float32) + + log_var2 = log_var2[0][p_y, p_x] + log_var2 = log_var2.cpu().numpy() + log_var2 = log_var2.reshape((-1)).astype(np.float32) + # assert proj_output.reshape((-1)).shape == log_var2.reshape((-1)).shape == pred_np.reshape((-1)).shape + + # map to original label + pred_np = to_orig_fn(pred_np) + + # save scan + path = os.path.join( + self.logdir, "sequences", path_seq, "predictions", path_name + ) + pred_np.tofile(path) + + path = os.path.join( + self.logdir, "sequences", path_seq, "log_var", path_name + ) + if not os.path.exists( + os.path.join(self.logdir, "sequences", path_seq, "log_var") + ): + os.makedirs( + os.path.join(self.logdir, "sequences", path_seq, "log_var") + ) + log_var2.tofile(path) + + proj_output = proj_output[0][p_y, p_x] + proj_output = proj_output.cpu().numpy() + proj_output = proj_output.reshape((-1)).astype(np.float32) + + path = os.path.join( + self.logdir, "sequences", path_seq, "uncert", path_name + ) + if not os.path.exists( + os.path.join(self.logdir, "sequences", path_seq, "uncert") + ): + os.makedirs( + os.path.join(self.logdir, "sequences", path_seq, "uncert") + ) + proj_output.tofile(path) + + print(total_time / total_frames) + else: + proj_output = self.model(proj_in) + + proj_argmax = proj_output[0].argmax(dim=0) + if torch.cuda.is_available(): + torch.cuda.synchronize() + res = time.time() - end + # print("Network seq", path_seq, "scan", path_name, + # "in", res, "sec") + end = time.time() + cnn.append(res) + + if torch.cuda.is_available(): + torch.cuda.synchronize() + res = time.time() - end + # print("Network seq", path_seq, "scan", path_name, + # "in", res, "sec") + end = time.time() + cnn.append(res) + + if self.post: + # knn postproc + unproj_argmax = self.post( + proj_range, unproj_range, proj_argmax, p_x, p_y + ) + else: + # put in original pointcloud using indexes + unproj_argmax = proj_argmax[p_y, p_x] + + # measure elapsed time + if torch.cuda.is_available(): + torch.cuda.synchronize() + res = time.time() - end + # print("KNN Infered seq", path_seq, "scan", path_name, + # "in", res, "sec") + knn.append(res) + end = time.time() + + # save scan + # get the first scan in batch and project scan + pred_np = unproj_argmax.cpu().numpy() + pred_np = pred_np.reshape((-1)).astype(np.int32) + + # map to original label + pred_np = to_orig_fn(pred_np) + + # save scan + path = os.path.join( + self.logdir, "sequences", path_seq, "predictions", path_name + ) + pred_np.tofile(path) diff --git a/aimet_zoo_torch/salsanext/evaluators/salsanext_quanteval.py b/aimet_zoo_torch/salsanext/evaluators/salsanext_quanteval.py new file mode 100755 index 0000000..6e45cfd --- /dev/null +++ b/aimet_zoo_torch/salsanext/evaluators/salsanext_quanteval.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R1732,C0209,W0612,C0412,C0303,C0330 +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" +This script applies and evaluates a pre-trained salsaNext model taken from +https://github.com/TiagoCortinhal/SalsaNext. +Such model is for the semantic segmentation task with the metric (mIoU) and semantic-kitti dataset. +For quantization instructions, please refer to zoo_torch/salsaNext/salsaNext.md +""" + +import os +import sys +import argparse +import shutil +import pathlib +import yaml +import numpy as np +from tqdm import tqdm +import torch + +from aimet_common.defs import QuantScheme +from aimet_torch.quantsim import QuantizationSimModel +from aimet_torch.model_validator.model_validator import ModelValidator +from aimet_torch.model_preparer import prepare_model +from aimet_torch import batch_norm_fold + +from aimet_zoo_torch.salsanext.models.tasks.semantic.modules.ioueval import iouEval +from aimet_zoo_torch.salsanext.models.common.laserscan import SemLaserScan +from aimet_zoo_torch.salsanext.models.tasks.semantic.dataset.kitti import ( + parser as parserModule, +) +from aimet_zoo_torch.salsanext.models.model_definition import SalsaNext as SalsaNext_MZ + +from evaluation_func import User, str2bool, save_to_log + + +parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) +grandparent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent.parent) +DATA = yaml.safe_load( + open(os.path.join(grandparent_dir, "models", "data_cfg.yaml"), "r", encoding="utf8") +) +ARCH = yaml.safe_load( + open(os.path.join(grandparent_dir, "models", "arch_cfg.yaml"), "r", encoding="utf8") +) +log_dir = os.path.join(parent_dir, "logs", "sequences") +os.makedirs(log_dir, mode=777, exist_ok=True) + + +# Set seed for reproducibility +def seed(seed_number): + """set all seeds""" + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = True + torch.manual_seed(seed_number) + torch.cuda.manual_seed(seed_number) + torch.cuda.manual_seed_all(seed_number) + + +def arguments(): + """ + Arguments to run the script, at least including: + --dataset, dataset folder + --model, pretrained model folder + --log, output log folder + + One example: + python salsaNext_quanteval.py --dataset /tf/semntic_kitti/datasets_segmentation/dataset --log ./logs --model ./pretrained + """ + parser = argparse.ArgumentParser("./salsaNext_quanteval.py") + + parser.add_argument( + "--model-config", + type=str, + required=False, + default="salsanext_w8a8", + choices=["salsanext_w8a8", "salsanext_w4a8"], + help="Dataset to train with. No Default", + ) + parser.add_argument( + "--dataset-path", + "-d", + type=str, + required=True, + help="Dataset to train with. No Default", + ) + parser.add_argument( + "--log", + "-l", + type=str, + required=False, + default="./logs", + help="Directory to put the predictions.", + ) + parser.add_argument( + "--model", + "-m", + type=str, + required=False, + default="./pretrained", + help="Directory to get the trained model.", + ) + parser.add_argument( + "--uncertainty", + "-u", + type=str2bool, + nargs="?", + const=True, + default=False, + help="Set this if you want to use the Uncertainty Version", + ) + parser.add_argument( + "--monte-carlo", "-c", type=int, default=30, help="Number of samplings per scan" + ) + parser.add_argument( + "--split", + "-s", + type=str, + required=False, + default="valid", + help="Split to evaluate on. One of " + + "train, valid, test" + + ". Defaults to %(default)s", + ) + parser.add_argument( + "--predictions", + "-p", + type=str, + required=False, + default=None, + help="Prediction dir. Same organization as dataset, but predictions in" + 'each sequences "prediction" directory. No Default. If no option is set' + " we look for the labels in the same directory as dataset", + ) + parser.add_argument( + "--data_cfg", + "-dc", + type=str, + required=False, + default="config/labels/semantic-kitti.yaml", + help="Dataset config file. Defaults to %(default)s", + ) + parser.add_argument( + "--limit", + "-li", + type=int, + required=False, + default=None, + help='Limit to the first "--limit" points of each scan. Useful for' + " evaluating single scan from aggregated pointcloud." + " Defaults to %(default)s", + ) + + FLAGS, _ = parser.parse_known_args() + + if FLAGS.predictions is None: + FLAGS.predictions = FLAGS.log + + print("input configuration") + print(FLAGS) + + return FLAGS + + +def infer_main(FLAGS, model_given): + """ + First step in eval_func() + Make the inference. Save the inference output. + """ + # print summary of what we will do + print("----------") + print("INTERFACE:") + print("dataset", FLAGS.dataset_path) + print("log", FLAGS.log) + print("model", FLAGS.model) + print("Uncertainty", FLAGS.uncertainty) + print("Monte Carlo Sampling", FLAGS.monte_carlo) + print("infering", FLAGS.split) + print("----------\n") + print("----------\n") + + # create log folder + try: + if os.path.isdir(FLAGS.log): + shutil.rmtree(FLAGS.log) + os.makedirs(FLAGS.log) + os.makedirs(os.path.join(FLAGS.log, "sequences")) + for seq in DATA["split"]["train"]: + seq = f"{int(seq):02d}" + print("train", seq) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) + for seq in DATA["split"]["valid"]: + seq = f"{int(seq):02d}" + print("valid", seq) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) + for seq in DATA["split"]["test"]: + seq = f"{int(seq):02d}" + print("test", seq) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) + except Exception as e: + print(e) + print("Error creating log directory. Check permissions!") + raise + + # does model folder exist? + os.makedirs(FLAGS.model, exist_ok=True) + + # create user and infer dataset + user = User( + ARCH, + DATA, + FLAGS.dataset_path, + FLAGS.log, + FLAGS.model, + FLAGS.split, + FLAGS.uncertainty, + FLAGS.monte_carlo, + model_given, + ) + user.infer() + + +def evaluate( + test_sequences, + splits, + pred, + remap_lut, + evaluator, + class_strings, + class_inv_remap, + ignore, +): + """run evaluation""" + # get scan paths + scan_names = [] + for sequence in test_sequences: + sequence = f"{int(sequence):02d}" + scan_paths = os.path.join( + FLAGS.dataset_path, "sequences", str(sequence), "velodyne" + ) + # populate the scan names + seq_scan_names = [ + os.path.join(dp, f) + for dp, dn, fn in os.walk(os.path.expanduser(scan_paths)) + for f in fn + if ".bin" in f + ] + seq_scan_names.sort() + scan_names.extend(seq_scan_names) + + # get label paths + label_names = [] + for sequence in test_sequences: + sequence = f"{int(sequence):02d}" + label_paths = os.path.join( + FLAGS.dataset_path, "sequences", str(sequence), "labels" + ) + # populate the label names + seq_label_names = [ + os.path.join(dp, f) + for dp, dn, fn in os.walk(os.path.expanduser(label_paths)) + for f in fn + if ".label" in f + ] + seq_label_names.sort() + label_names.extend(seq_label_names) + + # get predictions paths + pred_names = [] + for sequence in test_sequences: + sequence = f"{int(sequence):02d}" + pred_paths = os.path.join( + FLAGS.predictions, "sequences", sequence, "predictions" + ) + # populate the label names + seq_pred_names = [ + os.path.join(dp, f) + for dp, dn, fn in os.walk(os.path.expanduser(pred_paths)) + for f in fn + if ".label" in f + ] + seq_pred_names.sort() + pred_names.extend(seq_pred_names) + + assert len(label_names) == len(scan_names) and len(label_names) == len(pred_names) + + print("Evaluating sequences: ") + # open each file, get the tensor, and make the iou comparison + for scan_file, label_file, pred_file in tqdm( + zip(scan_names, label_names, pred_names), total=len(scan_names) + ): + # open label + label = SemLaserScan(project=False) + label.open_scan(scan_file) + label.open_label(label_file) + u_label_sem = remap_lut[label.sem_label] # remap to xentropy format + if FLAGS.limit is not None: + u_label_sem = u_label_sem[: FLAGS.limit] + + # open prediction + pred = SemLaserScan(project=False) + pred.open_scan(scan_file) + pred.open_label(pred_file) + u_pred_sem = remap_lut[pred.sem_label] # remap to xentropy format + if FLAGS.limit is not None: + u_pred_sem = u_pred_sem[: FLAGS.limit] + + # add single scan to evaluation + evaluator.addBatch(u_pred_sem, u_label_sem) + + # when I am done, print the evaluation + m_accuracy = evaluator.getacc() + m_jaccard, class_jaccard = evaluator.getIoU() + + print( + "{split} set:\n" + "Acc avg {m_accuracy:.3f}\n" + "IoU avg {m_jaccard:.3f}".format( + split=splits, m_accuracy=m_accuracy, m_jaccard=m_jaccard + ) + ) + + save_to_log( + FLAGS.predictions, + "pred.txt", + "{split} set:\n" + "Acc avg {m_accuracy:.3f}\n" + "IoU avg {m_jaccard:.3f}".format( + split=splits, m_accuracy=m_accuracy, m_jaccard=m_jaccard + ), + ) + # print also classwise + for i, jacc in enumerate(class_jaccard): + if i not in ignore: + print( + "IoU class {i:} [{class_str:}] = {jacc:.3f}".format( + i=i, class_str=class_strings[class_inv_remap[i]], jacc=jacc + ) + ) + save_to_log( + FLAGS.predictions, + "pred.txt", + "IoU class {i:} [{class_str:}] = {jacc:.3f}".format( + i=i, class_str=class_strings[class_inv_remap[i]], jacc=jacc + ), + ) + + # print for spreadsheet + print("*" * 80) + print("below is the final result") + for i, jacc in enumerate(class_jaccard): + if i not in ignore: + sys.stdout.write("{jacc:.3f}".format(jacc=jacc.item())) + sys.stdout.write(",") + sys.stdout.write("{jacc:.3f}".format(jacc=m_jaccard.item())) + sys.stdout.write(",") + sys.stdout.write("{acc:.3f}".format(acc=m_accuracy.item())) + sys.stdout.write("\n") + sys.stdout.flush() + + return m_jaccard.item() + + +def evaluate_main(FLAGS): + """ + second step in eval_func() + Function to evaluate the model, and output the results. + """ + splits = ["train", "valid", "test"] + # fill in real predictions dir + if FLAGS.predictions is None: + FLAGS.predictions = FLAGS.dataset_path + + # print summary of what we will do + print("*" * 80) + print("INTERFACE:") + print("Data: ", FLAGS.dataset_path) + print("Predictions: ", FLAGS.predictions) + print("Split: ", FLAGS.split) + print("Config: ", FLAGS.data_cfg) + print("Limit: ", FLAGS.limit) + print("*" * 80) + + # assert split + assert FLAGS.split in splits + + # get number of interest classes, and the label mappings + class_strings = DATA["labels"] + class_remap = DATA["learning_map"] + class_inv_remap = DATA["learning_map_inv"] + class_ignore = DATA["learning_ignore"] + nr_classes = len(class_inv_remap) + + # make lookup table for mapping + maxkey = 0 + for key, data in class_remap.items(): + if key > maxkey: + maxkey = key + # +100 hack making lut bigger just in case there are unknown labels + remap_lut = np.zeros((maxkey + 100), dtype=np.int32) + for key, data in class_remap.items(): + try: + remap_lut[key] = data + except IndexError: + print("Wrong key ", key) + # print(remap_lut) + + # create evaluator + ignore = [] + for cl, ign in class_ignore.items(): + if ign: + x_cl = int(cl) + ignore.append(x_cl) + print("Ignoring xentropy class ", x_cl, " in IoU evaluation") + + # create evaluator + device = torch.device("cpu") + evaluator = iouEval(nr_classes, device, ignore) + evaluator.reset() + + # get test set + if FLAGS.split is None: + for splits in ("train", "valid"): + mIoU = evaluate( + (DATA["split"][splits]), + splits, + FLAGS.predictions, + remap_lut, + evaluator, + class_strings, + class_inv_remap, + ignore, + ) + else: + mIoU = evaluate( + DATA["split"][FLAGS.split], + splits, + FLAGS.predictions, + remap_lut, + evaluator, + class_strings, + class_inv_remap, + ignore, + ) + + return mIoU + + +def eval_func(temp_model, FLAGS): + """ + Main function to evaluate the model, including two steps: + 1st: make the inferene, and save the prediction. + 2nd: load prediction, and further make the final evaluation. + """ + temp_model.eval() + infer_main(FLAGS, temp_model) + mIoU = evaluate_main(FLAGS) + return mIoU + + +class ModelConfig: + """ + parameters configuration for AIMET. + """ + + def __init__(self, FLAGS): + self.input_shape = (1, 5, 64, 2048) + self.config_file = "htp_quantsim_config_pt_pertensor.json" + self.param_bw = 8 + self.output_bw = 8 + for arg in vars(FLAGS): + setattr(self, arg, getattr(FLAGS, arg)) + + +def forward_func(model, cal_dataloader): + """ + The simplified forward function for model compute_encoding in AIMET. + """ + iterations = 0 + with torch.no_grad(): + idx = 0 + for i, ( + proj_in, + proj_mask, + _, + _, + path_seq, + path_name, + p_x, + p_y, + proj_range, + unproj_range, + _, + _, + _, + _, + npoints, + ) in enumerate(cal_dataloader): + if i > 20: + print(i) + proj_output = model(proj_in.cuda()) + idx += 1 + if idx > iterations: + break + else: + continue + return 0.5 + + +def main(FLAGS): + """ + The main function. + """ + seed(1234) + parser = parserModule.Parser( + root=FLAGS.dataset_path, + train_sequences=DATA["split"]["train"], + valid_sequences=DATA["split"]["valid"], + test_sequences=DATA["split"]["test"], + labels=DATA["labels"], + color_map=DATA["color_map"], + learning_map=DATA["learning_map"], + learning_map_inv=DATA["learning_map_inv"], + sensor=ARCH["dataset"]["sensor"], + max_points=ARCH["dataset"]["max_points"], + batch_size=1, + workers=ARCH["train"]["workers"], + gt=True, + shuffle_train=False, + ) + + # build the original FP32 model + salsanext_original = SalsaNext_MZ(model_config=FLAGS.model_config) + salsanext_original.from_pretrained(quantized=False) + temp_model_FP32 = salsanext_original.model + mIoU_FP32 = eval_func(temp_model_FP32, FLAGS) + + # Quant configuration + config = ModelConfig(FLAGS) + size_data = torch.rand(config.input_shape) + quant_config = salsanext_original.cfg["optimization_config"][ + "quantization_configuration" + ] + kwargs = { + "quant_scheme": QuantScheme.post_training_percentile, + "default_param_bw": quant_config["param_bw"], + "default_output_bw": quant_config["output_bw"], + "config_file": salsanext_original.path_aimet_config, + "dummy_input": size_data.cuda(), + } + + # Validator -> Preparer -> Validator + ModelValidator.validate_model(temp_model_FP32.cuda(), model_input=size_data.cuda()) + temp_model_FP32 = prepare_model(temp_model_FP32.eval()) + ModelValidator.validate_model(temp_model_FP32.cuda(), model_input=size_data.cuda()) + + # Fold Batch Norms + batch_norm_fold.fold_all_batch_norms(temp_model_FP32, config.input_shape) + + # Evaluate original model naively W8A8 quantized + sim = QuantizationSimModel(temp_model_FP32.cuda(), **kwargs) + cal_dataloader = parser.get_train_set() + sim.set_percentile_value(99.9) + sim.compute_encodings( + forward_pass_callback=forward_func, forward_pass_callback_args=cal_dataloader + ) + temp_model = sim.model + mIoU_INT8 = eval_func(temp_model, FLAGS) + + # Score w8a8/w4a8 model + salsanext_quantized = SalsaNext_MZ(model_config=FLAGS.model_config) + sim_reload = salsanext_quantized.get_quantsim(quantized=True) + mIoU_INT_pre_encoding = eval_func(sim_reload.model.eval(), FLAGS) + + print(f"Original Model | 32-bit Environment | mIoU: {mIoU_FP32:.3f}") + print(f"Original Model | 8-bit Environment | mIoU: {mIoU_INT8:.3f}") + print( + f"Optimized Model, load encoding | {FLAGS.model_config[-4:]} Environment | mIoU: {mIoU_INT_pre_encoding:.3f}" + ) + + +if __name__ == "__main__": + FLAGS = arguments() + main(FLAGS) diff --git a/aimet_zoo_torch/salsanext/models/SalsaNext.py b/aimet_zoo_torch/salsanext/models/SalsaNext.py new file mode 100644 index 0000000..ef28321 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/SalsaNext.py @@ -0,0 +1,248 @@ +# !/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class ResContextBlock(nn.Module): + def __init__(self, in_filters, out_filters): + super(ResContextBlock, self).__init__() + self.conv1 = nn.Conv2d(in_filters, out_filters, kernel_size=(1, 1), stride=1) + self.act1 = nn.LeakyReLU() + + self.conv2 = nn.Conv2d(out_filters, out_filters, (3,3), padding=1) + self.act2 = nn.LeakyReLU() + self.bn1 = nn.BatchNorm2d(out_filters) + + self.conv3 = nn.Conv2d(out_filters, out_filters, (3,3),dilation=2, padding=2) + self.act3 = nn.LeakyReLU() + self.bn2 = nn.BatchNorm2d(out_filters) + + + def forward(self, x): + + shortcut = self.conv1(x) + shortcut = self.act1(shortcut) + + resA = self.conv2(shortcut) + resA = self.act2(resA) + resA1 = self.bn1(resA) + + resA = self.conv3(resA1) + resA = self.act3(resA) + resA2 = self.bn2(resA) + + output = shortcut + resA2 + return output + + +class ResBlock(nn.Module): + def __init__(self, in_filters, out_filters, dropout_rate, kernel_size=(3, 3), stride=1, + pooling=True, drop_out=True): + super(ResBlock, self).__init__() + self.pooling = pooling + self.drop_out = drop_out + self.conv1 = nn.Conv2d(in_filters, out_filters, kernel_size=(1, 1), stride=stride) + self.act1 = nn.LeakyReLU() + + self.conv2 = nn.Conv2d(in_filters, out_filters, kernel_size=(3,3), padding=1) + self.act2 = nn.LeakyReLU() + self.bn1 = nn.BatchNorm2d(out_filters) + + self.conv3 = nn.Conv2d(out_filters, out_filters, kernel_size=(3,3),dilation=2, padding=2) + self.act3 = nn.LeakyReLU() + self.bn2 = nn.BatchNorm2d(out_filters) + + self.conv4 = nn.Conv2d(out_filters, out_filters, kernel_size=(2, 2), dilation=2, padding=1) + self.act4 = nn.LeakyReLU() + self.bn3 = nn.BatchNorm2d(out_filters) + + self.conv5 = nn.Conv2d(out_filters*3, out_filters, kernel_size=(1, 1)) + self.act5 = nn.LeakyReLU() + self.bn4 = nn.BatchNorm2d(out_filters) + + if pooling: + self.dropout = nn.Dropout2d(p=dropout_rate) + self.pool = nn.AvgPool2d(kernel_size=kernel_size, stride=2, padding=1) + else: + self.dropout = nn.Dropout2d(p=dropout_rate) + + def forward(self, x): + shortcut = self.conv1(x) + shortcut = self.act1(shortcut) + + resA = self.conv2(x) + resA = self.act2(resA) + resA1 = self.bn1(resA) + + resA = self.conv3(resA1) + resA = self.act3(resA) + resA2 = self.bn2(resA) + + resA = self.conv4(resA2) + resA = self.act4(resA) + resA3 = self.bn3(resA) + + concat = torch.cat((resA1,resA2,resA3),dim=1) + resA = self.conv5(concat) + resA = self.act5(resA) + resA = self.bn4(resA) + resA = shortcut + resA + + + if self.pooling: + if self.drop_out: + resB = self.dropout(resA) + else: + resB = resA + resB = self.pool(resB) + + return resB, resA + else: + if self.drop_out: + resB = self.dropout(resA) + else: + resB = resA + return resB + + +class UpBlock(nn.Module): + def __init__(self, in_filters, out_filters, dropout_rate, drop_out=True): + super(UpBlock, self).__init__() + self.drop_out = drop_out + self.in_filters = in_filters + self.out_filters = out_filters + + self.dropout1 = nn.Dropout2d(p=dropout_rate) + + self.dropout2 = nn.Dropout2d(p=dropout_rate) + + self.conv1 = nn.Conv2d(in_filters//4 + 2*out_filters, out_filters, (3,3), padding=1) + self.act1 = nn.LeakyReLU() + self.bn1 = nn.BatchNorm2d(out_filters) + + self.conv2 = nn.Conv2d(out_filters, out_filters, (3,3),dilation=2, padding=2) + self.act2 = nn.LeakyReLU() + self.bn2 = nn.BatchNorm2d(out_filters) + + self.conv3 = nn.Conv2d(out_filters, out_filters, (2,2), dilation=2,padding=1) + self.act3 = nn.LeakyReLU() + self.bn3 = nn.BatchNorm2d(out_filters) + + + self.conv4 = nn.Conv2d(out_filters*3,out_filters,kernel_size=(1,1)) + self.act4 = nn.LeakyReLU() + self.bn4 = nn.BatchNorm2d(out_filters) + + self.dropout3 = nn.Dropout2d(p=dropout_rate) + + self.pixel_shuffle = nn.PixelShuffle(2) + + def forward(self, x, skip): + upA = self.pixel_shuffle(x) + if self.drop_out: + upA = self.dropout1(upA) + + upB = torch.cat((upA,skip),dim=1) + if self.drop_out: + upB = self.dropout2(upB) + + upE = self.conv1(upB) + upE = self.act1(upE) + upE1 = self.bn1(upE) + + upE = self.conv2(upE1) + upE = self.act2(upE) + upE2 = self.bn2(upE) + + upE = self.conv3(upE2) + upE = self.act3(upE) + upE3 = self.bn3(upE) + + concat = torch.cat((upE1,upE2,upE3),dim=1) + upE = self.conv4(concat) + upE = self.act4(upE) + upE = self.bn4(upE) + if self.drop_out: + upE = self.dropout3(upE) + + return upE + + +class SalsaNext(nn.Module): + def __init__(self, nclasses): + super(SalsaNext, self).__init__() + self.nclasses = nclasses + + self.downCntx = ResContextBlock(5, 32) + self.downCntx2 = ResContextBlock(32, 32) + self.downCntx3 = ResContextBlock(32, 32) + + self.resBlock1 = ResBlock(32, 2 * 32, 0.2, pooling=True, drop_out=False) + self.resBlock2 = ResBlock(2 * 32, 2 * 2 * 32, 0.2, pooling=True) + self.resBlock3 = ResBlock(2 * 2 * 32, 2 * 4 * 32, 0.2, pooling=True) + self.resBlock4 = ResBlock(2 * 4 * 32, 2 * 4 * 32, 0.2, pooling=True) + self.resBlock5 = ResBlock(2 * 4 * 32, 2 * 4 * 32, 0.2, pooling=False) + + self.upBlock1 = UpBlock(2 * 4 * 32, 4 * 32, 0.2) + self.upBlock2 = UpBlock(4 * 32, 4 * 32, 0.2) + self.upBlock3 = UpBlock(4 * 32, 2 * 32, 0.2) + self.upBlock4 = UpBlock(2 * 32, 32, 0.2, drop_out=False) + + self.logits = nn.Conv2d(32, nclasses, kernel_size=(1, 1)) + + def forward(self, x): + downCntx = self.downCntx(x) + downCntx = self.downCntx2(downCntx) + downCntx = self.downCntx3(downCntx) + + down0c, down0b = self.resBlock1(downCntx) + down1c, down1b = self.resBlock2(down0c) + down2c, down2b = self.resBlock3(down1c) + down3c, down3b = self.resBlock4(down2c) + down5c = self.resBlock5(down3c) + + up4e = self.upBlock1(down5c,down3b) + up3e = self.upBlock2(up4e, down2b) + up2e = self.upBlock3(up3e, down1b) + up1e = self.upBlock4(up2e, down0b) + logits = self.logits(up1e) + + logits = logits + logits = F.softmax(logits, dim=1) + return logits diff --git a/aimet_zoo_torch/salsanext/models/__init__.py b/aimet_zoo_torch/salsanext/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/salsanext/models/arch_cfg.yaml b/aimet_zoo_torch/salsanext/models/arch_cfg.yaml new file mode 100644 index 0000000..463a934 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/arch_cfg.yaml @@ -0,0 +1,66 @@ +################################################################################ +# training parameters +################################################################################ +train: + loss: "xentropy" # must be either xentropy or iou + max_epochs: 150 + lr: 0.05 # sgd learning rate + wup_epochs: 1 # warmup during first XX epochs (can be float) + momentum: 0.9 # sgd momentum + lr_decay: 0.99 # learning rate decay per epoch after initial cycle (from min lr) + w_decay: 0.0001 # weight decay + batch_size: 1 # batch size + report_batch: 50 # every x batches, report loss + report_epoch: 1 # every x epochs, report validation set + epsilon_w: 0.001 # class weight w = 1 / (content + epsilon_w) + save_summary: False # Summary of weight histograms for tensorboard + save_scans: True # False doesn't save anything, True saves some + # sample images (one per batch of the last calculated batch) + # in log folder + show_scans: False # show scans during training + workers: 4 # number of threads to get data + +################################################################################ +# postproc parameters +################################################################################ +post: + CRF: + use: False + train: True + params: False # this should be a dict when in use + KNN: + use: False # This parameter default is false + params: + knn: 5 + search: 5 + sigma: 1.0 + cutoff: 1.0 + +################################################################################ +# classification head parameters +################################################################################ +# dataset (to find parser) +dataset: + labels: "kitti" + scans: "kitti" + max_points: 150000 # max of any scan in dataset + sensor: + name: "HDL64" + type: "spherical" # projective + fov_up: 3 + fov_down: -25 + img_prop: + width: 2048 + height: 64 + img_means: #range,x,y,z,signal + - 12.12 + - 10.88 + - 0.23 + - -1.04 + - 0.21 + img_stds: #range,x,y,z,signal + - 12.32 + - 11.47 + - 6.91 + - 0.86 + - 0.16 diff --git a/aimet_zoo_torch/salsanext/models/common/__init__.py b/aimet_zoo_torch/salsanext/models/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/salsanext/models/common/avgmeter.py b/aimet_zoo_torch/salsanext/models/common/avgmeter.py new file mode 100644 index 0000000..6f8b269 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/avgmeter.py @@ -0,0 +1,44 @@ +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count diff --git a/aimet_zoo_torch/salsanext/models/common/laserscan.py b/aimet_zoo_torch/salsanext/models/common/laserscan.py new file mode 100644 index 0000000..df7f43c --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/laserscan.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import time + +import numpy as np +import math +import random +from scipy.spatial.transform import Rotation as R + +class LaserScan: + """Class that contains LaserScan with x,y,z,r""" + EXTENSIONS_SCAN = ['.bin'] + + def __init__(self, project=False, H=64, W=1024, fov_up=3.0, fov_down=-25.0,DA=False,flip_sign=False,rot=False,drop_points=False): + self.project = project + self.proj_H = H + self.proj_W = W + self.proj_fov_up = fov_up + self.proj_fov_down = fov_down + self.DA = DA + self.flip_sign = flip_sign + self.rot = rot + self.drop_points = drop_points + + self.reset() + + def reset(self): + """ Reset scan members. """ + self.points = np.zeros((0, 3), dtype=np.float32) # [m, 3]: x, y, z + self.remissions = np.zeros((0, 1), dtype=np.float32) # [m ,1]: remission + + # projected range image - [H,W] range (-1 is no data) + self.proj_range = np.full((self.proj_H, self.proj_W), -1, + dtype=np.float32) + + # unprojected range (list of depths for each point) + self.unproj_range = np.zeros((0, 1), dtype=np.float32) + + # projected point cloud xyz - [H,W,3] xyz coord (-1 is no data) + self.proj_xyz = np.full((self.proj_H, self.proj_W, 3), -1, + dtype=np.float32) + + # projected remission - [H,W] intensity (-1 is no data) + self.proj_remission = np.full((self.proj_H, self.proj_W), -1, + dtype=np.float32) + + # projected index (for each pixel, what I am in the pointcloud) + # [H,W] index (-1 is no data) + self.proj_idx = np.full((self.proj_H, self.proj_W), -1, + dtype=np.int32) + + # for each point, where it is in the range image + self.proj_x = np.zeros((0, 1), dtype=np.int32) # [m, 1]: x + self.proj_y = np.zeros((0, 1), dtype=np.int32) # [m, 1]: y + + # mask containing for each pixel, if it contains a point or not + self.proj_mask = np.zeros((self.proj_H, self.proj_W), + dtype=np.int32) # [H,W] mask + + def size(self): + """ Return the size of the point cloud. """ + return self.points.shape[0] + + def __len__(self): + return self.size() + + def open_scan(self, filename): + """ Open raw scan and fill in attributes + """ + # reset just in case there was an open structure + self.reset() + + # check filename is string + if not isinstance(filename, str): + raise TypeError("Filename should be string type, " + "but was {type}".format(type=str(type(filename)))) + + # check extension is a laserscan + if not any(filename.endswith(ext) for ext in self.EXTENSIONS_SCAN): + raise RuntimeError("Filename extension is not valid scan file.") + + # if all goes well, open pointcloud + scan = np.fromfile(filename, dtype=np.float32) + scan = scan.reshape((-1, 4)) + + # put in attribute + points = scan[:, 0:3] # get xyz + remissions = scan[:, 3] # get remission + if self.drop_points is not False: + self.points_to_drop = np.random.randint(0, len(points)-1,int(len(points)*self.drop_points)) + points = np.delete(points,self.points_to_drop,axis=0) + remissions = np.delete(remissions,self.points_to_drop) + + self.set_points(points, remissions) + + def set_points(self, points, remissions=None): + """ Set scan attributes (instead of opening from file) + """ + # reset just in case there was an open structure + self.reset() + + # check scan makes sense + if not isinstance(points, np.ndarray): + raise TypeError("Scan should be numpy array") + + # check remission makes sense + if remissions is not None and not isinstance(remissions, np.ndarray): + raise TypeError("Remissions should be numpy array") + + # put in attribute + self.points = points # get + if self.flip_sign: + self.points[:, 1] = -self.points[:, 1] + if self.DA: + jitter_x = random.uniform(-5,5) + jitter_y = random.uniform(-3, 3) + jitter_z = random.uniform(-1, 0) + self.points[:, 0] += jitter_x + self.points[:, 1] += jitter_y + self.points[:, 2] += jitter_z + if self.rot: + self.points = self.points @ R.random(random_state=1234).as_dcm().T + if remissions is not None: + self.remissions = remissions # get remission + #if self.DA: + # self.remissions = self.remissions[::-1].copy() + else: + self.remissions = np.zeros((points.shape[0]), dtype=np.float32) + + # if projection is wanted, then do it and fill in the structure + if self.project: + self.do_range_projection() + + def do_range_projection(self): + """ Project a pointcloud into a spherical projection image.projection. + Function takes no arguments because it can be also called externally + if the value of the constructor was not set (in case you change your + mind about wanting the projection) + """ + # laser parameters + fov_up = self.proj_fov_up / 180.0 * np.pi # field of view up in rad + fov_down = self.proj_fov_down / 180.0 * np.pi # field of view down in rad + fov = abs(fov_down) + abs(fov_up) # get field of view total in rad + + # get depth of all points + depth = np.linalg.norm(self.points, 2, axis=1) + + # get scan components + scan_x = self.points[:, 0] + scan_y = self.points[:, 1] + scan_z = self.points[:, 2] + + # get angles of all points + yaw = -np.arctan2(scan_y, scan_x) + pitch = np.arcsin(scan_z / depth) + + # get projections in image coords + proj_x = 0.5 * (yaw / np.pi + 1.0) # in [0.0, 1.0] + proj_y = 1.0 - (pitch + abs(fov_down)) / fov # in [0.0, 1.0] + + # scale to image size using angular resolution + proj_x *= self.proj_W # in [0.0, W] + proj_y *= self.proj_H # in [0.0, H] + + # round and clamp for use as index + proj_x = np.floor(proj_x) + proj_x = np.minimum(self.proj_W - 1, proj_x) + proj_x = np.maximum(0, proj_x).astype(np.int32) # in [0,W-1] + self.proj_x = np.copy(proj_x) # store a copy in orig order + + proj_y = np.floor(proj_y) + proj_y = np.minimum(self.proj_H - 1, proj_y) + proj_y = np.maximum(0, proj_y).astype(np.int32) # in [0,H-1] + self.proj_y = np.copy(proj_y) # stope a copy in original order + + # copy of depth in original order + self.unproj_range = np.copy(depth) + + # order in decreasing depth + indices = np.arange(depth.shape[0]) + order = np.argsort(depth)[::-1] + depth = depth[order] + indices = indices[order] + points = self.points[order] + remission = self.remissions[order] + proj_y = proj_y[order] + proj_x = proj_x[order] + + # assing to images + self.proj_range[proj_y, proj_x] = depth + self.proj_xyz[proj_y, proj_x] = points + self.proj_remission[proj_y, proj_x] = remission + self.proj_idx[proj_y, proj_x] = indices + self.proj_mask = (self.proj_idx > 0).astype(np.int32) + + +class SemLaserScan(LaserScan): + """Class that contains LaserScan with x,y,z,r,sem_label,sem_color_label,inst_label,inst_color_label""" + EXTENSIONS_LABEL = ['.label'] + + def __init__(self, sem_color_dict=None, project=False, H=64, W=1024, fov_up=3.0, fov_down=-25.0, max_classes=300,DA=False,flip_sign=False,drop_points=False): + super(SemLaserScan, self).__init__(project, H, W, fov_up, fov_down,DA=DA,flip_sign=flip_sign,drop_points=drop_points) + self.reset() + + # make semantic colors + if sem_color_dict: + # if I have a dict, make it + max_sem_key = 0 + for key, data in sem_color_dict.items(): + if key + 1 > max_sem_key: + max_sem_key = key + 1 + self.sem_color_lut = np.zeros((max_sem_key + 100, 3), dtype=np.float32) + for key, value in sem_color_dict.items(): + self.sem_color_lut[key] = np.array(value, np.float32) / 255.0 + else: + # otherwise make random + max_sem_key = max_classes + self.sem_color_lut = np.random.uniform(low=0.0, + high=1.0, + size=(max_sem_key, 3)) + # force zero to a gray-ish color + self.sem_color_lut[0] = np.full((3), 0.1) + + # make instance colors + max_inst_id = 100000 + self.inst_color_lut = np.random.uniform(low=0.0, + high=1.0, + size=(max_inst_id, 3)) + # force zero to a gray-ish color + self.inst_color_lut[0] = np.full((3), 0.1) + + def reset(self): + """ Reset scan members. """ + super(SemLaserScan, self).reset() + + # semantic labels + self.sem_label = np.zeros((0, 1), dtype=np.int32) # [m, 1]: label + self.sem_label_color = np.zeros((0, 3), dtype=np.float32) # [m ,3]: color + + # instance labels + self.inst_label = np.zeros((0, 1), dtype=np.int32) # [m, 1]: label + self.inst_label_color = np.zeros((0, 3), dtype=np.float32) # [m ,3]: color + + # projection color with semantic labels + self.proj_sem_label = np.zeros((self.proj_H, self.proj_W), + dtype=np.int32) # [H,W] label + self.proj_sem_color = np.zeros((self.proj_H, self.proj_W, 3), + dtype=np.float) # [H,W,3] color + + # projection color with instance labels + self.proj_inst_label = np.zeros((self.proj_H, self.proj_W), + dtype=np.int32) # [H,W] label + self.proj_inst_color = np.zeros((self.proj_H, self.proj_W, 3), + dtype=np.float) # [H,W,3] color + + def open_label(self, filename): + """ Open raw scan and fill in attributes + """ + # check filename is string + if not isinstance(filename, str): + raise TypeError("Filename should be string type, " + "but was {type}".format(type=str(type(filename)))) + + # check extension is a laserscan + if not any(filename.endswith(ext) for ext in self.EXTENSIONS_LABEL): + raise RuntimeError("Filename extension is not valid label file.") + + # if all goes well, open label + label = np.fromfile(filename, dtype=np.int32) + label = label.reshape((-1)) + + if self.drop_points is not False: + label = np.delete(label,self.points_to_drop) + # set it + self.set_label(label) + + def set_label(self, label): + """ Set points for label not from file but from np + """ + # check label makes sense + if not isinstance(label, np.ndarray): + raise TypeError("Label should be numpy array") + + # only fill in attribute if the right size + if label.shape[0] == self.points.shape[0]: + self.sem_label = label & 0xFFFF # semantic label in lower half + self.inst_label = label >> 16 # instance id in upper half + else: + print("Points shape: ", self.points.shape) + print("Label shape: ", label.shape) + raise ValueError("Scan and Label don't contain same number of points") + + # sanity check + assert ((self.sem_label + (self.inst_label << 16) == label).all()) + + if self.project: + self.do_label_projection() + + def colorize(self): + """ Colorize pointcloud with the color of each semantic label + """ + self.sem_label_color = self.sem_color_lut[self.sem_label] + self.sem_label_color = self.sem_label_color.reshape((-1, 3)) + + self.inst_label_color = self.inst_color_lut[self.inst_label] + self.inst_label_color = self.inst_label_color.reshape((-1, 3)) + + def do_label_projection(self): + # only map colors to labels that exist + mask = self.proj_idx >= 0 + + # semantics + self.proj_sem_label[mask] = self.sem_label[self.proj_idx[mask]] + self.proj_sem_color[mask] = self.sem_color_lut[self.sem_label[self.proj_idx[mask]]] + + # instances + self.proj_inst_label[mask] = self.inst_label[self.proj_idx[mask]] + self.proj_inst_color[mask] = self.inst_color_lut[self.inst_label[self.proj_idx[mask]]] diff --git a/aimet_zoo_torch/salsanext/models/common/laserscanvis.py b/aimet_zoo_torch/salsanext/models/common/laserscanvis.py new file mode 100644 index 0000000..fc4db4c --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/laserscanvis.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + + +import vispy +from vispy.scene import visuals, SceneCanvas +import numpy as np +from matplotlib import pyplot as plt + + +class LaserScanVis: + """Class that creates and handles a visualizer for a pointcloud""" + + def __init__(self, scan, scan_names, label_names, offset=0, + semantics=True, instances=False): + self.scan = scan + self.scan_names = scan_names + self.label_names = label_names + self.offset = offset + self.semantics = semantics + self.instances = instances + # sanity check + if not self.semantics and self.instances: + print("Instances are only allowed in when semantics=True") + raise ValueError + + self.reset() + self.update_scan() + + def reset(self): + """ Reset. """ + # last key press (it should have a mutex, but visualization is not + # safety critical, so let's do things wrong) + self.action = "no" # no, next, back, quit are the possibilities + + # new canvas prepared for visualizing data + self.canvas = SceneCanvas(keys='interactive', show=True) + # interface (n next, b back, q quit, very simple) + self.canvas.events.key_press.connect(self.key_press) + self.canvas.events.draw.connect(self.draw) + # grid + self.grid = self.canvas.central_widget.add_grid() + + # laserscan part + self.scan_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.canvas.scene) + self.grid.add_widget(self.scan_view, 0, 0) + self.scan_vis = visuals.Markers() + self.scan_view.camera = 'turntable' + self.scan_view.add(self.scan_vis) + visuals.XYZAxis(parent=self.scan_view.scene) + # add semantics + if self.semantics: + print("Using semantics in visualizer") + self.sem_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.canvas.scene) + self.grid.add_widget(self.sem_view, 0, 1) + self.sem_vis = visuals.Markers() + self.sem_view.camera = 'turntable' + self.sem_view.add(self.sem_vis) + visuals.XYZAxis(parent=self.sem_view.scene) + # self.sem_view.camera.link(self.scan_view.camera) + + if self.instances: + print("Using instances in visualizer") + self.inst_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.canvas.scene) + self.grid.add_widget(self.inst_view, 0, 2) + self.inst_vis = visuals.Markers() + self.inst_view.camera = 'turntable' + self.inst_view.add(self.inst_vis) + visuals.XYZAxis(parent=self.inst_view.scene) + # self.inst_view.camera.link(self.scan_view.camera) + + # img canvas size + self.multiplier = 1 + self.canvas_W = 1024 + self.canvas_H = 64 + if self.semantics: + self.multiplier += 1 + if self.instances: + self.multiplier += 1 + + # new canvas for img + self.img_canvas = SceneCanvas(keys='interactive', show=True, + size=(self.canvas_W, self.canvas_H * self.multiplier)) + # grid + self.img_grid = self.img_canvas.central_widget.add_grid() + # interface (n next, b back, q quit, very simple) + self.img_canvas.events.key_press.connect(self.key_press) + self.img_canvas.events.draw.connect(self.draw) + + # add a view for the depth + self.img_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.img_canvas.scene) + self.img_grid.add_widget(self.img_view, 0, 0) + self.img_vis = visuals.Image(cmap='viridis') + self.img_view.add(self.img_vis) + + # add semantics + if self.semantics: + self.sem_img_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.img_canvas.scene) + self.img_grid.add_widget(self.sem_img_view, 1, 0) + self.sem_img_vis = visuals.Image(cmap='viridis') + self.sem_img_view.add(self.sem_img_vis) + + # add instances + if self.instances: + self.inst_img_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.img_canvas.scene) + self.img_grid.add_widget(self.inst_img_view, 2, 0) + self.inst_img_vis = visuals.Image(cmap='viridis') + self.inst_img_view.add(self.inst_img_vis) + + def get_mpl_colormap(self, cmap_name): + cmap = plt.get_cmap(cmap_name) + + # Initialize the matplotlib color map + sm = plt.cm.ScalarMappable(cmap=cmap) + + # Obtain linear color range + color_range = sm.to_rgba(np.linspace(0, 1, 256), bytes=True)[:, 2::-1] + + return color_range.reshape(256, 3).astype(np.float32) / 255.0 + + def update_scan(self): + # first open data + self.scan.open_scan(self.scan_names[self.offset]) + if self.semantics: + self.scan.open_label(self.label_names[self.offset]) + self.scan.colorize() + + # then change names + title = "scan " + str(self.offset) + " of " + str(len(self.scan_names)-1) + self.canvas.title = title + self.img_canvas.title = title + + # then do all the point cloud stuff + + # plot scan + power = 16 + # print() + range_data = np.copy(self.scan.unproj_range) + # print(range_data.max(), range_data.min()) + range_data = range_data**(1 / power) + # print(range_data.max(), range_data.min()) + viridis_range = ((range_data - range_data.min()) / + (range_data.max() - range_data.min()) * + 255).astype(np.uint8) + viridis_map = self.get_mpl_colormap("viridis") + viridis_colors = viridis_map[viridis_range] + self.scan_vis.set_data(self.scan.points, + face_color=viridis_colors[..., ::-1], + edge_color=viridis_colors[..., ::-1], + size=1) + + # plot semantics + if self.semantics: + self.sem_vis.set_data(self.scan.points, + face_color=self.scan.sem_label_color[..., ::-1], + edge_color=self.scan.sem_label_color[..., ::-1], + size=1) + + # plot instances + if self.instances: + self.inst_vis.set_data(self.scan.points, + face_color=self.scan.inst_label_color[..., ::-1], + edge_color=self.scan.inst_label_color[..., ::-1], + size=1) + + # now do all the range image stuff + # plot range image + data = np.copy(self.scan.proj_range) + # print(data[data > 0].max(), data[data > 0].min()) + data[data > 0] = data[data > 0]**(1 / power) + data[data < 0] = data[data > 0].min() + # print(data.max(), data.min()) + data = (data - data[data > 0].min()) / \ + (data.max() - data[data > 0].min()) + # print(data.max(), data.min()) + self.img_vis.set_data(data) + self.img_vis.update() + + if self.semantics: + self.sem_img_vis.set_data(self.scan.proj_sem_color[..., ::-1]) + self.sem_img_vis.update() + + if self.instances: + self.inst_img_vis.set_data(self.scan.proj_inst_color[..., ::-1]) + self.inst_img_vis.update() + + # interface + def key_press(self, event): + self.canvas.events.key_press.block() + self.img_canvas.events.key_press.block() + if event.key == 'N': + self.offset += 1 + if self.offset >= len(self.scan_names): + self.offset = 0 + self.update_scan() + elif event.key == 'B': + self.offset -= 1 + if self.offset <= 0: + self.offset = len(self.scan_names)-1 + self.update_scan() + elif event.key == 'Q' or event.key == 'Escape': + self.destroy() + + def draw(self, event): + if self.canvas.events.key_press.blocked(): + self.canvas.events.key_press.unblock() + if self.img_canvas.events.key_press.blocked(): + self.img_canvas.events.key_press.unblock() + + def destroy(self): + # destroy the visualization + self.canvas.close() + self.img_canvas.close() + vispy.app.quit() + + def run(self): + vispy.app.run() diff --git a/aimet_zoo_torch/salsanext/models/common/logger.py b/aimet_zoo_torch/salsanext/models/common/logger.py new file mode 100644 index 0000000..be506af --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/logger.py @@ -0,0 +1,79 @@ +# pylint: skip-file +# Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514 + +import numpy as np +import scipy.misc +import tensorflow as tf +from torch.utils.tensorboard import SummaryWriter +import torch +try: + from StringIO import StringIO # Python 2.7 +except ImportError: + from io import BytesIO # Python 3.x + + +class Logger(object): + + def __init__(self, log_dir): + """Create a summary writer logging to log_dir.""" + self.writer = tf.summary.FileWriter(log_dir) + + def scalar_summary(self, tag, value, step): + """Log a scalar variable.""" + summary = tf.Summary( + value=[tf.Summary.Value(tag=tag, simple_value=value)]) + self.writer.add_summary(summary, step) + self.writer.flush() + + def image_summary(self, tag, images, step): + """Log a list of images.""" + + img_summaries = [] + for i, img in enumerate(images): + # Write the image to a string + try: + s = StringIO() + except: + s = BytesIO() + scipy.misc.toimage(img).save(s, format="png") + + # Create an Image object + img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(), + height=img.shape[0], + width=img.shape[1]) + # Create a Summary value + img_summaries.append(tf.Summary.Value( + tag='%s/%d' % (tag, i), image=img_sum)) + + # Create and write Summary + summary = tf.Summary(value=img_summaries) + self.writer.add_summary(summary, step) + self.writer.flush() + + def histo_summary(self, tag, values, step, bins=1000): + """Log a histogram of the tensor of values.""" + + # Create a histogram using numpy + counts, bin_edges = np.histogram(values, bins=bins) + + # Fill the fields of the histogram proto + hist = tf.HistogramProto() + hist.min = float(np.min(values)) + hist.max = float(np.max(values)) + hist.num = int(np.prod(values.shape)) + hist.sum = float(np.sum(values)) + hist.sum_squares = float(np.sum(values ** 2)) + + # Drop the start of the first bin + bin_edges = bin_edges[1:] + + # Add bin edges and counts + for edge in bin_edges: + hist.bucket_limit.append(edge) + for c in counts: + hist.bucket.append(c) + + # Create and write Summary + summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)]) + self.writer.add_summary(summary, step) + self.writer.flush() diff --git a/aimet_zoo_torch/salsanext/models/common/summary.py b/aimet_zoo_torch/salsanext/models/common/summary.py new file mode 100644 index 0000000..c27b6e7 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/summary.py @@ -0,0 +1,116 @@ +# pylint: skip-file +### https://github.com/sksq96/pytorch-summary/blob/master/torchsummary/torchsummary.py +import torch +import torch.nn as nn +from torch.autograd import Variable + +from collections import OrderedDict +import numpy as np + + +def summary(model, input_size, batch_size=-1, device="cuda"): + def register_hook(module): + + def hook(module, input, output): + class_name = str(module.__class__).split(".")[-1].split("'")[0] + module_idx = len(summary) + + m_key = "%s-%i" % (class_name, module_idx + 1) + summary[m_key] = OrderedDict() + summary[m_key]["input_shape"] = list(input[0].size()) + summary[m_key]["input_shape"][0] = batch_size + if isinstance(output, (list, tuple)): + summary[m_key]["output_shape"] = [ + [-1] + list(o.size())[1:] for o in output + ] + else: + summary[m_key]["output_shape"] = list(output.size()) + summary[m_key]["output_shape"][0] = batch_size + + params = 0 + if hasattr(module, "weight") and hasattr(module.weight, "size"): + params += torch.prod(torch.LongTensor(list(module.weight.size()))) + summary[m_key]["trainable"] = module.weight.requires_grad + if hasattr(module, "bias") and hasattr(module.bias, "size"): + params += torch.prod(torch.LongTensor(list(module.bias.size()))) + summary[m_key]["nb_params"] = params + + if ( + not isinstance(module, nn.Sequential) + and not isinstance(module, nn.ModuleList) + and not (module == model) + ): + hooks.append(module.register_forward_hook(hook)) + + device = device.lower() + assert device in [ + "cuda", + "cpu", + ], "Input device is not valid, please specify 'cuda' or 'cpu'" + + if device == "cuda" and torch.cuda.is_available(): + dtype = torch.cuda.FloatTensor + else: + dtype = torch.FloatTensor + + # multiple inputs to the network + if isinstance(input_size, tuple): + input_size = [input_size] + + # batch_size of 2 for batchnorm + x = [torch.rand(2, *in_size).type(dtype) for in_size in input_size] + # message +=type(x[0])) + + # create properties + summary = OrderedDict() + hooks = [] + + # register hook + model.apply(register_hook) + + # make a forward pass + # message +=x.shape) + model(*x) + + # remove these hooks + for h in hooks: + h.remove() + message = "" + message += "----------------------------------------------------------------\n" + line_new = "{:>20} {:>25} {:>15}".format("Layer (type)", "Output Shape", "Param #") + message += line_new + "\n" + message += "================================================================\n" + total_params = 0 + total_output = 0 + trainable_params = 0 + for layer in summary: + # input_shape, output_shape, trainable, nb_params + line_new = "{:>20} {:>25} {:>15}".format( + layer, + str(summary[layer]["output_shape"]), + "{0:,}".format(summary[layer]["nb_params"]), + ) + total_params += summary[layer]["nb_params"] + total_output += np.prod(summary[layer]["output_shape"]) + if "trainable" in summary[layer]: + if summary[layer]["trainable"] == True: + trainable_params += summary[layer]["nb_params"] + message += line_new + "\n" + + # assume 4 bytes/number (float on cuda). + total_input_size = abs(np.prod(input_size) * batch_size * 4. / (1024 ** 2.)) + total_output_size = abs(2. * total_output * 4. / (1024 ** 2.)) # x2 for gradients + total_params_size = abs(total_params.numpy() * 4. / (1024 ** 2.)) + total_size = total_params_size + total_output_size + total_input_size + + message += "================================================================\n" + message += "Total params: {0:,}\n".format(total_params) + message += "Trainable params: {0:,}\n".format(trainable_params) + message += "Non-trainable params: {0:,}\n".format(total_params - trainable_params) + message += "----------------------------------------------------------------\n" + message += "Input size (MB): %0.2f\n" % total_input_size + message += "Forward/backward pass size (MB): %0.2f\n" % total_output_size + message += "Params size (MB): %0.2f\n" % total_params_size + message += "Estimated Total Size (MB): %0.2f\n" % total_size + message += "----------------------------------------------------------------\n" + return message diff --git a/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/__init__.py b/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/batchnorm.py b/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/batchnorm.py new file mode 100644 index 0000000..5323f2a --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/batchnorm.py @@ -0,0 +1,369 @@ +# -*- coding: utf-8 -*- +# pylint: skip-file +# File : batchnorm.py +# Author : Jiayuan Mao +# Email : maojiayuan@gmail.com +# Date : 27/01/2018 +# +# This file is part of Synchronized-BatchNorm-PyTorch. +# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch +# Distributed under MIT License. + +import collections + +import torch +import torch.nn.functional as F +from torch.nn.modules.batchnorm import _BatchNorm +from torch.nn.parallel._functions import ReduceAddCoalesced, Broadcast + +from .comm import SyncMaster +from .replicate import DataParallelWithCallback + +__all__ = ['SynchronizedBatchNorm1d', 'SynchronizedBatchNorm2d', + 'SynchronizedBatchNorm3d', 'convert_model'] + + +def _sum_ft(tensor): + """sum over the first and last dimention""" + return tensor.sum(dim=0).sum(dim=-1) + + +def _unsqueeze_ft(tensor): + """add new dementions at the front and the tail""" + return tensor.unsqueeze(0).unsqueeze(-1) + + +_ChildMessage = collections.namedtuple( + '_ChildMessage', ['sum', 'ssum', 'sum_size']) +_MasterMessage = collections.namedtuple('_MasterMessage', ['sum', 'inv_std']) + + +class _SynchronizedBatchNorm(_BatchNorm): + def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True): + super(_SynchronizedBatchNorm, self).__init__( + num_features, eps=eps, momentum=momentum, affine=affine) + + self._sync_master = SyncMaster(self._data_parallel_master) + + self._is_parallel = False + self._parallel_id = None + self._slave_pipe = None + + def forward(self, input): + # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation. + if not (self._is_parallel and self.training): + return F.batch_norm( + input, self.running_mean, self.running_var, self.weight, self.bias, + self.training, self.momentum, self.eps) + + # Resize the input to (B, C, -1). + input_shape = input.size() + input = input.view(input.size(0), self.num_features, -1) + + # Compute the sum and square-sum. + sum_size = input.size(0) * input.size(2) + input_sum = _sum_ft(input) + input_ssum = _sum_ft(input ** 2) + + # Reduce-and-broadcast the statistics. + if self._parallel_id == 0: + mean, inv_std = self._sync_master.run_master( + _ChildMessage(input_sum, input_ssum, sum_size)) + else: + mean, inv_std = self._slave_pipe.run_slave( + _ChildMessage(input_sum, input_ssum, sum_size)) + + # Compute the output. + if self.affine: + # MJY:: Fuse the multiplication for speed. + output = (input - _unsqueeze_ft(mean)) * \ + _unsqueeze_ft(inv_std * self.weight) + _unsqueeze_ft(self.bias) + else: + output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std) + + # Reshape it. + return output.view(input_shape) + + def __data_parallel_replicate__(self, ctx, copy_id): + self._is_parallel = True + self._parallel_id = copy_id + + # parallel_id == 0 means master device. + if self._parallel_id == 0: + ctx.sync_master = self._sync_master + else: + self._slave_pipe = ctx.sync_master.register_slave(copy_id) + + def _data_parallel_master(self, intermediates): + """Reduce the sum and square-sum, compute the statistics, and broadcast it.""" + + # Always using same "device order" makes the ReduceAdd operation faster. + # Thanks to:: Tete Xiao (http://tetexiao.com/) + intermediates = sorted(intermediates, key=lambda i: i[1].sum.get_device()) + + to_reduce = [i[1][:2] for i in intermediates] + to_reduce = [j for i in to_reduce for j in i] # flatten + target_gpus = [i[1].sum.get_device() for i in intermediates] + + sum_size = sum([i[1].sum_size for i in intermediates]) + sum_, ssum = ReduceAddCoalesced.apply(target_gpus[0], 2, *to_reduce) + mean, inv_std = self._compute_mean_std(sum_, ssum, sum_size) + + broadcasted = Broadcast.apply(target_gpus, mean, inv_std) + + outputs = [] + for i, rec in enumerate(intermediates): + outputs.append((rec[0], _MasterMessage(*broadcasted[i * 2:i * 2 + 2]))) + + return outputs + + def _compute_mean_std(self, sum_, ssum, size): + """Compute the mean and standard-deviation with sum and square-sum. This method + also maintains the moving average on the master device.""" + assert size > 1, 'BatchNorm computes unbiased standard-deviation, which requires size > 1.' + mean = sum_ / size + sumvar = ssum - sum_ * mean + unbias_var = sumvar / (size - 1) + bias_var = sumvar / size + + self.running_mean = (1 - self.momentum) * \ + self.running_mean + self.momentum * mean.data + self.running_var = (1 - self.momentum) * \ + self.running_var + self.momentum * unbias_var.data + + return mean, bias_var.clamp(self.eps) ** -0.5 + + +class SynchronizedBatchNorm1d(_SynchronizedBatchNorm): + r"""Applies Synchronized Batch Normalization over a 2d or 3d input that is seen as a + mini-batch. + + .. math:: + + y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta + + This module differs from the built-in PyTorch BatchNorm1d as the mean and + standard-deviation are reduced across all devices during training. + + For example, when one uses `nn.DataParallel` to wrap the network during + training, PyTorch's implementation normalize the tensor on each device using + the statistics only on that device, which accelerated the computation and + is also easy to implement, but the statistics might be inaccurate. + Instead, in this synchronized version, the statistics will be computed + over all training samples distributed on multiple devices. + + Note that, for one-GPU or CPU-only case, this module behaves exactly same + as the built-in PyTorch implementation. + + The mean and standard-deviation are calculated per-dimension over + the mini-batches and gamma and beta are learnable parameter vectors + of size C (where C is the input size). + + During training, this layer keeps a running estimate of its computed mean + and variance. The running sum is kept with a default momentum of 0.1. + + During evaluation, this running mean/variance is used for normalization. + + Because the BatchNorm is done over the `C` dimension, computing statistics + on `(N, L)` slices, it's common terminology to call this Temporal BatchNorm + + Args: + num_features: num_features from an expected input of size + `batch_size x num_features [x width]` + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Default: 0.1 + affine: a boolean value that when set to ``True``, gives the layer learnable + affine parameters. Default: ``True`` + + Shape: + - Input: :math:`(N, C)` or :math:`(N, C, L)` + - Output: :math:`(N, C)` or :math:`(N, C, L)` (same shape as input) + + Examples: + >>> # With Learnable Parameters + >>> m = SynchronizedBatchNorm1d(100) + >>> # Without Learnable Parameters + >>> m = SynchronizedBatchNorm1d(100, affine=False) + >>> input = torch.autograd.Variable(torch.randn(20, 100)) + >>> output = m(input) + """ + + def _check_input_dim(self, input): + if input.dim() != 2 and input.dim() != 3: + raise ValueError('expected 2D or 3D input (got {}D input)' + .format(input.dim())) + super(SynchronizedBatchNorm1d, self)._check_input_dim(input) + + +class SynchronizedBatchNorm2d(_SynchronizedBatchNorm): + r"""Applies Batch Normalization over a 4d input that is seen as a mini-batch + of 3d inputs + + .. math:: + + y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta + + This module differs from the built-in PyTorch BatchNorm2d as the mean and + standard-deviation are reduced across all devices during training. + + For example, when one uses `nn.DataParallel` to wrap the network during + training, PyTorch's implementation normalize the tensor on each device using + the statistics only on that device, which accelerated the computation and + is also easy to implement, but the statistics might be inaccurate. + Instead, in this synchronized version, the statistics will be computed + over all training samples distributed on multiple devices. + + Note that, for one-GPU or CPU-only case, this module behaves exactly same + as the built-in PyTorch implementation. + + The mean and standard-deviation are calculated per-dimension over + the mini-batches and gamma and beta are learnable parameter vectors + of size C (where C is the input size). + + During training, this layer keeps a running estimate of its computed mean + and variance. The running sum is kept with a default momentum of 0.1. + + During evaluation, this running mean/variance is used for normalization. + + Because the BatchNorm is done over the `C` dimension, computing statistics + on `(N, H, W)` slices, it's common terminology to call this Spatial BatchNorm + + Args: + num_features: num_features from an expected input of + size batch_size x num_features x height x width + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Default: 0.1 + affine: a boolean value that when set to ``True``, gives the layer learnable + affine parameters. Default: ``True`` + + Shape: + - Input: :math:`(N, C, H, W)` + - Output: :math:`(N, C, H, W)` (same shape as input) + + Examples: + >>> # With Learnable Parameters + >>> m = SynchronizedBatchNorm2d(100) + >>> # Without Learnable Parameters + >>> m = SynchronizedBatchNorm2d(100, affine=False) + >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45)) + >>> output = m(input) + """ + + def _check_input_dim(self, input): + if input.dim() != 4: + raise ValueError('expected 4D input (got {}D input)' + .format(input.dim())) + super(SynchronizedBatchNorm2d, self)._check_input_dim(input) + + +class SynchronizedBatchNorm3d(_SynchronizedBatchNorm): + r"""Applies Batch Normalization over a 5d input that is seen as a mini-batch + of 4d inputs + + .. math:: + + y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta + + This module differs from the built-in PyTorch BatchNorm3d as the mean and + standard-deviation are reduced across all devices during training. + + For example, when one uses `nn.DataParallel` to wrap the network during + training, PyTorch's implementation normalize the tensor on each device using + the statistics only on that device, which accelerated the computation and + is also easy to implement, but the statistics might be inaccurate. + Instead, in this synchronized version, the statistics will be computed + over all training samples distributed on multiple devices. + + Note that, for one-GPU or CPU-only case, this module behaves exactly same + as the built-in PyTorch implementation. + + The mean and standard-deviation are calculated per-dimension over + the mini-batches and gamma and beta are learnable parameter vectors + of size C (where C is the input size). + + During training, this layer keeps a running estimate of its computed mean + and variance. The running sum is kept with a default momentum of 0.1. + + During evaluation, this running mean/variance is used for normalization. + + Because the BatchNorm is done over the `C` dimension, computing statistics + on `(N, D, H, W)` slices, it's common terminology to call this Volumetric BatchNorm + or Spatio-temporal BatchNorm + + Args: + num_features: num_features from an expected input of + size batch_size x num_features x depth x height x width + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Default: 0.1 + affine: a boolean value that when set to ``True``, gives the layer learnable + affine parameters. Default: ``True`` + + Shape: + - Input: :math:`(N, C, D, H, W)` + - Output: :math:`(N, C, D, H, W)` (same shape as input) + + Examples: + >>> # With Learnable Parameters + >>> m = SynchronizedBatchNorm3d(100) + >>> # Without Learnable Parameters + >>> m = SynchronizedBatchNorm3d(100, affine=False) + >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45, 10)) + >>> output = m(input) + """ + + def _check_input_dim(self, input): + if input.dim() != 5: + raise ValueError('expected 5D input (got {}D input)' + .format(input.dim())) + super(SynchronizedBatchNorm3d, self)._check_input_dim(input) + + +def convert_model(module): + """Traverse the input module and its child recursively + and replace all instance of torch.nn.modules.batchnorm.BatchNorm*N*d + to SynchronizedBatchNorm*N*d + + Args: + module: the input module needs to be convert to SyncBN model + + Examples: + >>> import torch.nn as nn + >>> import torchvision + >>> # m is a standard pytorch model + >>> m = torchvision.models.resnet18(True) + >>> m = nn.DataParallel(m) + >>> # after convert, m is using SyncBN + >>> m = convert_model(m) + """ + if isinstance(module, torch.nn.DataParallel): + mod = module.module + mod = convert_model(mod) + mod = DataParallelWithCallback(mod) + return mod + + mod = module + for pth_module, sync_module in zip([torch.nn.modules.batchnorm.BatchNorm1d, + torch.nn.modules.batchnorm.BatchNorm2d, + torch.nn.modules.batchnorm.BatchNorm3d], + [SynchronizedBatchNorm1d, + SynchronizedBatchNorm2d, + SynchronizedBatchNorm3d]): + if isinstance(module, pth_module): + mod = sync_module(module.num_features, module.eps, + module.momentum, module.affine) + mod.running_mean = module.running_mean + mod.running_var = module.running_var + if module.affine: + mod.weight.data = module.weight.data.clone().detach() + mod.bias.data = module.bias.data.clone().detach() + + for name, child in module.named_children(): + mod.add_module(name, convert_model(child)) + + return mod diff --git a/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/comm.py b/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/comm.py new file mode 100644 index 0000000..497e420 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/comm.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +# pylint: skip-file +# File : comm.py +# Author : Jiayuan Mao +# Email : maojiayuan@gmail.com +# Date : 27/01/2018 +# +# This file is part of Synchronized-BatchNorm-PyTorch. +# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch +# Distributed under MIT License. + +import collections +import queue +import threading + +__all__ = ['FutureResult', 'SlavePipe', 'SyncMaster'] + + +class FutureResult(object): + """A thread-safe future implementation. Used only as one-to-one pipe.""" + + def __init__(self): + self._result = None + self._lock = threading.Lock() + self._cond = threading.Condition(self._lock) + + def put(self, result): + with self._lock: + assert self._result is None, 'Previous result has\'t been fetched.' + self._result = result + self._cond.notify() + + def get(self): + with self._lock: + if self._result is None: + self._cond.wait() + + res = self._result + self._result = None + return res + + +_MasterRegistry = collections.namedtuple('MasterRegistry', ['result']) +_SlavePipeBase = collections.namedtuple( + '_SlavePipeBase', ['identifier', 'queue', 'result']) + + +class SlavePipe(_SlavePipeBase): + """Pipe for master-slave communication.""" + + def run_slave(self, msg): + self.queue.put((self.identifier, msg)) + ret = self.result.get() + self.queue.put(True) + return ret + + +class SyncMaster(object): + """An abstract `SyncMaster` object. + + - During the replication, as the data parallel will trigger an callback of each module, all slave devices should + call `register(id)` and obtain an `SlavePipe` to communicate with the master. + - During the forward pass, master device invokes `run_master`, all messages from slave devices will be collected, + and passed to a registered callback. + - After receiving the messages, the master device should gather the information and determine to message passed + back to each slave devices. + """ + + def __init__(self, master_callback): + """ + + Args: + master_callback: a callback to be invoked after having collected messages from slave devices. + """ + self._master_callback = master_callback + self._queue = queue.Queue() + self._registry = collections.OrderedDict() + self._activated = False + + def __getstate__(self): + return {'master_callback': self._master_callback} + + def __setstate__(self, state): + self.__init__(state['master_callback']) + + def register_slave(self, identifier): + """ + Register an slave device. + + Args: + identifier: an identifier, usually is the device id. + + Returns: a `SlavePipe` object which can be used to communicate with the master device. + + """ + if self._activated: + assert self._queue.empty(), 'Queue is not clean before next initialization.' + self._activated = False + self._registry.clear() + future = FutureResult() + self._registry[identifier] = _MasterRegistry(future) + return SlavePipe(identifier, self._queue, future) + + def run_master(self, master_msg): + """ + Main entry for the master device in each forward pass. + The messages were first collected from each devices (including the master device), and then + an callback will be invoked to compute the message to be sent back to each devices + (including the master device). + + Args: + master_msg: the message that the master want to send to itself. This will be placed as the first + message when calling `master_callback`. For detailed usage, see `_SynchronizedBatchNorm` for an example. + + Returns: the message to be sent back to the master device. + + """ + self._activated = True + + intermediates = [(0, master_msg)] + for i in range(self.nr_slaves): + intermediates.append(self._queue.get()) + + results = self._master_callback(intermediates) + assert results[0][0] == 0, 'The first result should belongs to the master.' + + for i, res in results: + if i == 0: + continue + self._registry[i].result.put(res) + + for i in range(self.nr_slaves): + assert self._queue.get() is True + + return results[0][1] + + @property + def nr_slaves(self): + return len(self._registry) diff --git a/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/replicate.py b/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/replicate.py new file mode 100644 index 0000000..d70945d --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/sync_batchnorm/replicate.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# pylint: skip-file +# File : replicate.py +# Author : Jiayuan Mao +# Email : maojiayuan@gmail.com +# Date : 27/01/2018 +# +# This file is part of Synchronized-BatchNorm-PyTorch. +# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch +# Distributed under MIT License. + +import functools + +from torch.nn.parallel.data_parallel import DataParallel + +__all__ = [ + 'CallbackContext', + 'execute_replication_callbacks', + 'DataParallelWithCallback', + 'patch_replication_callback' +] + + +class CallbackContext(object): + pass + + +def execute_replication_callbacks(modules): + """ + Execute an replication callback `__data_parallel_replicate__` on each module created by original replication. + + The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` + + Note that, as all modules are isomorphism, we assign each sub-module with a context + (shared among multiple copies of this module on different devices). + Through this context, different copies can share some information. + + We guarantee that the callback on the master copy (the first copy) will be called ahead of calling the callback + of any slave copies. + """ + master_copy = modules[0] + nr_modules = len(list(master_copy.modules())) + ctxs = [CallbackContext() for _ in range(nr_modules)] + + for i, module in enumerate(modules): + for j, m in enumerate(module.modules()): + if hasattr(m, '__data_parallel_replicate__'): + m.__data_parallel_replicate__(ctxs[j], i) + + +class DataParallelWithCallback(DataParallel): + """ + Data Parallel with a replication callback. + + An replication callback `__data_parallel_replicate__` of each module will be invoked after being created by + original `replicate` function. + The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` + + Examples: + > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) + > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) + # sync_bn.__data_parallel_replicate__ will be invoked. + """ + + def replicate(self, module, device_ids): + modules = super(DataParallelWithCallback, + self).replicate(module, device_ids) + execute_replication_callbacks(modules) + return modules + + +def patch_replication_callback(data_parallel): + """ + Monkey-patch an existing `DataParallel` object. Add the replication callback. + Useful when you have customized `DataParallel` implementation. + + Examples: + > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) + > sync_bn = DataParallel(sync_bn, device_ids=[0, 1]) + > patch_replication_callback(sync_bn) + # this is equivalent to + > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) + > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) + """ + + assert isinstance(data_parallel, DataParallel) + + old_replicate = data_parallel.replicate + + @functools.wraps(old_replicate) + def new_replicate(module, device_ids): + modules = old_replicate(module, device_ids) + execute_replication_callbacks(modules) + return modules + + data_parallel.replicate = new_replicate diff --git a/aimet_zoo_torch/salsanext/models/common/visualization.py b/aimet_zoo_torch/salsanext/models/common/visualization.py new file mode 100644 index 0000000..2f26cc1 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/visualization.py @@ -0,0 +1,138 @@ +# pylint: skip-file +import os + +import matplotlib +import numpy as np +import pykitti + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yaml + +basedir = '' +sequence = '' +uncerts = '' +preds = '' +gt = '' +img = '' +lidar = '' +projected_uncert = '' +projected_preds = '' + +dataset = pykitti.odometry(basedir, sequence) + +EXTENSIONS_LABEL = ['.label'] +EXTENSIONS_LIDAR = ['.bin'] +EXTENSIONS_IMG = ['.png'] + + +def is_label(filename): + return any(filename.endswith(ext) for ext in EXTENSIONS_LABEL) + + +def is_lidar(filename): + return any(filename.endswith(ext) for ext in EXTENSIONS_LIDAR) + + +def is_img(filename): + return any(filename.endswith(ext) for ext in EXTENSIONS_IMG) + + +def get_mpl_colormap(cmap_name): + cmap = plt.get_cmap(cmap_name) + + # Initialize the matplotlib color map + sm = plt.cm.ScalarMappable(cmap=cmap) + + # Obtain linear color range + color_range = sm.to_rgba(np.linspace(0, 1, 256), bytes=True)[:, 2::-1] + + return color_range.reshape(256, 3).astype(np.float32) / 255.0 + + +path = os.path.join(basedir + 'sequences/' + sequence + uncerts) + +scan_uncert = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(path)) for f in fn if is_label(f)] +scan_uncert.sort() +path = os.path.join(basedir + 'sequences/' + sequence + preds) +scan_preds = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(path)) for f in fn if is_label(f)] +scan_preds.sort() + +path = os.path.join(basedir + 'sequences/' + sequence + gt) +scan_gt = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(path)) for f in fn if is_label(f)] +scan_gt.sort() + +color_map_dict = yaml.safe_load(open("color_map.yml"))['color_map'] +learning_map = yaml.safe_load(open("color_map.yml"))['learning_map'] +color_map = {} +uncert_mean = np.zeros(20) +total_points_per_class = np.zeros(20) +for key, value in color_map_dict.items(): + color_map[key] = np.array(value, np.float32) / 255.0 + + +def plot_and_save(label_uncert, label_name, lidar_name, cam2_image_name): + labels = np.fromfile(label_name, dtype=np.int32).reshape((-1)) + uncerts = np.fromfile(label_uncert, dtype=np.float32).reshape((-1)) + velo_points = np.fromfile(lidar_name, dtype=np.float32).reshape(-1, 4) + try: + cam2_image = plt.imread(cam2_image_name) + except IOError: + print('detect error img %s' % label_name) + + plt.imshow(cam2_image) + + if True: + + # Project points to camera. + cam2_points = dataset.calib.T_cam2_velo.dot(velo_points.T).T + + # Filter out points behind camera + idx = cam2_points[:, 2] > 0 + print(idx) + # velo_points_projected = velo_points[idx] + cam2_points = cam2_points[idx] + labels_projected = labels[idx] + uncert_projected = uncerts[idx] + + # Remove homogeneous z. + cam2_points = cam2_points[:, :3] / cam2_points[:, 2:3] + + # Apply instrinsics. + intrinsic_cam2 = dataset.calib.K_cam2 + cam2_points = intrinsic_cam2.dot(cam2_points.T).T[:, [1, 0]] + cam2_points = cam2_points.astype(int) + + for i in range(0, cam2_points.shape[0]): + u, v = cam2_points[i, :] + label = labels_projected[i] + uncert = uncert_projected[i] + if label > 0 and v > 0 and v < 1241 and u > 0 and u < 376: + uncert_mean[learning_map[label]] += uncert + total_points_per_class[learning_map[label]] += 1 + m_circle = plt.Circle((v, u), 1, + color=matplotlib.cm.viridis(uncert), + alpha=0.4, + # color=color_map[label][..., ::-1] + ) + plt.gcf().gca().add_artist(m_circle) + + plt.axis('off') + path = os.path.join(basedir + 'sequences/' + sequence + projected_uncert) + plt.savefig(path + label_name.split('/')[-1].split('.')[0] + '.png', bbox_inches='tight', transparent=True, + pad_inches=0) + + +# with futures.ProcessPoolExecutor() as pool: +for label_uncert, label_name, lidar_name, cam2_image_name in zip(scan_uncert, scan_preds, dataset.velo_files, + dataset.cam2_files): + print(label_name.split('/')[-1]) + # if label_name == '/SPACE/DATA/SemanticKITTI/dataset/sequences/13/predictions/preds/001032.label': + plot_and_save(label_uncert, label_name, lidar_name, cam2_image_name) +print(total_points_per_class) +print(uncert_mean) +if __name__ == "__main__": + pass diff --git a/aimet_zoo_torch/salsanext/models/common/warmupLR.py b/aimet_zoo_torch/salsanext/models/common/warmupLR.py new file mode 100644 index 0000000..1292ae1 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/common/warmupLR.py @@ -0,0 +1,76 @@ +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + + +import torch.optim.lr_scheduler as toptim + + +class warmupLR(toptim._LRScheduler): + """ Warmup learning rate scheduler. + Initially, increases the learning rate from 0 to the final value, in a + certain number of steps. After this number of steps, each step decreases + LR exponentially. + """ + + def __init__(self, optimizer, lr, warmup_steps, momentum, decay): + # cyclic params + self.optimizer = optimizer + self.lr = lr + self.warmup_steps = warmup_steps + self.momentum = momentum + self.decay = decay + + # cap to one + if self.warmup_steps < 1: + self.warmup_steps = 1 + + # cyclic lr + self.initial_scheduler = toptim.CyclicLR(self.optimizer, + base_lr=0, + max_lr=self.lr, + step_size_up=self.warmup_steps, + step_size_down=self.warmup_steps, + cycle_momentum=False, + base_momentum=self.momentum, + max_momentum=self.momentum) + + # our params + self.last_epoch = -1 # fix for pytorch 1.1 and below + self.finished = False # am i done + super().__init__(optimizer) + + def get_lr(self): + return [self.lr * (self.decay ** self.last_epoch) for lr in self.base_lrs] + + def step(self, epoch=None): + if self.finished or self.initial_scheduler.last_epoch >= self.warmup_steps: + if not self.finished: + self.base_lrs = [self.lr for lr in self.base_lrs] + self.finished = True + return super(warmupLR, self).step(epoch) + else: + return self.initial_scheduler.step(epoch) diff --git a/aimet_zoo_torch/salsanext/models/data_cfg.yaml b/aimet_zoo_torch/salsanext/models/data_cfg.yaml new file mode 100644 index 0000000..1d5df93 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/data_cfg.yaml @@ -0,0 +1,212 @@ +# This file is covered by the LICENSE file in the root of this project. +name: "kitti" +labels: + 0: "unlabeled" + 1: "outlier" + 10: "car" + 11: "bicycle" + 13: "bus" + 15: "motorcycle" + 16: "on-rails" + 18: "truck" + 20: "other-vehicle" + 30: "person" + 31: "bicyclist" + 32: "motorcyclist" + 40: "road" + 44: "parking" + 48: "sidewalk" + 49: "other-ground" + 50: "building" + 51: "fence" + 52: "other-structure" + 60: "lane-marking" + 70: "vegetation" + 71: "trunk" + 72: "terrain" + 80: "pole" + 81: "traffic-sign" + 99: "other-object" + 252: "moving-car" + 253: "moving-bicyclist" + 254: "moving-person" + 255: "moving-motorcyclist" + 256: "moving-on-rails" + 257: "moving-bus" + 258: "moving-truck" + 259: "moving-other-vehicle" +color_map: # bgr + 0: [0, 0, 0] + 1: [0, 0, 255] + 10: [245, 150, 100] + 11: [245, 230, 100] + 13: [250, 80, 100] + 15: [150, 60, 30] + 16: [255, 0, 0] + 18: [180, 30, 80] + 20: [255, 0, 0] + 30: [30, 30, 255] + 31: [200, 40, 255] + 32: [90, 30, 150] + 40: [255, 0, 255] + 44: [255, 150, 255] + 48: [75, 0, 75] + 49: [75, 0, 175] + 50: [0, 200, 255] + 51: [50, 120, 255] + 52: [0, 150, 255] + 60: [170, 255, 150] + 70: [0, 175, 0] + 71: [0, 60, 135] + 72: [80, 240, 150] + 80: [150, 240, 255] + 81: [0, 0, 255] + 99: [255, 255, 50] + 252: [245, 150, 100] + 256: [255, 0, 0] + 253: [200, 40, 255] + 254: [30, 30, 255] + 255: [90, 30, 150] + 257: [250, 80, 100] + 258: [180, 30, 80] + 259: [255, 0, 0] +content: # as a ratio with the total number of points + 0: 0.018889854628292943 + 1: 0.0002937197336781505 + 10: 0.040818519255974316 + 11: 0.00016609538710764618 + 13: 2.7879693665067774e-05 + 15: 0.00039838616015114444 + 16: 0.0 + 18: 0.0020633612104619787 + 20: 0.0016218197275284021 + 30: 0.00017698551338515307 + 31: 1.1065903904919655e-08 + 32: 5.532951952459828e-09 + 40: 0.1987493871255525 + 44: 0.014717169549888214 + 48: 0.14392298360372 + 49: 0.0039048553037472045 + 50: 0.1326861944777486 + 51: 0.0723592229456223 + 52: 0.002395131480328884 + 60: 4.7084144280367186e-05 + 70: 0.26681502148037506 + 71: 0.006035012012626033 + 72: 0.07814222006271769 + 80: 0.002855498193863172 + 81: 0.0006155958086189918 + 99: 0.009923127583046915 + 252: 0.001789309418528068 + 253: 0.00012709999297008662 + 254: 0.00016059776092534436 + 255: 3.745553104802113e-05 + 256: 0.0 + 257: 0.00011351574470342043 + 258: 0.00010157861367183268 + 259: 4.3840131989471124e-05 +# classes that are indistinguishable from single scan or inconsistent in +# ground truth are mapped to their closest equivalent +learning_map: + 0: 0 # "unlabeled" + 1: 0 # "outlier" mapped to "unlabeled" --------------------------mapped + 10: 1 # "car" + 11: 2 # "bicycle" + 13: 5 # "bus" mapped to "other-vehicle" --------------------------mapped + 15: 3 # "motorcycle" + 16: 5 # "on-rails" mapped to "other-vehicle" ---------------------mapped + 18: 4 # "truck" + 20: 5 # "other-vehicle" + 30: 6 # "person" + 31: 7 # "bicyclist" + 32: 8 # "motorcyclist" + 40: 9 # "road" + 44: 10 # "parking" + 48: 11 # "sidewalk" + 49: 12 # "other-ground" + 50: 13 # "building" + 51: 14 # "fence" + 52: 0 # "other-structure" mapped to "unlabeled" ------------------mapped + 60: 9 # "lane-marking" to "road" ---------------------------------mapped + 70: 15 # "vegetation" + 71: 16 # "trunk" + 72: 17 # "terrain" + 80: 18 # "pole" + 81: 19 # "traffic-sign" + 99: 0 # "other-object" to "unlabeled" ----------------------------mapped + 252: 1 # "moving-car" to "car" ------------------------------------mapped + 253: 7 # "moving-bicyclist" to "bicyclist" ------------------------mapped + 254: 6 # "moving-person" to "person" ------------------------------mapped + 255: 8 # "moving-motorcyclist" to "motorcyclist" ------------------mapped + 256: 5 # "moving-on-rails" mapped to "other-vehicle" --------------mapped + 257: 5 # "moving-bus" mapped to "other-vehicle" -------------------mapped + 258: 4 # "moving-truck" to "truck" --------------------------------mapped + 259: 5 # "moving-other"-vehicle to "other-vehicle" ----------------mapped +learning_map_inv: # inverse of previous map + 0: 0 # "unlabeled", and others ignored + 1: 10 # "car" + 2: 11 # "bicycle" + 3: 15 # "motorcycle" + 4: 18 # "truck" + 5: 20 # "other-vehicle" + 6: 30 # "person" + 7: 31 # "bicyclist" + 8: 32 # "motorcyclist" + 9: 40 # "road" + 10: 44 # "parking" + 11: 48 # "sidewalk" + 12: 49 # "other-ground" + 13: 50 # "building" + 14: 51 # "fence" + 15: 70 # "vegetation" + 16: 71 # "trunk" + 17: 72 # "terrain" + 18: 80 # "pole" + 19: 81 # "traffic-sign" +learning_ignore: # Ignore classes + 0: True # "unlabeled", and others ignored + 1: False # "car" + 2: False # "bicycle" + 3: False # "motorcycle" + 4: False # "truck" + 5: False # "other-vehicle" + 6: False # "person" + 7: False # "bicyclist" + 8: False # "motorcyclist" + 9: False # "road" + 10: False # "parking" + 11: False # "sidewalk" + 12: False # "other-ground" + 13: False # "building" + 14: False # "fence" + 15: False # "vegetation" + 16: False # "trunk" + 17: False # "terrain" + 18: False # "pole" + 19: False # "traffic-sign" +split: # sequence numbers + train: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 9 + - 10 + valid: + - 8 + test: + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 diff --git a/aimet_zoo_torch/salsanext/models/model_cards/salsanext_w4a8.json b/aimet_zoo_torch/salsanext/models/model_cards/salsanext_w4a8.json new file mode 100644 index 0000000..cc0b4c4 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/model_cards/salsanext_w4a8.json @@ -0,0 +1,24 @@ +{ + "name": "SalsaNext", + "framework": "pytorch", + "task": "semantic segmentation", + "model_args": {}, + "input_shape": [1, 5, 64, 2048], + "training_dataset": "SemanticKitti", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf", + "techniques": ["bath_norm_folding"] + } + }, + "artifacts": { + "url_pre_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/SalsaNext", + "url_post_opt_weights": "https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/download/torch_salsanext_models_update/SalsaNext_optimized_w4A8_model.pth", + "url_aimet_encodings": "https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/download/torch_salsanext_models_update/SalsaNext_optimized_w4A8_encoding.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.24/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} diff --git a/aimet_zoo_torch/salsanext/models/model_cards/salsanext_w8a8.json b/aimet_zoo_torch/salsanext/models/model_cards/salsanext_w8a8.json new file mode 100644 index 0000000..6f5c32c --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/model_cards/salsanext_w8a8.json @@ -0,0 +1,24 @@ +{ + "name": "SalsaNext", + "framework": "pytorch", + "task": "semantic segmentation", + "model_args": {}, + "input_shape": [1, 5, 64, 2048], + "training_dataset": "SemanticKitti", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "percentile", + "techniques": ["bath_norm_folding", "adaround"] + } + }, + "artifacts": { + "url_pre_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/SalsaNext", + "url_post_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/SalsaNext_optimized_model.pth", + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_salsanext/SalsaNext_optimized_encoding.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.24/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + } +} diff --git a/aimet_zoo_torch/salsanext/models/model_definition.py b/aimet_zoo_torch/salsanext/models/model_definition.py new file mode 100644 index 0000000..17fd45a --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/model_definition.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R1732 +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2022 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" SalsaNext """ + +import json +import os +import pathlib +from collections import OrderedDict +import yaml +import torch +from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim +from aimet_zoo_torch.common.downloader import Downloader +from aimet_zoo_torch.salsanext.models.tasks.semantic.modules.SalsaNext import ( + SalsaNext as SalsaNextBase, +) + + +class SalsaNext(Downloader): + """SalsaNext Semantic Segmentation parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + + def __init__(self, model_config=None): + """ + :param model_config: named model config from which to obtain model artifacts and arguments. + If provided, overwrites the other arguments passed to this object + """ + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.cfg = False + if model_config: + config_filepath = self.parent_dir + "/model_cards/" + model_config + ".json" + if os.path.exists(config_filepath): + with open(config_filepath, encoding='utf8') as f_in: + self.cfg = json.load(f_in) + if self.cfg: + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=self.parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + self.dummy_input = torch.rand(self.input_shape, device=torch.device("cuda")) + self.model = SalsaNextBase(nclasses=20) + self.DATA = yaml.safe_load( + open(os.path.join(self.parent_dir, "data_cfg.yaml"), "r", encoding='utf8') + ) + self.ARCH = yaml.safe_load( + open(os.path.join(self.parent_dir, "arch_cfg.yaml"), "r", encoding='utf8') + ) + + def from_pretrained(self, quantized=False): + """load pretrained weights""" + if not self.cfg: + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) + self._download_pre_opt_weights() + self._download_post_opt_weights() + self._download_aimet_config() + self._download_aimet_encodings() + if quantized: + self.model = torch.load(self.path_post_opt_weights) + else: + state_dict = torch.load(self.path_pre_opt_weights)["state_dict"] + new_dict = OrderedDict() + for key, values in state_dict.items(): + key = key[7:] + new_dict[key] = values + self.model.load_state_dict(new_dict, strict=True) + self.model.cuda() + self.model.eval() + + def get_quantsim(self, quantized=False): + """get quantsim object with pre-loaded encodings""" + if not self.cfg: + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) + if quantized: + self.from_pretrained(quantized=True) + else: + self.from_pretrained(quantized=False) + dummy_input = torch.rand(self.input_shape, device=torch.device("cuda")) + quant_config = self.cfg["optimization_config"]["quantization_configuration"] + kwargs = { + "quant_scheme": quant_config["quant_scheme"], + "default_param_bw": quant_config["param_bw"], + "default_output_bw": quant_config["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } + sim = QuantizationSimModel(self.model, **kwargs) + if quant_config["quant_scheme"] == "percentile": + sim.set_percentile_value(99.9) + if self.path_aimet_encodings and quantized: + load_encodings_to_sim(sim, self.path_aimet_encodings) + print("load_encodings_to_sim finished!") + if self.path_adaround_encodings and quantized: + sim.set_and_freeze_param_encodings(self.path_adaround_encodings) + print("set_and_freeze_param_encodings finished!") + sim.model.cuda() + sim.model.eval() + return sim + + def __call__(self, x): + return self.model(x) diff --git a/aimet_zoo_torch/salsanext/models/tasks/__init__.py b/aimet_zoo_torch/salsanext/models/tasks/__init__.py new file mode 100644 index 0000000..136d063 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/__init__.py @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/__init__.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/__init__.py new file mode 100644 index 0000000..baa8c5b --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/__init__.py @@ -0,0 +1,6 @@ +# pylint: skip-file +import sys + +TRAIN_PATH = "../../" +DEPLOY_PATH = "../../../deploy" +sys.path.insert(0, TRAIN_PATH) diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/config/labels/semantic-kitti-all.yaml b/aimet_zoo_torch/salsanext/models/tasks/semantic/config/labels/semantic-kitti-all.yaml new file mode 100644 index 0000000..36b0072 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/config/labels/semantic-kitti-all.yaml @@ -0,0 +1,224 @@ +# This file is covered by the LICENSE file in the root of this project. +name: "kitti" +labels: + 0: "unlabeled" + 1: "outlier" + 10: "car" + 11: "bicycle" + 13: "bus" + 15: "motorcycle" + 16: "on-rails" + 18: "truck" + 20: "other-vehicle" + 30: "person" + 31: "bicyclist" + 32: "motorcyclist" + 40: "road" + 44: "parking" + 48: "sidewalk" + 49: "other-ground" + 50: "building" + 51: "fence" + 52: "other-structure" + 60: "lane-marking" + 70: "vegetation" + 71: "trunk" + 72: "terrain" + 80: "pole" + 81: "traffic-sign" + 99: "other-object" + 252: "moving-car" + 253: "moving-bicyclist" + 254: "moving-person" + 255: "moving-motorcyclist" + 256: "moving-on-rails" + 257: "moving-bus" + 258: "moving-truck" + 259: "moving-other-vehicle" +color_map: # bgr + 0: [0, 0, 0] + 1: [0, 0, 255] + 10: [245, 150, 100] + 11: [245, 230, 100] + 13: [250, 80, 100] + 15: [150, 60, 30] + 16: [255, 0, 0] + 18: [180, 30, 80] + 20: [255, 0, 0] + 30: [30, 30, 255] + 31: [200, 40, 255] + 32: [90, 30, 150] + 40: [255, 0, 255] + 44: [255, 150, 255] + 48: [75, 0, 75] + 49: [75, 0, 175] + 50: [0, 200, 255] + 51: [50, 120, 255] + 52: [0, 150, 255] + 60: [170, 255, 150] + 70: [0, 175, 0] + 71: [0, 60, 135] + 72: [80, 240, 150] + 80: [150, 240, 255] + 81: [0, 0, 255] + 99: [255, 255, 50] + 252: [245, 150, 100] + 256: [255, 0, 0] + 253: [200, 40, 255] + 254: [30, 30, 255] + 255: [90, 30, 150] + 257: [250, 80, 100] + 258: [180, 30, 80] + 259: [255, 0, 0] +content: # as a ratio with the total number of points + 0: 0.018889854628292943 + 1: 0.0002937197336781505 + 10: 0.040818519255974316 + 11: 0.00016609538710764618 + 13: 2.7879693665067774e-05 + 15: 0.00039838616015114444 + 16: 0.0 + 18: 0.0020633612104619787 + 20: 0.0016218197275284021 + 30: 0.00017698551338515307 + 31: 1.1065903904919655e-08 + 32: 5.532951952459828e-09 + 40: 0.1987493871255525 + 44: 0.014717169549888214 + 48: 0.14392298360372 + 49: 0.0039048553037472045 + 50: 0.1326861944777486 + 51: 0.0723592229456223 + 52: 0.002395131480328884 + 60: 4.7084144280367186e-05 + 70: 0.26681502148037506 + 71: 0.006035012012626033 + 72: 0.07814222006271769 + 80: 0.002855498193863172 + 81: 0.0006155958086189918 + 99: 0.009923127583046915 + 252: 0.001789309418528068 + 253: 0.00012709999297008662 + 254: 0.00016059776092534436 + 255: 3.745553104802113e-05 + 256: 0.0 + 257: 0.00011351574470342043 + 258: 0.00010157861367183268 + 259: 4.3840131989471124e-05 +# classes that are indistinguishable from single scan or inconsistent in +# ground truth are mapped to their closest equivalent +learning_map: + 0: 0 # "unlabeled" + 1: 0 # "outlier" mapped to "unlabeled" --------------------------mapped + 10: 1 # "car" + 11: 2 # "bicycle" + 13: 5 # "bus" mapped to "other-vehicle" --------------------------mapped + 15: 3 # "motorcycle" + 16: 5 # "on-rails" mapped to "other-vehicle" ---------------------mapped + 18: 4 # "truck" + 20: 5 # "other-vehicle" + 30: 6 # "person" + 31: 7 # "bicyclist" + 32: 8 # "motorcyclist" + 40: 9 # "road" + 44: 10 # "parking" + 48: 11 # "sidewalk" + 49: 12 # "other-ground" + 50: 13 # "building" + 51: 14 # "fence" + 52: 0 # "other-structure" mapped to "unlabeled" ------------------mapped + 60: 9 # "lane-marking" to "road" ---------------------------------mapped + 70: 15 # "vegetation" + 71: 16 # "trunk" + 72: 17 # "terrain" + 80: 18 # "pole" + 81: 19 # "traffic-sign" + 99: 0 # "other-object" to "unlabeled" ----------------------------mapped + 252: 20 # "moving-car" + 253: 21 # "moving-bicyclist" + 254: 22 # "moving-person" + 255: 23 # "moving-motorcyclist" + 256: 24 # "moving-on-rails" mapped to "moving-other-vehicle" ------mapped + 257: 24 # "moving-bus" mapped to "moving-other-vehicle" -----------mapped + 258: 25 # "moving-truck" + 259: 24 # "moving-other-vehicle" +learning_map_inv: # inverse of previous map + 0: 0 # "unlabeled", and others ignored + 1: 10 # "car" + 2: 11 # "bicycle" + 3: 15 # "motorcycle" + 4: 18 # "truck" + 5: 20 # "other-vehicle" + 6: 30 # "person" + 7: 31 # "bicyclist" + 8: 32 # "motorcyclist" + 9: 40 # "road" + 10: 44 # "parking" + 11: 48 # "sidewalk" + 12: 49 # "other-ground" + 13: 50 # "building" + 14: 51 # "fence" + 15: 70 # "vegetation" + 16: 71 # "trunk" + 17: 72 # "terrain" + 18: 80 # "pole" + 19: 81 # "traffic-sign" + 20: 252 # "moving-car" + 21: 253 # "moving-bicyclist" + 22: 254 # "moving-person" + 23: 255 # "moving-motorcyclist" + 24: 259 # "moving-other-vehicle" + 25: 258 # "moving-truck" +learning_ignore: # Ignore classes + 0: True # "unlabeled", and others ignored + 1: False # "car" + 2: False # "bicycle" + 3: False # "motorcycle" + 4: False # "truck" + 5: False # "other-vehicle" + 6: False # "person" + 7: False # "bicyclist" + 8: False # "motorcyclist" + 9: False # "road" + 10: False # "parking" + 11: False # "sidewalk" + 12: False # "other-ground" + 13: False # "building" + 14: False # "fence" + 15: False # "vegetation" + 16: False # "trunk" + 17: False # "terrain" + 18: False # "pole" + 19: False # "traffic-sign" + 20: False # "moving-car" + 21: False # "moving-bicyclist" + 22: False # "moving-person" + 23: False # "moving-motorcyclist" + 24: False # "moving-other-vehicle" + 25: False # "moving-truck" +split: # sequence numbers + train: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 9 + - 10 + valid: + - 8 + test: + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/config/labels/semantic-kitti.yaml b/aimet_zoo_torch/salsanext/models/tasks/semantic/config/labels/semantic-kitti.yaml new file mode 100644 index 0000000..1d5df93 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/config/labels/semantic-kitti.yaml @@ -0,0 +1,212 @@ +# This file is covered by the LICENSE file in the root of this project. +name: "kitti" +labels: + 0: "unlabeled" + 1: "outlier" + 10: "car" + 11: "bicycle" + 13: "bus" + 15: "motorcycle" + 16: "on-rails" + 18: "truck" + 20: "other-vehicle" + 30: "person" + 31: "bicyclist" + 32: "motorcyclist" + 40: "road" + 44: "parking" + 48: "sidewalk" + 49: "other-ground" + 50: "building" + 51: "fence" + 52: "other-structure" + 60: "lane-marking" + 70: "vegetation" + 71: "trunk" + 72: "terrain" + 80: "pole" + 81: "traffic-sign" + 99: "other-object" + 252: "moving-car" + 253: "moving-bicyclist" + 254: "moving-person" + 255: "moving-motorcyclist" + 256: "moving-on-rails" + 257: "moving-bus" + 258: "moving-truck" + 259: "moving-other-vehicle" +color_map: # bgr + 0: [0, 0, 0] + 1: [0, 0, 255] + 10: [245, 150, 100] + 11: [245, 230, 100] + 13: [250, 80, 100] + 15: [150, 60, 30] + 16: [255, 0, 0] + 18: [180, 30, 80] + 20: [255, 0, 0] + 30: [30, 30, 255] + 31: [200, 40, 255] + 32: [90, 30, 150] + 40: [255, 0, 255] + 44: [255, 150, 255] + 48: [75, 0, 75] + 49: [75, 0, 175] + 50: [0, 200, 255] + 51: [50, 120, 255] + 52: [0, 150, 255] + 60: [170, 255, 150] + 70: [0, 175, 0] + 71: [0, 60, 135] + 72: [80, 240, 150] + 80: [150, 240, 255] + 81: [0, 0, 255] + 99: [255, 255, 50] + 252: [245, 150, 100] + 256: [255, 0, 0] + 253: [200, 40, 255] + 254: [30, 30, 255] + 255: [90, 30, 150] + 257: [250, 80, 100] + 258: [180, 30, 80] + 259: [255, 0, 0] +content: # as a ratio with the total number of points + 0: 0.018889854628292943 + 1: 0.0002937197336781505 + 10: 0.040818519255974316 + 11: 0.00016609538710764618 + 13: 2.7879693665067774e-05 + 15: 0.00039838616015114444 + 16: 0.0 + 18: 0.0020633612104619787 + 20: 0.0016218197275284021 + 30: 0.00017698551338515307 + 31: 1.1065903904919655e-08 + 32: 5.532951952459828e-09 + 40: 0.1987493871255525 + 44: 0.014717169549888214 + 48: 0.14392298360372 + 49: 0.0039048553037472045 + 50: 0.1326861944777486 + 51: 0.0723592229456223 + 52: 0.002395131480328884 + 60: 4.7084144280367186e-05 + 70: 0.26681502148037506 + 71: 0.006035012012626033 + 72: 0.07814222006271769 + 80: 0.002855498193863172 + 81: 0.0006155958086189918 + 99: 0.009923127583046915 + 252: 0.001789309418528068 + 253: 0.00012709999297008662 + 254: 0.00016059776092534436 + 255: 3.745553104802113e-05 + 256: 0.0 + 257: 0.00011351574470342043 + 258: 0.00010157861367183268 + 259: 4.3840131989471124e-05 +# classes that are indistinguishable from single scan or inconsistent in +# ground truth are mapped to their closest equivalent +learning_map: + 0: 0 # "unlabeled" + 1: 0 # "outlier" mapped to "unlabeled" --------------------------mapped + 10: 1 # "car" + 11: 2 # "bicycle" + 13: 5 # "bus" mapped to "other-vehicle" --------------------------mapped + 15: 3 # "motorcycle" + 16: 5 # "on-rails" mapped to "other-vehicle" ---------------------mapped + 18: 4 # "truck" + 20: 5 # "other-vehicle" + 30: 6 # "person" + 31: 7 # "bicyclist" + 32: 8 # "motorcyclist" + 40: 9 # "road" + 44: 10 # "parking" + 48: 11 # "sidewalk" + 49: 12 # "other-ground" + 50: 13 # "building" + 51: 14 # "fence" + 52: 0 # "other-structure" mapped to "unlabeled" ------------------mapped + 60: 9 # "lane-marking" to "road" ---------------------------------mapped + 70: 15 # "vegetation" + 71: 16 # "trunk" + 72: 17 # "terrain" + 80: 18 # "pole" + 81: 19 # "traffic-sign" + 99: 0 # "other-object" to "unlabeled" ----------------------------mapped + 252: 1 # "moving-car" to "car" ------------------------------------mapped + 253: 7 # "moving-bicyclist" to "bicyclist" ------------------------mapped + 254: 6 # "moving-person" to "person" ------------------------------mapped + 255: 8 # "moving-motorcyclist" to "motorcyclist" ------------------mapped + 256: 5 # "moving-on-rails" mapped to "other-vehicle" --------------mapped + 257: 5 # "moving-bus" mapped to "other-vehicle" -------------------mapped + 258: 4 # "moving-truck" to "truck" --------------------------------mapped + 259: 5 # "moving-other"-vehicle to "other-vehicle" ----------------mapped +learning_map_inv: # inverse of previous map + 0: 0 # "unlabeled", and others ignored + 1: 10 # "car" + 2: 11 # "bicycle" + 3: 15 # "motorcycle" + 4: 18 # "truck" + 5: 20 # "other-vehicle" + 6: 30 # "person" + 7: 31 # "bicyclist" + 8: 32 # "motorcyclist" + 9: 40 # "road" + 10: 44 # "parking" + 11: 48 # "sidewalk" + 12: 49 # "other-ground" + 13: 50 # "building" + 14: 51 # "fence" + 15: 70 # "vegetation" + 16: 71 # "trunk" + 17: 72 # "terrain" + 18: 80 # "pole" + 19: 81 # "traffic-sign" +learning_ignore: # Ignore classes + 0: True # "unlabeled", and others ignored + 1: False # "car" + 2: False # "bicycle" + 3: False # "motorcycle" + 4: False # "truck" + 5: False # "other-vehicle" + 6: False # "person" + 7: False # "bicyclist" + 8: False # "motorcyclist" + 9: False # "road" + 10: False # "parking" + 11: False # "sidewalk" + 12: False # "other-ground" + 13: False # "building" + 14: False # "fence" + 15: False # "vegetation" + 16: False # "trunk" + 17: False # "terrain" + 18: False # "pole" + 19: False # "traffic-sign" +split: # sequence numbers + train: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 9 + - 10 + valid: + - 8 + test: + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/dataset/kitti/__init__.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/dataset/kitti/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/dataset/kitti/parser.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/dataset/kitti/parser.py new file mode 100644 index 0000000..0d477e4 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/dataset/kitti/parser.py @@ -0,0 +1,462 @@ +# pylint: skip-file + + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +import os +import numpy as np +import torch +from torch.utils.data import Dataset +from aimet_zoo_torch.salsanext.models.common.laserscan import LaserScan, SemLaserScan +import torchvision + +import torch +import math +import random +from PIL import Image +try: + import accimage +except ImportError: + accimage = None +import numpy as np +import numbers +import types +from collections.abc import Sequence, Iterable +import warnings + + +EXTENSIONS_SCAN = ['.bin'] +EXTENSIONS_LABEL = ['.label'] + + +def is_scan(filename): + return any(filename.endswith(ext) for ext in EXTENSIONS_SCAN) + + +def is_label(filename): + return any(filename.endswith(ext) for ext in EXTENSIONS_LABEL) + + +def my_collate(batch): + data = [item[0] for item in batch] + project_mask = [item[1] for item in batch] + proj_labels = [item[2] for item in batch] + data = torch.stack(data,dim=0) + project_mask = torch.stack(project_mask,dim=0) + proj_labels = torch.stack(proj_labels, dim=0) + + to_augment =(proj_labels == 12).nonzero() + to_augment_unique_12 = torch.unique(to_augment[:, 0]) + + to_augment = (proj_labels == 5).nonzero() + to_augment_unique_5 = torch.unique(to_augment[:, 0]) + + to_augment = (proj_labels == 8).nonzero() + to_augment_unique_8 = torch.unique(to_augment[:, 0]) + + to_augment_unique = torch.cat((to_augment_unique_5,to_augment_unique_8,to_augment_unique_12),dim=0) + to_augment_unique = torch.unique(to_augment_unique) + + for k in to_augment_unique: + data = torch.cat((data,torch.flip(data[k.item()], [2]).unsqueeze(0)),dim=0) + proj_labels = torch.cat((proj_labels,torch.flip(proj_labels[k.item()], [1]).unsqueeze(0)),dim=0) + project_mask = torch.cat((project_mask,torch.flip(project_mask[k.item()], [1]).unsqueeze(0)),dim=0) + + return data, project_mask,proj_labels + +class SemanticKitti(Dataset): + + def __init__(self, root, # directory where data is + sequences, # sequences for this data (e.g. [1,3,4,6]) + labels, # label dict: (e.g 10: "car") + color_map, # colors dict bgr (e.g 10: [255, 0, 0]) + learning_map, # classes to learn (0 to N-1 for xentropy) + learning_map_inv, # inverse of previous (recover labels) + sensor, # sensor to parse scans from + max_points=150000, # max number of points present in dataset + gt=True, + transform=False): # send ground truth? + # save deats + self.root = os.path.join(root, "sequences") + self.sequences = sequences + self.labels = labels + self.color_map = color_map + self.learning_map = learning_map + self.learning_map_inv = learning_map_inv + self.sensor = sensor + self.sensor_img_H = sensor["img_prop"]["height"] + self.sensor_img_W = sensor["img_prop"]["width"] + self.sensor_img_means = torch.tensor(sensor["img_means"], + dtype=torch.float) + self.sensor_img_stds = torch.tensor(sensor["img_stds"], + dtype=torch.float) + self.sensor_fov_up = sensor["fov_up"] + self.sensor_fov_down = sensor["fov_down"] + self.max_points = max_points + self.gt = gt + self.transform = transform + + # get number of classes (can't be len(self.learning_map) because there + # are multiple repeated entries, so the number that matters is how many + # there are for the xentropy) + self.nclasses = len(self.learning_map_inv) + + # sanity checks + + # make sure directory exists + if os.path.isdir(self.root): + print("Sequences folder exists! Using sequences from %s" % self.root) + else: + raise ValueError("Sequences folder doesn't exist! Exiting...") + + # make sure labels is a dict + assert(isinstance(self.labels, dict)) + + # make sure color_map is a dict + assert(isinstance(self.color_map, dict)) + + # make sure learning_map is a dict + assert(isinstance(self.learning_map, dict)) + + # make sure sequences is a list + assert(isinstance(self.sequences, list)) + + # placeholder for filenames + self.scan_files = [] + self.label_files = [] + + # fill in with names, checking that all sequences are complete + for seq in self.sequences: + # to string + seq = '{0:02d}'.format(int(seq)) + + print("parsing seq {}".format(seq)) + + # get paths for each + scan_path = os.path.join(self.root, seq, "velodyne") + label_path = os.path.join(self.root, seq, "labels") + + # get files + scan_files = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(scan_path)) for f in fn if is_scan(f)] + label_files = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(label_path)) for f in fn if is_label(f)] + + # check all scans have labels + if self.gt: + assert(len(scan_files) == len(label_files)) + + # extend list + self.scan_files.extend(scan_files) + self.label_files.extend(label_files) + + # sort for correspondance + self.scan_files.sort() + self.label_files.sort() + + print("Using {} scans from sequences {}".format(len(self.scan_files), + self.sequences)) + + def __getitem__(self, index): + # get item in tensor shape + scan_file = self.scan_files[index] + if self.gt: + label_file = self.label_files[index] + + # open a semantic laserscan + DA = False + flip_sign = False + rot = False + drop_points = False + if self.transform: + if random.random() > 0.5: + if random.random() > 0.5: + DA = True + if random.random() > 0.5: + flip_sign = True + if random.random() > 0.5: + rot = True + drop_points = random.uniform(0, 0.5) + + if self.gt: + scan = SemLaserScan(self.color_map, + project=True, + H=self.sensor_img_H, + W=self.sensor_img_W, + fov_up=self.sensor_fov_up, + fov_down=self.sensor_fov_down, + DA=DA, + flip_sign=flip_sign, + drop_points=drop_points) + else: + scan = LaserScan(project=True, + H=self.sensor_img_H, + W=self.sensor_img_W, + fov_up=self.sensor_fov_up, + fov_down=self.sensor_fov_down, + DA=DA, + rot=rot, + flip_sign=flip_sign, + drop_points=drop_points) + + # open and obtain scan + scan.open_scan(scan_file) + if self.gt: + scan.open_label(label_file) + # map unused classes to used classes (also for projection) + scan.sem_label = self.map(scan.sem_label, self.learning_map) + scan.proj_sem_label = self.map(scan.proj_sem_label, self.learning_map) + + # make a tensor of the uncompressed data (with the max num points) + unproj_n_points = scan.points.shape[0] + unproj_xyz = torch.full((self.max_points, 3), -1.0, dtype=torch.float) + unproj_xyz[:unproj_n_points] = torch.from_numpy(scan.points) + unproj_range = torch.full([self.max_points], -1.0, dtype=torch.float) + unproj_range[:unproj_n_points] = torch.from_numpy(scan.unproj_range) + unproj_remissions = torch.full([self.max_points], -1.0, dtype=torch.float) + unproj_remissions[:unproj_n_points] = torch.from_numpy(scan.remissions) + if self.gt: + unproj_labels = torch.full([self.max_points], -1.0, dtype=torch.int32) + unproj_labels[:unproj_n_points] = torch.from_numpy(scan.sem_label) + else: + unproj_labels = [] + + # get points and labels + proj_range = torch.from_numpy(scan.proj_range).clone() + proj_xyz = torch.from_numpy(scan.proj_xyz).clone() + proj_remission = torch.from_numpy(scan.proj_remission).clone() + proj_mask = torch.from_numpy(scan.proj_mask) + if self.gt: + proj_labels = torch.from_numpy(scan.proj_sem_label).clone() + proj_labels = proj_labels * proj_mask + else: + proj_labels = [] + proj_x = torch.full([self.max_points], -1, dtype=torch.long) + proj_x[:unproj_n_points] = torch.from_numpy(scan.proj_x) + proj_y = torch.full([self.max_points], -1, dtype=torch.long) + proj_y[:unproj_n_points] = torch.from_numpy(scan.proj_y) + proj = torch.cat([proj_range.unsqueeze(0).clone(), + proj_xyz.clone().permute(2, 0, 1), + proj_remission.unsqueeze(0).clone()]) + proj = (proj - self.sensor_img_means[:, None, None] + ) / self.sensor_img_stds[:, None, None] + proj = proj * proj_mask.float() + + # get name and sequence + path_norm = os.path.normpath(scan_file) + path_split = path_norm.split(os.sep) + path_seq = path_split[-3] + path_name = path_split[-1].replace(".bin", ".label") + + # return + return proj, proj_mask, proj_labels, unproj_labels, path_seq, path_name, proj_x, proj_y, proj_range, unproj_range, proj_xyz, unproj_xyz, proj_remission, unproj_remissions, unproj_n_points + + def __len__(self): + return len(self.scan_files) + + @staticmethod + def map(label, mapdict): + # put label from original values to xentropy + # or vice-versa, depending on dictionary values + # make learning map a lookup table + maxkey = 0 + for key, data in mapdict.items(): + if isinstance(data, list): + nel = len(data) + else: + nel = 1 + if key > maxkey: + maxkey = key + # +100 hack making lut bigger just in case there are unknown labels + if nel > 1: + lut = np.zeros((maxkey + 100, nel), dtype=np.int32) + else: + lut = np.zeros((maxkey + 100), dtype=np.int32) + for key, data in mapdict.items(): + try: + lut[key] = data + except IndexError: + print("Wrong key ", key) + # do the mapping + return lut[label] + + +class Parser(): + # standard conv, BN, relu + def __init__(self, + root, # directory for data + train_sequences, # sequences to train + valid_sequences, # sequences to validate. + test_sequences, # sequences to test (if none, don't get) + labels, # labels in data + color_map, # color for each label + learning_map, # mapping for training labels + learning_map_inv, # recover labels from xentropy + sensor, # sensor to use + max_points, # max points in each scan in entire dataset + batch_size, # batch size for train and val + workers, # threads to load data + gt=True, # get gt? + shuffle_train=True): # shuffle training set? + super(Parser, self).__init__() + + # if I am training, get the dataset + self.root = root + self.train_sequences = train_sequences + self.valid_sequences = valid_sequences + self.test_sequences = test_sequences + self.labels = labels + self.color_map = color_map + self.learning_map = learning_map + self.learning_map_inv = learning_map_inv + self.sensor = sensor + self.max_points = max_points + self.batch_size = batch_size + self.workers = workers + self.gt = gt + self.shuffle_train = shuffle_train + + # number of classes that matters is the one for xentropy + self.nclasses = len(self.learning_map_inv) + + # Data loading code + self.train_dataset = SemanticKitti(root=self.root, + sequences=self.train_sequences, + labels=self.labels, + color_map=self.color_map, + learning_map=self.learning_map, + learning_map_inv=self.learning_map_inv, + sensor=self.sensor, + max_points=max_points, + transform=True, + gt=self.gt) + + self.trainloader = torch.utils.data.DataLoader(self.train_dataset, + batch_size=self.batch_size, + shuffle=self.shuffle_train, + num_workers=self.workers, + drop_last=True) + assert len(self.trainloader) > 0 + self.trainiter = iter(self.trainloader) + + self.valid_dataset = SemanticKitti(root=self.root, + sequences=self.valid_sequences, + labels=self.labels, + color_map=self.color_map, + learning_map=self.learning_map, + learning_map_inv=self.learning_map_inv, + sensor=self.sensor, + max_points=max_points, + gt=self.gt) + + self.validloader = torch.utils.data.DataLoader(self.valid_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.workers, + drop_last=True) + assert len(self.validloader) > 0 + self.validiter = iter(self.validloader) + + if self.test_sequences: + self.test_dataset = SemanticKitti(root=self.root, + sequences=self.test_sequences, + labels=self.labels, + color_map=self.color_map, + learning_map=self.learning_map, + learning_map_inv=self.learning_map_inv, + sensor=self.sensor, + max_points=max_points, + gt=False) + + self.testloader = torch.utils.data.DataLoader(self.test_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.workers, + drop_last=True) + assert len(self.testloader) > 0 + self.testiter = iter(self.testloader) + + def get_train_batch(self): + scans = self.trainiter.next() + return scans + + def get_train_set(self): + return self.trainloader + + def get_valid_batch(self): + scans = self.validiter.next() + return scans + + def get_valid_set(self): + return self.validloader + + def get_test_batch(self): + scans = self.testiter.next() + return scans + + def get_test_set(self): + return self.testloader + + def get_train_size(self): + return len(self.trainloader) + + def get_valid_size(self): + return len(self.validloader) + + def get_test_size(self): + return len(self.testloader) + + def get_n_classes(self): + return self.nclasses + + def get_original_class_string(self, idx): + return self.labels[idx] + + def get_xentropy_class_string(self, idx): + return self.labels[self.learning_map_inv[idx]] + + def to_original(self, label): + # put label in original values + return SemanticKitti.map(label, self.learning_map_inv) + + def to_xentropy(self, label): + # put label in xentropy values + return SemanticKitti.map(label, self.learning_map) + + def to_color(self, label): + # put label in original values + label = SemanticKitti.map(label, self.learning_map_inv) + # put label in color + return SemanticKitti.map(label, self.color_map) \ No newline at end of file diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/evaluate_iou.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/evaluate_iou.py new file mode 100644 index 0000000..4da7bc8 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/evaluate_iou.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + + +import argparse +import os +import yaml +import sys +import numpy as np +import torch +import __init__ as booger + +from tasks.semantic.modules.ioueval import iouEval +from common.laserscan import SemLaserScan + +# possible splits +splits = ['train','valid','test'] +def save_to_log(logdir,logfile,message): + f = open(logdir+'/'+logfile, "a") + f.write(message+'\n') + f.close() + return + +def eval(test_sequences,splits,pred): + # get scan paths + scan_names = [] + for sequence in test_sequences: + sequence = '{0:02d}'.format(int(sequence)) + scan_paths = os.path.join(FLAGS.dataset, "sequences", + str(sequence), "velodyne") + # populate the scan names + seq_scan_names = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(scan_paths)) for f in fn if ".bin" in f] + seq_scan_names.sort() + scan_names.extend(seq_scan_names) + # print(scan_names) + + # get label paths + label_names = [] + for sequence in test_sequences: + sequence = '{0:02d}'.format(int(sequence)) + label_paths = os.path.join(FLAGS.dataset, "sequences", + str(sequence), "labels") + # populate the label names + seq_label_names = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(label_paths)) for f in fn if ".label" in f] + seq_label_names.sort() + label_names.extend(seq_label_names) + # print(label_names) + + # get predictions paths + pred_names = [] + for sequence in test_sequences: + sequence = '{0:02d}'.format(int(sequence)) + pred_paths = os.path.join(FLAGS.predictions, "sequences", + sequence, "predictions") + # populate the label names + seq_pred_names = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(pred_paths)) for f in fn if ".label" in f] + seq_pred_names.sort() + pred_names.extend(seq_pred_names) + # print(pred_names) + + # check that I have the same number of files + # print("labels: ", len(label_names)) + # print("predictions: ", len(pred_names)) + assert (len(label_names) == len(scan_names) and + len(label_names) == len(pred_names)) + + # print("Evaluating sequences: ") + # open each file, get the tensor, and make the iou comparison + for scan_file, label_file, pred_file in zip(scan_names, label_names, pred_names): + # print("evaluating label ", label_file, "with", pred_file) + # open label + label = SemLaserScan(project=False) + label.open_scan(scan_file) + label.open_label(label_file) + u_label_sem = remap_lut[label.sem_label] # remap to xentropy format + if FLAGS.limit is not None: + u_label_sem = u_label_sem[:FLAGS.limit] + + # open prediction + pred = SemLaserScan(project=False) + pred.open_scan(scan_file) + pred.open_label(pred_file) + u_pred_sem = remap_lut[pred.sem_label] # remap to xentropy format + if FLAGS.limit is not None: + u_pred_sem = u_pred_sem[:FLAGS.limit] + + # add single scan to evaluation + evaluator.addBatch(u_pred_sem, u_label_sem) + + # when I am done, print the evaluation + m_accuracy = evaluator.getacc() + m_jaccard, class_jaccard = evaluator.getIoU() + + print('{split} set:\n' + 'Acc avg {m_accuracy:.3f}\n' + 'IoU avg {m_jaccard:.3f}'.format(split=splits, + m_accuracy=m_accuracy, + m_jaccard=m_jaccard)) + + save_to_log(FLAGS.predictions,'pred.txt','{split} set:\n' + 'Acc avg {m_accuracy:.3f}\n' + 'IoU avg {m_jaccard:.3f}'.format(split=splits, + m_accuracy=m_accuracy, + m_jaccard=m_jaccard)) + # print also classwise + for i, jacc in enumerate(class_jaccard): + if i not in ignore: + print('IoU class {i:} [{class_str:}] = {jacc:.3f}'.format( + i=i, class_str=class_strings[class_inv_remap[i]], jacc=jacc)) + save_to_log(FLAGS.predictions, 'pred.txt', 'IoU class {i:} [{class_str:}] = {jacc:.3f}'.format( + i=i, class_str=class_strings[class_inv_remap[i]], jacc=jacc)) + + # print for spreadsheet + print("*" * 80) + print("below can be copied straight for paper table") + for i, jacc in enumerate(class_jaccard): + if i not in ignore: + sys.stdout.write('{jacc:.3f}'.format(jacc=jacc.item())) + sys.stdout.write(",") + sys.stdout.write('{jacc:.3f}'.format(jacc=m_jaccard.item())) + sys.stdout.write(",") + sys.stdout.write('{acc:.3f}'.format(acc=m_accuracy.item())) + sys.stdout.write('\n') + sys.stdout.flush() + +if __name__ == '__main__': + parser = argparse.ArgumentParser("./evaluate_iou.py") + parser.add_argument( + '--dataset', '-d', + type=str, + required=True, + help='Dataset dir. No Default', + ) + parser.add_argument( + '--predictions', '-p', + type=str, + required=None, + help='Prediction dir. Same organization as dataset, but predictions in' + 'each sequences "prediction" directory. No Default. If no option is set' + ' we look for the labels in the same directory as dataset' + ) + parser.add_argument( + '--split', '-s', + type=str, + required=False, + choices=["train", "valid", "test"], + default=None, + help='Split to evaluate on. One of ' + + str(splits) + '. Defaults to %(default)s', + ) + parser.add_argument( + '--data_cfg', '-dc', + type=str, + required=False, + default="config/labels/semantic-kitti.yaml", + help='Dataset config file. Defaults to %(default)s', + ) + parser.add_argument( + '--limit', '-l', + type=int, + required=False, + default=None, + help='Limit to the first "--limit" points of each scan. Useful for' + ' evaluating single scan from aggregated pointcloud.' + ' Defaults to %(default)s', + ) + + FLAGS, unparsed = parser.parse_known_args() + + # fill in real predictions dir + if FLAGS.predictions is None: + FLAGS.predictions = FLAGS.dataset + + # print summary of what we will do + print("*" * 80) + print("INTERFACE:") + print("Data: ", FLAGS.dataset) + print("Predictions: ", FLAGS.predictions) + print("Split: ", FLAGS.split) + print("Config: ", FLAGS.data_cfg) + print("Limit: ", FLAGS.limit) + print("*" * 80) + + # assert split + assert (FLAGS.split in splits) + + # open data config file + try: + print("Opening data config file %s" % FLAGS.data_cfg) + DATA = yaml.safe_load(open(FLAGS.data_cfg, 'r')) + except Exception as e: + print(e) + print("Error opening data yaml file.") + quit() + + # get number of interest classes, and the label mappings + class_strings = DATA["labels"] + class_remap = DATA["learning_map"] + class_inv_remap = DATA["learning_map_inv"] + class_ignore = DATA["learning_ignore"] + nr_classes = len(class_inv_remap) + + # make lookup table for mapping + maxkey = 0 + for key, data in class_remap.items(): + if key > maxkey: + maxkey = key + # +100 hack making lut bigger just in case there are unknown labels + remap_lut = np.zeros((maxkey + 100), dtype=np.int32) + for key, data in class_remap.items(): + try: + remap_lut[key] = data + except IndexError: + print("Wrong key ", key) + # print(remap_lut) + + # create evaluator + ignore = [] + for cl, ign in class_ignore.items(): + if ign: + x_cl = int(cl) + ignore.append(x_cl) + print("Ignoring xentropy class ", x_cl, " in IoU evaluation") + + # create evaluator + device = torch.device("cpu") + evaluator = iouEval(nr_classes, device, ignore) + evaluator.reset() + + # get test set + if FLAGS.split is None: + for splits in ('train','valid'): + eval((DATA["split"][splits]),splits,FLAGS.predictions) + else: + eval(DATA["split"][FLAGS.split],splits,FLAGS.predictions) + + + + diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/infer.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/infer.py new file mode 100644 index 0000000..8d33e52 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/infer.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + + +import argparse +import subprocess +import datetime +import yaml +from shutil import copyfile +import os +import shutil +import __init__ as booger + +from tasks.semantic.modules.user import * +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y'): + return True + elif v.lower() in ('no', 'false', 'f', 'n'): + return False + else: + raise argparse.ArgumentTypeError('Boolean expected') + +if __name__ == '__main__': + splits = ["train", "valid", "test"] + parser = argparse.ArgumentParser("./infer.py") + parser.add_argument( + '--dataset', '-d', + type=str, + required=True, + help='Dataset to train with. No Default', + ) + parser.add_argument( + '--log', '-l', + type=str, + default=os.path.expanduser("~") + '/logs/' + + datetime.datetime.now().strftime("%Y-%-m-%d-%H:%M") + '/', + help='Directory to put the predictions. Default: ~/logs/date+time' + ) + parser.add_argument( + '--model', '-m', + type=str, + required=True, + default=None, + help='Directory to get the trained model.' + ) + + parser.add_argument( + '--uncertainty', '-u', + type=str2bool, nargs='?', + const=True, default=False, + help='Set this if you want to use the Uncertainty Version' + ) + + parser.add_argument( + '--monte-carlo', '-c', + type=int, default=30, + help='Number of samplings per scan' + ) + + + parser.add_argument( + '--split', '-s', + type=str, + required=False, + default=None, + help='Split to evaluate on. One of ' + + str(splits) + '. Defaults to %(default)s', + ) + FLAGS, unparsed = parser.parse_known_args() + + # print summary of what we will do + print("----------") + print("INTERFACE:") + print("dataset", FLAGS.dataset) + print("log", FLAGS.log) + print("model", FLAGS.model) + print("Uncertainty", FLAGS.uncertainty) + print("Monte Carlo Sampling", FLAGS.monte_carlo) + print("infering", FLAGS.split) + print("----------\n") + #print("Commit hash (training version): ", str( + # subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip())) + print("----------\n") + + # open arch config file + try: + print("Opening arch config file from %s" % FLAGS.model) + ARCH = yaml.safe_load(open(FLAGS.model + "/arch_cfg.yaml", 'r')) + except Exception as e: + print(e) + print("Error opening arch yaml file.") + quit() + + # open data config file + try: + print("Opening data config file from %s" % FLAGS.model) + DATA = yaml.safe_load(open(FLAGS.model + "/data_cfg.yaml", 'r')) + except Exception as e: + print(e) + print("Error opening data yaml file.") + quit() + + # create log folder + try: + if os.path.isdir(FLAGS.log): + shutil.rmtree(FLAGS.log) + os.makedirs(FLAGS.log) + os.makedirs(os.path.join(FLAGS.log, "sequences")) + for seq in DATA["split"]["train"]: + seq = '{0:02d}'.format(int(seq)) + print("train", seq) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) + for seq in DATA["split"]["valid"]: + seq = '{0:02d}'.format(int(seq)) + print("valid", seq) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) + for seq in DATA["split"]["test"]: + seq = '{0:02d}'.format(int(seq)) + print("test", seq) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq)) + os.makedirs(os.path.join(FLAGS.log, "sequences", seq, "predictions")) + except Exception as e: + print(e) + print("Error creating log directory. Check permissions!") + raise + + except Exception as e: + print(e) + print("Error creating log directory. Check permissions!") + quit() + + # does model folder exist? + if os.path.isdir(FLAGS.model): + print("model folder exists! Using model from %s" % (FLAGS.model)) + else: + print("model folder doesnt exist! Can't infer...") + quit() + + # create user and infer dataset + user = User(ARCH, DATA, FLAGS.dataset, FLAGS.log, FLAGS.model,FLAGS.split,FLAGS.uncertainty,FLAGS.monte_carlo) + user.infer() diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/Lovasz_Softmax.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/Lovasz_Softmax.py new file mode 100644 index 0000000..bda33d8 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/Lovasz_Softmax.py @@ -0,0 +1,148 @@ +# pylint: skip-file +""" + +MIT License + +Copyright (c) 2018 Maxim Berman +Copyright (c) 2020 Tiago Cortinhal, George Tzelepis and Eren Erdal Aksoy + + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +""" +import torch +import torch.nn as nn +from torch.autograd import Variable + + +try: + from itertools import ifilterfalse +except ImportError: + from itertools import filterfalse as ifilterfalse + + +def isnan(x): + return x != x + + +def mean(l, ignore_nan=False, empty=0): + """ + nanmean compatible with generators. + """ + l = iter(l) + if ignore_nan: + l = ifilterfalse(isnan, l) + try: + n = 1 + acc = next(l) + except StopIteration: + if empty == 'raise': + raise ValueError('Empty mean') + return empty + for n, v in enumerate(l, 2): + acc += v + if n == 1: + return acc + return acc / n + + +def lovasz_grad(gt_sorted): + """ + Computes gradient of the Lovasz extension w.r.t sorted errors + See Alg. 1 in paper + """ + p = len(gt_sorted) + gts = gt_sorted.sum() + intersection = gts - gt_sorted.float().cumsum(0) + union = gts + (1 - gt_sorted).float().cumsum(0) + jaccard = 1. - intersection / union + if p > 1: # cover 1-pixel case + jaccard[1:p] = jaccard[1:p] - jaccard[0:-1] + return jaccard + + +def lovasz_softmax(probas, labels, classes='present', per_image=False, ignore=None): + """ + Multi-class Lovasz-Softmax loss + probas: [B, C, H, W] Variable, class probabilities at each prediction (between 0 and 1). + Interpreted as binary (sigmoid) output with outputs of size [B, H, W]. + labels: [B, H, W] Tensor, ground truth labels (between 0 and C - 1) + classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average. + per_image: compute the loss per image instead of per batch + ignore: void class labels + """ + if per_image: + loss = mean(lovasz_softmax_flat(*flatten_probas(prob.unsqueeze(0), lab.unsqueeze(0), ignore), classes=classes) + for prob, lab in zip(probas, labels)) + else: + loss = lovasz_softmax_flat(*flatten_probas(probas, labels, ignore), classes=classes) + return loss + + +def lovasz_softmax_flat(probas, labels, classes='present'): + """ + Multi-class Lovasz-Softmax loss + probas: [P, C] Variable, class probabilities at each prediction (between 0 and 1) + labels: [P] Tensor, ground truth labels (between 0 and C - 1) + classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average. + """ + if probas.numel() == 0: + # only void pixels, the gradients should be 0 + return probas * 0. + C = probas.size(1) + losses = [] + class_to_sum = list(range(C)) if classes in ['all', 'present'] else classes + for c in class_to_sum: + fg = (labels == c).float() # foreground for class c + if (classes == 'present' and fg.sum() == 0): + continue + if C == 1: + if len(classes) > 1: + raise ValueError('Sigmoid output possible only with 1 class') + class_pred = probas[:, 0] + else: + class_pred = probas[:, c] + errors = (Variable(fg) - class_pred).abs() + errors_sorted, perm = torch.sort(errors, 0, descending=True) + perm = perm.data + fg_sorted = fg[perm] + losses.append(torch.dot(errors_sorted, Variable(lovasz_grad(fg_sorted)))) + return mean(losses) + + +def flatten_probas(probas, labels, ignore=None): + """ + Flattens predictions in the batch + """ + if probas.dim() == 3: + # assumes output of a sigmoid layer + B, H, W = probas.size() + probas = probas.view(B, 1, H, W) + B, C, H, W = probas.size() + probas = probas.permute(0, 2, 3, 1).contiguous().view(-1, C) # B * H * W, C = P, C + labels = labels.view(-1) + if ignore is None: + return probas, labels + valid = (labels != ignore) + vprobas = probas[valid.nonzero().squeeze()] + vlabels = labels[valid] + return vprobas, vlabels + + +class Lovasz_softmax(nn.Module): + def __init__(self, classes='present', per_image=False, ignore=None): + super(Lovasz_softmax, self).__init__() + self.classes = classes + self.per_image = per_image + self.ignore = ignore + + def forward(self, probas, labels): + return lovasz_softmax(probas, labels, self.classes, self.per_image, self.ignore) diff --git a/aimet_zoo_torch/salsaNext/models/SalsaNext.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/SalsaNext.py similarity index 76% rename from aimet_zoo_torch/salsaNext/models/SalsaNext.py rename to aimet_zoo_torch/salsanext/models/tasks/semantic/modules/SalsaNext.py index 2d7159e..694ad2f 100644 --- a/aimet_zoo_torch/salsaNext/models/SalsaNext.py +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/SalsaNext.py @@ -1,8 +1,44 @@ # !/usr/bin/env python3 +# pylint: skip-file # This is the updated file, based on the original code from https://github.com/TiagoCortinhal/SalsaNext, used for the AIMET. + + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + import imp -import __init__ as booger import torch import torch.nn as nn import torch.nn.functional as F diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/SalsaNextAdf.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/SalsaNextAdf.py new file mode 100644 index 0000000..a100437 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/SalsaNextAdf.py @@ -0,0 +1,271 @@ +# !/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +import imp + +import __init__ as booger +import torch +import torch.nn as nn +import torch.nn.functional as F +#from tasks.semantic.modules.ConcreteDropout import adf.Dropout +import tasks.semantic.modules.adf as adf + +#We need to define the variance. For now we are using the 1e-3 as the authors... +#Also for the gridsearch im not sure how to use it... +#What kind of metric should we use? + + +def keep_variance_fn(x): + return x + 2e-7 + +class ResContextBlock(nn.Module): + def __init__(self, in_filters, out_filters): + super(ResContextBlock, self).__init__() + self.conv1 = adf.Conv2d(in_filters, out_filters, kernel_size=(1, 1), stride=1) + self.act1 = adf.LeakyReLU() + + self.conv2 = adf.Conv2d(out_filters, out_filters, (3,3), padding=1) + self.act2 = adf.LeakyReLU() + self.bn1 = adf.BatchNorm2d(out_filters) + + self.conv3 = adf.Conv2d(out_filters, out_filters, (3,3),dilation=1, padding=1) + self.act3 = adf.LeakyReLU() + self.bn2 = adf.BatchNorm2d(out_filters) + + + def forward(self, x): + + shortcut = self.conv1(*x) + shortcut = self.act1(*shortcut) + + resA = self.conv2(*shortcut) + resA = self.act2(*resA) + resA1 = self.bn1(*resA) + + resA = self.conv3(*resA1) + resA = self.act3(*resA) + resA2 = self.bn2(*resA) + + output = shortcut[0] + resA2[0],shortcut[1] + resA2[1] + return output + + +class ResBlock(nn.Module): + def __init__(self, in_filters, out_filters, kernel_size=(3, 3), stride=1, + pooling=True, drop_out=True,p=0.2): + super(ResBlock, self).__init__() + self.pooling = pooling + self.drop_out = drop_out + self.p = p + + self.conv1 = adf.Conv2d(in_filters, out_filters, kernel_size=(1, 1), stride=stride) + self.act1 = adf.LeakyReLU() + + self.conv2 = adf.Conv2d(in_filters, out_filters, kernel_size=(3,3), padding=1) + self.act2 = adf.LeakyReLU() + self.bn1 = adf.BatchNorm2d(out_filters) + + self.conv3 = adf.Conv2d(out_filters, out_filters, kernel_size=(3,3),dilation=2, padding=2) + self.act3 = adf.LeakyReLU() + self.bn2 = adf.BatchNorm2d(out_filters) + + self.conv4 = adf.Conv2d(out_filters, out_filters, kernel_size=(2, 2), dilation=2, padding=1) + self.act4 = adf.LeakyReLU() + self.bn3 = adf.BatchNorm2d(out_filters) + + self.conv5 = adf.Conv2d(out_filters*3, out_filters, kernel_size=(1, 1)) + self.act5 = adf.LeakyReLU() + self.bn4 = adf.BatchNorm2d(out_filters) + + if pooling: + self.dropout = adf.Dropout(p=self.p, keep_variance_fn=keep_variance_fn) + self.pool = adf.AvgPool2d(keep_variance_fn,kernel_size=kernel_size) + else: + self.dropout = adf.Dropout(p=self.p, keep_variance_fn=keep_variance_fn) + + def forward(self, x): + shortcut = self.conv1(*x) + shortcut = self.act1(*shortcut) + + resA = self.conv2(*x) + resA = self.act2(*resA) + resA1 = self.bn1(*resA) + + resA = self.conv3(*resA1) + resA = self.act3(*resA) + resA2 = self.bn2(*resA) + + resA = self.conv4(*resA2) + resA = self.act4(*resA) + resA3 = self.bn3(*resA) + + concat_mean = torch.cat((resA1[0],resA2[0],resA3[0]),dim=1) + concat_var = torch.cat((resA1[1],resA2[1],resA3[1]),dim=1) + concat = concat_mean,concat_var + resA = self.conv5(*concat) + resA = self.act5(*resA) + resA = self.bn4(*resA) + resA = shortcut[0] + resA[0],shortcut[1] + resA[1] + + + if self.pooling: + if self.drop_out: + resB = self.dropout(*resA) + else: + resB = resA + resB = self.pool(*resB) + + return resB, resA + else: + if self.drop_out: + resB = self.dropout(*resA) + else: + resB = resA + return resB + + +class UpBlock(nn.Module): + def __init__(self, in_filters, out_filters,drop_out=True, p=0.2): + super(UpBlock, self).__init__() + self.drop_out = drop_out + self.in_filters = in_filters + self.out_filters = out_filters + self.p = p + + self.dropout1 = adf.Dropout(p=self.p, keep_variance_fn=keep_variance_fn) + self.dropout2 = adf.Dropout(p=self.p, keep_variance_fn=keep_variance_fn) + + self.conv1 = adf.Conv2d(in_filters//4 + 2*out_filters, out_filters, (3,3), padding=1) + self.act1 = adf.LeakyReLU() + self.bn1 = adf.BatchNorm2d(out_filters) + + self.conv2 = adf.Conv2d(out_filters, out_filters, (3,3),dilation=2, padding=2) + self.act2 = adf.LeakyReLU() + self.bn2 = adf.BatchNorm2d(out_filters) + + self.conv3 = adf.Conv2d(out_filters, out_filters, (2,2), dilation=2,padding=1) + self.act3 = adf.LeakyReLU() + self.bn3 = adf.BatchNorm2d(out_filters) + + self.conv4 = adf.Conv2d(out_filters*3,out_filters,kernel_size=(1,1)) + self.act4 = adf.LeakyReLU() + self.bn4 = adf.BatchNorm2d(out_filters) + self.dropout3 = adf.Dropout(p=self.p, keep_variance_fn=keep_variance_fn) + self.dropout4 = adf.Dropout(p=self.p, keep_variance_fn=keep_variance_fn) + + def forward(self, x, skip): + #Does Pixel-Shuffle need something in particular? Or can we apply it do the mean and var individually? + mean, var = x + upA_mean = nn.PixelShuffle(2)(mean) + upA_var = nn.PixelShuffle(2)(var) + upA = upA_mean, upA_var + + if self.drop_out: + upA = self.dropout1(*upA) + + upB_mean = torch.cat((upA[0],skip[0]),dim=1) + upB_var = torch.cat((upA[1], skip[1]), dim=1) + upB = upB_mean, upB_var + + if self.drop_out: + upB = self.dropout2(*upB) + + upE = self.conv1(*upB) + upE = self.act1(*upE) + upE1 = self.bn1(*upE) + + upE = self.conv2(*upE1) + upE = self.act2(*upE) + upE2 = self.bn2(*upE) + + upE = self.conv3(*upE2) + upE = self.act3(*upE) + upE3 = self.bn3(*upE) + + concat_mean = torch.cat((upE1[0],upE2[0],upE3[0]),dim=1) + concat_var = torch.cat((upE1[1], upE2[1], upE3[1]), dim=1) + concat = concat_mean, concat_var + if self.drop_out: + concat = self.dropout3(*concat) + upE = self.conv4(*concat) + upE = self.act4(*upE) + upE = self.bn4(*upE) + if self.drop_out: + upE = self.dropout4(*upE) + + return upE + + +class SalsaNextUncertainty(nn.Module): + def __init__(self, nclasses,p=0.2): + super(SalsaNextUncertainty, self).__init__() + self.nclasses = nclasses + self.p = p + + self.downCntx = ResContextBlock(5, 32) + self.downCntx2 = ResContextBlock(32, 32) + self.downCntx3 = ResContextBlock(32, 32) + + + self.resBlock1 = ResBlock(32, 2 * 32, pooling=True, drop_out=False,p=self.p) + self.resBlock2 = ResBlock(2 * 32, 4 * 32, pooling=True,p=self.p) + self.resBlock3 = ResBlock(4 * 32, 8 * 32, pooling=True,p=self.p) + self.resBlock4 = ResBlock(8 * 32, 8 * 32, pooling=True,p=self.p) + self.resBlock5 = ResBlock(8 * 32, 8 * 32, pooling=False,p=self.p) + + self.upBlock1 = UpBlock(8 * 32, 4 * 32,p=self.p) + self.upBlock2 = UpBlock(4 * 32, 4 * 32,p=self.p) + self.upBlock3 = UpBlock(4 * 32, 2 * 32,p=self.p) + self.upBlock4 = UpBlock(2 * 32, 32, drop_out=False,p=self.p) + + self.logits = adf.Conv2d(32, nclasses, kernel_size=(1, 1)) + + def forward(self, x): + inputs_mean = x + inputs_variance = torch.zeros_like(inputs_mean) + 2e-7 + x = inputs_mean, inputs_variance + + downCntx = self.downCntx(x) + downCntx = self.downCntx2(downCntx) + downCntx = self.downCntx3(downCntx) + + + down0c, down0b = self.resBlock1(downCntx) + down1c, down1b = self.resBlock2(down0c) + down2c, down2b = self.resBlock3(down1c) + down3c, down3b = self.resBlock4(down2c) + down5c = self.resBlock5(down3c) + + up4e = self.upBlock1(down5c,down3b) + up3e = self.upBlock2(up4e, down2b) + up2e = self.upBlock3(up3e, down1b) + up1 = self.upBlock4(up2e, down0b) + + logits = self.logits(*up1) + + return logits \ No newline at end of file diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/__init__.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/adf.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/adf.py new file mode 100644 index 0000000..f9ecfb1 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/adf.py @@ -0,0 +1,453 @@ +# pylint: skip-file +""" +MIT License + +Copyright (c) 2019 mattiasegu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + + +import operator +from collections import OrderedDict +from itertools import islice +from numbers import Number +import numpy as np +import torch +import torch.nn as nn +from torch.nn.parameter import Parameter +from torch.nn import functional as F +from torch.nn.modules.conv import _ConvNd +from torch.nn.modules.conv import _ConvTransposeMixin +from torch.nn.modules.utils import _pair + +def resize2D(inputs, size_targets, mode="bilinear"): + size_inputs = [inputs.size(2), inputs.size(3)] + + if all([size_inputs == size_targets]): + return inputs # nothing to do + elif any([size_targets < size_inputs]): + resized = F.adaptive_avg_pool2d(inputs, size_targets) # downscaling + else: + resized = F.upsample(inputs, size=size_targets, mode=mode) # upsampling + + # correct scaling + return resized + + +def resize2D_as(inputs, output_as, mode="bilinear"): + size_targets = [output_as.size(2), output_as.size(3)] + return resize2D(inputs, size_targets, mode=mode) + + +def normcdf(value, mu=0.0, stddev=1.0): + sinv = (1.0 / stddev) if isinstance(stddev, Number) else stddev.reciprocal() + return 0.5 * (1.0 + torch.erf((value - mu) * sinv / np.sqrt(2.0))) + + +def _normal_log_pdf(value, mu, stddev): + var = (stddev ** 2) + log_scale = np.log(stddev) if isinstance(stddev, Number) else torch.log(stddev) + return -((value - mu) ** 2) / (2.0*var) - log_scale - np.log(np.sqrt(2.0*np.pi)) + + +def normpdf(value, mu=0.0, stddev=1.0): + return torch.exp(_normal_log_pdf(value, mu, stddev)) + + +class AvgPool2d(nn.Module): + def __init__(self, keep_variance_fn=None,kernel_size=2): + super(AvgPool2d, self).__init__() + self._keep_variance_fn = keep_variance_fn + self.kernel_size = kernel_size + + def forward(self, inputs_mean, inputs_variance): + outputs_mean = F.avg_pool2d(inputs_mean, self.kernel_size,stride=2,padding=1) + outputs_variance = F.avg_pool2d(inputs_variance, self.kernel_size,stride=2,padding=1) + outputs_variance = outputs_variance / (inputs_mean.size(2) * inputs_mean.size(3)) + + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + + # TODO: avg pooling means that every neuron is multiplied by the same + # weight, that is 1/number of neurons in the channel + # outputs_variance*1/(H*W) should be enough already + + return outputs_mean, outputs_variance/(inputs_mean.shape[2]*inputs_mean.shape[3]) + + +class Softmax(nn.Module): + def __init__(self, dim=1, keep_variance_fn=None): + super(Softmax, self).__init__() + self.dim = dim + self._keep_variance_fn = keep_variance_fn + + def forward(self, features_mean, features_variance, eps=1e-5): + """Softmax function applied to a multivariate Gaussian distribution. + It works under the assumption that features_mean and features_variance + are the parameters of a the indepent gaussians that contribute to the + multivariate gaussian. + Mean and variance of the log-normal distribution are computed following + https://en.wikipedia.org/wiki/Log-normal_distribution.""" + + log_gaussian_mean = features_mean + 0.5 * features_variance + log_gaussian_variance = 2 * log_gaussian_mean + + log_gaussian_mean = torch.exp(log_gaussian_mean) + log_gaussian_variance = torch.exp(log_gaussian_variance) + log_gaussian_variance = log_gaussian_variance * (torch.exp(features_variance) - 1) + + constant = torch.sum(log_gaussian_mean, dim=self.dim) + eps + constant = constant.unsqueeze(self.dim) + outputs_mean = log_gaussian_mean / constant + outputs_variance = log_gaussian_variance / (constant ** 2) + + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return outputs_mean, outputs_variance + + +class ReLU(nn.Module): + def __init__(self, keep_variance_fn=None): + super(ReLU, self).__init__() + self._keep_variance_fn = keep_variance_fn + + def forward(self, features_mean, features_variance): + features_stddev = torch.sqrt(features_variance) + div = features_mean / features_stddev + pdf = normpdf(div) + cdf = normcdf(div) + outputs_mean = features_mean * cdf + features_stddev * pdf + outputs_variance = (features_mean ** 2 + features_variance) * cdf \ + + features_mean * features_stddev * pdf - outputs_mean ** 2 + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return outputs_mean, outputs_variance + + +class LeakyReLU(nn.Module): + def __init__(self, negative_slope=0.01, keep_variance_fn=None): + super(LeakyReLU, self).__init__() + self._keep_variance_fn = keep_variance_fn + self._negative_slope = negative_slope + + def forward(self, features_mean, features_variance): + features_stddev = torch.sqrt(features_variance) + div = features_mean / features_stddev + pdf = normpdf(div) + cdf = normcdf(div) + negative_cdf = 1.0 - cdf + mu_cdf = features_mean * cdf + stddev_pdf = features_stddev * pdf + squared_mean_variance = features_mean ** 2 + features_variance + mean_stddev_pdf = features_mean * stddev_pdf + mean_r = mu_cdf + stddev_pdf + variance_r = squared_mean_variance * cdf + mean_stddev_pdf - mean_r ** 2 + mean_n = - features_mean * negative_cdf + stddev_pdf + variance_n = squared_mean_variance * negative_cdf - mean_stddev_pdf - mean_n ** 2 + covxy = - mean_r * mean_n + outputs_mean = mean_r - self._negative_slope * mean_n + outputs_variance = variance_r \ + + self._negative_slope * self._negative_slope * variance_n \ + - 2.0 * self._negative_slope * covxy + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return outputs_mean, outputs_variance + + +class Dropout(nn.Module): + """ADF implementation of nn.Dropout2d""" + + def __init__(self, p: float = 0.5, keep_variance_fn=None, inplace=False): + super(Dropout, self).__init__() + self._keep_variance_fn = keep_variance_fn + self.inplace = inplace + if p < 0 or p > 1: + raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p)) + self.p = p + + def forward(self, inputs_mean, inputs_variance): + if self.training: + binary_mask = torch.ones_like(inputs_mean) + binary_mask = F.dropout2d(binary_mask, self.p, self.training, self.inplace) + + outputs_mean = inputs_mean * binary_mask + outputs_variance = inputs_variance * binary_mask ** 2 + + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return outputs_mean, outputs_variance + + outputs_variance = inputs_variance + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return inputs_mean, outputs_variance + + +class MaxPool2d(nn.Module): + def __init__(self, keep_variance_fn=None): + super(MaxPool2d, self).__init__() + self._keep_variance_fn = keep_variance_fn + + def _max_pool_internal(self, mu_a, mu_b, var_a, var_b): + stddev = torch.sqrt(var_a + var_b) + ab = mu_a - mu_b + alpha = ab / stddev + pdf = normpdf(alpha) + cdf = normcdf(alpha) + z_mu = stddev * pdf + ab * cdf + mu_b + z_var = ((mu_a + mu_b) * stddev * pdf + + (mu_a ** 2 + var_a) * cdf + + (mu_b ** 2 + var_b) * (1.0 - cdf) - z_mu ** 2) + if self._keep_variance_fn is not None: + z_var = self._keep_variance_fn(z_var) + return z_mu, z_var + + def _max_pool_1x2(self, inputs_mean, inputs_variance): + mu_a = inputs_mean[:, :, :, 0::2] + mu_b = inputs_mean[:, :, :, 1::2] + var_a = inputs_variance[:, :, :, 0::2] + var_b = inputs_variance[:, :, :, 1::2] + outputs_mean, outputs_variance = self._max_pool_internal( + mu_a, mu_b, var_a, var_b) + return outputs_mean, outputs_variance + + def _max_pool_2x1(self, inputs_mean, inputs_variance): + mu_a = inputs_mean[:, :, 0::2, :] + mu_b = inputs_mean[:, :, 1::2, :] + var_a = inputs_variance[:, :, 0::2, :] + var_b = inputs_variance[:, :, 1::2, :] + outputs_mean, outputs_variance = self._max_pool_internal( + mu_a, mu_b, var_a, var_b) + return outputs_mean, outputs_variance + + def forward(self, inputs_mean, inputs_variance): + z_mean, z_variance = self._max_pool_1x2(inputs_mean, inputs_variance) + outputs_mean, outputs_variance = self._max_pool_2x1(z_mean, z_variance) + return outputs_mean, outputs_variance + + +class Linear(nn.Module): + def __init__(self, in_features, out_features, bias=True, keep_variance_fn=None): + super(Linear, self).__init__() + self._keep_variance_fn = keep_variance_fn + self.in_features = in_features + self.out_features = out_features + self.weight = Parameter(torch.Tensor(out_features, in_features)) + if bias: + self.bias = Parameter(torch.Tensor(out_features)) + else: + self.register_parameter('bias', None) + + def forward(self, inputs_mean, inputs_variance): + outputs_mean = F.linear(inputs_mean, self.weight, self.bias) + outputs_variance = F.linear(inputs_variance, self.weight ** 2, None) + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return outputs_mean, outputs_variance + + +class BatchNorm2d(nn.Module): + _version = 2 + __constants__ = ['track_running_stats', 'momentum', 'eps', 'weight', 'bias', + 'running_mean', 'running_var', 'num_batches_tracked'] + + def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, + track_running_stats=True, keep_variance_fn=None): + super(BatchNorm2d, self).__init__() + self._keep_variance_fn = keep_variance_fn + self.num_features = num_features + self.eps = eps + self.momentum = momentum + self.affine = affine + self.track_running_stats = track_running_stats + if self.affine: + self.weight = Parameter(torch.Tensor(num_features)) + self.bias = Parameter(torch.Tensor(num_features)) + else: + self.register_parameter('weight', None) + self.register_parameter('bias', None) + if self.track_running_stats: + self.register_buffer('running_mean', torch.zeros(num_features)) + self.register_buffer('running_var', torch.ones(num_features)) + self.register_buffer('num_batches_tracked', torch.tensor(0, dtype=torch.long)) + else: + self.register_parameter('running_mean', None) + self.register_parameter('running_var', None) + self.register_parameter('num_batches_tracked', None) + self.reset_parameters() + + def reset_running_stats(self): + if self.track_running_stats: + self.running_mean.zero_() + self.running_var.fill_(1) + self.num_batches_tracked.zero_() + + def reset_parameters(self): + self.reset_running_stats() + if self.affine: + nn.init.uniform_(self.weight) + nn.init.zeros_(self.bias) + + def _check_input_dim(self, input): + raise NotImplementedError + + def forward(self, inputs_mean, inputs_variance): + + # exponential_average_factor is self.momentum set to + # (when it is available) only so that if gets updated + # in ONNX graph when this node is exported to ONNX. + if self.momentum is None: + exponential_average_factor = 0.0 + else: + exponential_average_factor = self.momentum + + if self.training and self.track_running_stats: + if self.num_batches_tracked is not None: + self.num_batches_tracked += 1 + if self.momentum is None: # use cumulative moving average + exponential_average_factor = 1.0 / float(self.num_batches_tracked) + else: # use exponential moving average + exponential_average_factor = self.momentum + + outputs_mean = F.batch_norm( + inputs_mean, self.running_mean, self.running_var, self.weight, self.bias, + self.training or not self.track_running_stats, + exponential_average_factor, self.eps) + outputs_variance = inputs_variance + weight = ((self.weight.unsqueeze(0)).unsqueeze(2)).unsqueeze(3) + outputs_variance = outputs_variance * weight ** 2 + """ + for i in range(outputs_variance.size(1)): + outputs_variance[:,i,:,:]=outputs_variance[:,i,:,:].clone()*self.weight[i]**2 + """ + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return outputs_mean, outputs_variance + + +class Conv2d(_ConvNd): + def __init__(self, in_channels, out_channels, kernel_size, stride=1, + padding=0, dilation=1, groups=1, bias=True, + keep_variance_fn=None, padding_mode='zeros'): + self._keep_variance_fn = keep_variance_fn + kernel_size = _pair(kernel_size) + stride = _pair(stride) + padding = _pair(padding) + dilation = _pair(dilation) + super(Conv2d, self).__init__( + in_channels, out_channels, kernel_size, stride, padding, dilation, + False, _pair(0), groups, bias, padding_mode) + + def forward(self, inputs_mean, inputs_variance): + outputs_mean = F.conv2d( + inputs_mean, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) + outputs_variance = F.conv2d( + inputs_variance, self.weight ** 2, None, self.stride, self.padding, self.dilation, self.groups) + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return outputs_mean, outputs_variance + + +class ConvTranspose2d(_ConvTransposeMixin, _ConvNd): + def __init__(self, in_channels, out_channels, kernel_size, stride=1, + padding=0, output_padding=0, groups=1, bias=True, dilation=1, + keep_variance_fn=None, padding_mode='zeros'): + self._keep_variance_fn = keep_variance_fn + kernel_size = _pair(kernel_size) + stride = _pair(stride) + padding = _pair(padding) + dilation = _pair(dilation) + output_padding = _pair(output_padding) + super(ConvTranspose2d, self).__init__( + in_channels, out_channels, kernel_size, stride, padding, dilation, + True, output_padding, groups, bias, padding_mode) + + def forward(self, inputs_mean, inputs_variance, output_size=None): + output_padding = self._output_padding(inputs_mean, output_size, self.stride, self.padding, self.kernel_size) + outputs_mean = F.conv_transpose2d( + inputs_mean, self.weight, self.bias, self.stride, self.padding, + output_padding, self.groups, self.dilation) + outputs_variance = F.conv_transpose2d( + inputs_variance, self.weight ** 2, None, self.stride, self.padding, + output_padding, self.groups, self.dilation) + if self._keep_variance_fn is not None: + outputs_variance = self._keep_variance_fn(outputs_variance) + return outputs_mean, outputs_variance + + +def concatenate_as(tensor_list, tensor_as, dim, mode="bilinear"): + means = [resize2D_as(x[0], tensor_as[0], mode=mode) for x in tensor_list] + variances = [resize2D_as(x[1], tensor_as[0], mode=mode) for x in tensor_list] + means = torch.cat(means, dim=dim) + variances = torch.cat(variances, dim=dim) + return means, variances + + +class Sequential(nn.Module): + def __init__(self, *args): + super(Sequential, self).__init__() + if len(args) == 1 and isinstance(args[0], OrderedDict): + for key, module in args[0].items(): + self.add_module(key, module) + else: + for idx, module in enumerate(args): + self.add_module(str(idx), module) + + def _get_item_by_idx(self, iterator, idx): + """Get the idx-th item of the iterator""" + size = len(self) + idx = operator.index(idx) + if not -size <= idx < size: + raise IndexError('index {} is out of range'.format(idx)) + idx %= size + return next(islice(iterator, idx, None)) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return Sequential(OrderedDict(list(self._modules.items())[idx])) + else: + return self._get_item_by_idx(self._modules.values(), idx) + + def __setitem__(self, idx, module): + key = self._get_item_by_idx(self._modules.keys(), idx) + return setattr(self, key, module) + + def __delitem__(self, idx): + if isinstance(idx, slice): + for key in list(self._modules.keys())[idx]: + delattr(self, key) + else: + key = self._get_item_by_idx(self._modules.keys(), idx) + delattr(self, key) + + def __len__(self): + return len(self._modules) + + def __dir__(self): + keys = super(Sequential, self).__dir__() + keys = [key for key in keys if not key.isdigit()] + return keys + + def forward(self, inputs, inputs_variance): + for module in self._modules.values(): + inputs, inputs_variance = module(inputs, inputs_variance) + + return inputs, inputs_variance + diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/ioueval.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/ioueval.py new file mode 100644 index 0000000..955fb6e --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/ioueval.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import numpy as np +import torch + + +class iouEval: + def __init__(self, n_classes, device, ignore=None): + self.n_classes = n_classes + self.device = device + # if ignore is larger than n_classes, consider no ignoreIndex + self.ignore = torch.tensor(ignore).long() + self.include = torch.tensor( + [n for n in range(self.n_classes) if n not in self.ignore]).long() + print("[IOU EVAL] IGNORE: ", self.ignore) + print("[IOU EVAL] INCLUDE: ", self.include) + self.reset() + + def num_classes(self): + return self.n_classes + + def reset(self): + self.conf_matrix = torch.zeros( + (self.n_classes, self.n_classes), device=self.device).long() + self.ones = None + self.last_scan_size = None # for when variable scan size is used + + def addBatch(self, x, y): # x=preds, y=targets + # if numpy, pass to pytorch + # to tensor + if isinstance(x, np.ndarray): + x = torch.from_numpy(np.array(x)).long().to(self.device) + if isinstance(y, np.ndarray): + y = torch.from_numpy(np.array(y)).long().to(self.device) + + # sizes should be "batch_size x H x W" + x_row = x.reshape(-1) # de-batchify + y_row = y.reshape(-1) # de-batchify + + # idxs are labels and predictions + idxs = torch.stack([x_row, y_row], dim=0) + + # ones is what I want to add to conf when I + if self.ones is None or self.last_scan_size != idxs.shape[-1]: + self.ones = torch.ones((idxs.shape[-1]), device=self.device).long() + self.last_scan_size = idxs.shape[-1] + + # make confusion matrix (cols = gt, rows = pred) + self.conf_matrix = self.conf_matrix.index_put_( + tuple(idxs), self.ones, accumulate=True) + + # print(self.tp.shape) + # print(self.fp.shape) + # print(self.fn.shape) + + def getStats(self): + # remove fp and fn from confusion on the ignore classes cols and rows + conf = self.conf_matrix.clone().double() + conf[self.ignore] = 0 + conf[:, self.ignore] = 0 + + # get the clean stats + tp = conf.diag() + fp = conf.sum(dim=1) - tp + fn = conf.sum(dim=0) - tp + return tp, fp, fn + + def getIoU(self): + tp, fp, fn = self.getStats() + intersection = tp + union = tp + fp + fn + 1e-15 + iou = intersection / union + iou_mean = (intersection[self.include] / union[self.include]).mean() + return iou_mean, iou # returns "iou mean", "iou per class" ALL CLASSES + + def getacc(self): + tp, fp, fn = self.getStats() + total_tp = tp.sum() + total = tp[self.include].sum() + fp[self.include].sum() + 1e-15 + acc_mean = total_tp / total + return acc_mean # returns "acc mean" diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/trainer.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/trainer.py new file mode 100644 index 0000000..3eb4d08 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/modules/trainer.py @@ -0,0 +1,670 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +import datetime +import os +import time +import imp +import cv2 +import torch +import torch.backends.cudnn as cudnn +import torch.nn as nn + +import torch.optim as optim +from matplotlib import pyplot as plt +from torch.autograd import Variable +from common.avgmeter import * +from common.logger import Logger +from common.sync_batchnorm.batchnorm import convert_model +from common.warmupLR import * +from tasks.semantic.modules.ioueval import * +from tasks.semantic.modules.SalsaNext import * +from tasks.semantic.modules.SalsaNextAdf import * +from tasks.semantic.modules.Lovasz_Softmax import Lovasz_softmax +import tasks.semantic.modules.adf as adf + +def keep_variance_fn(x): + return x + 1e-3 + +def one_hot_pred_from_label(y_pred, labels): + y_true = torch.zeros_like(y_pred) + ones = torch.ones_like(y_pred) + indexes = [l for l in labels] + y_true[torch.arange(labels.size(0)), indexes] = ones[torch.arange(labels.size(0)), indexes] + + return y_true + + +class SoftmaxHeteroscedasticLoss(torch.nn.Module): + def __init__(self): + super(SoftmaxHeteroscedasticLoss, self).__init__() + self.adf_softmax = adf.Softmax(dim=1, keep_variance_fn=keep_variance_fn) + + def forward(self, outputs, targets, eps=1e-5): + mean, var = self.adf_softmax(*outputs) + targets = torch.nn.functional.one_hot(targets, num_classes=20).permute(0,3,1,2).float() + + precision = 1 / (var + eps) + return torch.mean(0.5 * precision * (targets - mean) ** 2 + 0.5 * torch.log(var + eps)) + + +def save_to_log(logdir, logfile, message): + f = open(logdir + '/' + logfile, "a") + f.write(message + '\n') + f.close() + return + + +def save_checkpoint(to_save, logdir, suffix=""): + # Save the weights + torch.save(to_save, logdir + + "/SalsaNext" + suffix) + + +class Trainer(): + def __init__(self, ARCH, DATA, datadir, logdir, path=None,uncertainty=False): + # parameters + self.ARCH = ARCH + self.DATA = DATA + self.datadir = datadir + self.log = logdir + self.path = path + self.uncertainty = uncertainty + + self.batch_time_t = AverageMeter() + self.data_time_t = AverageMeter() + self.batch_time_e = AverageMeter() + self.epoch = 0 + + # put logger where it belongs + + self.info = {"train_update": 0, + "train_loss": 0, + "train_acc": 0, + "train_iou": 0, + "valid_loss": 0, + "valid_acc": 0, + "valid_iou": 0, + "best_train_iou": 0, + "best_val_iou": 0} + + # get the data + parserModule = imp.load_source("parserModule", + booger.TRAIN_PATH + '/tasks/semantic/dataset/' + + self.DATA["name"] + '/parser.py') + self.parser = parserModule.Parser(root=self.datadir, + train_sequences=self.DATA["split"]["train"], + valid_sequences=self.DATA["split"]["valid"], + test_sequences=None, + labels=self.DATA["labels"], + color_map=self.DATA["color_map"], + learning_map=self.DATA["learning_map"], + learning_map_inv=self.DATA["learning_map_inv"], + sensor=self.ARCH["dataset"]["sensor"], + max_points=self.ARCH["dataset"]["max_points"], + batch_size=self.ARCH["train"]["batch_size"], + workers=self.ARCH["train"]["workers"], + gt=True, + shuffle_train=True) + + # weights for loss (and bias) + + epsilon_w = self.ARCH["train"]["epsilon_w"] + content = torch.zeros(self.parser.get_n_classes(), dtype=torch.float) + for cl, freq in DATA["content"].items(): + x_cl = self.parser.to_xentropy(cl) # map actual class to xentropy class + content[x_cl] += freq + self.loss_w = 1 / (content + epsilon_w) # get weights + for x_cl, w in enumerate(self.loss_w): # ignore the ones necessary to ignore + if DATA["learning_ignore"][x_cl]: + # don't weigh + self.loss_w[x_cl] = 0 + print("Loss weights from content: ", self.loss_w.data) + + with torch.no_grad(): + if not self.uncertainty: + self.model = SalsaNext(self.parser.get_n_classes()) + else: + self.model = SalsaNextUncertainty(self.parser.get_n_classes()) + + self.tb_logger = Logger(self.log + "/tb") + + # GPU? + self.gpu = False + self.multi_gpu = False + self.n_gpus = 0 + self.model_single = self.model + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print("Training in device: ", self.device) + if torch.cuda.is_available() and torch.cuda.device_count() > 0: + cudnn.benchmark = True + cudnn.fastest = True + self.gpu = True + self.n_gpus = 1 + self.model.cuda() + if torch.cuda.is_available() and torch.cuda.device_count() > 1: + print("Let's use", torch.cuda.device_count(), "GPUs!") + self.model = nn.DataParallel(self.model) # spread in gpus + self.model = convert_model(self.model).cuda() # sync batchnorm + self.model_single = self.model.module # single model to get weight names + self.multi_gpu = True + self.n_gpus = torch.cuda.device_count() + + + self.criterion = nn.NLLLoss(weight=self.loss_w).to(self.device) + self.ls = Lovasz_softmax(ignore=0).to(self.device) + self.SoftmaxHeteroscedasticLoss = SoftmaxHeteroscedasticLoss().to(self.device) + # loss as dataparallel too (more images in batch) + if self.n_gpus > 1: + self.criterion = nn.DataParallel(self.criterion).cuda() # spread in gpus + self.ls = nn.DataParallel(self.ls).cuda() + self.SoftmaxHeteroscedasticLoss = nn.DataParallel(self.SoftmaxHeteroscedasticLoss).cuda() + self.optimizer = optim.SGD([{'params': self.model.parameters()}], + lr=self.ARCH["train"]["lr"], + momentum=self.ARCH["train"]["momentum"], + weight_decay=self.ARCH["train"]["w_decay"]) + + # Use warmup learning rate + # post decay and step sizes come in epochs and we want it in steps + steps_per_epoch = self.parser.get_train_size() + up_steps = int(self.ARCH["train"]["wup_epochs"] * steps_per_epoch) + final_decay = self.ARCH["train"]["lr_decay"] ** (1 / steps_per_epoch) + self.scheduler = warmupLR(optimizer=self.optimizer, + lr=self.ARCH["train"]["lr"], + warmup_steps=up_steps, + momentum=self.ARCH["train"]["momentum"], + decay=final_decay) + + if self.path is not None: + torch.nn.Module.dump_patches = True + w_dict = torch.load(path + "/SalsaNext", + map_location=lambda storage, loc: storage) + self.model.load_state_dict(w_dict['state_dict'], strict=True) + self.optimizer.load_state_dict(w_dict['optimizer']) + self.epoch = w_dict['epoch'] + 1 + self.scheduler.load_state_dict(w_dict['scheduler']) + print("dict epoch:", w_dict['epoch']) + self.info = w_dict['info'] + print("info", w_dict['info']) + + + def calculate_estimate(self, epoch, iter): + estimate = int((self.data_time_t.avg + self.batch_time_t.avg) * \ + (self.parser.get_train_size() * self.ARCH['train']['max_epochs'] - ( + iter + 1 + epoch * self.parser.get_train_size()))) + \ + int(self.batch_time_e.avg * self.parser.get_valid_size() * ( + self.ARCH['train']['max_epochs'] - (epoch))) + return str(datetime.timedelta(seconds=estimate)) + + @staticmethod + def get_mpl_colormap(cmap_name): + cmap = plt.get_cmap(cmap_name) + # Initialize the matplotlib color map + sm = plt.cm.ScalarMappable(cmap=cmap) + # Obtain linear color range + color_range = sm.to_rgba(np.linspace(0, 1, 256), bytes=True)[:, 2::-1] + return color_range.reshape(256, 1, 3) + + @staticmethod + def make_log_img(depth, mask, pred, gt, color_fn): + # input should be [depth, pred, gt] + # make range image (normalized to 0,1 for saving) + depth = (cv2.normalize(depth, None, alpha=0, beta=1, + norm_type=cv2.NORM_MINMAX, + dtype=cv2.CV_32F) * 255.0).astype(np.uint8) + out_img = cv2.applyColorMap( + depth, Trainer.get_mpl_colormap('viridis')) * mask[..., None] + # make label prediction + pred_color = color_fn((pred * mask).astype(np.int32)) + out_img = np.concatenate([out_img, pred_color], axis=0) + # make label gt + gt_color = color_fn(gt) + out_img = np.concatenate([out_img, gt_color], axis=0) + return (out_img).astype(np.uint8) + + @staticmethod + def save_to_log(logdir, logger, info, epoch, w_summary=False, model=None, img_summary=False, imgs=[]): + # save scalars + for tag, value in info.items(): + logger.scalar_summary(tag, value, epoch) + + # save summaries of weights and biases + if w_summary and model: + for tag, value in model.named_parameters(): + tag = tag.replace('.', '/') + logger.histo_summary(tag, value.data.cpu().numpy(), epoch) + if value.grad is not None: + logger.histo_summary( + tag + '/grad', value.grad.data.cpu().numpy(), epoch) + + if img_summary and len(imgs) > 0: + directory = os.path.join(logdir, "predictions") + if not os.path.isdir(directory): + os.makedirs(directory) + for i, img in enumerate(imgs): + name = os.path.join(directory, str(i) + ".png") + cv2.imwrite(name, img) + + def train(self): + + self.ignore_class = [] + for i, w in enumerate(self.loss_w): + if w < 1e-10: + self.ignore_class.append(i) + print("Ignoring class ", i, " in IoU evaluation") + self.evaluator = iouEval(self.parser.get_n_classes(), + self.device, self.ignore_class) + + # train for n epochs + for epoch in range(self.epoch, self.ARCH["train"]["max_epochs"]): + + # train for 1 epoch + acc, iou, loss, update_mean,hetero_l = self.train_epoch(train_loader=self.parser.get_train_set(), + model=self.model, + criterion=self.criterion, + optimizer=self.optimizer, + epoch=epoch, + evaluator=self.evaluator, + scheduler=self.scheduler, + color_fn=self.parser.to_color, + report=self.ARCH["train"]["report_batch"], + show_scans=self.ARCH["train"]["show_scans"]) + + # update info + self.info["train_update"] = update_mean + self.info["train_loss"] = loss + self.info["train_acc"] = acc + self.info["train_iou"] = iou + self.info["train_hetero"] = hetero_l + + # remember best iou and save checkpoint + state = {'epoch': epoch, 'state_dict': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'info': self.info, + 'scheduler': self.scheduler.state_dict() + } + save_checkpoint(state, self.log, suffix="") + + if self.info['train_iou'] > self.info['best_train_iou']: + print("Best mean iou in training set so far, save model!") + self.info['best_train_iou'] = self.info['train_iou'] + state = {'epoch': epoch, 'state_dict': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'info': self.info, + 'scheduler': self.scheduler.state_dict() + } + save_checkpoint(state, self.log, suffix="_train_best") + + if epoch % self.ARCH["train"]["report_epoch"] == 0: + # evaluate on validation set + print("*" * 80) + acc, iou, loss, rand_img,hetero_l = self.validate(val_loader=self.parser.get_valid_set(), + model=self.model, + criterion=self.criterion, + evaluator=self.evaluator, + class_func=self.parser.get_xentropy_class_string, + color_fn=self.parser.to_color, + save_scans=self.ARCH["train"]["save_scans"]) + + # update info + self.info["valid_loss"] = loss + self.info["valid_acc"] = acc + self.info["valid_iou"] = iou + self.info['valid_heteros'] = hetero_l + + # remember best iou and save checkpoint + if self.info['valid_iou'] > self.info['best_val_iou']: + print("Best mean iou in validation so far, save model!") + print("*" * 80) + self.info['best_val_iou'] = self.info['valid_iou'] + + # save the weights! + state = {'epoch': epoch, 'state_dict': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'info': self.info, + 'scheduler': self.scheduler.state_dict() + } + save_checkpoint(state, self.log, suffix="_valid_best") + + print("*" * 80) + + # save to log + Trainer.save_to_log(logdir=self.log, + logger=self.tb_logger, + info=self.info, + epoch=epoch, + w_summary=self.ARCH["train"]["save_summary"], + model=self.model_single, + img_summary=self.ARCH["train"]["save_scans"], + imgs=rand_img) + + print('Finished Training') + + return + + def train_epoch(self, train_loader, model, criterion, optimizer, epoch, evaluator, scheduler, color_fn, report=10, + show_scans=False): + losses = AverageMeter() + acc = AverageMeter() + iou = AverageMeter() + hetero_l = AverageMeter() + update_ratio_meter = AverageMeter() + + # empty the cache to train now + if self.gpu: + torch.cuda.empty_cache() + + # switch to train mode + model.train() + + end = time.time() + for i, (in_vol, proj_mask, proj_labels, _, path_seq, path_name, _, _, _, _, _, _, _, _, _) in enumerate(train_loader): + # measure data loading time + self.data_time_t.update(time.time() - end) + if not self.multi_gpu and self.gpu: + in_vol = in_vol.cuda() + #proj_mask = proj_mask.cuda() + if self.gpu: + proj_labels = proj_labels.cuda().long() + + # compute output + if self.uncertainty: + output = model(in_vol) + output_mean, output_var = adf.Softmax(dim=1, keep_variance_fn=keep_variance_fn)(*output) + hetero = self.SoftmaxHeteroscedasticLoss(output,proj_labels) + loss_m = criterion(output_mean.clamp(min=1e-8), proj_labels) + hetero + self.ls(output_mean, proj_labels.long()) + + hetero_l.update(hetero.mean().item(), in_vol.size(0)) + output = output_mean + else: + output = model(in_vol) + loss_m = criterion(torch.log(output.clamp(min=1e-8)), proj_labels) + self.ls(output, proj_labels.long()) + + optimizer.zero_grad() + if self.n_gpus > 1: + idx = torch.ones(self.n_gpus).cuda() + loss_m.backward(idx) + else: + loss_m.backward() + optimizer.step() + + # measure accuracy and record loss + loss = loss_m.mean() + with torch.no_grad(): + evaluator.reset() + argmax = output.argmax(dim=1) + evaluator.addBatch(argmax, proj_labels) + accuracy = evaluator.getacc() + jaccard, class_jaccard = evaluator.getIoU() + + losses.update(loss.item(), in_vol.size(0)) + acc.update(accuracy.item(), in_vol.size(0)) + iou.update(jaccard.item(), in_vol.size(0)) + + # measure elapsed time + self.batch_time_t.update(time.time() - end) + end = time.time() + + # get gradient updates and weights, so I can print the relationship of + # their norms + update_ratios = [] + for g in self.optimizer.param_groups: + lr = g["lr"] + for value in g["params"]: + if value.grad is not None: + w = np.linalg.norm(value.data.cpu().numpy().reshape((-1))) + update = np.linalg.norm(-max(lr, 1e-10) * + value.grad.cpu().numpy().reshape((-1))) + update_ratios.append(update / max(w, 1e-10)) + update_ratios = np.array(update_ratios) + update_mean = update_ratios.mean() + update_std = update_ratios.std() + update_ratio_meter.update(update_mean) # over the epoch + + if show_scans: + # get the first scan in batch and project points + mask_np = proj_mask[0].cpu().numpy() + depth_np = in_vol[0][0].cpu().numpy() + pred_np = argmax[0].cpu().numpy() + gt_np = proj_labels[0].cpu().numpy() + out = Trainer.make_log_img(depth_np, mask_np, pred_np, gt_np, color_fn) + + mask_np = proj_mask[1].cpu().numpy() + depth_np = in_vol[1][0].cpu().numpy() + pred_np = argmax[1].cpu().numpy() + gt_np = proj_labels[1].cpu().numpy() + out2 = Trainer.make_log_img(depth_np, mask_np, pred_np, gt_np, color_fn) + + out = np.concatenate([out, out2], axis=0) + cv2.imshow("sample_training", out) + cv2.waitKey(1) + if self.uncertainty: + + if i % self.ARCH["train"]["report_batch"] == 0: + print( 'Lr: {lr:.3e} | ' + 'Update: {umean:.3e} mean,{ustd:.3e} std | ' + 'Epoch: [{0}][{1}/{2}] | ' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f}) | ' + 'Data {data_time.val:.3f} ({data_time.avg:.3f}) | ' + 'Loss {loss.val:.4f} ({loss.avg:.4f}) | ' + 'Hetero {hetero_l.val:.4f} ({hetero_l.avg:.4f}) | ' + 'acc {acc.val:.3f} ({acc.avg:.3f}) | ' + 'IoU {iou.val:.3f} ({iou.avg:.3f}) | [{estim}]'.format( + epoch, i, len(train_loader), batch_time=self.batch_time_t, + data_time=self.data_time_t, loss=losses, hetero_l=hetero_l,acc=acc, iou=iou, lr=lr, + umean=update_mean, ustd=update_std, estim=self.calculate_estimate(epoch, i))) + + save_to_log(self.log, 'log.txt', 'Lr: {lr:.3e} | ' + 'Update: {umean:.3e} mean,{ustd:.3e} std | ' + 'Epoch: [{0}][{1}/{2}] | ' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f}) | ' + 'Data {data_time.val:.3f} ({data_time.avg:.3f}) | ' + 'Loss {loss.val:.4f} ({loss.avg:.4f}) | ' + 'Hetero {hetero.val:.4f} ({hetero.avg:.4f}) | ' + 'acc {acc.val:.3f} ({acc.avg:.3f}) | ' + 'IoU {iou.val:.3f} ({iou.avg:.3f}) | [{estim}]'.format( + epoch, i, len(train_loader), batch_time=self.batch_time_t, + data_time=self.data_time_t, loss=losses, hetero=hetero_l,acc=acc, iou=iou, lr=lr, + umean=update_mean, ustd=update_std, estim=self.calculate_estimate(epoch, i))) + else: + if i % self.ARCH["train"]["report_batch"] == 0: + print('Lr: {lr:.3e} | ' + 'Update: {umean:.3e} mean,{ustd:.3e} std | ' + 'Epoch: [{0}][{1}/{2}] | ' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f}) | ' + 'Data {data_time.val:.3f} ({data_time.avg:.3f}) | ' + 'Loss {loss.val:.4f} ({loss.avg:.4f}) | ' + 'acc {acc.val:.3f} ({acc.avg:.3f}) | ' + 'IoU {iou.val:.3f} ({iou.avg:.3f}) | [{estim}]'.format( + epoch, i, len(train_loader), batch_time=self.batch_time_t, + data_time=self.data_time_t, loss=losses, acc=acc, iou=iou, lr=lr, + umean=update_mean, ustd=update_std, estim=self.calculate_estimate(epoch, i))) + + save_to_log(self.log, 'log.txt', 'Lr: {lr:.3e} | ' + 'Update: {umean:.3e} mean,{ustd:.3e} std | ' + 'Epoch: [{0}][{1}/{2}] | ' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f}) | ' + 'Data {data_time.val:.3f} ({data_time.avg:.3f}) | ' + 'Loss {loss.val:.4f} ({loss.avg:.4f}) | ' + 'acc {acc.val:.3f} ({acc.avg:.3f}) | ' + 'IoU {iou.val:.3f} ({iou.avg:.3f}) | [{estim}]'.format( + epoch, i, len(train_loader), batch_time=self.batch_time_t, + data_time=self.data_time_t, loss=losses, acc=acc, iou=iou, lr=lr, + umean=update_mean, ustd=update_std, estim=self.calculate_estimate(epoch, i))) + + # step scheduler + scheduler.step() + + return acc.avg, iou.avg, losses.avg, update_ratio_meter.avg,hetero_l.avg + + def validate(self, val_loader, model, criterion, evaluator, class_func, color_fn, save_scans): + losses = AverageMeter() + jaccs = AverageMeter() + wces = AverageMeter() + acc = AverageMeter() + iou = AverageMeter() + hetero_l = AverageMeter() + rand_imgs = [] + + # switch to evaluate mode + model.eval() + evaluator.reset() + + # empty the cache to infer in high res + if self.gpu: + torch.cuda.empty_cache() + + with torch.no_grad(): + end = time.time() + for i, (in_vol, proj_mask, proj_labels, _, path_seq, path_name, _, _, _, _, _, _, _, _, _) in enumerate(val_loader): + if not self.multi_gpu and self.gpu: + in_vol = in_vol.cuda() + proj_mask = proj_mask.cuda() + if self.gpu: + proj_labels = proj_labels.cuda(non_blocking=True).long() + + # compute output + if self.uncertainty: + log_var, output, _ = model(in_vol) + log_out = torch.log(output.clamp(min=1e-8)) + mean = output.argmax(dim=1) + log_var = log_var.mean(dim=1) + hetero = self.SoftmaxHeteroscedasticLoss(mean.float(),proj_labels.float()).mean() + jacc = self.ls(output, proj_labels) + wce = criterion(log_out, proj_labels) + loss = wce + jacc + hetero_l.update(hetero.mean().item(), in_vol.size(0)) + else: + output = model(in_vol) + log_out = torch.log(output.clamp(min=1e-8)) + jacc = self.ls(output, proj_labels) + wce = criterion(log_out, proj_labels) + loss = wce + jacc + + # measure accuracy and record loss + argmax = output.argmax(dim=1) + evaluator.addBatch(argmax, proj_labels) + losses.update(loss.mean().item(), in_vol.size(0)) + jaccs.update(jacc.mean().item(),in_vol.size(0)) + + + wces.update(wce.mean().item(),in_vol.size(0)) + + + + if save_scans: + # get the first scan in batch and project points + mask_np = proj_mask[0].cpu().numpy() + depth_np = in_vol[0][0].cpu().numpy() + pred_np = argmax[0].cpu().numpy() + gt_np = proj_labels[0].cpu().numpy() + out = Trainer.make_log_img(depth_np, + mask_np, + pred_np, + gt_np, + color_fn) + rand_imgs.append(out) + + # measure elapsed time + self.batch_time_e.update(time.time() - end) + end = time.time() + + accuracy = evaluator.getacc() + jaccard, class_jaccard = evaluator.getIoU() + acc.update(accuracy.item(), in_vol.size(0)) + iou.update(jaccard.item(), in_vol.size(0)) + if self.uncertainty: + print('Validation set:\n' + 'Time avg per batch {batch_time.avg:.3f}\n' + 'Loss avg {loss.avg:.4f}\n' + 'Jaccard avg {jac.avg:.4f}\n' + 'WCE avg {wces.avg:.4f}\n' + 'Hetero avg {hetero.avg}:.4f\n' + 'Acc avg {acc.avg:.3f}\n' + 'IoU avg {iou.avg:.3f}'.format(batch_time=self.batch_time_e, + loss=losses, + jac=jaccs, + wces=wces, + hetero=hetero_l, + acc=acc, iou=iou)) + + save_to_log(self.log, 'log.txt', 'Validation set:\n' + 'Time avg per batch {batch_time.avg:.3f}\n' + 'Loss avg {loss.avg:.4f}\n' + 'Jaccard avg {jac.avg:.4f}\n' + 'WCE avg {wces.avg:.4f}\n' + 'Hetero avg {hetero.avg}:.4f\n' + 'Acc avg {acc.avg:.3f}\n' + 'IoU avg {iou.avg:.3f}'.format(batch_time=self.batch_time_e, + loss=losses, + jac=jaccs, + wces=wces, + hetero=hetero_l, + acc=acc, iou=iou)) + # print also classwise + for i, jacc in enumerate(class_jaccard): + print('IoU class {i:} [{class_str:}] = {jacc:.3f}'.format( + i=i, class_str=class_func(i), jacc=jacc)) + save_to_log(self.log, 'log.txt', 'IoU class {i:} [{class_str:}] = {jacc:.3f}'.format( + i=i, class_str=class_func(i), jacc=jacc)) + self.info["valid_classes/"+class_func(i)] = jacc + else: + + print('Validation set:\n' + 'Time avg per batch {batch_time.avg:.3f}\n' + 'Loss avg {loss.avg:.4f}\n' + 'Jaccard avg {jac.avg:.4f}\n' + 'WCE avg {wces.avg:.4f}\n' + 'Acc avg {acc.avg:.3f}\n' + 'IoU avg {iou.avg:.3f}'.format(batch_time=self.batch_time_e, + loss=losses, + jac=jaccs, + wces=wces, + acc=acc, iou=iou)) + + save_to_log(self.log, 'log.txt', 'Validation set:\n' + 'Time avg per batch {batch_time.avg:.3f}\n' + 'Loss avg {loss.avg:.4f}\n' + 'Jaccard avg {jac.avg:.4f}\n' + 'WCE avg {wces.avg:.4f}\n' + 'Acc avg {acc.avg:.3f}\n' + 'IoU avg {iou.avg:.3f}'.format(batch_time=self.batch_time_e, + loss=losses, + jac=jaccs, + wces=wces, + acc=acc, iou=iou)) + # print also classwise + for i, jacc in enumerate(class_jaccard): + print('IoU class {i:} [{class_str:}] = {jacc:.3f}'.format( + i=i, class_str=class_func(i), jacc=jacc)) + save_to_log(self.log, 'log.txt', 'IoU class {i:} [{class_str:}] = {jacc:.3f}'.format( + i=i, class_str=class_func(i), jacc=jacc)) + self.info["valid_classes/" + class_func(i)] = jacc + + + return acc.avg, iou.avg, losses.avg, rand_imgs, hetero_l.avg diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/postproc/KNN.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/postproc/KNN.py new file mode 100644 index 0000000..b446055 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/postproc/KNN.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def get_gaussian_kernel(kernel_size=3, sigma=2, channels=1): + # Create a x, y coordinate grid of shape (kernel_size, kernel_size, 2) + x_coord = torch.arange(kernel_size) + x_grid = x_coord.repeat(kernel_size).view(kernel_size, kernel_size) + y_grid = x_grid.t() + xy_grid = torch.stack([x_grid, y_grid], dim=-1).float() + + mean = (kernel_size - 1) / 2. + variance = sigma ** 2. + + # Calculate the 2-dimensional gaussian kernel which is + # the product of two gaussian distributions for two different + # variables (in this case called x and y) + gaussian_kernel = (1. / (2. * math.pi * variance)) * \ + torch.exp(-torch.sum((xy_grid - mean) ** 2., dim=-1) / (2 * variance)) + + # Make sure sum of values in gaussian kernel equals 1. + gaussian_kernel = gaussian_kernel / torch.sum(gaussian_kernel) + + # Reshape to 2d depthwise convolutional weight + gaussian_kernel = gaussian_kernel.view(kernel_size, kernel_size) + + return gaussian_kernel + + +class KNN(nn.Module): + def __init__(self, params, nclasses): + super().__init__() + print("*" * 80) + print("Cleaning point-clouds with kNN post-processing") + self.knn = params["knn"] + self.search = params["search"] + self.sigma = params["sigma"] + self.cutoff = params["cutoff"] + self.nclasses = nclasses + print("kNN parameters:") + print("knn:", self.knn) + print("search:", self.search) + print("sigma:", self.sigma) + print("cutoff:", self.cutoff) + print("nclasses:", self.nclasses) + print("*" * 80) + + def forward(self, proj_range, unproj_range, proj_argmax, px, py): + ''' Warning! Only works for un-batched pointclouds. + If they come batched we need to iterate over the batch dimension or do + something REALLY smart to handle unaligned number of points in memory + ''' + # get device + if proj_range.is_cuda: + device = torch.device("cuda") + else: + device = torch.device("cpu") + + # sizes of projection scan + H, W = proj_range.shape + + # number of points + P = unproj_range.shape + + # check if size of kernel is odd and complain + if (self.search % 2 == 0): + raise ValueError("Nearest neighbor kernel must be odd number") + + # calculate padding + pad = int((self.search - 1) / 2) + + # unfold neighborhood to get nearest neighbors for each pixel (range image) + proj_unfold_k_rang = F.unfold(proj_range[None, None, ...], + kernel_size=(self.search, self.search), + padding=(pad, pad)) + + # index with px, py to get ALL the pcld points + idx_list = py * W + px + unproj_unfold_k_rang = proj_unfold_k_rang[:, :, idx_list] + + # WARNING, THIS IS A HACK + # Make non valid (<0) range points extremely big so that there is no screwing + # up the nn self.search + unproj_unfold_k_rang[unproj_unfold_k_rang < 0] = float("inf") + + # now the matrix is unfolded TOTALLY, replace the middle points with the actual range points + center = int(((self.search * self.search) - 1) / 2) + unproj_unfold_k_rang[:, center, :] = unproj_range + + # now compare range + k2_distances = torch.abs(unproj_unfold_k_rang - unproj_range) + + # make a kernel to weigh the ranges according to distance in (x,y) + # I make this 1 - kernel because I want distances that are close in (x,y) + # to matter more + inv_gauss_k = ( + 1 - get_gaussian_kernel(self.search, self.sigma, 1)).view(1, -1, 1) + inv_gauss_k = inv_gauss_k.to(device).type(proj_range.type()) + + # apply weighing + k2_distances = k2_distances * inv_gauss_k + + # find nearest neighbors + _, knn_idx = k2_distances.topk( + self.knn, dim=1, largest=False, sorted=False) + + # do the same unfolding with the argmax + proj_unfold_1_argmax = F.unfold(proj_argmax[None, None, ...].float(), + kernel_size=(self.search, self.search), + padding=(pad, pad)).long() + unproj_unfold_1_argmax = proj_unfold_1_argmax[:, :, idx_list] + + # get the top k predictions from the knn at each pixel + knn_argmax = torch.gather( + input=unproj_unfold_1_argmax, dim=1, index=knn_idx) + + # fake an invalid argmax of classes + 1 for all cutoff items + if self.cutoff > 0: + knn_distances = torch.gather(input=k2_distances, dim=1, index=knn_idx) + knn_invalid_idx = knn_distances > self.cutoff + knn_argmax[knn_invalid_idx] = self.nclasses + + # now vote + # argmax onehot has an extra class for objects after cutoff + knn_argmax_onehot = torch.zeros( + (1, self.nclasses + 1, P[0]), device=device).type(proj_range.type()) + ones = torch.ones_like(knn_argmax).type(proj_range.type()) + knn_argmax_onehot = knn_argmax_onehot.scatter_add_(1, knn_argmax, ones) + + # now vote (as a sum over the onehot shit) (don't let it choose unlabeled OR invalid) + knn_argmax_out = knn_argmax_onehot[:, 1:-1].argmax(dim=1) + 1 + + # reshape again + knn_argmax_out = knn_argmax_out.view(P) + + return knn_argmax_out diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/postproc/__init__.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/postproc/__init__.py new file mode 100644 index 0000000..2b8657a --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/postproc/__init__.py @@ -0,0 +1,6 @@ +# pylint: skip-file +import sys + +TRAIN_PATH = "../" +DEPLOY_PATH = "../../deploy" +sys.path.insert(0, TRAIN_PATH) diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/train.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/train.py new file mode 100644 index 0000000..55c353a --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/train.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + + +import argparse +import datetime +import os +import shutil +from shutil import copyfile +import __init__ as booger +import yaml +from tasks.semantic.modules.trainer import * +from pip._vendor.distlib.compat import raw_input + +from tasks.semantic.modules.SalsaNextAdf import * +from tasks.semantic.modules.SalsaNext import * +#from tasks.semantic.modules.save_dataset_projected import * +import math +from decimal import Decimal + +def remove_exponent(d): + return d.quantize(Decimal(1)) if d == d.to_integral() else d.normalize() + +def millify(n, precision=0, drop_nulls=True, prefixes=[]): + millnames = ['', 'k', 'M', 'B', 'T', 'P', 'E', 'Z', 'Y'] + if prefixes: + millnames = [''] + millnames.extend(prefixes) + n = float(n) + millidx = max(0, min(len(millnames) - 1, + int(math.floor(0 if n == 0 else math.log10(abs(n)) / 3)))) + result = '{:.{precision}f}'.format(n / 10**(3 * millidx), precision=precision) + if drop_nulls: + result = remove_exponent(Decimal(result)) + return '{0}{dx}'.format(result, dx=millnames[millidx]) + + +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y'): + return True + elif v.lower() in ('no', 'false', 'f', 'n'): + return False + else: + raise argparse.ArgumentTypeError('Boolean expected') + +if __name__ == '__main__': + parser = argparse.ArgumentParser("./train.py") + parser.add_argument( + '--dataset', '-d', + type=str, + required=True, + help='Dataset to train with. No Default', + ) + parser.add_argument( + '--arch_cfg', '-ac', + type=str, + required=True, + help='Architecture yaml cfg file. See /config/arch for sample. No default!', + ) + parser.add_argument( + '--data_cfg', '-dc', + type=str, + required=False, + default='config/labels/semantic-kitti.yaml', + help='Classification yaml cfg file. See /config/labels for sample. No default!', + ) + parser.add_argument( + '--log', '-l', + type=str, + default="~/output", + help='Directory to put the log data. Default: ~/logs/date+time' + ) + parser.add_argument( + '--name', '-n', + type=str, + default="", + help='If you want to give an aditional discriptive name' + ) + parser.add_argument( + '--pretrained', '-p', + type=str, + required=False, + default=None, + help='Directory to get the pretrained model. If not passed, do from scratch!' + ) + parser.add_argument( + '--uncertainty', '-u', + type=str2bool, nargs='?', + const=True, default=False, + help='Set this if you want to use the Uncertainty Version' + ) + + FLAGS, unparsed = parser.parse_known_args() + FLAGS.log = FLAGS.log + '/logs/' + datetime.datetime.now().strftime("%Y-%-m-%d-%H:%M") + FLAGS.name + if FLAGS.uncertainty: + params = SalsaNextUncertainty(20) + pytorch_total_params = sum(p.numel() for p in params.parameters() if p.requires_grad) + else: + params = SalsaNext(20) + pytorch_total_params = sum(p.numel() for p in params.parameters() if p.requires_grad) + # print summary of what we will do + print("----------") + print("INTERFACE:") + print("dataset", FLAGS.dataset) + print("arch_cfg", FLAGS.arch_cfg) + print("data_cfg", FLAGS.data_cfg) + print("uncertainty", FLAGS.uncertainty) + print("Total of Trainable Parameters: {}".format(millify(pytorch_total_params,2))) + print("log", FLAGS.log) + print("pretrained", FLAGS.pretrained) + print("----------\n") + # print("Commit hash (training version): ", str( + # subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip())) + print("----------\n") + + # open arch config file + try: + print("Opening arch config file %s" % FLAGS.arch_cfg) + ARCH = yaml.safe_load(open(FLAGS.arch_cfg, 'r')) + except Exception as e: + print(e) + print("Error opening arch yaml file.") + quit() + + # open data config file + try: + print("Opening data config file %s" % FLAGS.data_cfg) + DATA = yaml.safe_load(open(FLAGS.data_cfg, 'r')) + except Exception as e: + print(e) + print("Error opening data yaml file.") + quit() + + # create log folder + try: + if FLAGS.pretrained == "": + FLAGS.pretrained = None + if os.path.isdir(FLAGS.log): + if os.listdir(FLAGS.log): + answer = raw_input("Log Directory is not empty. Do you want to proceed? [y/n] ") + if answer == 'n': + quit() + else: + shutil.rmtree(FLAGS.log) + os.makedirs(FLAGS.log) + else: + FLAGS.log = FLAGS.pretrained + print("Not creating new log file. Using pretrained directory") + except Exception as e: + print(e) + print("Error creating log directory. Check permissions!") + quit() + + # does model folder exist? + if FLAGS.pretrained is not None: + if os.path.isdir(FLAGS.pretrained): + print("model folder exists! Using model from %s" % (FLAGS.pretrained)) + else: + print("model folder doesnt exist! Start with random weights...") + else: + print("No pretrained directory found.") + + # copy all files to log folder (to remember what we did, and make inference + # easier). Also, standardize name to be able to open it later + try: + print("Copying files to %s for further reference." % FLAGS.log) + copyfile(FLAGS.arch_cfg, FLAGS.log + "/arch_cfg.yaml") + copyfile(FLAGS.data_cfg, FLAGS.log + "/data_cfg.yaml") + except Exception as e: + print(e) + print("Error copying files, check permissions. Exiting...") + quit() + + # create trainer and start the training + trainer = Trainer(ARCH, DATA, FLAGS.dataset, FLAGS.log, FLAGS.pretrained,FLAGS.uncertainty) + trainer.train() diff --git a/aimet_zoo_torch/salsanext/models/tasks/semantic/visualize.py b/aimet_zoo_torch/salsanext/models/tasks/semantic/visualize.py new file mode 100644 index 0000000..9605e91 --- /dev/null +++ b/aimet_zoo_torch/salsanext/models/tasks/semantic/visualize.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# pylint: skip-file + +# The MIT License +# +# Copyright (c) 2019 Tiago Cortinhal (Halmstad University, Sweden), George Tzelepis (Volvo Technology AB, Volvo Group Trucks Technology, Sweden) and Eren Erdal Aksoy (Halmstad University and Volvo Technology AB, Sweden) +# +# Copyright (c) 2019 Andres Milioto, Jens Behley, Cyrill Stachniss, Photogrammetry and Robotics Lab, University of Bonn. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + + +import argparse +import os +import yaml +import __init__ as booger + +from common.laserscan import LaserScan, SemLaserScan +from common.laserscanvis import LaserScanVis + +if __name__ == '__main__': + parser = argparse.ArgumentParser("./visualize.py") + parser.add_argument( + '--dataset', '-d', + type=str, + required=True, + help='Dataset to visualize. No Default', + ) + parser.add_argument( + '--config', '-c', + type=str, + required=False, + default="config/labels/semantic-kitti.yaml", + help='Dataset config file. Defaults to %(default)s', + ) + parser.add_argument( + '--sequence', '-s', + type=str, + default="00", + required=False, + help='Sequence to visualize. Defaults to %(default)s', + ) + parser.add_argument( + '--predictions', '-p', + type=str, + default=None, + required=False, + help='Alternate location for labels, to use predictions folder. ' + 'Must point to directory containing the predictions in the proper format ' + ' (see readme)' + 'Defaults to %(default)s', + ) + parser.add_argument( + '--ignore_semantics', '-i', + dest='ignore_semantics', + default=False, + action='store_true', + help='Ignore semantics. Visualizes uncolored pointclouds.' + 'Defaults to %(default)s', + ) + parser.add_argument( + '--offset', + type=int, + default=0, + required=False, + help='Sequence to start. Defaults to %(default)s', + ) + parser.add_argument( + '--ignore_safety', + dest='ignore_safety', + default=False, + action='store_true', + help='Normally you want the number of labels and ptcls to be the same,' + ', but if you are not done inferring this is not the case, so this disables' + ' that safety.' + 'Defaults to %(default)s', + ) + FLAGS, unparsed = parser.parse_known_args() + + # print summary of what we will do + print("*" * 80) + print("INTERFACE:") + print("Dataset", FLAGS.dataset) + print("Config", FLAGS.config) + print("Sequence", FLAGS.sequence) + print("Predictions", FLAGS.predictions) + print("ignore_semantics", FLAGS.ignore_semantics) + print("ignore_safety", FLAGS.ignore_safety) + print("offset", FLAGS.offset) + print("*" * 80) + + # open config file + try: + print("Opening config file %s" % FLAGS.config) + CFG = yaml.safe_load(open(FLAGS.config, 'r')) + except Exception as e: + print(e) + print("Error opening yaml file.") + quit() + + # fix sequence name + FLAGS.sequence = '{0:02d}'.format(int(FLAGS.sequence)) + + # does sequence folder exist? + scan_paths = os.path.join(FLAGS.dataset, "sequences", + FLAGS.sequence, "velodyne") + if os.path.isdir(scan_paths): + print("Sequence folder exists! Using sequence from %s" % scan_paths) + else: + print("Sequence folder doesn't exist! Exiting...") + quit() + + # populate the pointclouds + scan_names = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(scan_paths)) for f in fn] + scan_names.sort() + + # does sequence folder exist? + if not FLAGS.ignore_semantics: + if FLAGS.predictions is not None: + label_paths = os.path.join(FLAGS.predictions, "sequences", + FLAGS.sequence, "predictions") + else: + label_paths = os.path.join(FLAGS.dataset, "sequences", + FLAGS.sequence, "labels") + if os.path.isdir(label_paths): + print("Labels folder exists! Using labels from %s" % label_paths) + else: + print("Labels folder doesn't exist! Exiting...") + quit() + # populate the pointclouds + label_names = [os.path.join(dp, f) for dp, dn, fn in os.walk( + os.path.expanduser(label_paths)) for f in fn] + label_names.sort() + + # check that there are same amount of labels and scans + if not FLAGS.ignore_safety: + assert (len(label_names) == len(scan_names)) + + # create a scan + if FLAGS.ignore_semantics: + scan = LaserScan(project=True) # project all opened scans to spheric proj + else: + color_dict = CFG["color_map"] + scan = SemLaserScan(color_dict, project=True) + + # create a visualizer + semantics = not FLAGS.ignore_semantics + if not semantics: + label_names = None + vis = LaserScanVis(scan=scan, + scan_names=scan_names, + label_names=label_names, + offset=FLAGS.offset, + semantics=semantics, + instances=False) + + # print instructions + print("To navigate:") + print("\tb: back (previous scan)") + print("\tn: next (next scan)") + print("\tq: quit (exit program)") + + # run the visualizer + vis.run() diff --git a/aimet_zoo_torch/segnet/SegNet.md b/aimet_zoo_torch/segnet/SegNet.md new file mode 100644 index 0000000..4f84b2d --- /dev/null +++ b/aimet_zoo_torch/segnet/SegNet.md @@ -0,0 +1,57 @@ +# PyTorch SegNet +This document describes evaluation of optimized checkpoints for SegNet + +## Environment Setup + +### Setup AI Model Efficiency Toolkit (AIMET) +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.23/packaging/install.md) before proceeding further. +This model was tested with the `torch_gpu` variant of AIMET 1.23. + +### Additional Setup Dependencies +```bash + pip install scikit-image +``` + +### Experiment setup +- Clone the [aimet-model-zoo](https://github.com/quic/aimet-model-zoo.git) repo +```bash + git clone https://github.com/quic/aimet-model-zoo.git +``` +- Append the repo location to your `PYTHONPATH` with the following: +```bash + export PYTHONPATH=$PYTHONPATH: +``` + +### Dataset +This evaluation was designed for the CamVid dataset variant provided by SegNet authors [repository](https://github.com/alexgkendall/SegNet-Tutorial). +- Download and extract CamVid directory: +```bash + wget https://github.com/alexgkendall/SegNet-Tutorial/archive/refs/heads/master.zip + unzip master.zip && mv SegNet-Tutorial-master/CamVid . && rm -r SegNet-Tutorial-master +``` + +### Model checkpoints and configuration +- The SegNet model checkpoints can be downloaded from the [Releases](/../../releases) page. + +--- + +## Usage +To run evaluation with QuantSim in AIMET, use the following +```bash + python3 aimet-model-zoo/aimet_zoo_torch/segnet/evaluator/segnet_quanteval.py \ + --dataset-path \ + --model-config +``` + +Available model configurations are: +- segnet_w8a8 +- segnet_w4a8 + +--- + +## Quantization Configuration +- Weight quantization: 8 or 4 bits, per channel symmetric quantization +- Bias parameters are not quantized +- Activation quantization: 8 bits, asymmetric quantization +- Model inputs are quantized +- TF_enhanced was used as quantization scheme diff --git a/aimet_zoo_torch/segnet/__init__.py b/aimet_zoo_torch/segnet/__init__.py new file mode 100644 index 0000000..14f7177 --- /dev/null +++ b/aimet_zoo_torch/segnet/__init__.py @@ -0,0 +1,2 @@ +""" package for getting segnet original model and quantized model""" +from .model.model_definition import SegNet diff --git a/aimet_zoo_torch/segnet/dataloader/__init__.py b/aimet_zoo_torch/segnet/dataloader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/segnet/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/segnet/dataloader/dataloaders_and_eval_func.py new file mode 100644 index 0000000..5c77a16 --- /dev/null +++ b/aimet_zoo_torch/segnet/dataloader/dataloaders_and_eval_func.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +"""module for getting evaluation function of dataloader""" +from tqdm import tqdm +import torch +from torchmetrics.classification import MulticlassJaccardIndex #pylint:disable = import-error +from aimet_zoo_torch.segnet.model.datasets.camvid import CamVid + + +def camvid_train_dataloader(dataset_path): + """camvid dataset train dataloader""" + dataset = CamVid(dataset_path + '/train', dataset_path + '/trainannot') + dataloader = torch.utils.data.DataLoader(dataset, + batch_size = 1, shuffle = True, num_workers = 1) + return dataloader + +def camvid_test_dataloader(dataset_path): + """camvid dataset test dataloader""" + dataset = CamVid(dataset_path + '/test', dataset_path + '/testannot') + dataloader = torch.utils.data.DataLoader(dataset, + batch_size = 1, shuffle = False, num_workers = 1) + return dataloader + +def model_eval(dataloader, use_cuda): + """model evaluation using dataloader""" + def eval_func(model, N = -1): + model.eval() + metric = MulticlassJaccardIndex(num_classes = 12, + ignore_index = 11, average = 'none', + validate_args = True) + loss = torch.zeros(12, device = 'cpu') + #pylint:disable = chained-comparison + with torch.no_grad(): + for i, inputs in enumerate(tqdm(dataloader)): + if i >= N and N >= 0: + break + images, labels = inputs + labels = labels.squeeze(1) # remove channel dim + if use_cuda: + images = images.cuda() + output = model(images) + loss += metric(output.cpu(), labels) + loss = loss[:11] / len(dataloader) * 100 + return (loss, torch.mean(loss)) + return eval_func + +def get_dataloaders_and_eval_func(dataset_path, use_cuda): + """get dataloaders and evaluation function""" + train_loader = camvid_train_dataloader(dataset_path=dataset_path) + test_loader = camvid_test_dataloader(dataset_path=dataset_path) + return model_eval(train_loader, use_cuda), model_eval(test_loader, use_cuda) diff --git a/aimet_zoo_torch/segnet/evaluator/__init__.py b/aimet_zoo_torch/segnet/evaluator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/segnet/evaluator/segnet_quanteval.py b/aimet_zoo_torch/segnet/evaluator/segnet_quanteval.py new file mode 100644 index 0000000..13add5c --- /dev/null +++ b/aimet_zoo_torch/segnet/evaluator/segnet_quanteval.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" AIMET Quantsim evaluation code for SegNet """ + +# General imports +import argparse + +# PyTorch imports +import torch + +# SegNet imports +from aimet_zoo_torch.common.utils.utils import get_device +from aimet_zoo_torch.segnet import SegNet +from aimet_zoo_torch.segnet.dataloader.dataloaders_and_eval_func import get_dataloaders_and_eval_func + + +def arguments(): + """Parse input arguments""" + parser = argparse.ArgumentParser( + description="Evalustion script for PyTorch SegNet model quantization" + ) + parser.add_argument( + "--model-config", + help="Select the model configuration", + default="segnet_w8a8", + choices=["segnet_w8a8", "segnet_w4a8"], + type=str, + required=True, + ) + parser.add_argument( + "--dataset-path", + help="Path to CamVid dataset parent directory", + type=str, + required=True + ) + parser.add_argument("--use-cuda", help="Run evaluation on CUDA GPU", default=True, type=bool) + args = parser.parse_args() + return args + +def seed(seednum, use_cuda): + """random seed generator""" + torch.manual_seed(seednum) + if use_cuda: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.cuda.manual_seed(seednum) + torch.cuda.manual_seed_all(seednum) + +def main(): + """Run evaluations""" + args = arguments() + seed(0, args.use_cuda) + device = get_device(args) + + train_func, eval_func = get_dataloaders_and_eval_func(args.dataset_path, args.use_cuda) + + # Original Model + model = SegNet(model_config=args.model_config) + model.from_pretrained(quantized=False) + fp32_miou = eval_func(model.model.to(device)) + del model + + # Quantized Model + model = SegNet(model_config=args.model_config) + model.from_pretrained(quantized=True) + sim = model.get_quantsim(quantized=True) + sim.compute_encodings(train_func, -1) + quant_miou = eval_func(sim.model.to(device)) + del model + + print("FP32 mIoU = {:0.2f}%".format(fp32_miou[1].item())) + print("Quantized mIoU = {:0.2f}%".format(quant_miou[1].item())) + + +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/segnet/model/__init__.py b/aimet_zoo_torch/segnet/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/segnet/model/datasets/camvid.py b/aimet_zoo_torch/segnet/model/datasets/camvid.py new file mode 100644 index 0000000..f862266 --- /dev/null +++ b/aimet_zoo_torch/segnet/model/datasets/camvid.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +from torch.utils.data import Dataset +import torchvision.transforms as T +import torchvision.transforms.functional as TF +import os +import numpy as np +from skimage import io + + +SIZE = (360, 480) + +class CamVid(Dataset): + def __init__(self, dir, label_dir): + self.flist = [] + self.llist = [] + for f in os.listdir(dir): + if not f.startswith('.'): + image = io.imread(os.path.join(dir, f)) + label = io.imread(os.path.join(label_dir, f)).astype(np.int_) + + self.flist.extend([TF.resize(T.ToTensor()(image), + SIZE, T.InterpolationMode.BICUBIC)]) + self.llist.extend([TF.resize(T.ToTensor()(label), + SIZE, T.InterpolationMode.NEAREST)]) + + def __len__(self): + return len(self.flist) + + def __getitem__(self, i): + return (self.flist[i], self.llist[i]) + + # reference balancing + def weights_ref11(self): + return [0.2595, 0.1826, 4.5640, 0.1417, 0.9051, 0.3826, 9.6446, 1.8418, + 0.6823, 6.2478, 7.3614, + 1.0974] diff --git a/aimet_zoo_torch/segnet/model/model_cards/__init__.py b/aimet_zoo_torch/segnet/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/segnet/model/model_cards/segnet_w4a8.json b/aimet_zoo_torch/segnet/model/model_cards/segnet_w4a8.json new file mode 100644 index 0000000..409f7b2 --- /dev/null +++ b/aimet_zoo_torch/segnet/model/model_cards/segnet_w4a8.json @@ -0,0 +1,25 @@ +{ + "name": "SegNet", + "framework": "pytorch", + "task": "semantic segmentation", + "model_args": {}, + "input_shape": [null, 3, 360, 480], + "training_dataset": "CamVid", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "techniques": ["adaround"] + } + }, + "artifacts": { + "url_pre_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_segnet/SegNet.pth", + "url_post_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_segnet/segnet_w4a8_pc_state_dict.pth", + "url_adaround_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_segnet/segnet_w4a8_pc.encodings", + "url_aimet_encodings": null, + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} diff --git a/aimet_zoo_torch/segnet/model/model_cards/segnet_w8a8.json b/aimet_zoo_torch/segnet/model/model_cards/segnet_w8a8.json new file mode 100644 index 0000000..b328bc1 --- /dev/null +++ b/aimet_zoo_torch/segnet/model/model_cards/segnet_w8a8.json @@ -0,0 +1,25 @@ +{ + "name": "SegNet", + "framework": "pytorch", + "task": "semantic segmentation", + "model_args": {}, + "input_shape": [null, 3, 360, 480], + "training_dataset": "CamVid", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "techniques": ["ptq"] + } + }, + "artifacts": { + "url_pre_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_segnet/SegNet.pth", + "url_post_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_segnet/segnet_w8a8_pc_state_dict.pth", + "url_adaround_encodings": null, + "url_aimet_encodings": null, + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} diff --git a/aimet_zoo_torch/segnet/model/model_definition.py b/aimet_zoo_torch/segnet/model/model_definition.py new file mode 100644 index 0000000..f87b484 --- /dev/null +++ b/aimet_zoo_torch/segnet/model/model_definition.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +"""Class for downloading and setting up of optmized and original segnet model for AIMET model zoo""" +import os +import json +import pathlib +import torch +from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim#pylint:disable=import-error +from aimet_zoo_torch.common.downloader import Downloader +from aimet_zoo_torch.segnet.model.models.segnet import ( + get_seg_model, +) + +class SegNet(Downloader): + """SegNet parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + + def __init__(self, model_config=None, device=torch.device("cuda")): + """ + :param model_config: named model config from which to obtain model artifacts and arguments. + If provided, overwrites the other arguments passed to this object + """ + parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.device = device + self.cfg = False + if model_config: + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + if os.path.exists(config_filepath): + with open(config_filepath) as f_in: + self.cfg = json.load(f_in) + if self.cfg: + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + self.model = get_seg_model() + + def from_pretrained(self, quantized=False): + """load pretrained weights""" + if not self.cfg: + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) + self._download_pre_opt_weights() + self._download_post_opt_weights() + self._download_aimet_config() + self._download_aimet_encodings() + self._download_adaround_encodings() + if quantized: + self.model = torch.load(self.path_post_opt_weights) + self.model.to(self.device) + else: + state_dict = torch.load(self.path_pre_opt_weights) + self.model.load_state_dict(state_dict) + self.model.to(self.device) + self.model.eval() + + def get_quantsim(self, quantized=False): + """get quantsim object with pre-loaded encodings""" + if not self.cfg: + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) + if quantized: + self.from_pretrained(quantized=True) + else: + self.from_pretrained(quantized=False) + dummy_input = torch.rand(self.input_shape, device=self.device) + kwargs = { + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } + sim = QuantizationSimModel(self.model, **kwargs) + if self.path_aimet_encodings and quantized: + load_encodings_to_sim(sim, self.path_aimet_encodings) + print("load_encodings_to_sim finished!") + if self.path_adaround_encodings and quantized: + sim.set_and_freeze_param_encodings(self.path_adaround_encodings) + print("set_and_freeze_param_encodings finished!") + sim.model.to(self.device) + sim.model.eval() + return sim diff --git a/aimet_zoo_torch/segnet/model/models/segnet.py b/aimet_zoo_torch/segnet/model/models/segnet.py new file mode 100644 index 0000000..f561a5f --- /dev/null +++ b/aimet_zoo_torch/segnet/model/models/segnet.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +import torch.nn as nn +import torch.nn.functional as F + + +class SegNet(nn.Module): + def __init__(self, in_chn = 3, out_chn = 32, BN_momentum = 0.5): + super(SegNet, self).__init__() + + self.in_chn = in_chn + self.out_chn = out_chn + + self.MaxEn1 = nn.MaxPool2d(2, stride = 2, return_indices = True) + self.MaxEn2 = nn.MaxPool2d(2, stride = 2, return_indices = True) + self.MaxEn3 = nn.MaxPool2d(2, stride = 2, return_indices = True) + self.MaxEn4 = nn.MaxPool2d(2, stride = 2, return_indices = True) + self.MaxEn5 = nn.MaxPool2d(2, stride = 2, return_indices = True) + + self.en1 = nn.Sequential( + nn.Conv2d(self.in_chn, 64, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(64, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(64, 64, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(64, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + self.en2 = nn.Sequential( + nn.Conv2d(64, 128, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(128, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(128, 128, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(128, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + self.en3 = nn.Sequential( + nn.Conv2d(128, 256, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(256, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(256, 256, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(256, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(256, 256, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(256, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + self.en4 = nn.Sequential( + nn.Conv2d(256, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + self.en5 = nn.Sequential( + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + + # General Max Pool 2D/Upsampling for DECODING layers + #self.MaxDe = nn.MaxUnpool2d(2, stride = 2) + + self.dec5 = nn.Sequential( + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + self.dec4 = nn.Sequential( + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(512, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(512, 256, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(256, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + self.dec3 = nn.Sequential( + nn.Conv2d(256, 256, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(256, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(256, 256, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(256, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(256, 128, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(128, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + self.dec2 = nn.Sequential( + nn.Conv2d(128, 128, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(128, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(128, 64, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(64, momentum = BN_momentum), + nn.ReLU(inplace=True)) + + self.dec1 = nn.Sequential( + nn.Conv2d(64, 64, kernel_size=(3, 3), padding=(1, 1)), + nn.BatchNorm2d(64, momentum = BN_momentum), + nn.ReLU(inplace=True), + nn.Conv2d(64, self.out_chn, kernel_size=(3, 3), padding=(1, 1))) + + def forward(self, x): + # ENCODER + x = self.en1(x) + x, ind1 = self.MaxEn1(x) + size1 = x.size() + + x = self.en2(x) + x, ind2 = self.MaxEn2(x) + size2 = x.size() + + x = self.en3(x) + x, ind3 = self.MaxEn3(x) + size3 = x.size() + + x = self.en4(x) + x, ind4 = self.MaxEn4(x) + size4 = x.size() + + x = self.en5(x) + x, ind5 = self.MaxEn5(x) + size5 = x.size() + + # DECODER + #x = self.MaxDe(x, ind5, output_size = size4) + x = F.max_unpool2d(x, ind5, 2, stride = 2, output_size = size4) + x = self.dec5(x) + + #x = self.MaxDe(x, ind4, output_size = size3) + x = F.max_unpool2d(x, ind4, 2, stride = 2, output_size = size3) + x = self.dec4(x) + + #x = self.MaxDe(x, ind3, output_size = size2) + x = F.max_unpool2d(x, ind3, 2, stride = 2, output_size = size2) + x = self.dec3(x) + + #x = self.MaxDe(x, ind2, output_size = size1) + x = F.max_unpool2d(x, ind2, 2, stride = 2, output_size = size1) + x = self.dec2(x) + + #x = self.MaxDe(x, ind1) + x = F.max_unpool2d(x, ind1, 2, stride = 2) + x = self.dec1(x) + + x = F.softmax(x, dim = 1) + + return x + +def get_seg_model(): + model = SegNet(in_chn = 3, out_chn = 12, BN_momentum = 0.5) + return model diff --git a/aimet_zoo_torch/segnet/requirements.txt b/aimet_zoo_torch/segnet/requirements.txt new file mode 100644 index 0000000..5cfc385 --- /dev/null +++ b/aimet_zoo_torch/segnet/requirements.txt @@ -0,0 +1,2 @@ +scikit-image +torchmetrics diff --git a/aimet_zoo_torch/sesr/__init__.py b/aimet_zoo_torch/sesr/__init__.py index cfe6c0c..77b3ce0 100644 --- a/aimet_zoo_torch/sesr/__init__.py +++ b/aimet_zoo_torch/sesr/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import SESR \ No newline at end of file +""" package for getting SESR original model and quantized model""" +from .model.model_definition import SESR diff --git a/aimet_zoo_torch/sesr/evaluators/__init__.py b/aimet_zoo_torch/sesr/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/sesr/evaluators/sesr_quanteval.py b/aimet_zoo_torch/sesr/evaluators/sesr_quanteval.py index 46d0328..0479a55 100644 --- a/aimet_zoo_torch/sesr/evaluators/sesr_quanteval.py +++ b/aimet_zoo_torch/sesr/evaluators/sesr_quanteval.py @@ -8,25 +8,52 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET evaluation code for QuickSRNet ''' +""" AIMET evaluation code for QuickSRNet """ import argparse from aimet_zoo_torch.sesr import SESR from aimet_zoo_torch.common.super_resolution.psnr import evaluate_average_psnr -from aimet_zoo_torch.common.super_resolution.utils import load_dataset, pass_calibration_data +from aimet_zoo_torch.common.super_resolution.utils import ( + load_dataset, + pass_calibration_data, +) from aimet_zoo_torch.common.super_resolution.inference import run_model # add arguments def arguments(): """parses command line arguments""" - parser = argparse.ArgumentParser(description='Arguments for evaluating model') - parser.add_argument('--dataset-path', help='path to image evaluation dataset', type=str, required=True) - parser.add_argument('--model-config', help='model configuration to be tested', type=str, required=True) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) - parser.add_argument('--batch-size',help='batch_size for loading data',type=int,default=16) - parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) + parser = argparse.ArgumentParser(description="Arguments for evaluating model") + parser.add_argument( + "--dataset-path", + help="path to image evaluation dataset", + type=str, + required=True, + ) + parser.add_argument( + "--model-config", + help="model configuration to be tested", + type=str, + required=True, + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--batch-size", help="batch_size for loading data", type=int, default=16 + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU", type=bool, default=True + ) args = parser.parse_args() return args @@ -35,20 +62,24 @@ def main(): """executes evaluation""" args = arguments() - model_fp32 = SESR(model_config = args.model_config) + model_fp32 = SESR(model_config=args.model_config) model_fp32.from_pretrained(quantized=False) sim_fp32 = model_fp32.get_quantsim(quantized=False) - model_int8 = SESR(model_config = args.model_config) + model_int8 = SESR(model_config=args.model_config) model_int8.from_pretrained(quantized=True) sim_int8 = model_int8.get_quantsim(quantized=True) IMAGES_LR, IMAGES_HR = load_dataset(args.dataset_path, model_fp32.scaling_factor) - sim_fp32.compute_encodings(forward_pass_callback=pass_calibration_data, - forward_pass_callback_args=(IMAGES_LR, args.use_cuda)) - sim_int8.compute_encodings(forward_pass_callback=pass_calibration_data, - forward_pass_callback_args=(IMAGES_LR, args.use_cuda)) + sim_fp32.compute_encodings( + forward_pass_callback=pass_calibration_data, + forward_pass_callback_args=(IMAGES_LR, args.use_cuda), + ) + sim_int8.compute_encodings( + forward_pass_callback=pass_calibration_data, + forward_pass_callback_args=(IMAGES_LR, args.use_cuda), + ) # Run model inference on test images and get super-resolved images IMAGES_SR_original_fp32 = run_model(model_fp32, IMAGES_LR, args.use_cuda) @@ -58,13 +89,14 @@ def main(): # Get the average PSNR for all test-images avg_psnr = evaluate_average_psnr(IMAGES_SR_original_fp32, IMAGES_HR) - print(f'Original Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Original Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_original_int8, IMAGES_HR) - print(f'Original Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Original Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_optimized_fp32, IMAGES_HR) - print(f'Optimized Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Optimized Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_optimized_int8, IMAGES_HR) - print(f'Optimized Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Optimized Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}") -if __name__ == '__main__': - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/sesr/model/model_cards/__init__.py b/aimet_zoo_torch/sesr/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/sesr/model/model_definition.py b/aimet_zoo_torch/sesr/model/model_definition.py index 4f852b7..b5ce935 100644 --- a/aimet_zoo_torch/sesr/model/model_definition.py +++ b/aimet_zoo_torch/sesr/model/model_definition.py @@ -8,9 +8,13 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -import torch +"""Class for downloading and setting up of optmized and original SESR model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet + import json import os +import torch from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim from aimet_zoo_torch.common.downloader import Downloader from aimet_zoo_torch.common.super_resolution.models import SESRRelease @@ -18,44 +22,65 @@ class SESR(SESRRelease, Downloader): """ABPN parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" - def __init__(self, model_config=None, num_channels=16, scaling_factor=2, num_lblocks=3,**kwargs): + + def __init__( + self, + model_config=None, + num_channels=16, + scaling_factor=2, + num_lblocks=3, + **kwargs + ): """ :param model_config: named model config from which to obtain model artifacts and arguments. - If provided, overwrites the other arguments passed to this object + If provided, overwrites the other arguments passed to this object :param scaling_factor: scaling factor for LR-to-HR upscaling (2x, 3x, 4x... or 1.5x) - :param num_channels: number of feature channels for convolutional layers + :param num_channels: number of feature channels for convolutional layers """ - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) - self.scaling_factor = self.cfg['model_args']['scaling_factor'] if self.cfg else scaling_factor - self.num_channels = self.cfg['model_args']['num_channels'] if self.cfg else num_channels - self.num_lblocks = self.cfg['model_args']['num_lblocks'] if self.cfg else num_lblocks + self.scaling_factor = ( + self.cfg["model_args"]["scaling_factor"] if self.cfg else scaling_factor + ) + self.num_channels = ( + self.cfg["model_args"]["num_channels"] if self.cfg else num_channels + ) + self.num_lblocks = ( + self.cfg["model_args"]["num_lblocks"] if self.cfg else num_lblocks + ) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) - SESRRelease.__init__(self, - scaling_factor = self.scaling_factor, - num_channels = self.num_channels, - num_lblocks = self.num_lblocks, - **kwargs) - + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + SESRRelease.__init__( + self, + scaling_factor=self.scaling_factor, + num_channels=self.num_channels, + num_lblocks=self.num_lblocks, + **kwargs + ) def from_pretrained(self, quantized=False): """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() @@ -63,11 +88,11 @@ def from_pretrained(self, quantized=False): self._download_adaround_encodings() if quantized: self.collapse() - state_dict = torch.load(self.path_post_opt_weights)['state_dict'] + state_dict = torch.load(self.path_post_opt_weights)["state_dict"] self.load_state_dict(state_dict) self.cuda() else: - state_dict = torch.load(self.path_pre_opt_weights)['state_dict'] + state_dict = torch.load(self.path_pre_opt_weights)["state_dict"] self.load_state_dict(state_dict) self.cuda() self.eval() @@ -75,23 +100,32 @@ def from_pretrained(self, quantized=False): def get_quantsim(self, quantized=False): """get quantsim object with pre-loaded encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self, **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) sim.model.eval() - return sim \ No newline at end of file + return sim diff --git a/aimet_zoo_torch/srgan/evaluators/srgan_quanteval.py b/aimet_zoo_torch/srgan/evaluators/srgan_quanteval.py index 27ba09d..002d160 100644 --- a/aimet_zoo_torch/srgan/evaluators/srgan_quanteval.py +++ b/aimet_zoo_torch/srgan/evaluators/srgan_quanteval.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 # -*- mode: python -*- # ============================================================================= # @@-COPYRIGHT-START-@@ @@ -31,6 +31,7 @@ from aimet_torch import quantsim import codes.options.options as option +#pylint:disable = consider-using-from-import import codes.utils.util as util from codes.data.util import bgr2ycbcr from codes.data import create_dataset, create_dataloader @@ -41,12 +42,8 @@ def evaluate_generator( - generator, - test_loader, - options, - mode="y_channel", - output_dir=None, - device=None): + generator, test_loader, options, mode="y_channel", output_dir=None, device=None +): """ :param generator: an srgan model`s generator part, must be an nn.module :param test_loader: a pytorch dataloader @@ -62,8 +59,8 @@ def evaluate_generator( print("Testing on Y channel...") else: raise ValueError( - "evaluation mode not supported!" - "Must be one of `RGB` or `y_channel`") + "evaluation mode not supported! Must be one of `RGB` or `y_channel`" + ) psnr_values = [] ssim_values = [] @@ -102,8 +99,7 @@ def evaluate_generator( # calculate PSNR and SSIM if need_GT: gt_img = util.tensor2img(visuals["GT"]) - sr_img, gt_img = util.crop_border( - [sr_img, gt_img], options["scale"]) + sr_img, gt_img = util.crop_border([sr_img, gt_img], options["scale"]) if mode == "rgb": psnr = util.calculate_psnr(sr_img, gt_img) @@ -285,9 +281,7 @@ def parse_args(): "-bout", help="Default bitwidth (4-31) to use for quantizing layer inputs and outputs", default=8, - choices=range( - 4, - 32), + choices=range(4, 32), type=int, ) parser.add_argument( @@ -313,9 +307,9 @@ def parse_args(): return parser.parse_args() - class ModelConfig: """Adding hardcoded values into args from parseargs() and return config object""" + def __init__(self, args): self.yml = "./test_SRGAN.yml" self.quant_scheme = "tf_enhanced" @@ -388,7 +382,8 @@ def main(args): test_set_name = test_loader.dataset.opt["name"] print(f"Testing on dataset {test_set_name}") psnr_vals, ssim_vals = evaluate_generator( - sim.model, test_loader, opt, device=device, output_dir=config.output_dir) + sim.model, test_loader, opt, device=device, output_dir=config.output_dir + ) psnr_val = np.mean(psnr_vals) ssim_val = np.mean(ssim_vals) print( diff --git a/aimet_zoo_torch/ssd_mobilenetv2/__init__.py b/aimet_zoo_torch/ssd_mobilenetv2/__init__.py index 27720b7..c2b1181 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/__init__.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/__init__.py @@ -1 +1,5 @@ -from .model.model_definition import SSDMobileNetV2, create_mobilenetv2_ssd_lite_predictor \ No newline at end of file +""" package for creating and getting ssdmobilenetv2 original model and quantized model""" +from .model.model_definition import ( + SSDMobileNetV2, + create_mobilenetv2_ssd_lite_predictor, +) diff --git a/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/collation.py b/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/collation.py index 1101350..1003c77 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/collation.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/collation.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/generate_vocdata.py b/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/generate_vocdata.py index 4d1d7e9..10692d2 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/generate_vocdata.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/generate_vocdata.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/open_images.py b/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/open_images.py index 3fb2594..f99859e 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/open_images.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/open_images.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/voc_dataset.py b/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/voc_dataset.py index 4349289..0a4c7b0 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/voc_dataset.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/dataloader/datasets/voc_dataset.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py b/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py index 0af6e89..36fbb91 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py @@ -34,15 +34,17 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET QuantSim script on MobileNetV2-SSD Lite ''' -''' Currently We apply QuantSim on Batch Norm folded model ''' +""" AIMET QuantSim script on MobileNetV2-SSD Lite + Currently We apply QuantSim on Batch Norm folded model +""" import argparse import pathlib -import numpy as np import copy import os +import random import urllib.request +import numpy as np from tqdm import tqdm import torch @@ -52,277 +54,333 @@ # AIMET model zoo imports from aimet_zoo_torch.common.utils.utils import get_device -from aimet_zoo_torch.ssd_mobilenetv2 import SSDMobileNetV2, create_mobilenetv2_ssd_lite_predictor +from aimet_zoo_torch.ssd_mobilenetv2 import ( + SSDMobileNetV2, + create_mobilenetv2_ssd_lite_predictor, +) + def download_labels(): - """ - downloads model labels - """ - if not os.path.exists("./voc-model-labels.txt"): - urllib.request.urlretrieve( - "https://storage.googleapis.com/models-hao/voc-model-labels.txt", - "voc-model-labels.txt") + """ + downloads model labels + """ + if not os.path.exists("./voc-model-labels.txt"): + urllib.request.urlretrieve( + "https://storage.googleapis.com/models-hao/voc-model-labels.txt", + "voc-model-labels.txt", + ) + def arguments(): - """parses command line arguments""" - parser = argparse.ArgumentParser(description="SSD Evaluation on VOC Dataset.") - parser.add_argument("--model-config", type=str, help="Model configuration to load pre-trained weights from", default='ssd_mobilenetv2_w8a8', choices=['ssd_mobilenetv2_w8a8']) - parser.add_argument("--dataset-path", type=str, help="The root directory of dataset, e.g., my_path/VOCdevkit/VOC2007/") - parser.add_argument("--default-output-bw", type=int, default=8) - parser.add_argument("--default-param-bw", type=int, default=8) - parser.add_argument("--use-cuda", type=bool, default=True) - args = parser.parse_args() - return args + # pylint: disable = redefined-outer-name + """parses command line arguments""" + parser = argparse.ArgumentParser(description="SSD Evaluation on VOC Dataset.") + parser.add_argument( + "--model-config", + type=str, + help="Model configuration to load pre-trained weights from", + default="ssd_mobilenetv2_w8a8", + choices=["ssd_mobilenetv2_w8a8"], + ) + parser.add_argument( + "--dataset-path", + type=str, + help="The root directory of dataset, e.g., my_path/VOCdevkit/VOC2007/", + ) + parser.add_argument("--default-output-bw", type=int, default=8) + parser.add_argument("--default-param-bw", type=int, default=8) + parser.add_argument("--use-cuda", type=bool, default=True) + args = parser.parse_args() + return args class CalibrationDataset(Dataset): - """Calibration Dataset""" - def __init__(self, data_dict, device='cpu'): - """ - Args: - txt_file (string): Path to text file with location of images, label in img name - """ - self.data = data_dict - self.device = device + """Calibration Dataset""" + # pylint: disable = redefined-outer-name + def __init__(self, data_dict, device="cpu"): + """ + Args: + txt_file (string): Path to text file with location of images, label in img name + """ + self.data = data_dict + self.device = device - def __len__(self): - return len(self.data.ids) + def __len__(self): + return len(self.data.ids) - def __getitem__(self, idx): - image = self.data.get_image(idx) - label = self.data.get_annotation(idx) - return torch.Tensor(image).to(self.device), label + def __getitem__(self, idx): + image = self.data.get_image(idx) + label = self.data.get_annotation(idx) + return torch.Tensor(image).to(self.device), label def work_init(work_id): - """seed function to initialize workers""" - seed = torch.initial_seed() % 2**32 - random.seed(seed + work_id) - np.random.seed(seed + work_id) + """seed function to initialize workers""" + seed = torch.initial_seed() % 2**32 + random.seed(seed + work_id) + np.random.seed(seed + work_id) def model_eval(args, predictor, dataset): - aimet_dataset=copy.deepcopy(dataset) - aimet_dataset.ids=aimet_dataset.ids[:500] - calib_dataset = CalibrationDataset(aimet_dataset) - data_loader_kwargs = { 'worker_init_fn':work_init, 'num_workers' : 0} - batch_size = 1 - calib_dataloader = DataLoader(calib_dataset, batch_size, shuffle = False, pin_memory = True, **data_loader_kwargs) - calib = tqdm(calib_dataloader) - def func_quant(model, iterations=2000, device=torch.device('cuda' if torch.cuda.is_available() and args.use_cuda else 'cpu')): - model = model.to(device) - for i, samples in enumerate(calib): - image = samples[0] - image = predictor.transform(image.squeeze(0).numpy()) - image = image.unsqueeze(0).to(device) - model(image) - return func_quant + """ model evalution function""" + # pylint: disable = redefined-outer-name, unused-variable, unused-argument + aimet_dataset = copy.deepcopy(dataset) + aimet_dataset.ids = aimet_dataset.ids[:500] + calib_dataset = CalibrationDataset(aimet_dataset) + data_loader_kwargs = {"worker_init_fn": work_init, "num_workers": 0} + batch_size = 1 + calib_dataloader = DataLoader( + calib_dataset, batch_size, shuffle=False, pin_memory=True, **data_loader_kwargs + ) + calib = tqdm(calib_dataloader) + + def func_quant( + model, + iterations=2000, + device=torch.device( + "cuda" if torch.cuda.is_available() and args.use_cuda else "cpu" + ), + ): + model = model.to(device) + for i, samples in enumerate(calib): + image = samples[0] + image = predictor.transform(image.squeeze(0).numpy()) + image = image.unsqueeze(0).to(device) + model(image) + + return func_quant def group_annotation_by_class(dataset): - true_case_stat = {} - all_gt_boxes = {} - all_difficult_cases = {} - for i in range(len(dataset)): - image_id, annotation = dataset.get_annotation(i) - gt_boxes, classes, is_difficult = annotation - gt_boxes = torch.from_numpy(gt_boxes) - for i, difficult in enumerate(is_difficult): - class_index = int(classes[i]) - gt_box = gt_boxes[i] - if not difficult: - true_case_stat[class_index] = true_case_stat.get(class_index, 0) + 1 - - if class_index not in all_gt_boxes: - all_gt_boxes[class_index] = {} - if image_id not in all_gt_boxes[class_index]: - all_gt_boxes[class_index][image_id] = [] - all_gt_boxes[class_index][image_id].append(gt_box) - if class_index not in all_difficult_cases: - all_difficult_cases[class_index]={} - if image_id not in all_difficult_cases[class_index]: - all_difficult_cases[class_index][image_id] = [] - all_difficult_cases[class_index][image_id].append(difficult) - - for class_index in all_gt_boxes: - for image_id in all_gt_boxes[class_index]: - all_gt_boxes[class_index][image_id] = torch.stack(all_gt_boxes[class_index][image_id]) - for class_index in all_difficult_cases: - for image_id in all_difficult_cases[class_index]: - all_gt_boxes[class_index][image_id] = torch.tensor(all_gt_boxes[class_index][image_id]) - return true_case_stat, all_gt_boxes, all_difficult_cases - - -def compute_average_precision_per_class(num_true_cases, gt_boxes, difficult_cases, - prediction_file, iou_threshold, use_2007_metric): - with open(prediction_file) as f: - image_ids = [] - boxes = [] - scores = [] - for line in f: - t = line.rstrip().split(" ") - image_ids.append(t[0]) - scores.append(float(t[1])) - box = torch.tensor([float(v) for v in t[2:]]).unsqueeze(0) - box -= 1.0 # convert to python format where indexes start from 0 - boxes.append(box) - scores = np.array(scores) - sorted_indexes = np.argsort(-scores) - boxes = [boxes[i] for i in sorted_indexes] - image_ids = [image_ids[i] for i in sorted_indexes] - true_positive = np.zeros(len(image_ids)) - false_positive = np.zeros(len(image_ids)) - matched = set() - for i, image_id in enumerate(image_ids): - box = boxes[i] - if image_id not in gt_boxes: - false_positive[i] = 1 - continue - - gt_box = gt_boxes[image_id] - ious = box_utils.iou_of(box, gt_box) - max_iou = torch.max(ious).item() - max_arg = torch.argmax(ious).item() - if max_iou > iou_threshold: - if difficult_cases[image_id][max_arg] == 0: - if (image_id, max_arg) not in matched: - true_positive[i] = 1 - matched.add((image_id, max_arg)) - else: - false_positive[i] = 1 - else: - false_positive[i] = 1 - - true_positive = true_positive.cumsum() - false_positive = false_positive.cumsum() - precision = true_positive / (true_positive + false_positive) - recall = true_positive / num_true_cases - if use_2007_metric: - return measurements.compute_voc2007_average_precision(precision, recall) - else: - return measurements.compute_average_precision(precision, recall) + """ group annotation by class """ + # pylint: disable = redefined-outer-name, unused-variable, unused-argument + true_case_stat = {} + all_gt_boxes = {} + all_difficult_cases = {} + for i in range(len(dataset)): + image_id, annotation = dataset.get_annotation(i) + gt_boxes, classes, is_difficult = annotation + gt_boxes = torch.from_numpy(gt_boxes) + for i, difficult in enumerate(is_difficult): + class_index = int(classes[i]) + gt_box = gt_boxes[i] + if not difficult: + true_case_stat[class_index] = true_case_stat.get(class_index, 0) + 1 + + if class_index not in all_gt_boxes: + all_gt_boxes[class_index] = {} + if image_id not in all_gt_boxes[class_index]: + all_gt_boxes[class_index][image_id] = [] + all_gt_boxes[class_index][image_id].append(gt_box) + if class_index not in all_difficult_cases: + all_difficult_cases[class_index] = {} + if image_id not in all_difficult_cases[class_index]: + all_difficult_cases[class_index][image_id] = [] + all_difficult_cases[class_index][image_id].append(difficult) + #pylint:disable = consider-using-dict-items + for class_index in all_gt_boxes: + for image_id in all_gt_boxes[class_index]: + all_gt_boxes[class_index][image_id] = torch.stack( + all_gt_boxes[class_index][image_id] + ) + for class_index in all_difficult_cases: + for image_id in all_difficult_cases[class_index]: + all_gt_boxes[class_index][image_id] = torch.tensor( + all_gt_boxes[class_index][image_id] + ) + return true_case_stat, all_gt_boxes, all_difficult_cases + + +def compute_average_precision_per_class( + num_true_cases, + gt_boxes, + difficult_cases, + prediction_file, + iou_threshold, + use_2007_metric, + ): # pylint: disable = too-many-locals + """ compute average precision per class""" + with open(prediction_file) as f: + image_ids = [] + boxes = [] + scores = [] + for line in f: + t = line.rstrip().split(" ") + image_ids.append(t[0]) + scores.append(float(t[1])) + box = torch.tensor([float(v) for v in t[2:]]).unsqueeze(0) + box -= 1.0 # convert to python format where indexes start from 0 + boxes.append(box) + scores = np.array(scores) + sorted_indexes = np.argsort(-scores) + boxes = [boxes[i] for i in sorted_indexes] + image_ids = [image_ids[i] for i in sorted_indexes] + true_positive = np.zeros(len(image_ids)) + false_positive = np.zeros(len(image_ids)) + matched = set() + for i, image_id in enumerate(image_ids): + box = boxes[i] + if image_id not in gt_boxes: + false_positive[i] = 1 + continue + + gt_box = gt_boxes[image_id] + ious = box_utils.iou_of(box, gt_box) + max_iou = torch.max(ious).item() + max_arg = torch.argmax(ious).item() + if max_iou > iou_threshold: + if difficult_cases[image_id][max_arg] == 0: + if (image_id, max_arg) not in matched: + true_positive[i] = 1 + matched.add((image_id, max_arg)) + else: + false_positive[i] = 1 + else: + false_positive[i] = 1 + + true_positive = true_positive.cumsum() + false_positive = false_positive.cumsum() + precision = true_positive / (true_positive + false_positive) + recall = true_positive / num_true_cases + if use_2007_metric: + return measurements.compute_voc2007_average_precision(precision, recall) + return measurements.compute_average_precision(precision, recall) def evaluate_predictor(predictor): - ''' - :param predictor: - :return: Average precision per classes for the given predictor - ''' - - results = [] - for i in tqdm(range(len(dataset))): - image = dataset.get_image(i) - boxes, labels, probs = predictor.predict(image) - indexes = torch.ones(labels.size(0), 1, dtype=torch.float32) * i - results.append(torch.cat([ - indexes.reshape(-1, 1), - labels.reshape(-1, 1).float(), - probs.reshape(-1, 1), - boxes + 1.0 # matlab's indexes start from 1 - ], dim=1)) - results = torch.cat(results) - for class_index, class_name in enumerate(class_names): - if class_index == 0: - continue # ignore background - prediction_path = eval_path / f"det_test_{class_name}.txt" - with open(prediction_path, "w") as f: - sub = results[results[:, 1] == class_index, :] - for i in range(sub.size(0)): - tmp = sub[i, 2:].cpu() - prob_box = tmp.numpy() - image_id = dataset.ids[int(sub[i, 0])] - print( - image_id + " " + " ".join([str(v) for v in prob_box]), - file=f - ) - aps = [] - print("\n\nAverage Precision Per Class:") - for class_index, class_name in enumerate(class_names): - if class_index == 0: - continue - prediction_path = eval_path / f"det_test_{class_name}.txt" - ap = compute_average_precision_per_class( - true_case_stat[class_index], - all_gb_boxes[class_index], - all_difficult_cases[class_index], - prediction_path, - config.iou_threshold, - config.use_2007_metric - ) - aps.append(ap) - print(f"{class_name}: {ap}") - - print(f"\nAverage Precision Across All Classes:{sum(aps)/len(aps)}") - return aps - - -class ModelConfig(): - """Hardcoded model configurations""" - def __init__(self, args): - self.input_shape = (1, 3, 300, 300) - self.config_file = None - self.iou_threshold = 0.5 - self.nms_method = 'hard' - self.use_2007_metric = True - self.quantsim_config_file = 'default_config_per_channel.json' - for arg in vars(args): - setattr(self, arg, getattr(args, arg)) - - -if __name__ == '__main__': - args = arguments() - config = ModelConfig(args) - download_labels() - - eval_path = pathlib.Path('./eval_results') - eval_path.mkdir(exist_ok=True) - class_names = [name.strip() for name in open('voc-model-labels.txt').readlines()] - device = get_device(args) - dataset = VOCDataset(config.dataset_path, is_test=True) - true_case_stat, all_gb_boxes, all_difficult_cases = group_annotation_by_class(dataset) - - print('Initializing Original Model:') - model_fp32 = SSDMobileNetV2(model_config = args.model_config) - model_fp32.from_pretrained(quantized=False) - predictor_orig_fp32 = create_mobilenetv2_ssd_lite_predictor(model_fp32.model, nms_method='hard', device=device) - sim_fp32 = model_fp32.get_quantsim(quantized=False) - eval_func_fp32 = model_eval(config, predictor_orig_fp32, dataset) - predictor_sim_fp32 = create_mobilenetv2_ssd_lite_predictor(sim_fp32.model, nms_method=config.nms_method, device=device) - sim_fp32.compute_encodings(eval_func_fp32, (predictor_sim_fp32, 2000, device)) - - - print('Initializing Optimized Model') - model_int8 = SSDMobileNetV2(model_config = args.model_config) - model_int8.from_pretrained(quantized=True) - predictor_orig_int8 = create_mobilenetv2_ssd_lite_predictor(model_int8.model, nms_method=config.nms_method, device=device) - sim_int8 = model_int8.get_quantsim(quantized=True) - eval_func_int8 = model_eval(config, predictor_orig_int8, dataset) - predictor_sim_int8 = create_mobilenetv2_ssd_lite_predictor(sim_int8.model, nms_method=config.nms_method, device=device) - sim_int8.compute_encodings(eval_func_int8, (predictor_sim_int8, 2000, device)) - - # Original FP32 model on FP32 device - print('Computing Original Model on FP32 device') - aps = evaluate_predictor(predictor_orig_fp32) - mAP_fp32model_fp32env = sum(aps)/len(aps) - - # Original FP32 model on INT8 device - print('Computing Original Model on INT8 device') - aps = evaluate_predictor(predictor_sim_fp32) - mAP_fp32model_int8env = sum(aps)/len(aps) - - # Quantized INT8 model on FP32 device - print('Computing Optimized Model on FP32 device') - aps = evaluate_predictor(predictor_orig_int8) - mAP_int8model_fp32env = sum(aps)/len(aps) - - # Quantized INT8 model on INT8 device - print('Computing Optimized Model on INT8 device') - aps = evaluate_predictor(predictor_sim_int8) - mAP_int8model_int8env = sum(aps)/len(aps) - - print('\n\n') - print('## Evaluation Summary ##') - print(f'Original Model on FP32 device | mAP: {mAP_fp32model_fp32env:.4f}') - print(f'Original Model on INT8 device | mAP: {mAP_fp32model_int8env:.4f}') - print(f'Optimized Model on FP32 device | mAP: {mAP_int8model_fp32env:.4f}') - print(f'Optimized Model on INT8 device | mAP: {mAP_int8model_int8env:.4f}') + """ + :param predictor: + :return: Average precision per classes for the given predictor + """ + # pylint: disable = too-many-locals, redefined-outer-name + results = [] + for i in tqdm(range(len(dataset))): + image = dataset.get_image(i) + boxes, labels, probs = predictor.predict(image) + indexes = torch.ones(labels.size(0), 1, dtype=torch.float32) * i + results.append( + torch.cat( + [ + indexes.reshape(-1, 1), + labels.reshape(-1, 1).float(), + probs.reshape(-1, 1), + boxes + 1.0, # matlab's indexes start from 1 + ], + dim=1, + ) + ) + results = torch.cat(results) + for class_index, class_name in enumerate(class_names): + if class_index == 0: + continue # ignore background + prediction_path = eval_path / f"det_test_{class_name}.txt" + with open(prediction_path, "w") as f: + sub = results[results[:, 1] == class_index, :] + for i in range(sub.size(0)): + tmp = sub[i, 2:].cpu() + prob_box = tmp.numpy() + image_id = dataset.ids[int(sub[i, 0])] + print(image_id + " " + " ".join([str(v) for v in prob_box]), file=f) + aps = [] + print("\n\nAverage Precision Per Class:") + for class_index, class_name in enumerate(class_names): + if class_index == 0: + continue + prediction_path = eval_path / f"det_test_{class_name}.txt" + ap = compute_average_precision_per_class( + true_case_stat[class_index], + all_gb_boxes[class_index], + all_difficult_cases[class_index], + prediction_path, + config.iou_threshold, + config.use_2007_metric, + ) + aps.append(ap) + print(f"{class_name}: {ap}") + + print(f"\nAverage Precision Across All Classes:{sum(aps)/len(aps)}") + return aps + + +class ModelConfig: + """Hardcoded model configurations""" + #pylint: disable = redefined-outer-name + def __init__(self, args): + self.input_shape = (1, 3, 300, 300) + self.config_file = None + self.iou_threshold = 0.5 + self.nms_method = "hard" + self.use_2007_metric = True + self.quantsim_config_file = "default_config_per_channel.json" + for arg in vars(args): + setattr(self, arg, getattr(args, arg)) + + +if __name__ == "__main__": + args = arguments() + config = ModelConfig(args) + download_labels() + + eval_path = pathlib.Path("./eval_results") + eval_path.mkdir(exist_ok=True) + #pylint:disable = consider-using-with + class_names = [name.strip() for name in open("voc-model-labels.txt").readlines()] + device = get_device(args) + #pylint:disable = no-member + dataset = VOCDataset(config.dataset_path, is_test=True) + true_case_stat, all_gb_boxes, all_difficult_cases = group_annotation_by_class( + dataset + ) + + print("Initializing Original Model:") + model_fp32 = SSDMobileNetV2(model_config=args.model_config) + model_fp32.from_pretrained(quantized=False) + predictor_orig_fp32 = create_mobilenetv2_ssd_lite_predictor( + model_fp32.model, nms_method="hard", device=device + ) + sim_fp32 = model_fp32.get_quantsim(quantized=False) + eval_func_fp32 = model_eval(config, predictor_orig_fp32, dataset) + predictor_sim_fp32 = create_mobilenetv2_ssd_lite_predictor( + sim_fp32.model, nms_method=config.nms_method, device=device + ) + sim_fp32.compute_encodings(eval_func_fp32, (predictor_sim_fp32, 2000, device)) + + print("Initializing Optimized Model") + model_int8 = SSDMobileNetV2(model_config=args.model_config) + model_int8.from_pretrained(quantized=True) + predictor_orig_int8 = create_mobilenetv2_ssd_lite_predictor( + model_int8.model, nms_method=config.nms_method, device=device + ) + sim_int8 = model_int8.get_quantsim(quantized=True) + eval_func_int8 = model_eval(config, predictor_orig_int8, dataset) + predictor_sim_int8 = create_mobilenetv2_ssd_lite_predictor( + sim_int8.model, nms_method=config.nms_method, device=device + ) + sim_int8.compute_encodings(eval_func_int8, (predictor_sim_int8, 2000, device)) + + # Original FP32 model on FP32 device + print("Computing Original Model on FP32 device") + aps = evaluate_predictor(predictor_orig_fp32) + mAP_fp32model_fp32env = sum(aps) / len(aps) + + # Original FP32 model on INT8 device + print("Computing Original Model on INT8 device") + aps = evaluate_predictor(predictor_sim_fp32) + mAP_fp32model_int8env = sum(aps) / len(aps) + + # Quantized INT8 model on FP32 device + print("Computing Optimized Model on FP32 device") + aps = evaluate_predictor(predictor_orig_int8) + mAP_int8model_fp32env = sum(aps) / len(aps) + + # Quantized INT8 model on INT8 device + print("Computing Optimized Model on INT8 device") + aps = evaluate_predictor(predictor_sim_int8) + mAP_int8model_int8env = sum(aps) / len(aps) + + print("\n\n") + print("## Evaluation Summary ##") + print(f"Original Model on FP32 device | mAP: {mAP_fp32model_fp32env:.4f}") + print(f"Original Model on INT8 device | mAP: {mAP_fp32model_int8env:.4f}") + print(f"Optimized Model on FP32 device | mAP: {mAP_int8model_fp32env:.4f}") + print(f"Optimized Model on INT8 device | mAP: {mAP_int8model_int8env:.4f}") diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/model_cards/__init__.py b/aimet_zoo_torch/ssd_mobilenetv2/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/model_definition.py b/aimet_zoo_torch/ssd_mobilenetv2/model/model_definition.py index ef099fe..7f1a167 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/model_definition.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/model_definition.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- mode: python -*- # ============================================================================= diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/alexnet.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/alexnet.py index 8ad2a9a..ca47275 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/alexnet.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/alexnet.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenet.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenet.py index ef80ee2..63ce587 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenet.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenet.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenet_v2.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenet_v2.py index 3115836..89705f2 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenet_v2.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenet_v2.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenetv3.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenetv3.py index 61ed75d..c2a4d6e 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenetv3.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/mobilenetv3.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/multibox_loss.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/multibox_loss.py index b6b29a9..d68316f 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/multibox_loss.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/multibox_loss.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/scaled_l2_norm.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/scaled_l2_norm.py index 8f3b87d..3afbfe4 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/scaled_l2_norm.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/scaled_l2_norm.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/squeezenet.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/squeezenet.py index 7201725..db9664b 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/squeezenet.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/squeezenet.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/vgg.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/vgg.py index dfaa5a3..60348b6 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/vgg.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/nn/vgg.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/prunning/prunner.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/prunning/prunner.py index 360f886..3d24f65 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/prunning/prunner.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/prunning/prunner.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/mobilenetv1_ssd_config.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/mobilenetv1_ssd_config.py index 4c01126..732985a 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/mobilenetv1_ssd_config.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/mobilenetv1_ssd_config.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/squeezenet_ssd_config.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/squeezenet_ssd_config.py index aa2a75e..95c2e90 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/squeezenet_ssd_config.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/squeezenet_ssd_config.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/vgg_ssd_config.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/vgg_ssd_config.py index 6fb5d0c..6ec7a88 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/vgg_ssd_config.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/config/vgg_ssd_config.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/data_preprocessing.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/data_preprocessing.py index bf74bdc..2bb71a2 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/data_preprocessing.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/data_preprocessing.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/fpn_mobilenetv1_ssd.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/fpn_mobilenetv1_ssd.py index f5f8b4e..968173a 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/fpn_mobilenetv1_ssd.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/fpn_mobilenetv1_ssd.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/fpn_ssd.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/fpn_ssd.py index d9f85aa..62c2d0a 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/fpn_ssd.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/fpn_ssd.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenet_v2_ssd_lite.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenet_v2_ssd_lite.py index 833dd4b..db3c036 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenet_v2_ssd_lite.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenet_v2_ssd_lite.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv1_ssd.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv1_ssd.py index ede3065..410c581 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv1_ssd.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv1_ssd.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv1_ssd_lite.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv1_ssd_lite.py index f990686..20f9081 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv1_ssd_lite.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv1_ssd_lite.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv3_ssd_lite.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv3_ssd_lite.py index 828d815..579fac2 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv3_ssd_lite.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/mobilenetv3_ssd_lite.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/predictor.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/predictor.py index 5d04818..749a0ea 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/predictor.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/predictor.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/squeezenet_ssd_lite.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/squeezenet_ssd_lite.py index b4d10e3..960ebb1 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/squeezenet_ssd_lite.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/squeezenet_ssd_lite.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/ssd.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/ssd.py index 24244c9..046670c 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/ssd.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/ssd.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/vgg_ssd.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/vgg_ssd.py index a3d1391..d30b924 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/vgg_ssd.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/ssd/vgg_ssd.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/transforms/transforms.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/transforms/transforms.py index 10b60c0..d30fc5d 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/transforms/transforms.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/transforms/transforms.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/__init__.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/__init__.py index 0789bdb..a5dc313 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/__init__.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/__init__.py @@ -1 +1,2 @@ +#pylint: skip-file from .misc import * diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/box_utils.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/box_utils.py index 73fe383..fad4746 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/box_utils.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/box_utils.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/box_utils_numpy.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/box_utils_numpy.py index 792e06a..9f03943 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/box_utils_numpy.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/box_utils_numpy.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/measurements.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/measurements.py index 5ac9c08..b8b55d0 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/measurements.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/measurements.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/misc.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/misc.py index 91d5fe8..3a48f49 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/misc.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/misc.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/model_book.py b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/model_book.py index 8036ab1..b3dc3f1 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/model_book.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/model/vision/utils/model_book.py @@ -1,3 +1,4 @@ +# pylint: skip-file # ================================================================================= # # MIT License diff --git a/aimet_zoo_torch/ssd_res50/__init__.py b/aimet_zoo_torch/ssd_res50/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/ssd_res50/dataloader/__init__.py b/aimet_zoo_torch/ssd_res50/dataloader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/ssd_res50/evaluators/__init__.py b/aimet_zoo_torch/ssd_res50/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/ssd_res50/evaluators/ssd_res50_quanteval.py b/aimet_zoo_torch/ssd_res50/evaluators/ssd_res50_quanteval.py index 289919c..507e38b 100644 --- a/aimet_zoo_torch/ssd_res50/evaluators/ssd_res50_quanteval.py +++ b/aimet_zoo_torch/ssd_res50/evaluators/ssd_res50_quanteval.py @@ -36,32 +36,50 @@ """ ''' AIMET Quantsim evaluation code for SSD Res50 ''' - import argparse import numpy as np -import torch from tqdm import tqdm +import torch from torch.utils.data import DataLoader -from pycocotools.cocoeval import COCOeval + from aimet_torch.model_preparer import prepare_model from aimet_torch.model_validator.model_validator import ModelValidator - -from src.utils import generate_dboxes, Encoder -from src.transform import SSDTransformer from aimet_zoo_torch.ssd_res50.dataloader.dataset import CocoDataset from aimet_zoo_torch.ssd_res50.dataloader.dataset import collate_fn from aimet_zoo_torch.ssd_res50.model.model_definition import SSD_Res50 +from src.utils import generate_dboxes, Encoder # pylint:disable = import-error +from src.transform import SSDTransformer # pylint:disable = import-error +from pycocotools.cocoeval import COCOeval # pylint:disable = import-error def get_args(): + """argument parser""" + #pylint:disable = redefined-outer-name parser = argparse.ArgumentParser("Evaluation script for quantized SSD Res50 Model") - parser.add_argument('--model-config', help='model configuration to use', required=True, type=str, - default='ssd_res50_w8a8', choices=['ssd_res50_w8a8']) - parser.add_argument('--dataset-path', help='The path to COCO 2017 dataset', required=True) - parser.add_argument('--batch-size', default=1, type=int, help='The batch size for dataloaders') - parser.add_argument('--num-workers', default=0, type=int, help='The number of workers for dataloaders') - parser.add_argument('--use-cuda', help='Use GPU for evaluation', action="store_true") + parser.add_argument( + "--model-config", + help="model configuration to use", + required=True, + type=str, + default="ssd_res50_w8a8", + choices=["ssd_res50_w8a8"], + ) + parser.add_argument( + "--dataset-path", help="The path to COCO 2017 dataset", required=True + ) + parser.add_argument( + "--batch-size", default=1, type=int, help="The batch size for dataloaders" + ) + parser.add_argument( + "--num-workers", + default=0, + type=int, + help="The number of workers for dataloaders", + ) + parser.add_argument( + "--use-cuda", help="Use GPU for evaluation", action="store_true" + ) args = parser.parse_args() return args @@ -71,33 +89,46 @@ def ssd_res50_quanteval(args): """ Evaluation function for SSD Res50 quantized model """ + #pylint:disable = redefined-outer-name if args.use_cuda: if torch.cuda.is_available(): - device = torch.device('cuda') + device = torch.device("cuda") else: - raise RuntimeError('Trying to use cuda device while no available cuda device is found!') + raise RuntimeError( + "Trying to use cuda device while no available cuda device is found!" + ) else: - device = torch.device('cpu') + device = torch.device("cpu") model_downloader = SSD_Res50(model_config=args.model_config) model_downloader.from_pretrained(quantized=False) model_downloader.model.to(device=device) dboxes = generate_dboxes() - test_set = CocoDataset(args.dataset_path, 2017, "val", SSDTransformer(dboxes, (300, 300), val=True)) + test_set = CocoDataset( + args.dataset_path, 2017, "val", SSDTransformer(dboxes, (300, 300), val=True) + ) encoder = Encoder(dboxes) - test_params = {"batch_size": args.batch_size, - "shuffle": False, - "drop_last": False, - "num_workers": args.num_workers, - "collate_fn": collate_fn} + test_params = { + "batch_size": args.batch_size, + "shuffle": False, + "drop_last": False, + "num_workers": args.num_workers, + "collate_fn": collate_fn, + } test_loader = DataLoader(test_set, **test_params) shape = model_downloader.input_shape dummy_input = torch.randn(shape).to(device) - evaluate(model_downloader.model, test_loader, encoder, model_downloader.cfg["evaluation"], device) + evaluate( + model_downloader.model, + test_loader, + encoder, + model_downloader.cfg["evaluation"], + device, + ) print("\n######### Prepare Model Started ############\n") model_downloader.model = prepare_model(model_downloader.model) @@ -110,7 +141,9 @@ def ssd_res50_quanteval(args): sim = model_downloader.get_quantsim(quantized=True) print("\n######### Validation for QuantSim ############\n") - evaluate(sim.model, test_loader, encoder, model_downloader.cfg["evaluation"], device) + evaluate( + sim.model, test_loader, encoder, model_downloader.cfg["evaluation"], device + ) def evaluate(model, test_loader, encoder, args, device): @@ -125,13 +158,17 @@ def evaluate(model, test_loader, encoder, args, device): :return: Evaluation score in Average Precision """ + #pylint:disable = too-many-locals, bare-except, unused-variable, redefined-outer-name + #ignored due to third party licensed function model.eval() nms_threshold = args["nms_threshold"] detections = [] category_ids = test_loader.dataset.coco.getCatIds() - for nbatch, (img, img_id, img_size, _, _) in tqdm(enumerate(test_loader), total=len(test_loader)): + for nbatch, (img, img_id, img_size, _, _) in tqdm( + enumerate(test_loader), total=len(test_loader) + ): if img_id: img = img.to(device) @@ -144,7 +181,9 @@ def evaluate(model, test_loader, encoder, args, device): ploc_i = ploc[idx, :, :].unsqueeze(0) plabel_i = plabel[idx, :, :].unsqueeze(0) try: - result = encoder.decode_batch(ploc_i, plabel_i, nms_threshold, 200)[0] + result = encoder.decode_batch(ploc_i, plabel_i, nms_threshold, 200)[ + 0 + ] except: print("No object detected in idx: {}".format(idx)) continue @@ -152,13 +191,25 @@ def evaluate(model, test_loader, encoder, args, device): height, width = img_size[idx] loc, label, prob = [r.cpu().numpy() for r in result] for loc_, label_, prob_ in zip(loc, label, prob): - detections.append([img_id[idx], loc_[0] * width, loc_[1] * height, (loc_[2] - loc_[0]) * width, - (loc_[3] - loc_[1]) * height, prob_, - category_ids[label_ - 1]]) + detections.append( + [ + img_id[idx], + loc_[0] * width, + loc_[1] * height, + (loc_[2] - loc_[0]) * width, + (loc_[3] - loc_[1]) * height, + prob_, + category_ids[label_ - 1], + ] + ) detections = np.array(detections, dtype=np.float32) - coco_eval = COCOeval(test_loader.dataset.coco, test_loader.dataset.coco.loadRes(detections), iouType="bbox") + coco_eval = COCOeval( + test_loader.dataset.coco, + test_loader.dataset.coco.loadRes(detections), + iouType="bbox", + ) coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() diff --git a/aimet_zoo_torch/ssd_res50/model/__init__.py b/aimet_zoo_torch/ssd_res50/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/ssd_res50/model/model_cards/__init__.py b/aimet_zoo_torch/ssd_res50/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/ssd_res50/model/model_cards/ssd_res50_w8a8.json b/aimet_zoo_torch/ssd_res50/model/model_cards/ssd_res50_w8a8.json index 69b6f03..30741fd 100644 --- a/aimet_zoo_torch/ssd_res50/model/model_cards/ssd_res50_w8a8.json +++ b/aimet_zoo_torch/ssd_res50/model/model_cards/ssd_res50_w8a8.json @@ -20,7 +20,7 @@ "url_pre_opt_weights": "https://drive.google.com/u/0/uc?id=1NGh8D7zAStasdLvLT1dRRiYFZRr0K4j3", "url_post_opt_weights": null, "url_adaround_encodings": null, - "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/SSD_Res50/SSD_Res50_torch.encodings", + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_ssd_res50/SSD_Res50_torch.encodings", "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" } } \ No newline at end of file diff --git a/aimet_zoo_torch/vit/__init__.py b/aimet_zoo_torch/vit/__init__.py index d36be2b..f69062e 100644 --- a/aimet_zoo_torch/vit/__init__.py +++ b/aimet_zoo_torch/vit/__init__.py @@ -1 +1,2 @@ +""" package for getting vit original model and quantized model""" from .model.model_definition import vit diff --git a/aimet_zoo_torch/vit/dataloader/__init__.py b/aimet_zoo_torch/vit/dataloader/__init__.py index 03d7e3f..9d2befd 100644 --- a/aimet_zoo_torch/vit/dataloader/__init__.py +++ b/aimet_zoo_torch/vit/dataloader/__init__.py @@ -1,2 +1,3 @@ +"""modules for getting dataloaders and dataset""" from .dataloaders import get_dataloaders from .dataloaders import get_dataset diff --git a/aimet_zoo_torch/vit/dataloader/utils/__init__.py b/aimet_zoo_torch/vit/dataloader/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/vit/evaluators/__init__.py b/aimet_zoo_torch/vit/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/vit/model/__init__.py b/aimet_zoo_torch/vit/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/vit/model/huggingface/__init__.py b/aimet_zoo_torch/vit/model/huggingface/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/vit/model/huggingface/baseline_models/__init__.py b/aimet_zoo_torch/vit/model/huggingface/baseline_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/vit/model/huggingface/baseline_models/vit/__init__.py b/aimet_zoo_torch/vit/model/huggingface/baseline_models/vit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/vit/model/model_cards/__init__.py b/aimet_zoo_torch/vit/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/vit/model/model_cards/vit_w8a8.json b/aimet_zoo_torch/vit/model/model_cards/vit_w8a8.json index 5bc7bc8..5c4de72 100644 --- a/aimet_zoo_torch/vit/model/model_cards/vit_w8a8.json +++ b/aimet_zoo_torch/vit/model/model_cards/vit_w8a8.json @@ -28,6 +28,6 @@ "tar_url_pre_opt_weights":null, "tar_url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/imgnet_vit_5e4_clamp_rl.tar.gz", "url_aimet_encodings": null, - "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/torch_vit/default_config.json" + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.24/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/vit/model/model_definition.py b/aimet_zoo_torch/vit/model/model_definition.py index 82ade3d..352c333 100644 --- a/aimet_zoo_torch/vit/model/model_definition.py +++ b/aimet_zoo_torch/vit/model/model_definition.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- mode: python -*- -#pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912,W1203,W1201,R0201 # ============================================================================= # @@-COPYRIGHT-START-@@ # @@ -14,7 +14,6 @@ import csv from collections import defaultdict import torch -import datasets from transformers import AutoConfig as Config from transformers import AutoFeatureExtractor as FeatureExtractor from aimet_torch.quantsim import QuantizationSimModel @@ -24,11 +23,12 @@ from aimet_zoo_torch.vit.model.huggingface.baseline_models.vit.modeling_vit import ( ViTForImageClassification as VitModel, ) - +import datasets class vit(Downloader): - """ model vit configuration class """ + """model vit configuration class""" + def __init__(self, model_config=None, quantized=False): """ dataloader @@ -87,7 +87,7 @@ def get_model_from_pretrained(self, dataset): feature_extractor = FeatureExtractor.from_pretrained( model_name_or_path, ) - + # pylint:disable = unused-variable interpolate = False higher_resolution = self.cfg["model_args"]["higher_resolution"] == "True" ignore_mismatched_sizes = ( @@ -146,26 +146,18 @@ def get_quantsim(self, dataloader, eval_function): metric = datasets.load_metric("accuracy") dummy_input = self._get_dummy_input(dataloader) + quant_scheme_config = self.cfg["optimization_config"]["quantization_configuration"]["quant_scheme"] if ( - self.cfg["optimization_config"]["quantization_configuration"][ - "quant_scheme" - ] - == "tf" + quant_scheme_config == "tf" ): quant_scheme = QuantScheme.post_training_tf elif ( - self.cfg["optimization_config"]["quantization_configuration"][ - "quant_scheme" - ] - == "tf_enhanced" + quant_scheme_config == "tf_enhanced" ): quant_scheme = QuantScheme.post_training_tf_enhanced elif ( - self.cfg["optimization_config"]["quantization_configuration"][ - "quant_scheme" - ] - == "tf_range_learning" + quant_scheme_config == "tf_range_learning" ): quant_scheme = QuantScheme.training_range_learning_with_tf_init @@ -191,6 +183,7 @@ def get_quantsim(self, dataloader, eval_function): self._load_encoding_data(quant_sim, model_name_or_path) # remove dropout quantizers disable_list = [] + #pylint:disable = protected-access, unused-variable for name, module in quant_sim.model.named_modules(): if isinstance(module, QcQuantizeWrapper) and isinstance( module._module_to_wrap, torch.nn.Dropout diff --git a/aimet_zoo_torch/xlsr/__init__.py b/aimet_zoo_torch/xlsr/__init__.py index 63b6264..d3baae3 100644 --- a/aimet_zoo_torch/xlsr/__init__.py +++ b/aimet_zoo_torch/xlsr/__init__.py @@ -1 +1,2 @@ -from .model.model_definition import XLSR \ No newline at end of file +""" package for getting xlsr original model and quantized model""" +from .model.model_definition import XLSR diff --git a/aimet_zoo_torch/xlsr/evaluators/__init__.py b/aimet_zoo_torch/xlsr/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/xlsr/evaluators/xlsr_quanteval.py b/aimet_zoo_torch/xlsr/evaluators/xlsr_quanteval.py index db0a363..7083cc6 100644 --- a/aimet_zoo_torch/xlsr/evaluators/xlsr_quanteval.py +++ b/aimet_zoo_torch/xlsr/evaluators/xlsr_quanteval.py @@ -8,25 +8,52 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET evaluation code for QuickSRNet ''' +""" AIMET evaluation code for QuickSRNet """ import argparse from aimet_zoo_torch.xlsr import XLSR from aimet_zoo_torch.common.super_resolution.psnr import evaluate_average_psnr -from aimet_zoo_torch.common.super_resolution.utils import load_dataset, pass_calibration_data +from aimet_zoo_torch.common.super_resolution.utils import ( + load_dataset, + pass_calibration_data, +) from aimet_zoo_torch.common.super_resolution.inference import run_model # add arguments def arguments(): """parses command line arguments""" - parser = argparse.ArgumentParser(description='Arguments for evaluating model') - parser.add_argument('--dataset-path', help='path to image evaluation dataset', type=str, required=True) - parser.add_argument('--model-config', help='model configuration to be tested', type=str, required=True) - parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) - parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) - parser.add_argument('--batch-size',help='batch_size for loading data',type=int,default=16) - parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) + parser = argparse.ArgumentParser(description="Arguments for evaluating model") + parser.add_argument( + "--dataset-path", + help="path to image evaluation dataset", + type=str, + required=True, + ) + parser.add_argument( + "--model-config", + help="model configuration to be tested", + type=str, + required=True, + ) + parser.add_argument( + "--default-output-bw", + help="Default output bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--default-param-bw", + help="Default parameter bitwidth for quantization.", + type=int, + default=8, + ) + parser.add_argument( + "--batch-size", help="batch_size for loading data", type=int, default=16 + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU", type=bool, default=True + ) args = parser.parse_args() return args @@ -35,20 +62,24 @@ def main(): """executes evaluation""" args = arguments() - model_fp32 = XLSR(model_config = args.model_config) + model_fp32 = XLSR(model_config=args.model_config) model_fp32.from_pretrained(quantized=False) sim_fp32 = model_fp32.get_quantsim(quantized=False) - model_int8 = XLSR(model_config = args.model_config) + model_int8 = XLSR(model_config=args.model_config) model_int8.from_pretrained(quantized=True) sim_int8 = model_int8.get_quantsim(quantized=True) IMAGES_LR, IMAGES_HR = load_dataset(args.dataset_path, model_fp32.scaling_factor) - sim_fp32.compute_encodings(forward_pass_callback=pass_calibration_data, - forward_pass_callback_args=(IMAGES_LR, args.use_cuda)) - sim_int8.compute_encodings(forward_pass_callback=pass_calibration_data, - forward_pass_callback_args=(IMAGES_LR, args.use_cuda)) + sim_fp32.compute_encodings( + forward_pass_callback=pass_calibration_data, + forward_pass_callback_args=(IMAGES_LR, args.use_cuda), + ) + sim_int8.compute_encodings( + forward_pass_callback=pass_calibration_data, + forward_pass_callback_args=(IMAGES_LR, args.use_cuda), + ) # Run model inference on test images and get super-resolved images IMAGES_SR_original_fp32 = run_model(model_fp32, IMAGES_LR, args.use_cuda) @@ -58,13 +89,14 @@ def main(): # Get the average PSNR for all test-images avg_psnr = evaluate_average_psnr(IMAGES_SR_original_fp32, IMAGES_HR) - print(f'Original Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Original Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_original_int8, IMAGES_HR) - print(f'Original Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Original Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_optimized_fp32, IMAGES_HR) - print(f'Optimized Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Optimized Model | FP32 Environment | Avg. PSNR: {avg_psnr:.3f}") avg_psnr = evaluate_average_psnr(IMAGES_SR_optimized_int8, IMAGES_HR) - print(f'Optimized Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}') + print(f"Optimized Model | INT8 Environment | Avg. PSNR: {avg_psnr:.3f}") -if __name__ == '__main__': - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/xlsr/model/model_cards/__init__.py b/aimet_zoo_torch/xlsr/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/xlsr/model/model_definition.py b/aimet_zoo_torch/xlsr/model/model_definition.py index 73f56d2..c1b446d 100644 --- a/aimet_zoo_torch/xlsr/model/model_definition.py +++ b/aimet_zoo_torch/xlsr/model/model_definition.py @@ -7,10 +7,13 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +"""Class for downloading and setting up of optmized and original xlsr model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet -import torch import json import os +import torch from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim from aimet_zoo_torch.common.downloader import Downloader from aimet_zoo_torch.common.super_resolution.models import XLSRRelease @@ -18,50 +21,63 @@ class XLSR(XLSRRelease, Downloader): """ABPN parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + def __init__(self, model_config=None, scaling_factor=2, **kwargs): """ :param model_config: named model config from which to obtain model artifacts and arguments. - If provided, overwrites the other arguments passed to this object + If provided, overwrites the other arguments passed to this object :param scaling_factor: scaling factor for LR-to-HR upscaling (2x, 3x, 4x... or 1.5x) - :param num_channels: number of feature channels for convolutional layers + :param num_channels: number of feature channels for convolutional layers """ - parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) self.cfg = False if model_config: - config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) - self.scaling_factor = self.cfg['model_args']['scaling_factor'] if self.cfg else scaling_factor + self.scaling_factor = ( + self.cfg["model_args"]["scaling_factor"] if self.cfg else scaling_factor + ) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) - XLSRRelease.__init__(self, - scaling_factor = self.cfg['model_args']['scaling_factor'] if self.cfg else scaling_factor, - **kwargs) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + XLSRRelease.__init__( + self, + scaling_factor=self.cfg["model_args"]["scaling_factor"] + if self.cfg + else scaling_factor, + **kwargs + ) def from_pretrained(self, quantized=False): """load pretrained weights""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() self._download_aimet_encodings() self._download_adaround_encodings() if quantized: - state_dict = torch.load(self.path_post_opt_weights)['state_dict'] + state_dict = torch.load(self.path_post_opt_weights)["state_dict"] self.load_state_dict(state_dict) self.cuda() else: - state_dict = torch.load(self.path_pre_opt_weights)['state_dict'] + state_dict = torch.load(self.path_pre_opt_weights)["state_dict"] self.load_state_dict(state_dict) self.cuda() self.eval() @@ -69,23 +85,32 @@ def from_pretrained(self, quantized=False): def get_quantsim(self, quantized=False): """get quantsim object with pre-loaded encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) if quantized: self.from_pretrained(quantized=True) else: self.from_pretrained(quantized=False) - device = torch.device('cuda') - dummy_input = torch.rand(self.input_shape, device = device) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } sim = QuantizationSimModel(self, **kwargs) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) sim.model.eval() - return sim \ No newline at end of file + return sim diff --git a/aimet_zoo_torch/yolox/__init__.py b/aimet_zoo_torch/yolox/__init__.py index 8e557dc..daf0617 100755 --- a/aimet_zoo_torch/yolox/__init__.py +++ b/aimet_zoo_torch/yolox/__init__.py @@ -5,5 +5,5 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - -from .model.model_definition import YOLOX \ No newline at end of file +""" loading yolox model downloader class""" +from .model.model_definition import YOLOX diff --git a/aimet_zoo_torch/yolox/dataloader/__init__.py b/aimet_zoo_torch/yolox/dataloader/__init__.py index 8f9de2f..3d4f546 100755 --- a/aimet_zoo_torch/yolox/dataloader/__init__.py +++ b/aimet_zoo_torch/yolox/dataloader/__init__.py @@ -4,4 +4,4 @@ # Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. # # @@-COPYRIGHT-END-@@ -# ============================================================================= \ No newline at end of file +# ============================================================================= diff --git a/aimet_zoo_torch/yolox/dataloader/data/__init__.py b/aimet_zoo_torch/yolox/dataloader/data/__init__.py index b74a347..fc95858 100755 --- a/aimet_zoo_torch/yolox/dataloader/data/__init__.py +++ b/aimet_zoo_torch/yolox/dataloader/data/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii, Inc. and its affiliates. -#pylint: skip-file +# pylint: skip-file from .data_augment import ValTransform -from .datasets import COCODataset \ No newline at end of file +from .datasets import COCODataset diff --git a/aimet_zoo_torch/yolox/dataloader/data/data_augment.py b/aimet_zoo_torch/yolox/dataloader/data/data_augment.py index a5fd488..a53416e 100755 --- a/aimet_zoo_torch/yolox/dataloader/data/data_augment.py +++ b/aimet_zoo_torch/yolox/dataloader/data/data_augment.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # ============================================================================= diff --git a/aimet_zoo_torch/yolox/dataloader/data/dataloading.py b/aimet_zoo_torch/yolox/dataloader/data/dataloading.py index f6fedb1..5932c4e 100755 --- a/aimet_zoo_torch/yolox/dataloader/data/dataloading.py +++ b/aimet_zoo_torch/yolox/dataloader/data/dataloading.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii, Inc. and its affiliates. diff --git a/aimet_zoo_torch/yolox/dataloader/data/datasets/__init__.py b/aimet_zoo_torch/yolox/dataloader/data/datasets/__init__.py index 5b46258..ca1e25a 100755 --- a/aimet_zoo_torch/yolox/dataloader/data/datasets/__init__.py +++ b/aimet_zoo_torch/yolox/dataloader/data/datasets/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii, Inc. and its affiliates. -#pylint: skip-file +# pylint: skip-file from .coco import COCODataset diff --git a/aimet_zoo_torch/yolox/dataloader/data/datasets/coco.py b/aimet_zoo_torch/yolox/dataloader/data/datasets/coco.py index ecb65d5..ec6c678 100755 --- a/aimet_zoo_torch/yolox/dataloader/data/datasets/coco.py +++ b/aimet_zoo_torch/yolox/dataloader/data/datasets/coco.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii, Inc. and its affiliates. diff --git a/aimet_zoo_torch/yolox/dataloader/data/datasets/datasets_wrapper.py b/aimet_zoo_torch/yolox/dataloader/data/datasets/datasets_wrapper.py index 7b2d06b..a519169 100755 --- a/aimet_zoo_torch/yolox/dataloader/data/datasets/datasets_wrapper.py +++ b/aimet_zoo_torch/yolox/dataloader/data/datasets/datasets_wrapper.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii, Inc. and its affiliates. diff --git a/aimet_zoo_torch/yolox/dataloader/data/samplers.py b/aimet_zoo_torch/yolox/dataloader/data/samplers.py index cdac7fd..56668b8 100755 --- a/aimet_zoo_torch/yolox/dataloader/data/samplers.py +++ b/aimet_zoo_torch/yolox/dataloader/data/samplers.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii, Inc. and its affiliates. diff --git a/aimet_zoo_torch/yolox/dataloader/dataloaders.py b/aimet_zoo_torch/yolox/dataloader/dataloaders.py index ecf0a5e..6595baf 100755 --- a/aimet_zoo_torch/yolox/dataloader/dataloaders.py +++ b/aimet_zoo_torch/yolox/dataloader/dataloaders.py @@ -5,16 +5,17 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - +""" yolox module for getting data loader""" from torch.utils.data import SequentialSampler, DataLoader from .data import COCODataset, ValTransform def get_data_loader(dataset_path, img_size, batch_size, num_workers): + """function to get coco 2017 dataset dataloader""" dataset = COCODataset( data_dir=dataset_path, - json_file='instances_val2017.json', + json_file="instances_val2017.json", name="images/val2017", img_size=img_size, preproc=ValTransform(legacy=False), diff --git a/aimet_zoo_torch/yolox/evaluators/__init__.py b/aimet_zoo_torch/yolox/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/yolox/evaluators/coco_evaluator.py b/aimet_zoo_torch/yolox/evaluators/coco_evaluator.py index 118f2f4..b202bcc 100755 --- a/aimet_zoo_torch/yolox/evaluators/coco_evaluator.py +++ b/aimet_zoo_torch/yolox/evaluators/coco_evaluator.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # ============================================================================= diff --git a/aimet_zoo_torch/yolox/evaluators/yolox_quanteval.py b/aimet_zoo_torch/yolox/evaluators/yolox_quanteval.py index a220e41..93f13bd 100755 --- a/aimet_zoo_torch/yolox/evaluators/yolox_quanteval.py +++ b/aimet_zoo_torch/yolox/evaluators/yolox_quanteval.py @@ -8,7 +8,8 @@ # @@-COPYRIGHT-END-@@ # ============================================================================= -''' AIMET Quantsim code for YOLOX ''' +#pylint:disable = unused-variable, redefined-outer-name +""" AIMET Quantsim code for YOLOX """ # General Python related imports from __future__ import absolute_import @@ -23,8 +24,6 @@ # Torch related imports import torch -# AIMET related imports -from aimet_torch.model_validator.model_validator import ModelValidator # AIMET model zoo related imports: model construction, dataloader, evaluation from aimet_zoo_torch.yolox import YOLOX @@ -33,7 +32,7 @@ def seed(seed_number): - ''' Set seed for reproducibility ''' + """Set seed for reproducibility""" torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = True torch.manual_seed(seed_number) @@ -42,13 +41,14 @@ def seed(seed_number): def eval_func(model, dataloader, img_size): - ''' define evaluation func to evaluate model with data_loader ''' + """define evaluation func to evaluate model with data_loader""" evaluator = COCOEvaluator(dataloader, img_size) return evaluator.evaluate(model) def forward_pass(decoder, model, data_loader): - ''' forward pass for compute encodings ''' + """forward pass for compute encodings""" + #pylint:disable = no-member tensor_type = torch.cuda.FloatTensor model = model.eval() @@ -61,44 +61,66 @@ def forward_pass(decoder, model, data_loader): def read_model_configs_from_model_card(model_card): - ''' read necessary params from model card ''' + """read necessary params from model card""" parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent.parent) - config_filepath = os.path.join(parent_dir, 'model', 'model_cards', f'{model_card}.json') - + config_filepath = os.path.join( + parent_dir, "model", "model_cards", f"{model_card}.json" + ) + if not os.path.exists(config_filepath): - raise NotImplementedError('Model_config file doesn\'t exist') - + raise NotImplementedError("Model_config file doesn't exist") + with open(config_filepath) as f_in: cfg = json.load(f_in) - input_shape = tuple(x if x != None else 1 for x in cfg['input_shape']) - default_param_bw = cfg['optimization_config']['quantization_configuration']['param_bw'] + input_shape = tuple(x if x is not None else 1 for x in cfg["input_shape"]) + default_param_bw = cfg["optimization_config"]["quantization_configuration"][ + "param_bw" + ] return input_shape, default_param_bw -def arguments(): - ''' argument parser ''' - parser = argparse.ArgumentParser(description='Evaluation script for PyTorch YOLOX models.') - parser.add_argument('--model-config', help='Select the model configuration', type=str, default="yolox_s", choices=["yolox_s", "yolox_l"]) - parser.add_argument('--dataset-path', help='Path to COCO2017 parent folder containing val2017', type=str, required=True) - parser.add_argument('--batch-size', help='Data batch size for a model', type=int, default=64) - args = parser.parse_args() +def arguments(raw_args): + """argument parser""" + parser = argparse.ArgumentParser( + description="Evaluation script for PyTorch YOLOX models." + ) + parser.add_argument( + "--model-config", + help="Select the model configuration", + type=str, + default="yolox_s", + choices=["yolox_s", "yolox_l"], + ) + parser.add_argument( + "--dataset-path", + help="Path to COCO2017 parent folder containing val2017", + type=str, + required=True, + ) + parser.add_argument( + "--batch-size", help="Data batch size for a model", type=int, default=64 + ) + args = parser.parse_args(raw_args) return args -def main(args): - ''' main function for quantization evaluation ''' +def main(raw_args=None): + """main function for quantization evaluation""" + args = arguments(raw_args) seed(1234) - input_shape, default_param_bw = read_model_configs_from_model_card(args.model_config) + input_shape, default_param_bw = read_model_configs_from_model_card( + args.model_config + ) img_size = (input_shape[-2], input_shape[-1]) # Get Dataloader dataloader = get_data_loader( - dataset_path=args.dataset_path, + dataset_path=args.dataset_path, img_size=img_size, - batch_size=args.batch_size, - num_workers=4 + batch_size=args.batch_size, + num_workers=4, ) # Load original model @@ -106,7 +128,7 @@ def main(args): model.from_pretrained(quantized=False) model_orig = model.model - print('Evaluating Original FP32 Model') + print("Evaluating Original FP32 Model") mAP_orig_fp32 = eval_func(model_orig, dataloader, img_size) del model_orig torch.cuda.empty_cache() @@ -116,7 +138,7 @@ def main(args): forward_func = partial(forward_pass, None) sim_orig.compute_encodings(forward_func, forward_pass_callback_args=dataloader) - print('Evaluating Original W8A8 Model') + print("Evaluating Original W8A8 Model") mAP_orig_int8 = eval_func(sim_orig.model, dataloader, img_size) del sim_orig torch.cuda.empty_cache() @@ -124,16 +146,19 @@ def main(args): # Load optimized model sim_optim = model.get_quantsim(quantized=True) - print('Evaluating Optimized W8A8 Model') + print("Evaluating Optimized W8A8 Model") mAP_optim_int8 = eval_func(sim_optim.model, dataloader, img_size) del sim_optim torch.cuda.empty_cache() - print(f'Original Model | 32-bit Environment | mAP: {100*mAP_orig_fp32:.2f}%') - print(f'Original Model | {default_param_bw}-bit Environment | mAP: {100*mAP_orig_int8:.2f}%') - print(f'Optimized Model | {default_param_bw}-bit Environment | mAP: {100*mAP_optim_int8:.2f}%') + print(f"Original Model | 32-bit Environment | mAP: {100*mAP_orig_fp32:.2f}%") + print( + f"Original Model | {default_param_bw}-bit Environment | mAP: {100*mAP_orig_int8:.2f}%" + ) + print( + f"Optimized Model | {default_param_bw}-bit Environment | mAP: {100*mAP_optim_int8:.2f}%" + ) -if __name__ == '__main__': - args = arguments() - main(args) \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/yolox/model/__init__.py b/aimet_zoo_torch/yolox/model/__init__.py index 8f9de2f..3d4f546 100755 --- a/aimet_zoo_torch/yolox/model/__init__.py +++ b/aimet_zoo_torch/yolox/model/__init__.py @@ -4,4 +4,4 @@ # Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. # # @@-COPYRIGHT-END-@@ -# ============================================================================= \ No newline at end of file +# ============================================================================= diff --git a/aimet_zoo_torch/yolox/model/model_cards/__init__.py b/aimet_zoo_torch/yolox/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/yolox/model/model_definition.py b/aimet_zoo_torch/yolox/model/model_definition.py index a4b64fd..d517cca 100755 --- a/aimet_zoo_torch/yolox/model/model_definition.py +++ b/aimet_zoo_torch/yolox/model/model_definition.py @@ -7,7 +7,9 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - +"""Class for downloading and setting up of optmized and original yolox model for AIMET model zoo""" +# pylint:disable = import-error, wrong-import-order +# adding this due to docker image not setup yet # General Python related imports import json import os @@ -29,31 +31,42 @@ class YOLOX(Downloader): """YOLOX objection detection parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + def __init__(self, model_config: str = None): parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = False - self.device = torch.device('cuda') + self.device = torch.device("cuda") if model_config: - config_filepath = os.path.join(parent_dir, 'model_cards', f'{model_config}.json') + config_filepath = os.path.join( + parent_dir, "model_cards", f"{model_config}.json" + ) if os.path.exists(config_filepath): with open(config_filepath) as f_in: self.cfg = json.load(f_in) if self.cfg: - Downloader.__init__(self, - url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], - url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], - url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], - url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], - url_aimet_config = self.cfg['artifacts']['url_aimet_config'], - model_dir = parent_dir, - model_config = model_config) - self.input_shape = tuple(x if x != None else 1 for x in self.cfg['input_shape']) + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) self.dummy_input = torch.rand(self.input_shape, device=self.device) self.prepared_model = None + self.model = None def from_pretrained(self, quantized=False): + """get pretrained model from downloading or coping""" if not self.cfg: - raise NotImplementedError('There are no pretrained weights available for the model_config passed') + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) self._download_pre_opt_weights() self._download_post_opt_weights() self._download_aimet_config() @@ -62,7 +75,9 @@ def from_pretrained(self, quantized=False): if quantized: self.model = model_entrypoint(self.cfg["name"]) fold_all_batch_norms(self.prepared_model.to(self.device), self.input_shape) - state_dict = torch.load(self.path_post_opt_weights, map_location=self.device) + state_dict = torch.load( + self.path_post_opt_weights, map_location=self.device + ) self.prepared_model.load_state_dict(state_dict) self.model = self.prepared_model else: @@ -74,28 +89,40 @@ def from_pretrained(self, quantized=False): ModelValidator.validate_model(self.model.to(self.device), self.dummy_input) def get_quantsim(self, quantized=False): + """" to get quantsim object for model from loading/computing proper encodings""" if not self.cfg: - raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + raise NotImplementedError( + "There is no Quantization Simulation available for the model_config passed" + ) self.from_pretrained(quantized=quantized) - + kwargs = { - 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], - 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], - 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], - 'config_file': self.path_aimet_config, - 'dummy_input': self.dummy_input} + "quant_scheme": self.cfg["optimization_config"][ + "quantization_configuration" + ]["quant_scheme"], + "default_param_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["param_bw"], + "default_output_bw": self.cfg["optimization_config"][ + "quantization_configuration" + ]["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": self.dummy_input, + } sim = QuantizationSimModel(self.model.to(self.device), **kwargs) if self.cfg["name"].endswith("s"): sim.set_percentile_value(99.9942) elif self.cfg["name"].endswith("l"): sim.set_percentile_value(99.99608) else: - raise ValueError("Currently only YOLOX-s (small) and YOLOX-l (large) model are allowed.") + raise ValueError( + "Currently only YOLOX-s (small) and YOLOX-l (large) model are allowed." + ) if self.path_aimet_encodings and quantized: load_encodings_to_sim(sim, self.path_aimet_encodings) - print('load_encodings_to_sim finished!') + print("load_encodings_to_sim finished!") if self.path_adaround_encodings and quantized: sim.set_and_freeze_param_encodings(self.path_adaround_encodings) - print('set_and_freeze_param_encodings finished!') + print("set_and_freeze_param_encodings finished!") return sim diff --git a/aimet_zoo_torch/yolox/model/yolo_x/__init__.py b/aimet_zoo_torch/yolox/model/yolo_x/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/yolox/model/yolo_x/models/__init__.py b/aimet_zoo_torch/yolox/model/yolo_x/models/__init__.py index 3e8a754..3dbe0bc 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/models/__init__.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/models/__init__.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii Inc. All rights reserved. -#pylint: skip-file +# pylint: skip-file from .darknet import CSPDarknet, Darknet from .yolo_head import YOLOXHead from .yolo_pafpn import YOLOPAFPN -from .yolox import YOLOX \ No newline at end of file +from .yolox import YOLOX diff --git a/aimet_zoo_torch/yolox/model/yolo_x/models/build.py b/aimet_zoo_torch/yolox/model/yolo_x/models/build.py index 5adddae..f674db9 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/models/build.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/models/build.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii Inc. All rights reserved. diff --git a/aimet_zoo_torch/yolox/model/yolo_x/models/darknet.py b/aimet_zoo_torch/yolox/model/yolo_x/models/darknet.py index 1b68b9b..258ade5 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/models/darknet.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/models/darknet.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python # -*- encoding: utf-8 -*- # Copyright (c) Megvii Inc. All rights reserved. diff --git a/aimet_zoo_torch/yolox/model/yolo_x/models/network_blocks.py b/aimet_zoo_torch/yolox/model/yolo_x/models/network_blocks.py index c6f8041..ab4738a 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/models/network_blocks.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/models/network_blocks.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python # -*- encoding: utf-8 -*- # Copyright (c) Megvii Inc. All rights reserved. diff --git a/aimet_zoo_torch/yolox/model/yolo_x/models/yolo_head.py b/aimet_zoo_torch/yolox/model/yolo_x/models/yolo_head.py index d67273f..448973b 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/models/yolo_head.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/models/yolo_head.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii Inc. All rights reserved. diff --git a/aimet_zoo_torch/yolox/model/yolo_x/models/yolo_pafpn.py b/aimet_zoo_torch/yolox/model/yolo_x/models/yolo_pafpn.py index c9ed963..951b38a 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/models/yolo_pafpn.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/models/yolo_pafpn.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python # -*- encoding: utf-8 -*- # Copyright (c) Megvii Inc. All rights reserved. diff --git a/aimet_zoo_torch/yolox/model/yolo_x/models/yolox.py b/aimet_zoo_torch/yolox/model/yolo_x/models/yolox.py index 1201cca..416f160 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/models/yolox.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/models/yolox.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python # -*- encoding: utf-8 -*- # ============================================================================= diff --git a/aimet_zoo_torch/yolox/model/yolo_x/utils/__init__.py b/aimet_zoo_torch/yolox/model/yolo_x/utils/__init__.py index 7017045..ee344dc 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/utils/__init__.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/utils/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii Inc. All rights reserved. -#pylint: skip-file +# pylint: skip-file -from .boxes import bboxes_iou, postprocess, xyxy2xywh \ No newline at end of file +from .boxes import bboxes_iou, postprocess, xyxy2xywh diff --git a/aimet_zoo_torch/yolox/model/yolo_x/utils/boxes.py b/aimet_zoo_torch/yolox/model/yolo_x/utils/boxes.py index 9732536..17f3b9d 100755 --- a/aimet_zoo_torch/yolox/model/yolo_x/utils/boxes.py +++ b/aimet_zoo_torch/yolox/model/yolo_x/utils/boxes.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # ============================================================================= diff --git a/aimet_zoo_torch/yolox/model/yolox_model.py b/aimet_zoo_torch/yolox/model/yolox_model.py index d85ddb3..a218cc8 100755 --- a/aimet_zoo_torch/yolox/model/yolox_model.py +++ b/aimet_zoo_torch/yolox/model/yolox_model.py @@ -1,3 +1,4 @@ +# pylint: skip-file #!/usr/bin/env python3 # -*- coding:utf-8 -*- # Copyright (c) Megvii Inc. All rights reserved. diff --git a/packaging/aimet_zoo_tensorflow_pyproject.toml b/packaging/aimet_zoo_tensorflow_pyproject.toml index a14699c..6a69784 100644 --- a/packaging/aimet_zoo_tensorflow_pyproject.toml +++ b/packaging/aimet_zoo_tensorflow_pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "aimet_zoo_tensorflow" -version = "1.1.0" +version = "1.3.0" description = "Collection of popular TensorFlow neural network models and evaluators to quantize floating-point models using the AI Model Efficiency ToolKit (AIMET)." readme = "README.md" requires-python = ">= 3.6" diff --git a/packaging/aimet_zoo_torch_pyproject.toml b/packaging/aimet_zoo_torch_pyproject.toml index 55b98d5..8d6461f 100644 --- a/packaging/aimet_zoo_torch_pyproject.toml +++ b/packaging/aimet_zoo_torch_pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "aimet_zoo_torch" -version = "1.1.0" +version = "1.3.0" description = "Collection of popular PyTorch neural network models and evaluators to quantize floating-point models using the AI Model Efficiency ToolKit (AIMET)." readme = "README.md" requires-python = ">= 3.6" diff --git a/packaging/pylint_model_zoo.cmake b/packaging/pylint_model_zoo.cmake index e74678d..59630ee 100644 --- a/packaging/pylint_model_zoo.cmake +++ b/packaging/pylint_model_zoo.cmake @@ -14,25 +14,54 @@ file(MAKE_DIRECTORY "${SOURCE_DIR}/packaging/results") message(STATUS "Source Directory: ${SOURCE_DIR}") message(STATUS "Result Directory: ${src_results_dir}") -# TODO: temporarily set a list containing both tf and torch -# This should move to being selectable betrween tf and/or torch -set(zoo-pylint_folder_list "aimet_zoo_tensorflow;aimet_zoo_torch") + +if(ENABLE_TENSORFLOW) + # Add AIMET Tensorflow package to package array list + list(APPEND pylint_list "aimet_zoo_tensorflow") +endif() + +if(ENABLE_TORCH) + # Add AIMET Torch package to package array list + list(APPEND pylint_list "aimet_zoo_torch") +endif() + +message(STATUS "Linting to proceed for: ${pylint_list}") + set(MSG_FORMAT "--msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'") set(pylint_failed 0) -foreach(zoo_pylint_folder IN LISTS zoo-pylint_folder_list) +foreach(zoo_pylint_folder IN LISTS pylint_list) message(STATUS "Aimet Model Zoo Variant: ${zoo_pylint_folder}") message(STATUS "Aimet Model Zoo Path Verification: ${SOURCE_DIR}/${zoo_pylint_folder}") + execute_process ( COMMAND pylint --rcfile=${SOURCE_DIR}/.pylintrc -r n ${MSG_FORMAT} ${SOURCE_DIR}/${zoo_pylint_folder} OUTPUT_VARIABLE pylint_complete RESULT_VARIABLE pylint_return ) - message(STATUS "Return Code is: ${pylint_return} -- Please correct the errors below") + message(STATUS "Return Code is: ${pylint_return}") + if(${pylint_return} AND NOT ${pylint_return} EQUAL "0") - message( WARNING "Pylint failed for ${zoo_pylint_folder}") + if(${pylint_return} EQUAL "1") + message(STATUS "Return Code is: ${pylint_return} -- fatal message issued") + elseif(${pylint_return} EQUAL "2") + message(STATUS "Return Code is: ${pylint_return} -- error message issued") + elseif(${pylint_return} EQUAL "4") + message(STATUS "Return Code is: ${pylint_return} -- warning message issued") + elseif(${pylint_return} EQUAL "8") + message(STATUS "Return Code is: ${pylint_return} -- refactor message issued") + elseif(${pylint_return} EQUAL "16") + message(STATUS "Return Code is: ${pylint_return} -- convention message issued") + elseif(${pylint_return} EQUAL "32") + message(STATUS "Return Code is: ${pylint_return} -- usage error") + else() + message(STATUS "Return Code is: ${pylint_return} -- multiple messages issued") + endif() + set(pylint_failed 1) endif() + file(WRITE "${src_results_dir}/pylint_${zoo_pylint_folder}" ${pylint_complete}) message(${pylint_complete}) + endforeach() diff --git a/packaging/version.txt b/packaging/version.txt index 1cc5f65..f0bb29e 100644 --- a/packaging/version.txt +++ b/packaging/version.txt @@ -1 +1 @@ -1.1.0 \ No newline at end of file +1.3.0