From 1b663e713e02e3e0ec69b19cf1161291a3f35597 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Tue, 16 Jun 2020 15:19:22 +0000 Subject: [PATCH 01/19] renamed checkerboard job name --- tests/cicd/main_build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index e4e23083..8afcd9a7 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -114,15 +114,15 @@ jobs: echo "cv_lib unit test job passed" ################################################################################################### -# Stage 3: Dutch F3 patch models on checkerboard test set: +# Stage 3: Patch models on checkerboard test set: # deconvnet, unet, HRNet patch depth, HRNet section depth # CAUTION: reverted these builds to single-GPU leaving new multi-GPU code in to be reverted later ################################################################################################### -- job: checkerboard_dutchf3_patch +- job: checkerboard_patch dependsOn: cv_lib_unit_tests_job timeoutInMinutes: 60 - displayName: Checkerboard Dutch F3 patch local + displayName: Checkerboard patch local pool: name: deepseismicagentpool steps: @@ -304,7 +304,7 @@ jobs: ################################################################################################### - job: F3_block_training_and_evaluation_local_notebook - dependsOn: checkerboard_dutchf3_patch + dependsOn: checkerboard_patch timeoutInMinutes: 5 displayName: F3 block training and evaluation local notebook pool: From c838bf2cb0b7034f83760f89fa7659bd15efc3e4 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Tue, 16 Jun 2020 23:39:08 +0000 Subject: [PATCH 02/19] restructured default outputs from test.py to be dumped under output dir and not debug dir --- experiments/interpretation/dutchf3_patch/local/test.py | 6 +++--- tests/cicd/src/check_performance.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index e79ff15f..99141d9e 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -275,10 +275,10 @@ def _evaluate_split( try: output_dir = generate_path( - f"debug/{config.OUTPUT_DIR}_test_{split}", git_branch(), git_hash(), config.MODEL.NAME, current_datetime(), + f"{config.OUTPUT_DIR}/test/{split}", git_branch(), git_hash(), config.MODEL.NAME, current_datetime(), ) except: - output_dir = generate_path(f"debug/{config.OUTPUT_DIR}_test_{split}", config.MODEL.NAME, current_datetime(),) + output_dir = generate_path(f"{config.OUTPUT_DIR}/test/{split}", config.MODEL.NAME, current_datetime(),) running_metrics_split = runningScore(n_classes) @@ -363,7 +363,7 @@ def test(*options, cfg=None, debug=False): load_log_configuration(config.LOG_CONFIG) logger = logging.getLogger(__name__) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - log_dir, model_name = os.path.split(config.TEST.MODEL_PATH) + log_dir, _ = os.path.split(config.TEST.MODEL_PATH) # load model: model = getattr(models, config.MODEL.NAME).get_seg_model(config) diff --git a/tests/cicd/src/check_performance.py b/tests/cicd/src/check_performance.py index 863ff86c..52f27d92 100644 --- a/tests/cicd/src/check_performance.py +++ b/tests/cicd/src/check_performance.py @@ -43,7 +43,7 @@ def main(args): if args.test: metrics_dict["Pixel Accuracy"] = "Pixel Acc: " metrics_dict["Mean IoU"] = "Mean IoU: " - else: + else: # validation metrics_dict["Pixel Accuracy"] = "pixacc" metrics_dict["Mean IoU"] = "mIoU" @@ -56,9 +56,9 @@ def main(args): logging.info("all done") -""" GLOBAL VARIABLES """ -# tolerance within which values are compared -ABS_TOL = 1e-2 +# """ GLOBAL VARIABLES """ +# # tolerance within which values are compared +# ABS_TOL = 1e-2 """ cmd-line arguments """ parser.add_argument("--infile", help="Location of the file which has the metrics", type=str, required=True) From 2ea84d277fc12f9cda0394f7eda73243ee1069f8 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Wed, 17 Jun 2020 02:02:11 +0000 Subject: [PATCH 03/19] test.py output re-org --- experiments/interpretation/dutchf3_patch/local/test.py | 2 +- interpretation/deepseismic_interpretation/dutchf3/data.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 99141d9e..ec5d4a71 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -230,7 +230,7 @@ def _patch_label_2d( # dump the data right before it's being put into the model and after scoring if debug: - outdir = f"debug/batch_{split}" + outdir = f"debug/test/batch_{split}" generate_path(outdir) for i in range(batch.shape[0]): path_prefix = f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}" diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index 83b4635b..6c9ec627 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -153,7 +153,7 @@ def __getitem__(self, index): im, lbl = _transform_WH_to_HW(im), _transform_WH_to_HW(lbl) if self.debug and "test" in self.split: - outdir = f"debug/sectionLoader_{self.split}_raw" + outdir = f"debug/test/sectionLoader_{self.split}_raw" generate_path(outdir) path_prefix = f"{outdir}/index_{index}_section_{section_name}" image_to_disk(im, path_prefix + "_img.png", self.MIN, self.MAX) @@ -167,7 +167,7 @@ def __getitem__(self, index): im, lbl = self.transform(im, lbl) if self.debug and "test" in self.split: - outdir = f"debug/sectionLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + outdir = f"debug/test/sectionLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" generate_path(outdir) path_prefix = f"{outdir}/index_{index}_section_{section_name}" image_to_disk(np.array(im[0]), path_prefix + "_img.png", self.MIN, self.MAX) @@ -397,7 +397,7 @@ def __getitem__(self, index): # dump images before augmentation if self.debug: - outdir = f"debug/testSectionLoaderWithDepth_{self.split}_raw" + outdir = f"debug/test/testSectionLoaderWithDepth_{self.split}_raw" generate_path(outdir) # this needs to take the first dimension of image (no depth) but lbl only has 1 dim path_prefix = f"{outdir}/index_{index}_section_{section_name}" @@ -416,7 +416,7 @@ def __getitem__(self, index): # dump images and labels to disk after augmentation if self.debug: outdir = ( - f"debug/testSectionLoaderWithDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + f"debug/test/testSectionLoaderWithDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" ) generate_path(outdir) path_prefix = f"{outdir}/index_{index}_section_{section_name}" From 3191782e75c7dc027f5403525bff0fc52bf7f1d4 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Wed, 17 Jun 2020 13:08:34 +0000 Subject: [PATCH 04/19] removed outdated variable from check_performance.py --- tests/cicd/src/check_performance.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/cicd/src/check_performance.py b/tests/cicd/src/check_performance.py index 52f27d92..a60fd67a 100644 --- a/tests/cicd/src/check_performance.py +++ b/tests/cicd/src/check_performance.py @@ -56,9 +56,6 @@ def main(args): logging.info("all done") -# """ GLOBAL VARIABLES """ -# # tolerance within which values are compared -# ABS_TOL = 1e-2 """ cmd-line arguments """ parser.add_argument("--infile", help="Location of the file which has the metrics", type=str, required=True) From 40744af0a6815415ee9271e96fec7ceab2099f27 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Wed, 17 Jun 2020 18:53:42 +0000 Subject: [PATCH 05/19] intermediate work --- .../dutchf3_patch/local/train.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index a37e26e1..275642bd 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -90,9 +90,6 @@ def run(*options, cfg=None, debug=False, input=None): # Set CUDNN benchmark mode: torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK - # We will write the model under outputs / config_file_name / model_dir - config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] - # Fix random seeds: torch.manual_seed(config.SEED) if torch.cuda.is_available(): @@ -129,6 +126,9 @@ def run(*options, cfg=None, debug=False, input=None): else: train_aug = val_aug = basic_aug + + + # Training and Validation Loaders: TrainPatchLoader = get_patch_loader(config) logging.info(f"Using {TrainPatchLoader}") @@ -142,8 +142,8 @@ def run(*options, cfg=None, debug=False, input=None): ) logger.info(train_set) - n_classes = train_set.n_classes + val_set = TrainPatchLoader( config, split="val", @@ -152,10 +152,16 @@ def run(*options, cfg=None, debug=False, input=None): debug=debug, ) logger.info(val_set) - - if debug: + data_flow_dict = dict() + + data_flow_dict['train_patch_loader_length'] = len(train_set) + data_flow_dict['validation_patch_loader_length'] = len(val_set) + data_flow_dict['train_input_shape'] = train_set.seismic.shape + data_flow_dict['train_label_shape'] = train_set.labels.shape + data_flow_dict['n_classes'] = n_classes + logger.info("Running in debug mode..") train_range = min(config.TRAIN.BATCH_SIZE_PER_GPU * config.NUM_DEBUG_BATCHES, len(train_set)) logging.info(f"train range in debug mode {train_range}") @@ -163,8 +169,9 @@ def run(*options, cfg=None, debug=False, input=None): valid_range = min(config.VALIDATION.BATCH_SIZE_PER_GPU, len(val_set)) val_set = data.Subset(val_set, range(valid_range)) - - + data_flow_dict['train_length_subset'] = len(train_set) + data_flow_dict['validation_length_subset'] = len(val_set) + train_loader = data.DataLoader( train_set, batch_size=config.TRAIN.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, shuffle=True ) @@ -172,6 +179,14 @@ def run(*options, cfg=None, debug=False, input=None): val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=1 ) # config.WORKERS) + if debug: + data_flow_dict['train_loader_length'] = len(train_loader) + data_flow_dict['validation_loader_length'] = len(val_loader) + + fname = f"data_flow_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" + with open(fname, 'w') as f: + json.dump(data_flow_dict, f, indent=4) + # Model: model = getattr(models, config.MODEL.NAME).get_seg_model(config) device = "cuda" if torch.cuda.is_available() else "cpu" From d275c8af7df8d748ff1745028fd59ac693743886 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Wed, 17 Jun 2020 18:55:14 +0000 Subject: [PATCH 06/19] intermediate work --- experiments/interpretation/dutchf3_patch/local/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index 275642bd..99c7d65a 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -183,7 +183,7 @@ def run(*options, cfg=None, debug=False, input=None): data_flow_dict['train_loader_length'] = len(train_loader) data_flow_dict['validation_loader_length'] = len(val_loader) - fname = f"data_flow_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" + fname = f"data_flow_train_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" with open(fname, 'w') as f: json.dump(data_flow_dict, f, indent=4) From 62c0a522799e6b90025a73ae768484abd3bbbd8d Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Thu, 18 Jun 2020 14:34:10 +0000 Subject: [PATCH 07/19] bunch of intermediate works --- .../dutchf3_patch/local/test.py | 44 ++++++++++++++++--- .../dutchf3_patch/local/train.py | 7 +-- .../dutchf3/data.py | 5 +-- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index ec5d4a71..57222dc8 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -251,7 +251,7 @@ def _patch_label_2d( def _evaluate_split( - split, section_aug, model, pre_processing, output_processing, device, running_metrics_overall, config, debug=False, + split, section_aug, model, pre_processing, output_processing, device, running_metrics_overall, config, data_flow, debug=False, ): logger = logging.getLogger(__name__) @@ -267,11 +267,24 @@ def _evaluate_split( n_classes = test_set.n_classes + if debug: + data_flow[split] = dict() + data_flow[split]['test_section_loader_length'] = len(test_set) + data_flow[split]['test_input_shape'] = test_set.seismic.shape + data_flow[split]['test_label_shape'] = test_set.labels.shape + data_flow[split]['n_classes'] = n_classes + + test_loader = data.DataLoader(test_set, batch_size=1, num_workers=config.WORKERS, shuffle=False) if debug: + data_flow[split]['test_loader_length'] = len(test_loader) logger.info("Running in Debug/Test mode") - test_loader = take(2, test_loader) + take_n = 2 + test_loader = take(take_n, test_loader) + data_flow[split]['take_n_sections'] = take_n + pred_list, gt_list, img_list = [], [], [] + try: output_dir = generate_path( @@ -281,14 +294,15 @@ def _evaluate_split( output_dir = generate_path(f"{config.OUTPUT_DIR}/test/{split}", config.MODEL.NAME, current_datetime(),) running_metrics_split = runningScore(n_classes) + # evaluation mode: with torch.no_grad(): # operations inside don't track history model.eval() - total_iteration = 0 + # total_iteration = 0 for i, (images, labels) in enumerate(test_loader): logger.info(f"split: {split}, section: {i}") - total_iteration = total_iteration + 1 + # total_iteration = total_iteration + 1 outputs = _patch_label_2d( model, images, @@ -306,6 +320,11 @@ def _evaluate_split( pred = outputs.detach().max(1)[1].numpy() gt = labels.numpy() + if debug: + pred_list.append((pred.shape, len(np.unique(pred)))) + gt_list.append((gt.shape, len(np.unique(gt)))) + img_list.append(images.numpy().shape) + running_metrics_split.update(gt, pred) running_metrics_overall.update(gt, pred) @@ -313,6 +332,11 @@ def _evaluate_split( mask_to_disk(pred.squeeze(), os.path.join(output_dir, f"{i}_pred.png"), n_classes) mask_to_disk(gt.squeeze(), os.path.join(output_dir, f"{i}_gt.png"), n_classes) + if debug: + data_flow[split]['pred_shape'] = pred_list + data_flow[split]['gt_shape'] = gt_list + data_flow[split]['img_shape'] = img_list + # get scores score, class_iou = running_metrics_split.get_scores() @@ -396,6 +420,7 @@ def test(*options, cfg=None, debug=False): output_processing = _output_processing_pipeline(config) splits = ["test1", "test2"] if "Both" in config.TEST.SPLIT else [config.TEST.SPLIT] + data_flow = dict() for sdx, split in enumerate(splits): labels = np.load(path.join(config.DATASET.ROOT, "test_once", split + "_labels.npy")) section_file = path.join(config.DATASET.ROOT, "splits", "section_" + split + ".txt") @@ -409,12 +434,22 @@ def test(*options, cfg=None, debug=False): device, running_metrics_overall, config, + data_flow, debug=debug, ) + if debug: + config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] + + fname = f"data_flow_test_{config_file_name}_{config.TRAIN.DEPTH}_{config.TRAIN.MODEL_DIR}.json" + with open(fname, 'w') as f: + json.dump(data_flow, f, indent=1) + # FINAL TEST RESULTS: score, class_iou = running_metrics_overall.get_scores() + print(score, class_iou) + logger.info("--------------- FINAL RESULTS -----------------") logger.info(f'Pixel Acc: {score["Pixel Acc: "]:.4f}') @@ -434,7 +469,6 @@ def test(*options, cfg=None, debug=False): np.savetxt(path.join(log_dir, "confusion.csv"), confusion, delimiter=" ") if debug: - config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] fname = f"metrics_test_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" with open(fname, "w") as fid: json.dump( diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index 99c7d65a..7fca8999 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -126,9 +126,6 @@ def run(*options, cfg=None, debug=False, input=None): else: train_aug = val_aug = basic_aug - - - # Training and Validation Loaders: TrainPatchLoader = get_patch_loader(config) logging.info(f"Using {TrainPatchLoader}") @@ -183,9 +180,9 @@ def run(*options, cfg=None, debug=False, input=None): data_flow_dict['train_loader_length'] = len(train_loader) data_flow_dict['validation_loader_length'] = len(val_loader) - fname = f"data_flow_train_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" + fname = f"data_flow_train_{config_file_name}_{config.TRAIN.DEPTH}_{config.TRAIN.MODEL_DIR}.json" with open(fname, 'w') as f: - json.dump(data_flow_dict, f, indent=4) + json.dump(data_flow_dict, f, indent=2) # Model: model = getattr(models, config.MODEL.NAME).get_seg_model(config) diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index 6c9ec627..a998da6a 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -773,9 +773,6 @@ def __repr__(self): "patch": TrainPatchLoaderWithDepth, } -_TRAIN_SECTION_LOADERS = {"section": TrainSectionLoaderWithDepth} - - def get_patch_loader(cfg): assert str(cfg.TRAIN.DEPTH).lower() in [ "section", @@ -785,6 +782,7 @@ def get_patch_loader(cfg): Valid values: section, patch, none." return _TRAIN_PATCH_LOADERS.get(cfg.TRAIN.DEPTH, TrainPatchLoader) +_TRAIN_SECTION_LOADERS = {"section": TrainSectionLoaderWithDepth} def get_section_loader(cfg): assert str(cfg.TRAIN.DEPTH).lower() in [ @@ -797,7 +795,6 @@ def get_section_loader(cfg): _TEST_LOADERS = {"section": TestSectionLoaderWithDepth} - def get_test_loader(cfg): logger = logging.getLogger(__name__) logger.info(f"Test loader {cfg.TRAIN.DEPTH}") From 8ab4eaa9e06b6eab94a6c7d67f8fca8cc44c2cad Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Thu, 18 Jun 2020 16:15:14 +0000 Subject: [PATCH 08/19] changing args for different trainings --- .../dutchf3_patch/local/test.py | 2 +- .../dutchf3_patch/local/train.py | 2 +- tests/cicd/main_build.yml | 4 + tests/cicd/src/check_data_flow.py | 98 +++++++++++++++++++ 4 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 tests/cicd/src/check_data_flow.py diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 57222dc8..523c4162 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -441,7 +441,7 @@ def test(*options, cfg=None, debug=False): if debug: config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] - fname = f"data_flow_test_{config_file_name}_{config.TRAIN.DEPTH}_{config.TRAIN.MODEL_DIR}.json" + fname = f"data_flow_test_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" with open(fname, 'w') as f: json.dump(data_flow, f, indent=1) diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index 7fca8999..0e2a263b 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -180,7 +180,7 @@ def run(*options, cfg=None, debug=False, input=None): data_flow_dict['train_loader_length'] = len(train_loader) data_flow_dict['validation_loader_length'] = len(val_loader) - fname = f"data_flow_train_{config_file_name}_{config.TRAIN.DEPTH}_{config.TRAIN.MODEL_DIR}.json" + fname = f"data_flow_train_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" with open(fname, 'w') as f: json.dump(data_flow_dict, f, indent=2) diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 8afcd9a7..16ee1527 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -148,6 +148,7 @@ jobs: 'TRAIN.END_EPOCH' 2 'TRAIN.SNAPSHOTS' 1 \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TRAIN.DEPTH' 'none' \ + 'TRAIN.BATCH_SIZE_PER_GPU' 16 'VALIDATION.BATCH_SIZE_PER_GPU' 32 \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ 'WORKERS' 1 \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } @@ -158,6 +159,7 @@ jobs: 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TRAIN.DEPTH' 'section' \ + 'TRAIN.BATCH_SIZE_PER_GPU' 16 'VALIDATION.BATCH_SIZE_PER_GPU' 32 \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ 'WORKERS' 1 \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } @@ -168,6 +170,7 @@ jobs: 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TRAIN.DEPTH' 'section' \ + 'TRAIN.BATCH_SIZE_PER_GPU' 16 'VALIDATION.BATCH_SIZE_PER_GPU' 32 \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ 'WORKERS' 1 \ --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } @@ -178,6 +181,7 @@ jobs: 'TRAIN.END_EPOCH' 2 'TRAIN.SNAPSHOTS' 1 \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TRAIN.DEPTH' 'section' \ + 'TRAIN.BATCH_SIZE_PER_GPU' 16 'VALIDATION.BATCH_SIZE_PER_GPU' 32 \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ 'WORKERS' 1 \ diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py new file mode 100644 index 00000000..5213cb83 --- /dev/null +++ b/tests/cicd/src/check_data_flow.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +""" Please see the def main() function for code description.""" +import json +import math + +""" libraries """ + +import numpy as np +import sys +import os + +np.set_printoptions(linewidth=200) +import logging + +# toggle to WARNING when running in production, or use CLI +logging.getLogger().setLevel(logging.DEBUG) +# logging.getLogger().setLevel(logging.WARNING) +import argparse + +parser = argparse.ArgumentParser() + +""" useful information when running from a GIT folder.""" +myname = os.path.realpath(__file__) +mypath = os.path.dirname(myname) +myname = os.path.basename(myname) + + +def main(args): + """ + + Tests to ensure proper data flow throughout the experiments. + + """ + + logging.info("loading data") + + with open(args.infile, "r") as fp: + data = json.load(fp) + + + + if args.step=='test': + pass + elif args.step=='train': + if args.train_depth=='none': + assert tuple(data['train_input_shape'])==(200, 200, 200) + + elif args.train_depth=='section': + assert tuple(data['train_input_shape'])==(200, 3, 200, 200) + elif args.train_depth=='patch': + pass + assert tuple(data['train_label_shape'])==(200, 200, 200) + assert data['train_patch_loader_length']==64 + assert data['validation_patch_loader_length']==1280 + assert data['train_length_subset']==64 + assert data['validation_length_subset']==32 + assert data['train_loader_length']==4 + assert data['validation_loader_length']==1 + assert data['n_classes']==2 + + + + + + logging.info("all done") + + + +""" cmd-line arguments """ +STEPS = ["test", "train"] +TRAIN_DEPTH = ["none", "patch", "section"] + +parser.add_argument("--infile", help="Location of the file which has the metrics", type=str, required=True) +parser.add_argument("--step", choices=STEPS, type=str, required=True, help="Data flow checks for test or training pipeline") +parser.add_argument("--train_depth", choices=TRAIN_DEPTH, type=str, required=True, help="Train depth flag, to check the dimensions") +""" main wrapper with profiler """ +if __name__ == "__main__": + main(parser.parse_args()) + +# pretty printing of the stack +""" + try: + logging.info('before main') + main(parser.parse_args()) + logging.info('after main') + except: + for frame in traceback.extract_tb(sys.exc_info()[2]): + fname,lineno,fn,text = frame + print ("Error in %s on line %d" % (fname, lineno)) +""" +# optionally enable profiling information +# import cProfile +# name = +# cProfile.run('main.run()', name + '.prof') +# import pstats +# p = pstats.Stats(name + '.prof') +# p.sort_stats('cumulative').print_stats(10) +# p.sort_stats('time').print_stats() From 54be704393673e1deabf425ac49a308d6df648c2 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Thu, 18 Jun 2020 19:39:35 +0000 Subject: [PATCH 09/19] final to run dev_build" --- tests/cicd/main_build.yml | 18 ++++++++ tests/cicd/src/check_data_flow.py | 70 ++++++++++++++++++------------- 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 16ee1527..a62cbcd6 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -199,6 +199,16 @@ jobs: # Remove the temporary directory rm -r "$dir" + + + set -e + python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_patch_deconvnet_no_depth.json --step train --train_depth none + python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_unet_section_depth.json --step train --train_depth section + python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_seresnet_unet_section_depth.json --step train --train_depth section + python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_hrnet_section_depth.json --step train --train_depth section + set +e + + # check validation set performance set -e @@ -290,6 +300,14 @@ jobs: # Remove the temporary directory rm -r "$dir" + # check data flow for test + set -e + python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_patch_deconvnet_no_depth.json --step test --train_depth none + python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_unet_section_depth.json --step test --train_depth section + python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_seresnet_unet_section_depth.json --step test --train_depth section + python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_hrnet_section_depth.json --step test --train_depth section + set +e + # check test set performance set -e # TODO: enable this after investigating reproducibility problem of patch_deconvnet for small-size training data diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py index 5213cb83..eee48fb8 100644 --- a/tests/cicd/src/check_data_flow.py +++ b/tests/cicd/src/check_data_flow.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 """ Please see the def main() function for code description.""" import json -import math + """ libraries """ import numpy as np -import sys import os np.set_printoptions(linewidth=200) @@ -37,18 +36,54 @@ def main(args): with open(args.infile, "r") as fp: data = json.load(fp) - + # Note: these are specific to the setup in + # main_build.yml for train.py + # and get_data_for_builds.sh and prepare_dutchf3.py and prepare_dutchf3.py if args.step=='test': - pass + + for test_key in data.keys(): + print(test_key) + if args.train_depth=='none': + print(args.train_depth) + assert tuple(data[test_key]['test_input_shape'])==(200, 200, 200) + for img in data[test_key]["img_shape"]: + assert tuple(img)==(1, 1, 200, 200) #dimenstion + + elif args.train_depth=='section': + print(args.train_depth) + assert tuple(data[test_key]['test_input_shape'])==(200, 3, 200, 200) + for img in data[test_key]["img_shape"]: + assert tuple(img)==(1, 3, 200, 200) #dimenstion + + + assert len(data[test_key]["pred_shape"])==data[test_key]["take_n_sections"] + assert len(data[test_key]["gt_shape"])==data[test_key]["take_n_sections"] + assert len(data[test_key]["img_shape"])==data[test_key]["take_n_sections"] + + assert tuple(data[test_key]["test_label_shape"])==(200, 200, 200) + + assert data[test_key]["test_section_loader_length"]==400 + assert data[test_key]["n_classes"]==2 + assert data[test_key]["test_loader_length"]==400 + + for pred, gt in zip(data[test_key]["pred_shape"], data[test_key]["gt_shape"]): + assert tuple(pred[0])==(1, 200, 200) #dimenstion + assert pred[1]<=2 # unique classes + + assert tuple(gt[0])==(1, 1, 200, 200) #dimenstion + assert gt[1]<=2 # unique classes + + + elif args.step=='train': if args.train_depth=='none': assert tuple(data['train_input_shape'])==(200, 200, 200) - elif args.train_depth=='section': assert tuple(data['train_input_shape'])==(200, 3, 200, 200) elif args.train_depth=='patch': - pass + raise Exception('Must be added') + assert tuple(data['train_label_shape'])==(200, 200, 200) assert data['train_patch_loader_length']==64 assert data['validation_patch_loader_length']==1280 @@ -57,11 +92,7 @@ def main(args): assert data['train_loader_length']==4 assert data['validation_loader_length']==1 assert data['n_classes']==2 - - - - logging.info("all done") @@ -77,22 +108,3 @@ def main(args): if __name__ == "__main__": main(parser.parse_args()) -# pretty printing of the stack -""" - try: - logging.info('before main') - main(parser.parse_args()) - logging.info('after main') - except: - for frame in traceback.extract_tb(sys.exc_info()[2]): - fname,lineno,fn,text = frame - print ("Error in %s on line %d" % (fname, lineno)) -""" -# optionally enable profiling information -# import cProfile -# name = -# cProfile.run('main.run()', name + '.prof') -# import pstats -# p = pstats.Stats(name + '.prof') -# p.sort_stats('cumulative').print_stats(10) -# p.sort_stats('time').print_stats() From f4de533d313643c6a0f786b4a51794943e0f9246 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Thu, 18 Jun 2020 19:51:52 +0000 Subject: [PATCH 10/19] remove print statements --- tests/cicd/src/check_data_flow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py index eee48fb8..f0c86a7c 100644 --- a/tests/cicd/src/check_data_flow.py +++ b/tests/cicd/src/check_data_flow.py @@ -43,18 +43,18 @@ def main(args): if args.step=='test': for test_key in data.keys(): - print(test_key) if args.train_depth=='none': - print(args.train_depth) assert tuple(data[test_key]['test_input_shape'])==(200, 200, 200) for img in data[test_key]["img_shape"]: assert tuple(img)==(1, 1, 200, 200) #dimenstion elif args.train_depth=='section': - print(args.train_depth) assert tuple(data[test_key]['test_input_shape'])==(200, 3, 200, 200) for img in data[test_key]["img_shape"]: assert tuple(img)==(1, 3, 200, 200) #dimenstion + + elif args.train_depth=='patch': + raise Exception('Must be added') assert len(data[test_key]["pred_shape"])==data[test_key]["take_n_sections"] From 1b81e83516bed2ef20f0bf9e8ccf81f806c795c8 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Thu, 18 Jun 2020 20:08:06 +0000 Subject: [PATCH 11/19] removed print statement --- experiments/interpretation/dutchf3_patch/local/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 523c4162..7c0a8445 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -448,8 +448,6 @@ def test(*options, cfg=None, debug=False): # FINAL TEST RESULTS: score, class_iou = running_metrics_overall.get_scores() - print(score, class_iou) - logger.info("--------------- FINAL RESULTS -----------------") logger.info(f'Pixel Acc: {score["Pixel Acc: "]:.4f}') From 7b07598c2fcc089eb69b932cf7d2c7a361ede692 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Fri, 19 Jun 2020 13:32:04 +0000 Subject: [PATCH 12/19] removed suppressed lines --- experiments/interpretation/dutchf3_patch/local/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 7c0a8445..3c6af170 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -299,10 +299,8 @@ def _evaluate_split( # evaluation mode: with torch.no_grad(): # operations inside don't track history model.eval() - # total_iteration = 0 for i, (images, labels) in enumerate(test_loader): logger.info(f"split: {split}, section: {i}") - # total_iteration = total_iteration + 1 outputs = _patch_label_2d( model, images, From 08681660ec0bd66fbc97ad9ba4e4973755fa4a4e Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Fri, 19 Jun 2020 15:53:11 +0000 Subject: [PATCH 13/19] added assertion error msg --- tests/cicd/src/check_data_flow.py | 71 +++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py index f0c86a7c..2d434ea4 100644 --- a/tests/cicd/src/check_data_flow.py +++ b/tests/cicd/src/check_data_flow.py @@ -44,35 +44,70 @@ def main(args): for test_key in data.keys(): if args.train_depth=='none': - assert tuple(data[test_key]['test_input_shape'])==(200, 200, 200) - for img in data[test_key]["img_shape"]: - assert tuple(img)==(1, 1, 200, 200) #dimenstion - + expected_test_input_shape = (200, 200, 200) + expected_img = (1, 1, 200, 200) + elif args.train_depth=='section': - assert tuple(data[test_key]['test_input_shape'])==(200, 3, 200, 200) - for img in data[test_key]["img_shape"]: - assert tuple(img)==(1, 3, 200, 200) #dimenstion + expected_test_input_shape = (200, 3, 200, 200) + expected_img = (1, 3, 200, 200) elif args.train_depth=='patch': + expected_test_input_shape = 'TBD' + expected_img = 'TBD' raise Exception('Must be added') + msg = f"Expected {expected_test_input_shape} for shape, received {tuple(data[test_key]['test_input_shape'])} instead, in {args.infile.split('.')[0]}" + assert tuple(data[test_key]['test_input_shape'])==expected_test_input_shape, msg + + expected_test_label_shape = (200, 200, 200) + msg = f"Expected {expected_test_label_shape} for shape, received {tuple(data[test_key]['test_label_shape'])} instead, in {args.infile.split('.')[0]}" + assert tuple(data[test_key]['test_label_shape'])==expected_test_label_shape, msg + + for img in data[test_key]["img_shape"]: + # dimension + msg = f"Expected {expected_img} for shape, received {tuple(img)} instead, in {args.infile.split('.')[0]}" + assert tuple(img)==expected_img, msg + + msg = f"Expected {data[test_key]["take_n_sections"]} number of items, received {len(data[test_key]["pred_shape"])} instead, in {args.infile.split('.')[0]}" + assert len(data[test_key]["pred_shape"])==data[test_key]["take_n_sections"], msg + + msg = f"Expected {data[test_key]["take_n_sections"]} number of items, received {len(data[test_key]["gt_shape"])} instead, in {args.infile.split('.')[0]}" + assert len(data[test_key]["gt_shape"])==data[test_key]["take_n_sections"], msg + + msg = f"Expected {data[test_key]["take_n_sections"]} number of items, received {len(data[test_key]["img_shape"])} instead, in {args.infile.split('.')[0]}" + assert len(data[test_key]["img_shape"])==data[test_key]["take_n_sections"], msg + - assert len(data[test_key]["pred_shape"])==data[test_key]["take_n_sections"] - assert len(data[test_key]["gt_shape"])==data[test_key]["take_n_sections"] - assert len(data[test_key]["img_shape"])==data[test_key]["take_n_sections"] + expected_len = 400 + msg = f"Expected {expected_len} for test section loader length, received {data[test_key]["test_section_loader_length"]} instead, in {args.infile.split('.')[0]}" + assert data[test_key]["test_section_loader_length"]==expected_len, msg - assert tuple(data[test_key]["test_label_shape"])==(200, 200, 200) + msg = f"Expected {expected_len} for test loader length, received {data[test_key]["test_loader_length"]} instead, in {args.infile.split('.')[0]}" + assert data[test_key]["test_loader_length"]==expected_len, msg - assert data[test_key]["test_section_loader_length"]==400 - assert data[test_key]["n_classes"]==2 - assert data[test_key]["test_loader_length"]==400 + expected_n_classes = 2 + msg = f"Expected {expected_n_classes} for test loader length, received {data[test_key]["n_classes"]} instead, in {args.infile.split('.')[0]}" + assert data[test_key]["n_classes"]==expected_n_classes, msg + + expected_pred = (1, 200, 200) + expected_gt = (1, 1, 200, 200) for pred, gt in zip(data[test_key]["pred_shape"], data[test_key]["gt_shape"]): - assert tuple(pred[0])==(1, 200, 200) #dimenstion - assert pred[1]<=2 # unique classes + #dimenstion + msg = f"Expected {expected_pred} for prediction shape, received {tuple(pred[0])} instead, in {args.infile.split('.')[0]}" + assert tuple(pred[0])==expected_pred, msg + + # unique classes + msg = f"Expected up to {expected_n_classes} unique prediction classes, received {pred[1]} instead, in {args.infile.split('.')[0]}" + assert pred[1]<=expected_n_classes, msg + + #dimenstion + msg = f"Expected {expected_gt} for ground truth mask shape, received {tuple(gt[0])} instead, in {args.infile.split('.')[0]}" + assert tuple(gt[0])==expected_gt, msg - assert tuple(gt[0])==(1, 1, 200, 200) #dimenstion - assert gt[1]<=2 # unique classes + # unique classes + msg = f"Expected up to {expected_n_classes} unique ground truth classes, received {gt[1]} instead, in {args.infile.split('.')[0]}" + assert gt[1]<=expected_n_classes, msg From dad93a5bb55a9db5d99a4c24222f056a454e61b7 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Fri, 19 Jun 2020 16:47:46 +0000 Subject: [PATCH 14/19] added assertion error msg, one intential bug to test --- tests/cicd/src/check_data_flow.py | 46 +++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py index 2d434ea4..4f7dd137 100644 --- a/tests/cicd/src/check_data_flow.py +++ b/tests/cicd/src/check_data_flow.py @@ -113,20 +113,48 @@ def main(args): elif args.step=='train': if args.train_depth=='none': - assert tuple(data['train_input_shape'])==(200, 200, 200) + expected_shape_in = (200, 200, 200) elif args.train_depth=='section': - assert tuple(data['train_input_shape'])==(200, 3, 200, 200) + expected_shape_in = (200, 3, 200, 200) elif args.train_depth=='patch': + expected_shape_in = 'TBD' raise Exception('Must be added') - assert tuple(data['train_label_shape'])==(200, 200, 200) - assert data['train_patch_loader_length']==64 - assert data['validation_patch_loader_length']==1280 - assert data['train_length_subset']==64 - assert data['validation_length_subset']==32 - assert data['train_loader_length']==4 + msg = f"Expected {expected_shape_in} for shape, received {tuple(data['train_input_shape'])} instead, in {args.infile.split('.')[0]}" + assert tuple(data['train_input_shape'])==expected_shape_in, msg + + expected_shape_label = (200, 200, 200) + msg = f"Expected {expected_shape_label} for shape, received {tuple(data['train_label_shape'])} instead, in {args.infile.split('.')[0]}" + assert tuple(data['train_label_shape'])==expected_shape_label, msg + + + expected_len = 64 + msg = f"Expected {expected_len} for train patch loader length, received {data['train_patch_loader_length']} instead, in {args.infile.split('.')[0]}" + assert data['train_patch_loader_length']==expected_len, msg + + expected_len = 1280 + msg = f"Expected {expected_len} for validation patch loader length, received {data['validation_patch_loader_length']} instead, in {args.infile.split('.')[0]}" + assert data['validation_patch_loader_length']==expected_len + + expected_len = 64 + msg = f"Expected {expected_len} for train subset length, received {data['train_length_subset']} instead, in {args.infile.split('.')[0]}" + assert data['train_length_subset']==expected_len + + expected_len = 32 + msg = f"Expected {expected_len} for validation subset length, received {data['validation_length_subset']} instead, in {args.infile.split('.')[0]}" + assert data['validation_length_subset']==expected_len + + expected_len = 55 #4 + msg = f"Expected {expected_len} for train loader length, received {data['train_loader_length']} instead, in {args.infile.split('.')[0]}" + assert data['train_loader_length']==expected_len + + expected_len = 4 + msg = f"Expected {expected_len} for train loader length, received {data['train_loader_length']} instead, in {args.infile.split('.')[0]}" assert data['validation_loader_length']==1 - assert data['n_classes']==2 + + expected_n_classes = 2 + msg = f"Expected {expected_n_classes} for number of classes, received {data['n_classes']} instead, in {args.infile.split('.')[0]}" + assert data['n_classes']==expected_n_classes logging.info("all done") From 5594a2a6f6813b0ab628f37c76b7b215e032559e Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Fri, 19 Jun 2020 18:17:59 +0000 Subject: [PATCH 15/19] testing a stupid bug --- tests/cicd/src/check_data_flow.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py index 4f7dd137..3adee873 100644 --- a/tests/cicd/src/check_data_flow.py +++ b/tests/cicd/src/check_data_flow.py @@ -64,30 +64,34 @@ def main(args): assert tuple(data[test_key]['test_label_shape'])==expected_test_label_shape, msg for img in data[test_key]["img_shape"]: - # dimension msg = f"Expected {expected_img} for shape, received {tuple(img)} instead, in {args.infile.split('.')[0]}" assert tuple(img)==expected_img, msg - msg = f"Expected {data[test_key]["take_n_sections"]} number of items, received {len(data[test_key]["pred_shape"])} instead, in {args.infile.split('.')[0]}" + # ----------------------------------------------- + exp_n_section = data[test_key]["take_n_sections"] + pred_shape_len = len(data[test_key]["pred_shape"]) + msg = f"Expected {exp_n_section} number of items, received {pred_shape_len} instead, in {args.infile.split('.')[0]}" assert len(data[test_key]["pred_shape"])==data[test_key]["take_n_sections"], msg - msg = f"Expected {data[test_key]["take_n_sections"]} number of items, received {len(data[test_key]["gt_shape"])} instead, in {args.infile.split('.')[0]}" + gt_shape_len = len(data[test_key]["gt_shape"]) + msg = f"Expected {exp_n_section} number of items, received {gt_shape_len} instead, in {args.infile.split('.')[0]}" assert len(data[test_key]["gt_shape"])==data[test_key]["take_n_sections"], msg - msg = f"Expected {data[test_key]["take_n_sections"]} number of items, received {len(data[test_key]["img_shape"])} instead, in {args.infile.split('.')[0]}" + img_shape_len = len(data[test_key]["img_shape"]) + msg = f"Expected {exp_n_section} number of items, received {img_shape_len} instead, in {args.infile.split('.')[0]}" assert len(data[test_key]["img_shape"])==data[test_key]["take_n_sections"], msg - expected_len = 400 - msg = f"Expected {expected_len} for test section loader length, received {data[test_key]["test_section_loader_length"]} instead, in {args.infile.split('.')[0]}" - assert data[test_key]["test_section_loader_length"]==expected_len, msg + # expected_len = 400 + # msg = f"Expected {expected_len} for test section loader length, received {data[test_key]["test_section_loader_length"]} instead, in {args.infile.split('.')[0]}" + # assert data[test_key]["test_section_loader_length"]==expected_len, msg - msg = f"Expected {expected_len} for test loader length, received {data[test_key]["test_loader_length"]} instead, in {args.infile.split('.')[0]}" - assert data[test_key]["test_loader_length"]==expected_len, msg + # msg = f"Expected {expected_len} for test loader length, received {data[test_key]["test_loader_length"]} instead, in {args.infile.split('.')[0]}" + # assert data[test_key]["test_loader_length"]==expected_len, msg expected_n_classes = 2 - msg = f"Expected {expected_n_classes} for test loader length, received {data[test_key]["n_classes"]} instead, in {args.infile.split('.')[0]}" - assert data[test_key]["n_classes"]==expected_n_classes, msg + # msg = f"Expected {expected_n_classes} for test loader length, received {data[test_key]["n_classes"]} instead, in {args.infile.split('.')[0]}" + # assert data[test_key]["n_classes"]==expected_n_classes, msg expected_pred = (1, 200, 200) expected_gt = (1, 1, 200, 200) From 1c14a91a3d63809b22dc4af13dee2e466ec2b3e3 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Fri, 19 Jun 2020 18:43:53 +0000 Subject: [PATCH 16/19] debug --- tests/cicd/main_build.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 2c533497..58286c7f 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -202,10 +202,10 @@ jobs: set -e - python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_patch_deconvnet_no_depth.json --step train --train_depth none - python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_unet_section_depth.json --step train --train_depth section - python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_seresnet_unet_section_depth.json --step train --train_depth section - python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_hrnet_section_depth.json --step train --train_depth section + python ../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_patch_deconvnet_no_depth.json --step train --train_depth none + python ../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_unet_section_depth.json --step train --train_depth section + python ../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_seresnet_unet_section_depth.json --step train --train_depth section + python ../../../tests/cicd/src/check_data_flow.py --infile data_flow_train_hrnet_section_depth.json --step train --train_depth section set +e @@ -302,10 +302,10 @@ jobs: # check data flow for test set -e - python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_patch_deconvnet_no_depth.json --step test --train_depth none - python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_unet_section_depth.json --step test --train_depth section - python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_seresnet_unet_section_depth.json --step test --train_depth section - python ../../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_hrnet_section_depth.json --step test --train_depth section + python ../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_patch_deconvnet_no_depth.json --step test --train_depth none + python ../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_unet_section_depth.json --step test --train_depth section + python ../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_seresnet_unet_section_depth.json --step test --train_depth section + python ../../../tests/cicd/src/check_data_flow.py --infile data_flow_test_hrnet_section_depth.json --step test --train_depth section set +e # check test set performance From e2a5ae842620a04fc6e3a165e1579cb1d93d5801 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Fri, 19 Jun 2020 18:59:49 +0000 Subject: [PATCH 17/19] omg --- tests/cicd/src/check_data_flow.py | 35 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py index 3adee873..0d17c2d5 100644 --- a/tests/cicd/src/check_data_flow.py +++ b/tests/cicd/src/check_data_flow.py @@ -71,27 +71,30 @@ def main(args): exp_n_section = data[test_key]["take_n_sections"] pred_shape_len = len(data[test_key]["pred_shape"]) msg = f"Expected {exp_n_section} number of items, received {pred_shape_len} instead, in {args.infile.split('.')[0]}" - assert len(data[test_key]["pred_shape"])==data[test_key]["take_n_sections"], msg + assert pred_shape_len==exp_n_section, msg gt_shape_len = len(data[test_key]["gt_shape"]) msg = f"Expected {exp_n_section} number of items, received {gt_shape_len} instead, in {args.infile.split('.')[0]}" - assert len(data[test_key]["gt_shape"])==data[test_key]["take_n_sections"], msg + assert gt_shape_len==exp_n_section, msg img_shape_len = len(data[test_key]["img_shape"]) msg = f"Expected {exp_n_section} number of items, received {img_shape_len} instead, in {args.infile.split('.')[0]}" - assert len(data[test_key]["img_shape"])==data[test_key]["take_n_sections"], msg + assert img_shape_len==exp_n_section, msg - # expected_len = 400 - # msg = f"Expected {expected_len} for test section loader length, received {data[test_key]["test_section_loader_length"]} instead, in {args.infile.split('.')[0]}" - # assert data[test_key]["test_section_loader_length"]==expected_len, msg + expected_len = 400 + lhs_assertion = data[test_key]["test_section_loader_length"] + msg = f"Expected {expected_len} for test section loader length, received {lhs_assertion} instead, in {args.infile.split('.')[0]}" + assert lhs_assertion==expected_len, msg - # msg = f"Expected {expected_len} for test loader length, received {data[test_key]["test_loader_length"]} instead, in {args.infile.split('.')[0]}" - # assert data[test_key]["test_loader_length"]==expected_len, msg + lhs_assertion = data[test_key]["test_loader_length"] + msg = f"Expected {expected_len} for test loader length, received {lhs_assertion} instead, in {args.infile.split('.')[0]}" + assert lhs_assertion==expected_len, msg expected_n_classes = 2 - # msg = f"Expected {expected_n_classes} for test loader length, received {data[test_key]["n_classes"]} instead, in {args.infile.split('.')[0]}" - # assert data[test_key]["n_classes"]==expected_n_classes, msg + lhs_assertion = data[test_key]["n_classes"] + msg = f"Expected {expected_n_classes} for test loader length, received {lhs_assertion} instead, in {args.infile.split('.')[0]}" + assert lhs_assertion==expected_n_classes, msg expected_pred = (1, 200, 200) expected_gt = (1, 1, 200, 200) @@ -138,27 +141,27 @@ def main(args): expected_len = 1280 msg = f"Expected {expected_len} for validation patch loader length, received {data['validation_patch_loader_length']} instead, in {args.infile.split('.')[0]}" - assert data['validation_patch_loader_length']==expected_len + assert data['validation_patch_loader_length']==expected_len, msg expected_len = 64 msg = f"Expected {expected_len} for train subset length, received {data['train_length_subset']} instead, in {args.infile.split('.')[0]}" - assert data['train_length_subset']==expected_len + assert data['train_length_subset']==expected_len, msg expected_len = 32 msg = f"Expected {expected_len} for validation subset length, received {data['validation_length_subset']} instead, in {args.infile.split('.')[0]}" - assert data['validation_length_subset']==expected_len + assert data['validation_length_subset']==expected_len, msg expected_len = 55 #4 msg = f"Expected {expected_len} for train loader length, received {data['train_loader_length']} instead, in {args.infile.split('.')[0]}" - assert data['train_loader_length']==expected_len + assert data['train_loader_length']==expected_len, msg expected_len = 4 msg = f"Expected {expected_len} for train loader length, received {data['train_loader_length']} instead, in {args.infile.split('.')[0]}" - assert data['validation_loader_length']==1 + assert data['validation_loader_length']==1, msg expected_n_classes = 2 msg = f"Expected {expected_n_classes} for number of classes, received {data['n_classes']} instead, in {args.infile.split('.')[0]}" - assert data['n_classes']==expected_n_classes + assert data['n_classes']==expected_n_classes, msg logging.info("all done") From 2a72450154ecc56303fc4aeff0ba00d0980c199f Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Fri, 19 Jun 2020 19:15:10 +0000 Subject: [PATCH 18/19] final --- tests/cicd/src/check_data_flow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py index 0d17c2d5..af2ac637 100644 --- a/tests/cicd/src/check_data_flow.py +++ b/tests/cicd/src/check_data_flow.py @@ -151,13 +151,13 @@ def main(args): msg = f"Expected {expected_len} for validation subset length, received {data['validation_length_subset']} instead, in {args.infile.split('.')[0]}" assert data['validation_length_subset']==expected_len, msg - expected_len = 55 #4 + expected_len = 4 msg = f"Expected {expected_len} for train loader length, received {data['train_loader_length']} instead, in {args.infile.split('.')[0]}" assert data['train_loader_length']==expected_len, msg - expected_len = 4 + expected_len = 1 msg = f"Expected {expected_len} for train loader length, received {data['train_loader_length']} instead, in {args.infile.split('.')[0]}" - assert data['validation_loader_length']==1, msg + assert data['validation_loader_length']==expected_len, msg expected_n_classes = 2 msg = f"Expected {expected_n_classes} for number of classes, received {data['n_classes']} instead, in {args.infile.split('.')[0]}" From ebaf999fe71863f955aaa3fe47f528ff15bdf6f0 Mon Sep 17 00:00:00 2001 From: Fatemeh Date: Fri, 19 Jun 2020 19:50:38 +0000 Subject: [PATCH 19/19] trigger build --- tests/cicd/src/check_data_flow.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cicd/src/check_data_flow.py b/tests/cicd/src/check_data_flow.py index af2ac637..33d07741 100644 --- a/tests/cicd/src/check_data_flow.py +++ b/tests/cicd/src/check_data_flow.py @@ -117,7 +117,6 @@ def main(args): assert gt[1]<=expected_n_classes, msg - elif args.step=='train': if args.train_depth=='none': expected_shape_in = (200, 200, 200)