Skip to content
This repository has been archived by the owner on Nov 16, 2023. It is now read-only.

Commit

Permalink
Merge branch 'seresnet' of github.com:microsoft/seismic-deeplearning …
Browse files Browse the repository at this point in the history
…into seresnet
  • Loading branch information
maxkazmsft committed Jun 17, 2020
2 parents 9bdcc04 + 4a47d56 commit 2a58589
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 190 deletions.
11 changes: 9 additions & 2 deletions experiments/interpretation/dutchf3_patch/local/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,12 +152,19 @@ def run(*options, cfg=None, debug=False, input=None):
debug=debug,
)
logger.info(val_set)



if debug:
logger.info("Running in debug mode..")
train_set = data.Subset(train_set, range(config.TRAIN.BATCH_SIZE_PER_GPU * config.NUM_DEBUG_BATCHES))
val_set = data.Subset(val_set, range(config.VALIDATION.BATCH_SIZE_PER_GPU))
train_range = min(config.TRAIN.BATCH_SIZE_PER_GPU * config.NUM_DEBUG_BATCHES, len(train_set))
logging.info(f"train range in debug mode {train_range}")
train_set = data.Subset(train_set, range(train_range))
valid_range = min(config.VALIDATION.BATCH_SIZE_PER_GPU, len(val_set))
val_set = data.Subset(val_set, range(valid_range))



train_loader = data.DataLoader(
train_set, batch_size=config.TRAIN.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, shuffle=True
)
Expand Down
89 changes: 60 additions & 29 deletions scripts/gen_checkerboard.py → scripts/gen_synthetic_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,59 +127,87 @@ def main(args):

logging.info("loading data")

train_seismic = np.load(os.path.join(args.dataroot, "train", "train_seismic.npy"))
train_labels = np.load(os.path.join(args.dataroot, "train", "train_labels.npy"))
test1_seismic = np.load(os.path.join(args.dataroot, "test_once", "test1_seismic.npy"))
test1_labels = np.load(os.path.join(args.dataroot, "test_once", "test1_labels.npy"))
test2_seismic = np.load(os.path.join(args.dataroot, "test_once", "test2_seismic.npy"))
test2_labels = np.load(os.path.join(args.dataroot, "test_once", "test2_labels.npy"))

assert train_seismic.shape == train_labels.shape
assert train_seismic.min() == WHITE
assert train_seismic.max() == BLACK
assert train_labels.min() == 0
# this is the number of classes in Alaudah's Dutch F3 dataset
assert train_labels.max() == 5

assert test1_seismic.shape == test1_labels.shape
assert test1_seismic.min() == WHITE
assert test1_seismic.max() == BLACK
assert test1_labels.min() == 0
# this is the number of classes in Alaudah's Dutch F3 dataset
assert test1_labels.max() == 5

assert test2_seismic.shape == test2_labels.shape
assert test2_seismic.min() == WHITE
assert test2_seismic.max() == BLACK
assert test2_labels.min() == 0
# this is the number of classes in Alaudah's Dutch F3 dataset
assert test2_labels.max() == 5

# TODO: extend this to binary and gradient
if args.type!='checkerboard':
assert args.based_on=='dutch_f3'

logging.info(f"synthetic data generation based on {args.based_on}")


if args.based_on=='dutch_f3':

train_seismic = np.load(os.path.join(args.dataroot, "train", "train_seismic.npy"))
train_labels = np.load(os.path.join(args.dataroot, "train", "train_labels.npy"))
test1_seismic = np.load(os.path.join(args.dataroot, "test_once", "test1_seismic.npy"))
test1_labels = np.load(os.path.join(args.dataroot, "test_once", "test1_labels.npy"))
test2_seismic = np.load(os.path.join(args.dataroot, "test_once", "test2_seismic.npy"))
test2_labels = np.load(os.path.join(args.dataroot, "test_once", "test2_labels.npy"))

assert train_seismic.shape == train_labels.shape
assert train_seismic.min() == WHITE
assert train_seismic.max() == BLACK
assert train_labels.min() == 0
# this is the number of classes in Alaudah's Dutch F3 dataset
assert train_labels.max() == 5

assert test1_seismic.shape == test1_labels.shape
assert test1_seismic.min() == WHITE
assert test1_seismic.max() == BLACK
assert test1_labels.min() == 0
# this is the number of classes in Alaudah's Dutch F3 dataset
assert test1_labels.max() == 5

assert test2_seismic.shape == test2_labels.shape
assert test2_seismic.min() == WHITE
assert test2_seismic.max() == BLACK
assert test2_labels.min() == 0
# this is the number of classes in Alaudah's Dutch F3 dataset
assert test2_labels.max() == 5
elif args.based_on=='fixed_box_number':
logging.info(f"box_number is {args.box_number}")
logging.info(f"box_size is {args.box_size}")
# Note: this assumes the data is 3D, opening up higher dimensions, this (and other parts of this scrpit)
# must be refactored
synthetic_shape = (int(args.box_number * args.box_size),) * 3
train_seismic = np.ones(synthetic_shape, dtype=float)
train_labels = np.ones(synthetic_shape, dtype=int)

test1_seismic = train_seismic
test1_labels = train_labels
test2_seismic = train_seismic
test2_labels = train_labels


if args.type == "checkerboard":

logging.info("train checkerbox")
n_inlines, n_crosslines, n_depth = train_seismic.shape
checkerboard_train_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size)
checkerboard_train_seismic = checkerboard_train_seismic.astype(train_seismic.dtype)
checkerboard_train_labels = checkerboard_train_seismic.astype(train_labels.dtype)
# labels are integers and start from zero
checkerboard_train_labels[checkerboard_train_seismic < WHITE_LABEL] = WHITE_LABEL

logging.info(f"training data shape {checkerboard_train_seismic.shape}")
# create checkerbox
logging.info("test1 checkerbox")
n_inlines, n_crosslines, n_depth = test1_seismic.shape

checkerboard_test1_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size)
checkerboard_test1_seismic = checkerboard_test1_seismic.astype(test1_seismic.dtype)
checkerboard_test1_labels = checkerboard_test1_seismic.astype(test1_labels.dtype)
# labels are integers and start from zero
checkerboard_test1_labels[checkerboard_test1_seismic < WHITE_LABEL] = WHITE_LABEL

logging.info(f"test1 data shape {checkerboard_test1_seismic.shape}")
logging.info("test2 checkerbox")
n_inlines, n_crosslines, n_depth = test2_seismic.shape

checkerboard_test2_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size)
checkerboard_test2_seismic = checkerboard_test2_seismic.astype(test2_seismic.dtype)
checkerboard_test2_labels = checkerboard_test2_seismic.astype(test2_labels.dtype)
# labels are integers and start from zero
checkerboard_test2_labels[checkerboard_test2_seismic < WHITE_LABEL] = WHITE_LABEL
logging.info(f"test2 data shape {checkerboard_test2_seismic.shape}")

# substitute gradient dataset instead of checkerboard
elif args.type == "gradient":
Expand Down Expand Up @@ -257,10 +285,13 @@ def main(args):
BLACK_LABEL = BLACK
TYPES = ["checkerboard", "gradient", "binary"]
GRADIENT_DIR = ["inline", "crossline", "depth"]
METHODS = ['dutch_f3', 'fixed_box_number']

parser.add_argument("--dataroot", help="Root location of the input data", type=str, required=True)
parser.add_argument("--dataout", help="Root location of the output data", type=str, required=True)
parser.add_argument("--box_size", help="Size of the bounding box", type=int, required=False, default=100)
parser.add_argument("--based_on", help="This determines the shape of synthetic data array", type=str, required=False, choices=METHODS, default='dutch_f3')
parser.add_argument("--box_number", help="Number of boxes", type=int, required=False, default=2)
parser.add_argument(
"--type", help="Type of data to generate", type=str, required=False, choices=TYPES, default="checkerboard",
)
Expand Down
171 changes: 14 additions & 157 deletions tests/cicd/main_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,8 @@ jobs:
pids=
# export CUDA_VISIBLE_DEVICES=0
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \
'NUM_DEBUG_BATCHES' 50 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'NUM_DEBUG_BATCHES' 64 \
'TRAIN.END_EPOCH' 2 'TRAIN.SNAPSHOTS' 1 \
'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \
'TRAIN.DEPTH' 'none' \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \
Expand All @@ -153,7 +154,8 @@ jobs:
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=1
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \
'NUM_DEBUG_BATCHES' 10 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'NUM_DEBUG_BATCHES' 64 \
'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \
'TRAIN.DEPTH' 'section' \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \
Expand All @@ -162,7 +164,8 @@ jobs:
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=2
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \
'NUM_DEBUG_BATCHES' 50 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'NUM_DEBUG_BATCHES' 64 \
'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \
'TRAIN.DEPTH' 'section' \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \
Expand All @@ -171,7 +174,8 @@ jobs:
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=3
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \
'NUM_DEBUG_BATCHES' 5 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'NUM_DEBUG_BATCHES' 64 \
'TRAIN.END_EPOCH' 2 'TRAIN.SNAPSHOTS' 1 \
'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \
'TRAIN.DEPTH' 'section' \
'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \
Expand All @@ -194,7 +198,8 @@ jobs:
# check validation set performance
set -e
python ../../../../tests/cicd/src/check_performance.py --infile metrics_patch_deconvnet_no_depth.json
# TODO: enable this after investigating reproducibility problem of patch_deconvnet for small-size training data
# python ../../../../tests/cicd/src/check_performance.py --infile metrics_patch_deconvnet_no_depth.json
python ../../../../tests/cicd/src/check_performance.py --infile metrics_unet_section_depth.json
python ../../../../tests/cicd/src/check_performance.py --infile metrics_seresnet_unet_section_depth.json
# TODO: enable HRNet test set metrics when we debug HRNet
Expand Down Expand Up @@ -283,7 +288,8 @@ jobs:
# check test set performance
set -e
python ../../../../tests/cicd/src/check_performance.py --infile metrics_test_patch_deconvnet_no_depth.json --test
# TODO: enable this after investigating reproducibility problem of patch_deconvnet for small-size training data
# python ../../../../tests/cicd/src/check_performance.py --infile metrics_test_patch_deconvnet_no_depth.json --test
python ../../../../tests/cicd/src/check_performance.py --infile metrics_test_unet_section_depth.json --test
python ../../../../tests/cicd/src/check_performance.py --infile metrics_test_seresnet_unet_section_depth.json --test
# TODO: enable HRNet test set metrics when we debug HRNet
Expand All @@ -292,162 +298,13 @@ jobs:
echo "PASSED"
###################################################################################################
# Stage 3: Dutch F3 patch models: deconvnet, unet, HRNet patch depth, HRNet section depth
# CAUTION: reverted these builds to single-GPU leaving new multi-GPU code in to be reverted later
###################################################################################################

- job: dutchf3_patch
dependsOn: checkerboard_dutchf3_patch
timeoutInMinutes: 60
displayName: Dutch F3 patch local
pool:
name: deepseismicagentpool
steps:
- bash: |
source activate seismic-interpretation
# disable auto error handling as we flag it manually
set +e
cd experiments/interpretation/dutchf3_patch/local
# Create a temporary directory to store the statuses
dir=$(mktemp -d)
pids=
# export CUDA_VISIBLE_DEVICES=0
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'TRAIN.DEPTH' 'none' \
'TRAIN.BATCH_SIZE_PER_GPU' 2 'VALIDATION.BATCH_SIZE_PER_GPU' 2 \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \
'WORKERS' 1 \
--cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=1
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'TRAIN.DEPTH' 'section' \
'TRAIN.BATCH_SIZE_PER_GPU' 2 'VALIDATION.BATCH_SIZE_PER_GPU' 2 \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \
'WORKERS' 1 \
--cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=2
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'TRAIN.DEPTH' 'section' \
'TRAIN.BATCH_SIZE_PER_GPU' 2 'VALIDATION.BATCH_SIZE_PER_GPU' 2 \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \
'WORKERS' 1 \
--cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=3
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'TRAIN.DEPTH' 'section' \
'TRAIN.BATCH_SIZE_PER_GPU' 2 'VALIDATION.BATCH_SIZE_PER_GPU' 2 \
'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \
'WORKERS' 1 \
--cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
wait $pids || exit 1
# check if any of the models had an error during execution
# Get return information for each pid
for file in "$dir"/*; do
printf 'PID %d returned %d\n' "${file##*/}" "$(<"$file")"
[[ "$(<"$file")" -ne "0" ]] && exit 1 || echo "pass"
done
# Remove the temporary directory
rm -r "$dir"
echo "All models finished training - start scoring"
# Create a temporary directory to store the statuses
dir=$(mktemp -d)
pids=
# export CUDA_VISIBLE_DEVICES=0
# find the latest model which we just trained
# if we're running on a build VM
model_dir=$(ls -td output/patch_deconvnet/no_depth/* | head -1)
# if we're running in a checked out git repo
[[ -z ${model_dir} ]] && model_dir=$(ls -td output/$(git rev-parse --abbrev-ref HEAD)/*/patch_deconvnet/no_depth/* | head -1)
model=$(ls -t ${model_dir}/*.pth | head -1)
# try running the test script
{ python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \
'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'no_depth' \
'TEST.MODEL_PATH' ${model} \
'WORKERS' 1 \
--cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=1
# find the latest model which we just trained
# if we're running on a build VM
model_dir=$(ls -td output/unet/section_depth/* | head -1)
# if we're running in a checked out git repo
[[ -z ${model_dir} ]] && model_dir=$(ls -td output/$(git rev-parse --abbrev-ref HEAD)/*/unet/section_depth* | head -1)
model=$(ls -t ${model_dir}/*.pth | head -1)
# try running the test script
{ python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \
'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \
'TEST.MODEL_PATH' ${model} \
'WORKERS' 1 \
--cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=2
# find the latest model which we just trained
# if we're running on a build VM
model_dir=$(ls -td output/seresnet_unet/section_depth/* | head -1)
# if we're running in a checked out git repo
[[ -z ${model_dir} ]] && model_dir=$(ls -td output/$(git rev-parse --abbrev-ref HEAD)/*/seresnet_unet/section_depth/* | head -1)
model=$(ls -t ${model_dir}/*.pth | head -1)
# try running the test script
{ python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \
'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \
'TEST.MODEL_PATH' ${model} \
'WORKERS' 1 \
--cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
# export CUDA_VISIBLE_DEVICES=3
# find the latest model which we just trained
# if we're running on a build VM
model_dir=$(ls -td output/hrnet/section_depth/* | head -1)
# if we're running in a checked out git repo
[[ -z ${model_dir} ]] && model_dir=$(ls -td output/$(git rev-parse --abbrev-ref HEAD)/*/hrnet/section_depth/* | head -1)
model=$(ls -t ${model_dir}/*.pth | head -1)
# try running the test script
{ python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \
'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \
'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \
'TEST.MODEL_PATH' ${model} \
'WORKERS' 1 \
--cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
# wait for completion
wait $pids || exit 1
# check if any of the models had an error during execution
# Get return information for each pid
for file in "$dir"/*; do
printf 'PID %d returned %d\n' "${file##*/}" "$(<"$file")"
[[ "$(<"$file")" -ne "0" ]] && exit 1 || echo "pass"
done
# Remove the temporary directory
rm -r "$dir"
echo "PASSED"
###################################################################################################
# Stage 5: Notebook tests
# Stage 4: Notebook tests
###################################################################################################

- job: F3_block_training_and_evaluation_local_notebook
dependsOn: dutchf3_patch
dependsOn: checkerboard_dutchf3_patch
timeoutInMinutes: 5
displayName: F3 block training and evaluation local notebook
pool:
Expand Down
Loading

0 comments on commit 2a58589

Please sign in to comment.