Skip to content
This repository has been archived by the owner on Nov 16, 2023. It is now read-only.

Correctness single GPU switch #290

Merged
merged 7 commits into from
Apr 29, 2020
Merged

Correctness single GPU switch #290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d0aa884
resolved rebase conflict
maxkazmsft Apr 20, 2020
8ed4af6
resolved merge conflict
maxkazmsft Apr 16, 2020
5641445
resolved rebase conflict
maxkazmsft Apr 20, 2020
19abb24
resolved merge conflict
maxkazmsft Apr 16, 2020
2332aae
resolved merge conflict
maxkazmsft Apr 22, 2020
784e3aa
resolved merge confict 2
maxkazmsft Apr 28, 2020
d5f3a1a
reverted multi-GPU builds to run on single GPU
maxkazmsft Apr 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 0 additions & 34 deletions contrib/experiments/interpretation/penobscot/local/logging.conf
View file
Open in desktop

This file was deleted.

33 changes: 17 additions & 16 deletions tests/cicd/main_build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ jobs:

###################################################################################################
# Stage 3: Dutch F3 patch models: deconvnet, unet, HRNet patch depth, HRNet section depth
# CAUTION: reverted these builds to single-GPU leaving new multi-GPU code in to be reverted later
###################################################################################################

- job: dutchf3_patch
Expand All @@ -113,30 +114,30 @@ jobs:
dir=$(mktemp -d)

pids=
export CUDA_VISIBLE_DEVICES=0
# export CUDA_VISIBLE_DEVICES=0
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'TRAIN.DEPTH' 'none' \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \
--cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } &
--cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
export CUDA_VISIBLE_DEVICES=1
# export CUDA_VISIBLE_DEVICES=1
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'TRAIN.DEPTH' 'section' \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \
--cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } &
--cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
export CUDA_VISIBLE_DEVICES=2
# export CUDA_VISIBLE_DEVICES=2
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'TRAIN.DEPTH' 'section' \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \
--cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } &
--cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
export CUDA_VISIBLE_DEVICES=3
# export CUDA_VISIBLE_DEVICES=3
{ python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \
'TRAIN.DEPTH' 'section' \
'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \
'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \
--cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } &
--cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"

wait $pids || exit 1
Expand All @@ -157,36 +158,36 @@ jobs:
dir=$(mktemp -d)

pids=
export CUDA_VISIBLE_DEVICES=0
# export CUDA_VISIBLE_DEVICES=0
# find the latest model which we just trained
model_dir=$(ls -td output/patch_deconvnet/no_depth/* | head -1)
model=$(ls -t ${model_dir}/*.pth | head -1)
# try running the test script
{ python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \
'TEST.MODEL_PATH' ${model} \
--cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } &
--cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
export CUDA_VISIBLE_DEVICES=1
# export CUDA_VISIBLE_DEVICES=1
# find the latest model which we just trained
model_dir=$(ls -td output/unet/section_depth/* | head -1)
model=$(ls -t ${model_dir}/*.pth | head -1)

# try running the test script
{ python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \
'TEST.MODEL_PATH' ${model} \
--cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } &
--cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
export CUDA_VISIBLE_DEVICES=2
# export CUDA_VISIBLE_DEVICES=2
# find the latest model which we just trained
model_dir=$(ls -td output/seresnet_unet/section_depth/* | head -1)
model=$(ls -t ${model_dir}/*.pth | head -1)

# try running the test script
{ python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \
'TEST.MODEL_PATH' ${model} \
--cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } &
--cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"
export CUDA_VISIBLE_DEVICES=3
# export CUDA_VISIBLE_DEVICES=3
# find the latest model which we just trained
model_dir=$(ls -td output/hrnet/section_depth/* | head -1)
model=$(ls -t ${model_dir}/*.pth | head -1)
Expand All @@ -195,7 +196,7 @@ jobs:
{ python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \
'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \
'TEST.MODEL_PATH' ${model} \
--cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } &
--cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }
pids+=" $!"

# wait for completion
Expand Down