From 194cf31ed5c4867e7b8665b4ffbb2c4a92956b02 Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Thu, 28 May 2020 11:30:09 -0400 Subject: [PATCH] fixed number of classes for main build tests (#327) * fixed number of classes for main build tests * corrected DATASET.ROOT in builds * added dev build script --- scripts/dev_build.py | 124 ++++++++++++++++++++++++++++++++++++++ tests/cicd/main_build.yml | 8 +-- 2 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 scripts/dev_build.py diff --git a/scripts/dev_build.py b/scripts/dev_build.py new file mode 100644 index 00000000..76b1d7e0 --- /dev/null +++ b/scripts/dev_build.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" Please see the def main() function for code description.""" + +""" libraries """ + +import numpy as np +import os +import sys +import yaml +import subprocess + +np.set_printoptions(linewidth=200) +import logging + +# toggle to WARNING when running in production, or use CLI +logging.getLogger().setLevel(logging.DEBUG) +# logging.getLogger().setLevel(logging.WARNING) +import argparse + +parser = argparse.ArgumentParser() + +""" useful information when running from a GIT folder.""" +myname = os.path.realpath(__file__) +mypath = os.path.dirname(myname) +myname = os.path.basename(myname) + +def main(args): + """ + + Runs main build jobs on your local VM. By default setup job is not run, + add --setup to run it (destroys existing environment and creates a new one, along with all the data) + + """ + + logging.info("loading data") + + with open(args.file) as file: + + list = yaml.load(file, Loader=yaml.FullLoader) + logging.info(f"Loaded {file}") + + # run single job + job_names = [x["job"] for x in list["jobs"]] if not args.job else args.job.split(',') + + if not args.setup and "setup" in job_names: + job_names.remove("setup") + + job_list = list["jobs"] + + # copy existing environment + current_env = os.environ.copy() + # modify for conda to work + # TODO: not sure why on DS VM this does not get picked up from the standard environment + current_env["PATH"] = PATH_PREFIX+":"+current_env["PATH"] + + for job in job_list: + job_name = job["job"] + if job_name not in job_names: + continue + + bash = job["steps"][0]["bash"] + + logging.info(f"Running job {job_name}") + + try: + completed = subprocess.run( + # 'set -e && source activate seismic-interpretation && which python && pytest --durations=0 cv_lib/tests/', + bash, + check=True, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + executable=current_env["SHELL"], + env=current_env, + cwd=os.getcwd() + ) + except subprocess.CalledProcessError as err: + logging.info(f'ERROR: \n{err}') + decoded_stdout = err.stdout.decode('utf-8') + log_file = "dev_build.latest_error.log" + logging.info(f"Have {len(err.stdout)} output bytes in {log_file}") + with open(log_file, 'w') as log_file: + log_file.write(decoded_stdout) + sys.exit() + else: + logging.info(f"returncode: {completed.returncode}") + logging.info(f"Have {len(completed.stdout)} output bytes: \n{completed.stdout.decode('utf-8')}") + + logging.info(f"Everything ran! You can try running the same jobs {job_names} on the build VM now") + +""" GLOBAL VARIABLES """ +PATH_PREFIX = "/data/anaconda/envs/seismic-interpretation/bin:/data/anaconda/bin" + +parser.add_argument( + "--file", help="Which yaml file you'd like to read which specifies build info", type=str, required=True +) +parser.add_argument( + "--job", help="CVS list of the job names which you would like to run", type=str, default=None, required=False +) +parser.add_argument("--setup", help="Add setup job", action="store_true") + +""" main wrapper with profiler """ +if __name__ == "__main__": + main(parser.parse_args()) + +# pretty printing of the stack +""" + try: + logging.info('before main') + main(parser.parse_args()) + logging.info('after main') + except: + for frame in traceback.extract_tb(sys.exc_info()[2]): + fname,lineno,fn,text = frame + print ("Error in %s on line %d" % (fname, lineno)) +""" +# optionally enable profiling information +# import cProfile +# name = +# cProfile.run('main.run()', name + '.prof') +# import pstats +# p = pstats.Stats(name + '.prof') +# p.sort_stats('cumulative').print_stats(10) +# p.sort_stats('time').print_stats() diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 827eddc4..3971407c 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -188,9 +188,9 @@ jobs: [[ -z ${model_dir} ]] && model_dir=$(ls -td output/$(git rev-parse --abbrev-ref HEAD)/*/patch_deconvnet/no_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script - { python test.py 'DATASET.ROOT' '/data/test/gradient/data' \ + { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'no_depth' \ - 'DATASET.NUM_CLASSES' 7 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0,1.0, 1.0,1.0, 1.0,1.0]' \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TEST.MODEL_PATH' ${model} \ 'WORKERS' 1 \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } @@ -203,9 +203,9 @@ jobs: [[ -z ${model_dir} ]] && model_dir=$(ls -td output/$(git rev-parse --abbrev-ref HEAD)/*/unet/section_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script - { python test.py 'DATASET.ROOT' '/data/test/binary/data' \ + { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ - 'DATASET.NUM_CLASSES' 7 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0,1.0, 1.0,1.0, 1.0,1.0]' \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TEST.MODEL_PATH' ${model} \ 'WORKERS' 1 \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; }