diff --git a/Production/crab/configs/2018/QCD.txt b/Production/crab/configs/2018/QCD.txt index ae5e0c9d371..cc4d2e56ad0 100644 --- a/Production/crab/configs/2018/QCD.txt +++ b/Production/crab/configs/2018/QCD.txt @@ -18,8 +18,8 @@ QCD_Pt_1400to1800_ext1 /QCD_Pt_1400to1800_TuneCP5_13TeV_pythia8/RunIIAutumn18Min QCD_Pt_1800to2400 /QCD_Pt_1800to2400_TuneCP5_13TeV_pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1/MINIAODSIM QCD_Pt_1800to2400_ext1 /QCD_Pt_1800to2400_TuneCP5_13TeV_pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15_ext1-v2/MINIAODSIM QCD_Pt_2400to3200 /QCD_Pt_2400to3200_TuneCP5_13TeV_pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1/MINIAODSIM +QCD_Pt_2400to3200_ext1 /QCD_Pt_2400to3200_TuneCP5_13TeV_pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15_ext1-v2/MINIAODSIM QCD_Pt_3200toInf /QCD_Pt_3200toInf_TuneCP5_13TeV_pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1/MINIAODSIM -QCD_Pt_3200toInf_ext1 /QCD_Pt_2400to3200_TuneCP5_13TeV_pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15_ext1-v2/MINIAODSIM QCD_Pt_15to7000_Flat2018_ext1 /QCD_Pt-15to7000_TuneCP5_Flat2018_13TeV_pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15_ext1-v1/MINIAODSIM QCD_Pt_15to7000_Flat_ext1 /QCD_Pt-15to7000_TuneCP5_Flat_13TeV_pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15_ext1-v1/MINIAODSIM diff --git a/Production/scripts/move_crab_outputs.py b/Production/scripts/move_crab_outputs.py new file mode 100644 index 00000000000..8f7c101b380 --- /dev/null +++ b/Production/scripts/move_crab_outputs.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# Move crab outputs into the permanent location. + +import argparse +import os +import shutil +import sys + +parser = argparse.ArgumentParser(description='Move crab outputs into the permanent location.', + formatter_class = lambda prog: argparse.HelpFormatter(prog,width=90)) +parser.add_argument('--input', required=True, type=str, help="crab outputs") +parser.add_argument('--output', required=True, type=str, help="new location") +parser.add_argument('--dry-run', action="store_true", required=False, help="Do not perform actual moving.") +args = parser.parse_args() + +dry_run_msg = '' +if args.dry_run: + dry_run_msg = ' (DRY RUN) ' +print('Moving crab outputs from "{}" to "{}"{}'.format(args.input, args.output, dry_run_msg)) + +error_flag = False +def report_error(message): + global error_flag + print("ERROR: {}".format(message)) + error_flag = True + +class Dataset: + job_name_prefix = 'crab_' + def __init__(self, input_path): + self.base_path, self.job_name = os.path.split(input_path) + if not self.job_name.startswith(Dataset.job_name_prefix): + raise RuntimeError('Unexpected job name "{}" in "{}"'.format(self.job_name, self.base_path)) + self.name = self.job_name[len(Dataset.job_name_prefix):] + job_id_dirs = os.listdir(input_path) + if len(job_id_dirs) == 0: + raise RuntimeError('No crab jobs were found for "{}"'.format(input_path)) + if len(job_id_dirs) > 1: + raise RuntimeError('Multiple crab jobs were found for "{}"'.format(input_path)) + job_id_dir = job_id_dirs[0] + job_id_dir_path = os.path.join(input_path, job_id_dir) + self.root_files = {} + for chunk_dir in os.listdir(job_id_dir_path): + chunk_dir_path = os.path.join(job_id_dir_path, chunk_dir) + if not os.path.isdir(chunk_dir_path): + raise RuntimeError('Unexpected file "{}" in "{}"'.format(chunk_dir, job_id_dir_path)) + for root_file in os.listdir(chunk_dir_path): + root_file_path = os.path.join(chunk_dir_path, root_file) + if not os.path.isfile(root_file_path) or not root_file.endswith('.root'): + raise RuntimeError('Unexpected file "{}" in "{}"'.format(root_file, chunk_dir_path)) + if root_file in self.root_files: + raise RuntimeError('Duplicated file "{}" in "{}"'.format(root_file, input_path)) + self.root_files[root_file] = root_file_path + +datasets = {} +for process_dir in sorted(os.listdir(args.input)): + process_dir_path = os.path.join(args.input, process_dir) + if not os.path.isdir(process_dir_path): + report_error('Unexpected file "{}" in "{}"'.format(process_dir, args.input)) + continue + for job_dir in sorted(os.listdir(process_dir_path)): + job_dir_path = os.path.join(process_dir_path, job_dir) + if not os.path.isdir(job_dir_path): + report_error('Unexpected file "{}" in "{}"'.format(job_dir, process_dir_path)) + continue + try: + ds = Dataset(job_dir_path) + if ds.name in datasets: + report_error('Duplicated dataset name = "".'.format(ds.name)) + datasets[ds.name] = ds + except RuntimeError as e: + report_error(e.message) + +if error_flag: + print("Stopping due to errors.") + sys.exit(1) + +try: + for ds_name, ds in datasets.items(): + print('Moving {}...'.format(ds_name)), + ds_output = os.path.join(args.output, ds_name) + if os.path.exists(ds_output): + raise RuntimeError('Output path "{}" already exists'.format(ds_output)) + if not args.dry_run: + os.makedirs(ds_output) + for root_file, root_file_path in ds.root_files.items(): + root_file_output_path = os.path.join(ds_output, root_file) + shutil.move(root_file_path, root_file_output_path) + print('done') +except RuntimeError as e: + report_error(e.message)