-
Notifications
You must be signed in to change notification settings - Fork 1
/
delphes.pbs
63 lines (51 loc) · 2.04 KB
/
delphes.pbs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#!/bin/bash
# Set the number of processing elements (PEs) or cores
# Set the number of PEs per node
# xe is for normal nodes, xk is for GPU nodes
#PBS -l nodes=1:ppn=16:xe
# Set the wallclock time
#PBS -l walltime=3:00:00
# Use shifter queue
#PBS -l gres=shifter
# Set the PBS_JOBNAME
#PBS -N delphes
# Set the job stdout and stderr
#PBS -e "${PBS_JOBNAME}.${PBS_JOBID}.err"
#PBS -o "${PBS_JOBNAME}.${PBS_JOBID}.out"
# Set email notification on termination or abort
#PBS -m ea
#PBS -M matthew.feickert@cern.ch
# Set allocation to charge
#PBS -A bbdz
# Ensure shifter enabled
module load shifter
PHYSICS_PROCESS="drell-yan"
OUTPUT_BASE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/${PBS_JOBNAME}"
OUTPUT_PATH="${OUTPUT_BASE_PATH}/${PBS_JOBID}"
mkdir -p "${OUTPUT_PATH}"
INPUT_FILE_PATH="/mnt/c/scratch/sciteam/${USER}/${PHYSICS_PROCESS}/madgraph/nevents_10e4_run/Events/run_01/tag_1_pythia8_events.hepmc"
# $HOME is /u/sciteam/${USER}
SHIFTER_IMAGE="scailfin/delphes-python-centos:3.5.0"
shifterimg pull "${SHIFTER_IMAGE}"
# The need to edit the contents of LD_LIBRARY_PATH is to remove NVIDIA libraries
# that place versions of libOpenGL in LD_LIBRARY_PATH that conflict with the
# Delphes Docker image and give a symbol lookup error.
# c.f. https://bluewaters.ncsa.illinois.edu/shifter#remarks-on-running-apps
# c.f. https://jira.ncsa.illinois.edu/browse/BWAPPS-7234
aprun \
--bypass-app-transfer \
--pes-per-node 1 \
--cpu-binding none \
-- shifter \
--clearenv \
--image="${SHIFTER_IMAGE}" \
--volume="${OUTPUT_BASE_PATH}":/root/data \
--volume=/mnt/a/"${HOME}":/mnt/a/"${HOME}" \
--workdir=/root/data \
-- /bin/bash -c 'export LD_LIBRARY_PATH=$(echo -e "${LD_LIBRARY_PATH//\:/\\n}" | grep -v /opt/cray/nvidia/390.46-1_1.0502.2481.1.1.gem/lib64 | tr "\n" ":") && \
export PATH="/usr/local/venv/bin:${PATH}" && \
printf "\n# printenv:\n" && printenv && printf "\n\n" && \
DelphesHepMC2 \
/usr/local/venv/cards/delphes_card_ATLAS.tcl \
delphes_output.root \
'"${INPUT_FILE_PATH}"