fix: merge changes

scripts/dohadd.py

@@ -0,0 +1,25 @@
+import os, sys
+from glob import glob
+
+indir = sys.argv[1]
+
+systs = os.listdir(indir)
+
+outfile = open("hadd.sh", "w")
+
+for syst in systs:
+    roots = glob(f"{indir}/{syst}/*/*.root")
+    if len(roots) == 0:
+        print(f"Skipping {indir}/{syst}. Not the right directory structure.")
+        continue
+    samps = os.listdir(f"{indir}/{syst}")
+    for samp in samps:
+        if len(glob(f"{indir}/{syst}/{samp}/*.root")) == 0:
+            continue
+        outfile.write(
+            f"hadd -v 0 {indir}/{syst}/{samp}.root {indir}/{syst}/{samp}/*.root\n"
+        )
+
+print(
+    "Now run `parallel :::: hadd.sh` from an environment with ROOT installed. E.g. \nconda activate rootenv\nparallel :::: hadd.sh\nconda activate btv_coffea"
+)

scripts/fetch.py

@@ -62,6 +62,23 @@
     action="store_true",
     default=False,
 )
+parser.add_argument(
+    "-r",
+    "--redirector",
+    help="xrootd ridirector in case sites are not found",
+    choices=["infn", "fnal", "cern"],
+    default="infn",
+)
+parser.add_argument(
+    "-j", "--ncpus", help="Number of CPUs to use for validation", default="4"
+)
+parser.add_argument(
+    "--skipvalidation",
+    action="store_true",
+    help="If true, the readability of files will not be validated.",
+    default=False,
+)
+
 parser.add_argument("--campaign", help="campaign info", default=None, type=str)
 
 
@@ -210,9 +227,14 @@ def getFilesFromDas(args):
 
         if xrd is None:
             print(
-                f"No SITE available in the whitelist for file {dsname}, change to global redirector"
+                f"No SITE available in the whitelist for file {dsname}, change to global redirector: {args.redirector}"
             )
-            xrd = "root://xrootd-cms.infn.it//"
+            redirector = {
+                "infn": "root://xrootd-cms.infn.it//",
+                "fnal": "root://cmsxrootd.fnal.gov/",
+                "cern": "root://cms-xrd-global.cern.ch/",
+            }
+            xrd = redirector[args.redirector]
         if args.limit is not None:
             flist = flist[: args.limit]
         if dsname not in fdict:
@@ -335,7 +357,7 @@ def remove_bad_files(sample_dict, outname, remove_bad=True):
         _rmap = p_map(
             validate,
             sample_dict[sample],
-            num_cpus=4,
+            num_cpus=int(args.ncpus),
             desc=f"Validating {sample[:20]}...",
         )
 
@@ -419,7 +441,8 @@ def main(args):
             empty = False
     assert empty, "you have empty lists"
     output_file = "./%s" % (args.output)
-    # fdict = remove_bad_files(fdict, args.output, True)  # remove bad files
+    if not args.skipvalidation:
+        fdict = remove_bad_files(fdict, args.output, True)  # remove bad files
     with open(output_file, "w") as fp:
         json.dump(fdict, fp, indent=4)
         print("The file is saved at: ", output_file)

scripts/plotdataMC.py

@@ -1,5 +1,5 @@
 import numpy as np
-import argparse, os, arrow, glob, re
+import argparse, os, arrow, glob, re, sys
 from coffea.util import load
 import matplotlib.pyplot as plt
 from matplotlib.offsetbox import AnchoredText
@@ -195,8 +195,12 @@
 else:
     var_set = args.variable.split(",")
 for index, discr in enumerate(var_set):
-    if not isinstance(collated["mc"][discr], hist.hist.Hist):
-        continue
+    try:
+        if not isinstance(collated["mc"][discr], hist.hist.Hist):
+            continue
+    except:
+        print(f"{discr} not found. Variable must be in", collated["mc"].keys())
+        sys.exit(1)
     ## remove empty
     if (
         discr not in collated["mc"].keys()

src/BTVNanoCommissioning/data/lumiMasks/Cert_Collisions2022_eraCD_355862_357900_Golden.json

@@ -0,0 +1,178 @@
+{
+  "355862": [[121, 133]],
+  "355863": [[1, 14]],
+  "355870": [[31, 67]],
+  "355871": [[1, 5]],
+  "355872": [
+    [1, 738],
+    [758, 995],
+    [997, 1217]
+  ],
+  "355892": [[14, 197]],
+  "355912": [[43, 200]],
+  "355913": [[1, 106]],
+  "355921": [[38, 442]],
+  "355933": [[75, 448]],
+  "355942": [[24, 189], [193, 213]],
+  "355988": [[43, 80], [85, 90]],
+  "355989": [[1, 24]],
+  "355998": [[1, 35]],
+  "355999": [[1, 9]],
+  "356004": [[1, 19]],
+  "356005": [[1, 187]],
+  "356043": [[1, 65]],
+  "356071": [[37, 191]],
+  "356074": [[1, 26]],
+  "356075": [[1, 125]],
+  "356076": [[1, 153]],
+  "356077": [[1, 472]],
+  "356135": [[46, 71]],
+  "356309": [[61, 184]],
+  "356316": [[45, 185]],
+  "356322": [[1, 19]],
+  "356323": [[1, 67], [69, 650]],
+  "356371": [[41, 50], [67, 72]],
+  "356375": [[35, 77], [101, 125]],
+  "356378": [
+    [8, 208],
+    [210, 219],
+    [221, 304]
+  ],
+  "356381": [[1, 1193]],
+  "356383": [[1, 33]],
+  "356385": [[1, 30]],
+  "356386": [[1, 122]],
+  "356426": [[39, 60]],
+  "356428": [[1, 300]],
+  "356433": [[1, 310]],
+  "356434": [[1, 13]],
+  "356435": [[1, 3], [8, 8]],
+  "356446": [[10, 623]],
+  "356523": [[32, 410], [412, 898]],
+  "356531": [[1, 56]],
+  "356563": [
+    [36, 113],
+    [117, 164],
+    [168, 177],
+    [181, 191],
+    [193, 194],
+    [199, 343]
+  ],
+  "356568": [[42, 64]],
+  "356569": [[1, 251]],
+  "356570": [[1, 98]],
+  "356576": [[58, 240]],
+  "356578": [[1, 865]],
+  "356580": [[1, 51]],
+  "356582": [[7, 104]],
+  "356614": [[1, 10], [16, 19], [27, 62]],
+  "356615": [[1, 1297]],
+  "356619": [[1, 173]],
+  "356810": [[44, 163]],
+  "356811": [[1, 44]],
+  "356812": [[1, 107]],
+  "356813": [[1, 54]],
+  "356814": [
+    [1, 305],
+    [307, 309],
+    [311, 366],
+    [368, 672]
+  ],
+  "356815": [[1, 54], [176, 219]],
+  "356824": [[1, 66]],
+  "356908": [[1, 26]],
+  "356919": [[29, 116]],
+  "356937": [[20, 138]],
+  "356946": [[1, 129]],
+  "356947": [[1, 350]],
+  "356948": [[1, 88]],
+  "356949": [[1, 94]],
+  "356951": [[1, 274]],
+  "356954": [[1, 364]],
+  "356955": [[1, 380]],
+  "356956": [[1, 109]],
+  "356968": [[81, 252]],
+  "356969": [[1, 236]],
+  "356970": [[1, 366]],
+  "356998": [[1, 5]],
+  "356999": [[1, 58]],
+  "357000": [[1, 50]],
+  "357001": [[1, 183]],
+  "357079": [[1, 22]],
+  "357080": [[1, 616]],
+  "357081": [[1, 759]],
+  "357101": [[54, 103]],
+  "357102": [[1, 13], [43, 134]],
+  "357104": [[1, 4]],
+  "357106": [[1, 60]],
+  "357112": [[1, 519]],
+  "357268": [[70, 143]],
+  "357271": [[1, 20], [22, 1570]],
+  "357328": [[44, 105]],
+  "357329": [[1, 668]],
+  "357330": [[1, 157]],
+  "357331": [[1, 23]],
+  "357332": [[1, 430]],
+  "357333": [[1, 207]],
+  "357401": [[48, 664]],
+  "357406": [[50, 174]],
+  "357438": [[35, 230]],
+  "357440": [[1, 354]],
+  "357441": [[1, 83]],
+  "357442": [[1, 1373]],
+  "357447": [[40, 50]],
+  "357472": [[34, 60]],
+  "357478": [[43, 50]],
+  "357479": [[1, 1046]],
+  "357482": [[1, 5], [21, 220]],
+  "357538": [[39, 63]],
+  "357542": [[1, 11], [13, 252]],
+  "357550": [[1, 36]],
+  "357610": [[63, 253]],
+  "357611": [[1, 412]],
+  "357612": [[1, 736]],
+  "357613": [[1, 256]],
+  "357688": [[1, 380]],
+  "357696": [[31, 319], [341, 410]],
+  "357697": [[1, 39]],
+  "357698": [[1, 63]],
+  "357699": [[1, 30]],
+  "357700": [[1, 757]],
+  "357701": [[1, 310]],
+  "357705": [[1, 202]],
+  "357706": [[1, 161]],
+  "357720": [[32, 92]],
+  "357732": [[30, 157]],
+  "357734": [[1, 300]],
+  "357735": [
+    [1, 387],
+    [389, 821],
+    [823, 947],
+    [949, 1126]
+  ],
+  "357754": [[29, 145]],
+  "357756": [[1, 425]],
+  "357757": [[1, 9]],
+  "357758": [[1, 85]],
+  "357759": [[1, 70]],
+  "357766": [[10, 124]],
+  "357777": [[1, 85]],
+  "357778": [[1, 359]],
+  "357779": [[1, 74]],
+  "357781": [[1, 10]],
+  "357802": [[43, 206]],
+  "357803": [[1, 153]],
+  "357804": [[1, 23]],
+  "357805": [[1, 88]],
+  "357806": [[1, 56]],
+  "357807": [[1, 244]],
+  "357808": [[1, 17]],
+  "357809": [[1, 41]],
+  "357812": [[1, 50]],
+  "357813": [[1, 293]],
+  "357814": [[1, 212]],
+  "357815": [[1, 944]],
+  "357898": [[1, 313]],
+  "357899": [[1, 637]],
+  "357900": [[1, 516]]
+}