Merge pull request cms-sw#130 from gpetruc/CMG_MiniAOD_Lite_V6_0_from…

…-CMSSW_7_0_6

improvements to friend tree production, and validation scripts

CMGTools/TTHAnalysis/cfg/run_susyAlphaT_cfg.py

@@ -195,7 +195,7 @@
 #    comp.name = 'TTJets'
     #    comp.files = [ '/store/mc/Spring14miniaod/TT_Tune4C_13TeV-pythia8-tauola/MINIAODSIM/PU20bx25_POSTLS170_V5-v1/00000/063013AD-9907-E411-8135-0026189438BD.root' ]
 
-    comp.files = [ '/afs/cern.ch/user/m/mbaber/WORK/private/CSA14Samples/TT_Tune4C_13TeV-pythia8_PU20bx25.root' ]
+    comp.files = [ '/afs/cern.ch/user/m/mbaber/WORK/public/CSA14Samples/TT_Tune4C_13TeV-pythia8_PU20bx25.root' ]
 
     selectedComponents = [comp]
     comp.splitFactor = 1

CMGTools/TTHAnalysis/cfg/validate_multilep.sh

@@ -0,0 +1,6 @@
+X=$PWD/$1; 
+if test \! -d $X/TTHToWW_PUS14; then echo "Did not find TTHToWW_PUS14 in $X"; exit 1; fi
+test -L $PWD/Reference_TTHToWW_PUS14 || ln -sd $PWD/Reference_TTHToWW_PUS14 $X/ -v;
+( cd ../python/plotter; 
+  python mcPlots.py -f --s2v --tree treeProducerSusyMultilepton  -P $X bins/validation_mca.txt bins/validation.txt bins/validation_plots.txt --pdir plots/70X/validation -p ref_ttHWW,ttHWW -u -e --plotmode=nostack --showRatio --maxRatioRange 0.65 1.35 --flagDifferences
+)

CMGTools/TTHAnalysis/macros/prepareEventVariablesFriendTree.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 from CMGTools.TTHAnalysis.treeReAnalyzer import *
 from glob import glob
-import os.path
+import os.path, re
 
 MODULES = []
 
@@ -13,17 +13,17 @@
 #MODULES.append( ('3l_mva', FinalMVA_3L()) )
 #from CMGTools.TTHAnalysis.tools.bbvars import bbVars
 #MODULES.append( ('bbvars', bbVars()) )
-#from CMGTools.TTHAnalysis.tools.finalMVA_susy_2lss import FinalMVA_SUSY_2LSS
-#MODULES.append( ('finalMVA_susy_2lss', FinalMVA_SUSY_2LSS()) )
+from CMGTools.TTHAnalysis.tools.finalMVA_susy_2lss import FinalMVA_SUSY_2LSS
+MODULES.append( ('finalMVA_susy_2lss', FinalMVA_SUSY_2LSS()) )
 #from CMGTools.TTHAnalysis.tools.ttbarEventReco_2lss import TTEventReco_MC
 #MODULES.append( ('ttreco_mc', TTEventReco_MC()) )
 #from CMGTools.TTHAnalysis.tools.ttbarEventReco_2lss import TTEventReco
 #MODULES.append( ('ttreco', TTEventReco(sortersToUse={"BestGuess":"", "BestBySum4NoTJJb":"_bySum4"})) )
 #MODULES.append( ('ttreco', TTEventReco(sortersToUse={"BestGuess":"","ByGuessLL2B":"_byLL"})) )
-#from CMGTools.TTHAnalysis.tools.MuonMVAFriend import MuonMVAFriend
-#MODULES.append( ('', MuonMVAFriend("BPH",     "/afs/cern.ch/work/g/gpetrucc/micro/cmg/CMSSW_7_0_9/src/CMGTools/TTHAnalysis/macros/leptons/train70XBPH_BDTG.weights.xml", label="BPH")) )
-#MODULES.append( ('', MuonMVAFriend("BPHCalo", "/afs/cern.ch/work/g/gpetrucc/micro/cmg/CMSSW_7_0_9/src/CMGTools/TTHAnalysis/macros/leptons/train70XBPHCalo_BDTG.weights.xml", label="BPHCalo")) )
-#MODULES.append( ('', MuonMVAFriend("Full",    "/afs/cern.ch/work/g/gpetrucc/micro/cmg/CMSSW_7_0_9/src/CMGTools/TTHAnalysis/macros/leptons/train70XFull_BDTG.weights.xml", label="Full")) )
+from CMGTools.TTHAnalysis.tools.MuonMVAFriend import MuonMVAFriend
+MODULES.append( ('MuMVAId', MuonMVAFriend("BPH",     "/afs/cern.ch/work/g/gpetrucc/TREES_70X_240914/0_muMVAId_v1/train70XBPH_BDTG.weights.xml", label="BPH")) )
+MODULES.append( ('MuMVAId', MuonMVAFriend("BPHCalo", "/afs/cern.ch/work/g/gpetrucc/TREES_70X_240914/0_muMVAId_v1/train70XBPHCalo_BDTG.weights.xml", label="BPHCalo")) )
+MODULES.append( ('MuMVAId', MuonMVAFriend("Full",    "/afs/cern.ch/work/g/gpetrucc/TREES_70X_240914/0_muMVAId_v1/train70XFull_BDTG.weights.xml", label="Full")) )
 from CMGTools.TTHAnalysis.tools.LepMVAFriend import LepMVAFriend
 MODULES.append( ('LepMVAFriend', LepMVAFriend(("/afs/cern.ch/work/g/gpetrucc/TREES_70X_240914/0_lepMVA_v1/%s_BDTG.weights.xml",
                                                "/afs/cern.ch/work/g/gpetrucc/TREES_70X_240914/0_lepMVA_v1/%s_BDTG.weights.xml"))) )
@@ -64,7 +64,9 @@ def analyze(self,event):
 
 from optparse import OptionParser
 parser = OptionParser(usage="%prog [options] <TREE_DIR> <OUT>")
+parser.add_option("-m", "--modules", dest="modules",  type="string", default=[], action="append", help="Run these modules");
 parser.add_option("-d", "--dataset", dest="datasets",  type="string", default=[], action="append", help="Process only this dataset (or dataset if specified multiple times)");
+parser.add_option("-D", "--dm", "--dataset-match", dest="datasetMatches",  type="string", default=[], action="append", help="Process only this dataset (or dataset if specified multiple times): REGEXP");
 parser.add_option("-c", "--chunk",   dest="chunks",    type="int",    default=[], action="append", help="Process only these chunks (works only if a single dataset is selected with -d)");
 parser.add_option("-N", "--events",  dest="chunkSize", type="int",    default=500000, help="Default chunk size when splitting trees");
 parser.add_option("-j", "--jobs",    dest="jobs",      type="int",    default=1, help="Use N threads");
@@ -92,6 +94,11 @@ def analyze(self,event):
         short = os.path.basename(D)
         if options.datasets != []:
             if short not in options.datasets: continue
+        if options.datasetMatches != []:
+            found = False
+            for dm in  options.datasetMatches:
+                if re.match(dm,short): found = True
+            if not found: continue
         data = ("DoubleMu" in short or "MuEG" in short or "DoubleElectron" in short or "SingleMu" in short)
         f = ROOT.TFile.Open(fname);
         t = f.Get(options.tree)
@@ -122,6 +129,7 @@ def analyze(self,event):
     friendPost =  "".join(["  -F  %s %s " % (fn,ft) for fn,ft in options.friendTrees])
     friendPost += "".join([" --FM %s %s " % (fn,ft) for fn,ft in options.friendTreesMC])
     friendPost += "".join([" --FD %s %s " % (fn,ft) for fn,ft in options.friendTreesData])
+    friendPost += "".join(["  -m  '%s'  " % m for m in options.modules])
     for (name,fin,fout,data,range,chunk) in jobs:
         if chunk != -1:
             print "{base} -d {data} -c {chunk} {post}".format(base=basecmd, data=name, chunk=chunk, post=friendPost)
@@ -152,6 +160,14 @@ def _runIt(myargs):
         return (name,(nev,0))
     print "==== %s starting (%d entries) ====" % (name, nev)
     booker = Booker(fout)
+    modulesToRun = MODULES
+    if options.modules != []:
+        toRun = {}
+        for m,v in MODULES:
+            for pat in options.modules:
+                if re.match(pat,m):
+                    toRun[m] = True 
+        modulesToRun = [ (m,v) for (m,v) in MODULES if m in toRun ]
     el = EventLoop([ VariableProducer(options.treeDir,booker,MODULES), ])
     el.loop([tb], eventRange=range)
     booker.done()

CMGTools/TTHAnalysis/python/plotter/bins/validation.txt

@@ -0,0 +1,2 @@
+>= 2 good leptons: nLepGood10 >= 2
+pt2011: LepGood1_pt>20 && LepGood2_pt>10

CMGTools/TTHAnalysis/python/plotter/bins/validation_mca.txt

@@ -0,0 +1,2 @@
+ttHWW    : TTHToWW_PUS14 : 1 ; FillColor=ROOT.kOrange+10
+ref_ttHWW+ : Reference_TTHToWW_PUS14 : 1 ; FillColor=ROOT.kAzure+2, Label="Ref."

CMGTools/TTHAnalysis/python/plotter/bins/validation_plots.txt

@@ -0,0 +1,46 @@
+nLep: nLepGood: [1.5,2.5,3.5,4.5,5.5]; XTitle="N(lep)"
+nLep10: nLepGood10: [1.5,2.5,3.5,4.5,5.5]; XTitle="N(lep)"
+nLep20: nLepGood20: [1.5,2.5,3.5,4.5,5.5]; XTitle="N(lep)"
+met:   met_pt: 16,0,800    ; XTitle="E_{T}^{miss}  [GeV]"
+met_phi:   met_phi: 8,-3.15,3.15    ; XTitle="phi of E_{T}^{miss}"
+lep1pt:  LepGood1_pt: 15,0,250; XTitle="Leading lepton p_{T}  [GeV]", Legend='TR'
+lep2pt:  LepGood2_pt: 15,0,100; XTitle="Second lepton p_{T}  [GeV]", Legend='TR'
+lep3pt:  LepGood3_pt: 10,0,50; XTitle="Third lepton p_{T}  [GeV]", Legend='TR'
+lep4pt:  LepGood4_pt: 10,0,50; XTitle="Fourth lepton p_{T}  [GeV]", Legend='TR'
+lep1relIso03:  LepGood1_relIso03: 15,0,.5;  Legend='TR'
+lep2relIso03:  LepGood2_relIso03: 15,0,.5;  Legend='TR'
+lep1sip3d:  LepGood1_sip3d: 15,0,5;  Legend='TR'
+lep2sip3d:  LepGood2_sip3d: 15,0,5;  Legend='TR'
+lep1tightId:  LepGood1_tightId: [-0.5,0.5,1.5];  Legend='TL'
+lep2tightId:  LepGood2_tightId: [-0.5,0.5,1.5];  Legend='TL'
+lep1mvaTTH:  LepGood1_mvaTTH: 20,-1,1;  Legend='TL'
+lep2mvaTTH:  LepGood2_mvaTTH: 20,-1,1;  Legend='TL'
+lep1mcMatchId:  LepGood1_mcMatchId: 28,-1.5,26.5;  Legend='TR'
+lep2mcMatchId:  LepGood2_mcMatchId: 28,-1.5,26.5;  Legend='TR'
+lep1mcMatchAny:  LepGood1_mcMatchAny: [-1.5,-0.5,0.5,1.5,2.5,3.5];  Legend='TR'
+lep2mcMatchAny:  LepGood2_mcMatchAny: [-1.5,-0.5,0.5,1.5,2.5,3.5];  Legend='TR'
+nJet25 : nJet25 : [-0.5,0.5,1.5,2.5,3.5,4.5,5.5,6.5,7.5,8.5,9.5] ; XTitle='N(jet\, p_{T} > 25)'
+nJet40 : nJet40 : [-0.5,0.5,1.5,2.5,3.5,4.5,5.5,6.5,7.5,8.5,9.5] ; XTitle='N(jet\, p_{T} > 40)'
+jet1pt:  Jet1_pt: 15,0,400; XTitle="Leading jet p_{T}  [GeV]", Legend='TR'
+jet2pt:  Jet2_pt: 15,0,200; XTitle="Second jet p_{T}  [GeV]", Legend='TR'
+jet3pt:  Jet3_pt: 10,0,150; XTitle="Third jet p_{T}  [GeV]", Legend='TR'
+jet4pt:  Jet4_pt: 10,0,100; XTitle="Fourth jet p_{T}  [GeV]", Legend='TR'
+jet1rawPt:  Jet1_rawPt: 15,0,400;  Legend='TR'
+jet2rawPt:  Jet2_rawPt: 15,0,200;  Legend='TR'
+jet1btagCSV:  Jet1_btagCSV: 12,0.0,1.5;  Legend='TR'
+jet2btagCSV:  Jet2_btagCSV: 12,0.0,1.5;  Legend='TR'
+jet1mcPt:  Jet1_mcPt: 15,0,400;  Legend='TR'
+jet2mcPt:  Jet2_mcPt: 15,0,200;  Legend='TR'
+jet1mcFlavour:  Jet1_mcFlavour: 29,-6.5,22.5;  Legend='TR'
+jet2mcFlavour:  Jet2_mcFlavour: 29,-6.5,22.5;  Legend='TR'
+jet1mcMatchId:  Jet1_mcMatchId: 28,-1.5,26.5;  Legend='TR'
+jet2mcMatchId:  Jet2_mcMatchId: 28,-1.5,26.5;  Legend='TR'
+htJet25:   htJet25: 15,0,2000    ; XTitle="H_{T}(lep + jet p_{T} > 25)  [GeV]"
+htJet40:   htJet40: 15,0,2000    ; XTitle="H_{T}(lep + jet p_{T} > 40)  [GeV]"
+htJet40j:   htJet40j: 15,0,2000    ; XTitle="H_{T}(jet p_{T} > 40)  [GeV]"
+mhtJet25:   mhtJet25: 15,0,800    ; XTitle="H_{T}^{miss}(lep + jet p_{T} > 25)  [GeV]"
+mhtJet40:   mhtJet40: 15,0,800    ; XTitle="H_{T}^{miss}(lep + jet p_{T} > 40)  [GeV]"
+nBJet25 : nBJetMedium25 : [-0.5,0.5,1.5,2.5,3.5,4.5,5.5,6.5] ; XTitle='N(b-jet\, p_{T} > 25\, CSVM)'
+nBJet40 : nBJetMedium40 : [-0.5,0.5,1.5,2.5,3.5,4.5,5.5,6.5] ; XTitle='N(b-jet\, p_{T} > 40\, CSVM)'
+minMllAFAS: minMllAFAS: 20,0,200; XTitle="min m(ll')  [GeV]"
+mZ1: mZ1: 20,0,200; XTitle="best mZ4  [GeV]"

CMGTools/TTHAnalysis/python/plotter/mcPlots.py

@@ -275,10 +275,22 @@ def doNormFit(pspec,pmap,mca):
                         htot.SetBinError(b, hypot(htot.GetBinError(b), pmap[p].GetBinContent(b)*syst))
 
 def doRatioHists(pspec,pmap,total,totalSyst,maxRange,fitRatio=False):
-    if "data" not in pmap: return (None,None,None,None)
+    numkey = "data" 
+    if "data" not in pmap: 
+        if len(pmap) == 4 and 'signal' in pmap and 'background' in pmap:
+            # do this first
+            total.GetXaxis().SetLabelOffset(999) ## send them away
+            total.GetXaxis().SetTitleOffset(999) ## in outer space
+            total.GetYaxis().SetLabelSize(0.05)
+            # then we can overwrite total with background
+            numkey = 'signal'
+            total     = pmap['background']
+            totalSyst = pmap['background']
+        else:    
+            return (None,None,None,None)
     ratio = None
-    if hasattr(pmap['data'], 'poissonGraph'):
-        ratio = pmap["data"].poissonGraph.Clone("data_div"); 
+    if hasattr(pmap[numkey], 'poissonGraph'):
+        ratio = pmap[numkey].poissonGraph.Clone("data_div"); 
         for i in xrange(ratio.GetN()):
             x    = ratio.GetX()[i]
             div  = total.GetBinContent(total.GetXaxis().FindBin(x))
@@ -287,7 +299,7 @@ def doRatioHists(pspec,pmap,total,totalSyst,maxRange,fitRatio=False):
                                    ratio.GetErrorYlow(i)/div  if div > 0 else 0, 
                                    ratio.GetErrorYhigh(i)/div if div > 0 else 0) 
     else:
-        ratio = pmap["data"].Clone("data_div"); 
+        ratio = pmap[numkey].Clone("data_div"); 
         ratio.Divide(total)
     unity  = totalSyst.Clone("sim_div");
     unity0 = total.Clone("sim_div");
@@ -531,7 +543,7 @@ def run(self,mca,cuts,plots,makeStack=True,makeCanvas=True):
                 dir.WriteTObject(stack)
                 # 
                 if not makeCanvas and not self._options.printPlots: continue
-                doRatio = self._options.showRatio and 'data' in pmap and ("TH2" not in total.ClassName())
+                doRatio = self._options.showRatio and ('data' in pmap or (self._options.plotmode != "stack" and len(pmap) == 4)) and ("TH2" not in total.ClassName())
                 islog = pspec.hasOption('Logy'); 
                 # define aspect ratio
                 if doRatio: ROOT.gStyle.SetPaperSize(20.,25.)
@@ -614,6 +626,16 @@ def run(self,mca,cuts,plots,makeStack=True,makeCanvas=True):
                     if sfitnorm != None:
                         sfitnorm.SetDirectory(dir); dir.WriteTObject(sfitnorm)
                         reMax(total,sfitnorm,islog)
+                if options.flagDifferences and len(pmap) == 4:
+                    new = pmap['signal']
+                    ref = pmap['background']
+                    if "TH1" in new.ClassName():
+                        for b in xrange(1,new.GetNbinsX()+1):
+                            if new.GetBinContent(b) != ref.GetBinContent(b):
+                                print "Plot: difference found in %s, bin %d" % (pspec.name, b)
+                                p1.SetFillColor(ROOT.kYellow-10)
+                                if p2: p2.SetFillColor(ROOT.kYellow-10)
+                                break
                 if makeCanvas: dir.WriteTObject(c1)
                 rdata,rnorm,rnorm2,rline = (None,None,None,None)
                 if doRatio:
@@ -678,6 +700,7 @@ def addPlotMakerOptions(parser):
     parser.add_option("--poisson", dest="poisson", action="store_true", default=False, help="Draw Poisson error bars")
     parser.add_option("--select-plot", "--sP", dest="plotselect", action="append", default=[], help="Select only these plots out of the full file")
     parser.add_option("--exclude-plot", "--xP", dest="plotexclude", action="append", default=[], help="Exclude these plots from the full file")
+    parser.add_option("--flagDifferences", dest="flagDifferences", action="store_true", default=False, help="Flag plots that are different (when using only two processes, and plotmode nostack")
 
 if __name__ == "__main__":
     from optparse import OptionParser

CMGTools/TTHAnalysis/python/tools/MuonMVAFriend.py

@@ -52,7 +52,6 @@ def __call__(self,lep):
 
 class MuonMVAFriend:
     def __init__(self,name,path,label=""):
-        print path
         self.mva = MuonMVA(name,path) 
         self.label = label
     def listBranches(self):