Merge pull request lsoffi#2 from tomcornelis/RunIIfinal_lfincoMerge

Run i ifinal lfinco merge

crab/submitWithVariableThresholds.py

@@ -0,0 +1,125 @@
+#!/bin/env python
+
+### Version of submission ###
+submitVersion = "L1Matching"
+
+
+
+defaultArgs   = ['isMC=False','doEleID=False','doPhoID=False','doTrigger=True', 'GT=101X_dataRun2_Prompt_v9']
+#mainOutputDir = '/store/user/tomc/tnpTuples/%s' % submitVersion # Change to your own
+mainOutputDir = '/store/group/phys_egamma/lfinco/new/2018/%s' % submitVersion
+from WMCore.Configuration import Configuration
+from CRABClient.UserUtilities import config
+config = config()
+
+config.General.requestName            = ''
+config.General.transferLogs           = False
+config.JobType.pluginName             = 'Analysis'
+
+config.JobType.psetName               = '../../python/TnPTreeProducer_cfg.py'
+config.JobType.sendExternalFolder     = True
+
+config.Data.inputDataset              = ''
+config.Data.inputDBS                  = 'global'
+config.Data.publication               = False
+config.Data.allowNonValidInputDataset = True
+config.Site.storageSite               = 'T2_CH_CERN'
+
+
+if __name__ == '__main__':
+
+  import urllib, os, glob
+  def download(url, destination):
+    print 'Downloading from %s' % url
+    try:    os.makedirs(destination)
+    except: pass
+    urllib.urlretrieve(url, os.path.join(destination, url.split('/')[-1]))
+
+  from FWCore.PythonUtilities.LumiList import LumiList
+  def subtractLumis(json, jsonToSubtract):
+    print 'Subtracting %s from %s' % (jsonToSubtract, json)
+    lumis = LumiList(filename = json) - LumiList(filename = jsonToSubtract)
+    lumis.writeJSON(fileName=json)
+
+  def mergeLumis(json, jsonToMerge):
+    print 'Merging %s into %s' % (jsonToMerge, json)
+    lumis = LumiList(filename = json) + LumiList(filename = jsonToMerge)
+    lumis.writeJSON(fileName=json)
+
+  def getSeedsForDoubleEle(year): # note: only getting DoubleEle seeds here
+    prescalePage = 'https://tomc.web.cern.ch/tomc/triggerPrescales/%s/' % year
+    hltTrigger   = 'HLT_Ele23_Ele12_CaloIdL_TrackIdL_IsoVL'
+    dirToStore   = os.path.join('prescaleInformation', year, hltTrigger)
+    download(prescalePage + hltTrigger + '.php', dirToStore)
+    with open(os.path.join(dirToStore, hltTrigger + '.php')) as f:
+      for line in f:
+        if 'prescale1' in line and 'L1_DoubleEG' in line:
+          download(prescalePage + line.split('>')[0].split('=')[-1], dirToStore)
+
+    jsonForThresholds = {}
+    for json in glob.glob(os.path.join(dirToStore, '*.json')):
+      leg1 = int(json.split('L1_DoubleEG_')[-1].split('_')[0].replace('LooseIso', ''))
+      leg2 = int(json.split('L1_DoubleEG_')[-1].split('_')[1].replace('LooseIso', ''))
+      if (leg1, leg2) in jsonForThresholds: mergeLumis(jsonForThresholds[(leg1, leg2)], json) # this theshold pair already exists, so we merge them into the existing one
+      else:                                 jsonForThresholds[(leg1, leg2)] = json
+
+    thresholdsToSubtract = []
+    for thresholds in sorted(jsonForThresholds.keys()):  # sorting from low to high thresholds
+      print
+      print 'Preparing json for thresholds %s' % str(thresholds)
+      json = jsonForThresholds[thresholds]
+      for t in thresholdsToSubtract:
+        subtractLumis(json, jsonForThresholds[t])
+      if not len(LumiList(filename = json)):
+        print "empty json"
+        continue
+      yield thresholds[0], thresholds[1], json
+      thresholdsToSubtract.append(thresholds)
+
+
+
+  from CRABAPI.RawCommand import crabCommand
+  from CRABClient.ClientExceptions import ClientException
+  from httplib import HTTPException
+
+  # We want to put all the CRAB project directories from the tasks we submit here into one common directory.
+  # That's why we need to set this parameter (here or above in the configuration file, it does not matter, we will not overwrite it).
+  config.General.workArea = 'crab_%s' % submitVersion
+
+  def submit(config, sample, leg1threshold, leg2threshold, json):
+    print  sample.split('/')[-2] + '_%s' % json.split('L1_DoubleEG_')[-1].split('_prescale')[0]
+    config.General.requestName  = sample.split('/')[-2] + '_%s' % json.split('L1_DoubleEG_')[-1].split('_prescale')[0]+'_retry'
+    config.Data.inputDataset    = sample
+    config.Data.outLFNDirBase   = '%s/%s/' % (mainOutputDir,'data')
+    config.Data.splitting       = 'LumiBased'
+    config.Data.lumiMask        = json
+    config.Data.unitsPerJob     = 100
+    config.JobType.pyCfgParams  = defaultArgs + ['L1Threshold=%s' % leg1threshold]
+
+    try:
+      crabCommand('submit', config = config)
+    except HTTPException as hte:
+      print "Failed submitting task: %s" % (hte.headers)
+    except ClientException as cle:
+      print "Failed submitting task: %s" % (cle)
+
+  from multiprocessing import Process
+  def submitWrapper(config, sample, leg1threshold, leg2threshold, json):
+    p = Process(target=submit, args=(config, sample, leg1threshold, leg2threshold, json))
+    p.start()
+    p.join()
+
+
+  for leg1, leg2, json in getSeedsForDoubleEle('2018'):
+    print 'Submitting for (%s, %s)' % (leg1, leg2)
+    #print LumiList(filename = json)
+    # Crab fails on this on second iteration, of course with only a very cryptic error message
+    # Not sure how to workaround this
+    submit(config, '/EGamma/Run2018A-17Sep2018-v2/MINIAOD', leg1, leg2, json)
+    submit(config, '/EGamma/Run2018B-17Sep2018-v1/MINIAOD', leg1, leg2, json)
+    submit(config, '/EGamma/Run2018C-17Sep2018-v1/MINIAOD', leg1, leg2, json)
+    submit(config, '/EGamma/Run2018D-PromptReco-v2/MINIAOD', leg1, leg2, json)
+
+
+
+

crab/submitWithVariableThresholds_MC.py

@@ -0,0 +1,97 @@
+#!/bin/env python
+
+### Version of submission ###
+submitVersion = "L1Matching"
+
+
+
+defaultArgs   = ['isMC=True','doEleID=False','doPhoID=False','doTrigger=True', 'GT=102X_upgrade2018_realistic_v12']
+#mainOutputDir = '/store/user/tomc/tnpTuples/%s' % submitVersion # Change to your own
+mainOutputDir = '/store/group/phys_egamma/lfinco/new/2018/%s' % submitVersion
+from WMCore.Configuration import Configuration
+from CRABClient.UserUtilities import config
+config = config()
+
+config.General.requestName            = '' 
+config.General.transferLogs           = False
+config.JobType.pluginName             = 'Analysis'
+
+config.JobType.psetName               = '../../python/TnPTreeProducer_cfg.py'
+config.JobType.sendExternalFolder     = True
+
+config.Data.inputDataset              = ''
+config.Data.inputDBS                  = 'global'
+config.Data.publication               = False
+config.Data.allowNonValidInputDataset = True
+config.Site.storageSite               = 'T2_CH_CERN'#'T2_BE_IIHE' 
+
+
+
+if __name__ == '__main__':
+
+  import urllib, os, glob
+  def download(url, destination):
+    try:    os.makedirs(destination)
+    except: pass
+    urllib.urlretrieve(url, os.path.join(destination, url.split('/')[-1])) 
+
+  def getSeedsForDoubleEle(year): # note: only getting DoubleEle seeds here
+    prescalePage = 'https://tomc.web.cern.ch/tomc/triggerPrescales/%s/' % year
+    hltTrigger   = 'HLT_Ele23_Ele12_CaloIdL_TrackIdL_IsoVL'
+    dirToStore   = os.path.join('prescaleInformation', year, hltTrigger)
+    download(prescalePage + hltTrigger + '.php', dirToStore) 
+    with open(os.path.join(dirToStore, hltTrigger + '.php')) as f:
+      for line in f:
+        if 'prescale1' in line and 'L1_DoubleEG' in line: 
+          download(prescalePage + line.split('>')[0].split('=')[-1], dirToStore)
+
+    for json in glob.glob(os.path.join(dirToStore, '*.json')):
+      leg1 = int(json.split('L1_DoubleEG_')[-1].split('_')[0].replace('LooseIso', ''))
+      leg2 = int(json.split('L1_DoubleEG_')[-1].split('_')[1].replace('LooseIso', ''))
+      yield leg1, leg2, json
+
+
+  from CRABAPI.RawCommand import crabCommand
+  from CRABClient.ClientExceptions import ClientException
+  from httplib import HTTPException
+
+  # We want to put all the CRAB project directories from the tasks we submit here into one common directory.
+  # That's why we need to set this parameter (here or above in the configuration file, it does not matter, we will not overwrite it).
+  config.General.workArea = 'crab_%s' % submitVersion
+
+  def submit(config, sample, leg1threshold, leg2threshold):
+    config.General.requestName  = sample.split('/')[-3] + '_%s_%s' % (leg1threshold, leg2threshold)
+    config.Data.inputDataset    = sample#'/EGamma/Run2018A-PromptReco-v1/MINIAOD'
+    config.Data.outLFNDirBase   = '%s/%s/' % (mainOutputDir,'mc')
+    config.Data.splitting       = 'FileBased'
+    #config.Data.lumiMask        = json
+    config.Data.unitsPerJob     = 5
+    config.JobType.pyCfgParams  = defaultArgs + ['L1Threshold=%s' % leg1threshold]
+    #config.JobType.pyCfgParams  = defaultArgs + ['leg1Threshold=%s' % 22, 'leg2Threshold=%s' % 0] 
+
+    try:
+      crabCommand('submit', config = config)
+    except HTTPException as hte:
+      print "Failed submitting task: %s" % (hte.headers)
+    except ClientException as cle:
+      print "Failed submitting task: %s" % (cle)
+
+  from multiprocessing import Process
+  def submitWrapper(config, sample, leg1threshold, leg2threshold):
+    p = Process(target=submit, args=(config, sample, leg1threshold, leg2threshold))
+    p.start()
+    p.join()
+
+  #for leg1, leg2, json in getSeedsForDoubleEle('2018'):
+    # Crab fails on this on second iteration, of course with only a very cryptic error message
+    # Not sure how to workaround this
+    #submit(config, '/EGamma/Run2018A-PromptReco-v1/MINIAOD', leg1, leg2, json)
+    #submit(config, '/EGamma/Run2018A-17Sep2018/MINIAOD', leg1, leg2, json)
+  #submit(config, '/DYJetsToLL_M-50_TuneCP5_13TeV-madgraphMLM-pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1/MINIAODSIM', 22, 0)
+  #submit(config, '/DYToEE_M-50_NNPDF31_TuneCP5_13TeV-powheg-pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1/MINIAODSIM', 22, 0)
+  submit(config, '/DYJetsToLL_M-50_TuneCP5_13TeV-madgraphMLM-pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1/MINIAODSIM', 25, 0)
+  submit(config, '/DYToEE_M-50_NNPDF31_TuneCP5_13TeV-powheg-pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1/MINIAODSIM', 25, 0)
+
+
+
+

plugins/MiniAODL1Stage2CandProducer.cc

@@ -0,0 +1,14 @@
+#include "MiniAODL1Stage2CandProducer.h"
+
+#include "DataFormats/PatCandidates/interface/Electron.h"
+#include "DataFormats/PatCandidates/interface/Photon.h"
+
+typedef MiniAODL1Stage2CandProducer<pat::Electron> PatElectronL1Stage2CandProducer;
+DEFINE_FWK_MODULE(PatElectronL1Stage2CandProducer);
+
+typedef MiniAODL1Stage2CandProducer<reco::GsfElectron> GsfElectronL1Stage2CandProducer;
+DEFINE_FWK_MODULE(GsfElectronL1Stage2CandProducer);
+
+//typedef MiniAODL1CandProducer<pat::Photon> PatPhotonL1CandProducer;
+//DEFINE_FWK_MODULE(PatPhotonL1CandProducer);
+

plugins/MiniAODL1Stage2CandProducer.h

@@ -0,0 +1,135 @@
+#ifndef _MINIADOL1STAGE2CANDPRODUCER_H_
+#define _MINIADOL1STAGE2CANDPRODUCER_H_
+
+#include "FWCore/Framework/interface/Frameworkfwd.h"
+#include "FWCore/Framework/interface/Event.h"
+#include "FWCore/Framework/interface/MakerMacros.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/Framework/interface/EDProducer.h"
+
+//#include "DataFormats/L1Trigger/interface/L1EmParticle.h"                                                                                                          
+//#include "DataFormats/L1Trigger/interface/L1EmParticleFwd.h"                                                                                                       
+#include "DataFormats/L1Trigger/interface/EGamma.h" //for stage 2 L1                                                                                                 
+#include <DataFormats/Math/interface/deltaR.h>
+
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+
+#include <string>
+#include <vector>
+
+template <class T>
+class MiniAODL1Stage2CandProducer : public edm::EDProducer {
+
+  typedef std::vector<T> TCollection;
+  typedef edm::Ref<TCollection> TRef;
+  typedef edm::RefVector<TCollection> TRefVector;
+
+ public:
+  MiniAODL1Stage2CandProducer(const edm::ParameterSet&);
+  ~MiniAODL1Stage2CandProducer();
+
+  bool l1OfflineMatching(const std::vector<l1t::EGamma>& triggerObjects,
+                         math::XYZTLorentzVector refP4, float dRmin, float dRminEE, int& index);
+ private:
+  /// compare two l1Extra in et                                                                                                                                      
+  struct ComparePt {
+    bool operator()( const l1t::EGamma& t1, const l1t::EGamma& t2 ) const {
+      return t1.et() > t2.et();
+    }
+  };
+  ComparePt ptComparator;
+
+  virtual void produce(edm::Event&, const edm::EventSetup&) override;
+
+  edm::EDGetTokenT<TRefVector> inputs_;
+  edm::EDGetTokenT<l1t::EGammaBxCollection> l1ObjectsToken_;
+  float minET_;
+  float dRMatch_;
+  float dRMatchEE_;
+
+};
+
+template <class T>
+MiniAODL1Stage2CandProducer<T>::MiniAODL1Stage2CandProducer(const edm::ParameterSet& iConfig ) :
+
+
+  inputs_(consumes<TRefVector>(iConfig.getParameter<edm::InputTag>("inputs"))),
+  l1ObjectsToken_(consumes<l1t::EGammaBxCollection> ( iConfig.getParameter<edm::InputTag>("objects"))),
+  minET_(iConfig.getParameter<double>("minET")),
+  dRMatch_(iConfig.getParameter<double>("dRmatch"))
+{
+
+  if(iConfig.exists("dRmatchEE"))
+    dRMatchEE_ = (iConfig.getParameter<double>("dRmatchEE"));
+  else
+    dRMatchEE_ = dRMatch_; //for backwards compatibility
+
+  produces<TRefVector>();
+}
+
+template <class T>
+MiniAODL1Stage2CandProducer<T>::~MiniAODL1Stage2CandProducer()
+{}
+
+template <class T>
+void MiniAODL1Stage2CandProducer<T>::produce(edm::Event &iEvent, const edm::EventSetup &eventSetup) {
+
+  edm::Handle<l1t::EGammaBxCollection> l1ObjectsH;
+  edm::Handle<TRefVector> inputs;
+
+  iEvent.getByToken(l1ObjectsToken_, l1ObjectsH);
+  iEvent.getByToken(inputs_, inputs);
+
+   //Merge L1 objects and sort by et
+  std::vector<l1t::EGamma> mergedL1;
+  for(auto it=l1ObjectsH->begin(0); it!=l1ObjectsH->end(0); it++){
+    mergedL1.push_back(*it);
+    // std::cout << "L1: "  << endl;
+  }
+
+  std::sort(mergedL1.begin(), mergedL1.end(), ptComparator);
+
+  // Create the output collection                                                                                                                                    
+  // std::auto_ptr<TRefVector> outColRef(new TRefVector);
+  std::unique_ptr<TRefVector> outColRef(new TRefVector);
+
+  for (size_t i=0; i<inputs->size(); i++) {
+    TRef ref = (*inputs)[i];
+    int index = -1;
+
+    if (l1OfflineMatching(mergedL1, ref->p4(), dRMatch_, dRMatchEE_, index)) {
+      outColRef->push_back(ref);
+    }
+  }
+
+  //iEvent.put(outColRef);
+  iEvent.put(std::move(outColRef));
+}
+
+template <class T>
+bool MiniAODL1Stage2CandProducer<T>::l1OfflineMatching(const std::vector<l1t::EGamma>& l1Objects,
+						       math::XYZTLorentzVector refP4, float dRmin, float dRminEE, int& index) {
+
+
+  index = 0;
+  //for (auto it=l1Objects.begin(0); it != l1Objects.end(0); it++) { //bx 0 only considered 
+  for (auto it=l1Objects.begin(); it != l1Objects.end(); it++) {
+    if (it->et() < minET_)
+      continue;
+
+    float dR = deltaR(refP4, it->p4());
+    if(fabs(refP4.eta()) < 1.5)
+      if (dR < dRmin)
+	return true;
+    if(dR < dRminEE) //allow for looser requirements in EE (not needed for stage2)
+      return true;
+    index++;
+  }
+
+  return false;
+
+}
+
+
+#endif
+

plugins/MiniAODStage2L1CandProducer.cc

@@ -0,0 +1,11 @@
+#include "MiniAODL1CandProducer.h"
+
+#include "DataFormats/PatCandidates/interface/Electron.h"
+#include "DataFormats/PatCandidates/interface/Photon.h"
+
+typedef MiniAODL1CandProducer<pat::Electron> PatElectronL1CandProducer;
+DEFINE_FWK_MODULE(PatElectronL1CandProducer);
+
+typedef MiniAODL1CandProducer<pat::Photon> PatPhotonL1CandProducer;
+DEFINE_FWK_MODULE(PatPhotonL1CandProducer);
+