From c38022db35429a81db27c9b63bf09266b4613af3 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Thu, 1 Oct 2020 22:01:37 +0200 Subject: [PATCH] Add configuration parameter to allow disabling the module deletion --- FWCore/Framework/interface/EventProcessor.h | 1 + FWCore/Framework/src/EventProcessor.cc | 37 +++++++++++-------- .../Framework/test/run_module_delete_tests.sh | 2 + .../test/test_module_delete_disable_cfg.py | 26 +++++++++++++ FWCore/ParameterSet/python/Config.py | 2 + .../src/validateTopLevelParameterSets.cc | 4 ++ 6 files changed, 56 insertions(+), 16 deletions(-) create mode 100644 FWCore/Framework/test/test_module_delete_disable_cfg.py diff --git a/FWCore/Framework/interface/EventProcessor.h b/FWCore/Framework/interface/EventProcessor.h index d7ef9d735c4fc..82fd029b31ee7 100644 --- a/FWCore/Framework/interface/EventProcessor.h +++ b/FWCore/Framework/interface/EventProcessor.h @@ -362,6 +362,7 @@ namespace edm { ExcludedDataMap eventSetupDataToExcludeFromPrefetching_; bool printDependencies_ = false; + bool deleteNonConsumedUnscheduledModules_ = true; }; // class EventProcessor //-------------------------------------------------------------------- diff --git a/FWCore/Framework/src/EventProcessor.cc b/FWCore/Framework/src/EventProcessor.cc index 9e4d2bf5842ec..8c575ebd9182c 100644 --- a/FWCore/Framework/src/EventProcessor.cc +++ b/FWCore/Framework/src/EventProcessor.cc @@ -399,6 +399,8 @@ namespace edm { IllegalParameters::setThrowAnException(optionsPset.getUntrackedParameter("throwIfIllegalParameter")); printDependencies_ = optionsPset.getUntrackedParameter("printDependencies"); + deleteNonConsumedUnscheduledModules_ = + optionsPset.getUntrackedParameter("deleteNonConsumedUnscheduledModules"); // Now do general initialization ScheduleItems items; @@ -433,6 +435,8 @@ namespace edm { nStreams = 1; nConcurrentLumis = 1; nConcurrentRuns = 1; + // in presence of looper do not delete modules + deleteNonConsumedUnscheduledModules_ = false; } espController_->setMaxConcurrentIOVs(nStreams, nConcurrentLumis); @@ -548,26 +552,27 @@ namespace edm { schedule_->convertCurrentProcessAlias(processConfiguration_->processName()); pathsAndConsumesOfModules_.initialize(schedule_.get(), preg()); - // in presence of looper do not delete modules - bool const deleteModules = not looper_; - std::vector consumedBySubProcesses; - for_all(subProcesses_, [&consumedBySubProcesses, deleteModules](auto& subProcess) { - auto c = subProcess.keepOnlyConsumedUnscheduledModules(deleteModules); - if (consumedBySubProcesses.empty()) { - consumedBySubProcesses = std::move(c); - } else if (not c.empty()) { - std::vector tmp; - tmp.reserve(consumedBySubProcesses.size() + c.size()); - std::merge( - consumedBySubProcesses.begin(), consumedBySubProcesses.end(), c.begin(), c.end(), std::back_inserter(tmp)); - std::swap(consumedBySubProcesses, tmp); - } - }); + for_all(subProcesses_, + [&consumedBySubProcesses, deleteModules = deleteNonConsumedUnscheduledModules_](auto& subProcess) { + auto c = subProcess.keepOnlyConsumedUnscheduledModules(deleteModules); + if (consumedBySubProcesses.empty()) { + consumedBySubProcesses = std::move(c); + } else if (not c.empty()) { + std::vector tmp; + tmp.reserve(consumedBySubProcesses.size() + c.size()); + std::merge(consumedBySubProcesses.begin(), + consumedBySubProcesses.end(), + c.begin(), + c.end(), + std::back_inserter(tmp)); + std::swap(consumedBySubProcesses, tmp); + } + }); // Note: all these may throw checkForModuleDependencyCorrectness(pathsAndConsumesOfModules_, printDependencies_); - if (deleteModules) { + if (deleteNonConsumedUnscheduledModules_) { if (auto const unusedModules = nonConsumedUnscheduledModules(pathsAndConsumesOfModules_, consumedBySubProcesses); not unusedModules.empty()) { pathsAndConsumesOfModules_.removeModules(unusedModules); diff --git a/FWCore/Framework/test/run_module_delete_tests.sh b/FWCore/Framework/test/run_module_delete_tests.sh index 2540d379210e5..3738258fd6611 100755 --- a/FWCore/Framework/test/run_module_delete_tests.sh +++ b/FWCore/Framework/test/run_module_delete_tests.sh @@ -15,3 +15,5 @@ cmsRun $TEST_DIR/test_module_delete_looper_cfg.py || die "module deletetion test echo "module deletion test with looper succeeded" cmsRun $TEST_DIR/test_module_delete_dependencygraph_cfg.py || die "module deletetion test with DependencyGraph failed" $? echo "module deletion test with DependencyGraph succeeded" +cmsRun $TEST_DIR/test_module_delete_disable_cfg.py || die "module deletetion test with disabling the deletion failed" $? +echo "module deletion test with disabling the deletion succeeded" diff --git a/FWCore/Framework/test/test_module_delete_disable_cfg.py b/FWCore/Framework/test/test_module_delete_disable_cfg.py new file mode 100644 index 0000000000000..6c53b453a9b1e --- /dev/null +++ b/FWCore/Framework/test/test_module_delete_disable_cfg.py @@ -0,0 +1,26 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTMODULEDELETE") + +process.maxEvents.input = 1 +process.options.deleteNonConsumedUnscheduledModules = False +process.source = cms.Source("EmptySource") + +process.intEventProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) +process.intEventConsumer = cms.EDAnalyzer("IntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intEventProducer"), + valueMustMatch = cms.untracked.int32(1) +) +process.intProducerNotConsumed = cms.EDProducer("edmtest::MustRunIntProducer", + ivalue = cms.int32(1), + mustRunEvent = cms.bool(False) +) + +process.t = cms.Task( + process.intEventProducer, + process.intProducerNotConsumed +) +process.p = cms.Path( + process.intEventConsumer, + process.t +) diff --git a/FWCore/ParameterSet/python/Config.py b/FWCore/ParameterSet/python/Config.py index 218dc66ec151d..712b87eb94a53 100644 --- a/FWCore/ParameterSet/python/Config.py +++ b/FWCore/ParameterSet/python/Config.py @@ -227,6 +227,7 @@ def defaultOptions_(): forceEventSetupCacheClearOnNewRun = untracked.bool(False), throwIfIllegalParameter = untracked.bool(True), printDependencies = untracked.bool(False), + deleteNonConsumedUnscheduledModules = untracked.bool(True), sizeOfStackForThreadsInKB = optional.untracked.uint32, Rethrow = untracked.vstring(), SkipEvent = untracked.vstring(), @@ -2008,6 +2009,7 @@ def testProcessDumpPython(self): SkipEvent = cms.untracked.vstring(), allowUnscheduled = cms.obsolete.untracked.bool, canDeleteEarly = cms.untracked.vstring(), + deleteNonConsumedUnscheduledModules = cms.untracked.bool(True), emptyRunLumiMode = cms.obsolete.untracked.string, eventSetup = cms.untracked.PSet( forceNumberOfConcurrentIOVs = cms.untracked.PSet( diff --git a/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc b/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc index e512232fbb042..f59e4d6a55680 100644 --- a/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc +++ b/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc @@ -41,6 +41,10 @@ namespace edm { description.addUntracked("throwIfIllegalParameter", true) ->setComment("Set false to disable exception throws when configuration validation detects illegal parameters"); description.addUntracked("printDependencies", false)->setComment("Print data dependencies between modules"); + description.addUntracked("deleteNonConsumedUnscheduledModules", true) + ->setComment( + "Delete modules that are unscheduled, i.e. only in Tasks, whose products are not consumed by any other " + "otherwise-running module"); // No default for this one because the parameter value is // actually used in the main function in cmsRun.cpp before