Add configuration parameter to allow disabling the module deletion

cms-sw · Oct 2, 2020 · c38022d · c38022d
1 parent 9c053f9
commit c38022d
Show file tree

Hide file tree

Showing 6 changed files with 56 additions and 16 deletions.
diff --git a/FWCore/Framework/interface/EventProcessor.h b/FWCore/Framework/interface/EventProcessor.h
@@ -362,6 +362,7 @@ namespace edm {
     ExcludedDataMap eventSetupDataToExcludeFromPrefetching_;
 
     bool printDependencies_ = false;
+    bool deleteNonConsumedUnscheduledModules_ = true;
   };  // class EventProcessor
 
   //--------------------------------------------------------------------

diff --git a/FWCore/Framework/src/EventProcessor.cc b/FWCore/Framework/src/EventProcessor.cc
@@ -399,6 +399,8 @@ namespace edm {
     IllegalParameters::setThrowAnException(optionsPset.getUntrackedParameter<bool>("throwIfIllegalParameter"));
 
     printDependencies_ = optionsPset.getUntrackedParameter<bool>("printDependencies");
+    deleteNonConsumedUnscheduledModules_ =
+        optionsPset.getUntrackedParameter<bool>("deleteNonConsumedUnscheduledModules");
 
     // Now do general initialization
     ScheduleItems items;
@@ -433,6 +435,8 @@ namespace edm {
       nStreams = 1;
       nConcurrentLumis = 1;
       nConcurrentRuns = 1;
+      // in presence of looper do not delete modules
+      deleteNonConsumedUnscheduledModules_ = false;
     }
     espController_->setMaxConcurrentIOVs(nStreams, nConcurrentLumis);
 
@@ -548,26 +552,27 @@ namespace edm {
     schedule_->convertCurrentProcessAlias(processConfiguration_->processName());
     pathsAndConsumesOfModules_.initialize(schedule_.get(), preg());
 
-    // in presence of looper do not delete modules
-    bool const deleteModules = not looper_;
-
     std::vector<ModuleProcessName> consumedBySubProcesses;
-    for_all(subProcesses_, [&consumedBySubProcesses, deleteModules](auto& subProcess) {
-      auto c = subProcess.keepOnlyConsumedUnscheduledModules(deleteModules);
-      if (consumedBySubProcesses.empty()) {
-        consumedBySubProcesses = std::move(c);
-      } else if (not c.empty()) {
-        std::vector<ModuleProcessName> tmp;
-        tmp.reserve(consumedBySubProcesses.size() + c.size());
-        std::merge(
-            consumedBySubProcesses.begin(), consumedBySubProcesses.end(), c.begin(), c.end(), std::back_inserter(tmp));
-        std::swap(consumedBySubProcesses, tmp);
-      }
-    });
+    for_all(subProcesses_,
+            [&consumedBySubProcesses, deleteModules = deleteNonConsumedUnscheduledModules_](auto& subProcess) {
+              auto c = subProcess.keepOnlyConsumedUnscheduledModules(deleteModules);
+              if (consumedBySubProcesses.empty()) {
+                consumedBySubProcesses = std::move(c);
+              } else if (not c.empty()) {
+                std::vector<ModuleProcessName> tmp;
+                tmp.reserve(consumedBySubProcesses.size() + c.size());
+                std::merge(consumedBySubProcesses.begin(),
+                           consumedBySubProcesses.end(),
+                           c.begin(),
+                           c.end(),
+                           std::back_inserter(tmp));
+                std::swap(consumedBySubProcesses, tmp);
+              }
+            });
 
     // Note: all these may throw
     checkForModuleDependencyCorrectness(pathsAndConsumesOfModules_, printDependencies_);
-    if (deleteModules) {
+    if (deleteNonConsumedUnscheduledModules_) {
       if (auto const unusedModules = nonConsumedUnscheduledModules(pathsAndConsumesOfModules_, consumedBySubProcesses);
           not unusedModules.empty()) {
         pathsAndConsumesOfModules_.removeModules(unusedModules);

diff --git a/FWCore/Framework/test/run_module_delete_tests.sh b/FWCore/Framework/test/run_module_delete_tests.sh
@@ -15,3 +15,5 @@ cmsRun $TEST_DIR/test_module_delete_looper_cfg.py || die "module deletetion test
 echo "module deletion test with looper succeeded"
 cmsRun $TEST_DIR/test_module_delete_dependencygraph_cfg.py || die "module deletetion test with DependencyGraph failed" $?
 echo "module deletion test with DependencyGraph succeeded"
+cmsRun $TEST_DIR/test_module_delete_disable_cfg.py || die "module deletetion test with disabling the deletion failed" $?
+echo "module deletion test with disabling the deletion succeeded"
diff --git a/FWCore/Framework/test/test_module_delete_disable_cfg.py b/FWCore/Framework/test/test_module_delete_disable_cfg.py
@@ -0,0 +1,26 @@
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process("TESTMODULEDELETE")
+
+process.maxEvents.input = 1
+process.options.deleteNonConsumedUnscheduledModules = False
+process.source = cms.Source("EmptySource")
+
+process.intEventProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(1))
+process.intEventConsumer = cms.EDAnalyzer("IntTestAnalyzer",
+    moduleLabel = cms.untracked.InputTag("intEventProducer"),
+    valueMustMatch = cms.untracked.int32(1)
+)
+process.intProducerNotConsumed = cms.EDProducer("edmtest::MustRunIntProducer",
+    ivalue = cms.int32(1),
+    mustRunEvent = cms.bool(False)
+)
+
+process.t = cms.Task(
+    process.intEventProducer,
+    process.intProducerNotConsumed
+)
+process.p = cms.Path(
+    process.intEventConsumer,
+    process.t
+)
diff --git a/FWCore/ParameterSet/python/Config.py b/FWCore/ParameterSet/python/Config.py
@@ -227,6 +227,7 @@ def defaultOptions_():
                               forceEventSetupCacheClearOnNewRun = untracked.bool(False),
                               throwIfIllegalParameter = untracked.bool(True),
                               printDependencies = untracked.bool(False),
+                              deleteNonConsumedUnscheduledModules = untracked.bool(True),
                               sizeOfStackForThreadsInKB = optional.untracked.uint32,
                               Rethrow = untracked.vstring(),
                               SkipEvent = untracked.vstring(),
@@ -2008,6 +2009,7 @@ def testProcessDumpPython(self):
     SkipEvent = cms.untracked.vstring(),
     allowUnscheduled = cms.obsolete.untracked.bool,
     canDeleteEarly = cms.untracked.vstring(),
+    deleteNonConsumedUnscheduledModules = cms.untracked.bool(True),
     emptyRunLumiMode = cms.obsolete.untracked.string,
     eventSetup = cms.untracked.PSet(
         forceNumberOfConcurrentIOVs = cms.untracked.PSet(

diff --git a/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc b/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc
@@ -41,6 +41,10 @@ namespace edm {
     description.addUntracked<bool>("throwIfIllegalParameter", true)
         ->setComment("Set false to disable exception throws when configuration validation detects illegal parameters");
     description.addUntracked<bool>("printDependencies", false)->setComment("Print data dependencies between modules");
+    description.addUntracked<bool>("deleteNonConsumedUnscheduledModules", true)
+        ->setComment(
+            "Delete modules that are unscheduled, i.e. only in Tasks, whose products are not consumed by any other "
+            "otherwise-running module");
 
     // No default for this one because the parameter value is
     // actually used in the main function in cmsRun.cpp before