Skip to content

Commit

Permalink
Print the labels of ConditionalTask modules that are not consumed in …
Browse files Browse the repository at this point in the history
…any of the associated Paths

The original goal was to delete such modules from the job (similarly
to non-consumed unscheduled modules), but that turned out to be
complicated (as there could be modules that should be treated like
this that _are_ consumed by some other module). Printing out the
labels should suffice the remaining HLT needs for ConditionalTask.
  • Loading branch information
makortel committed Nov 15, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 7d99d30 commit e5c6296
Showing 6 changed files with 203 additions and 13 deletions.
9 changes: 6 additions & 3 deletions FWCore/Framework/interface/StreamSchedule.h
Original file line number Diff line number Diff line change
@@ -316,7 +316,8 @@ namespace edm {
bool ignoreFilters,
PathWorkers& out,
std::vector<std::string> const& endPathNames,
ConditionalTaskHelper const& conditionalTaskHelper);
ConditionalTaskHelper const& conditionalTaskHelper,
std::unordered_set<std::string>& allConditionalModules);
void fillTrigPath(ParameterSet& proc_pset,
ProductRegistry& preg,
PreallocationConfiguration const* prealloc,
@@ -325,15 +326,17 @@ namespace edm {
std::string const& name,
TrigResPtr,
std::vector<std::string> const& endPathNames,
ConditionalTaskHelper const& conditionalTaskHelper);
ConditionalTaskHelper const& conditionalTaskHelper,
std::unordered_set<std::string>& allConditionalModules);
void fillEndPath(ParameterSet& proc_pset,
ProductRegistry& preg,
PreallocationConfiguration const* prealloc,
std::shared_ptr<ProcessConfiguration const> processConfiguration,
int bitpos,
std::string const& name,
std::vector<std::string> const& endPathNames,
ConditionalTaskHelper const& conditionalTaskHelper);
ConditionalTaskHelper const& conditionalTaskHelper,
std::unordered_set<std::string>& allConditionalModules);

void addToAllWorkers(Worker* w);

76 changes: 66 additions & 10 deletions FWCore/Framework/src/StreamSchedule.cc
Original file line number Diff line number Diff line change
@@ -377,6 +377,7 @@ namespace edm {

ConditionalTaskHelper conditionalTaskHelper(
proc_pset, preg, &prealloc, processConfiguration, workerManager_, pathNames);
std::unordered_set<std::string> conditionalModules;

int trig_bitpos = 0;
trig_paths_.reserve(pathNames.size());
@@ -389,7 +390,8 @@ namespace edm {
trig_name,
results(),
endPathNames,
conditionalTaskHelper);
conditionalTaskHelper,
conditionalModules);
++trig_bitpos;
hasPath = true;
}
@@ -406,8 +408,15 @@ namespace edm {
int bitpos = 0;
end_paths_.reserve(endPathNames.size());
for (auto const& end_path_name : endPathNames) {
fillEndPath(
proc_pset, preg, &prealloc, processConfiguration, bitpos, end_path_name, endPathNames, conditionalTaskHelper);
fillEndPath(proc_pset,
preg,
&prealloc,
processConfiguration,
bitpos,
end_path_name,
endPathNames,
conditionalTaskHelper,
conditionalModules);
++bitpos;
}

@@ -454,6 +463,31 @@ namespace edm {
}
}
number_of_unscheduled_modules_ = unscheduledLabels.size();

// Print conditional modules that were not consumed in none of their associated Paths
if (streamID.value() == 0 and not conditionalModules.empty()) {
// Intersection of unscheduled and ConditionalTask modules gives
// directly the set of conditional modules that were not
// consumed by anything in the Paths associated to the
// corresponding ConditionalTask.
std::vector<std::string_view> labelsToPrint;
std::copy_if(
unscheduledLabels.begin(),
unscheduledLabels.end(),
std::back_inserter(labelsToPrint),
[&conditionalModules](auto const& lab) { return conditionalModules.find(lab) != conditionalModules.end(); });

if (not labelsToPrint.empty()) {
edm::LogWarning log("NonConsumedConditionalModules");
log << "The following modules were part of some ConditionalTask, but were not\n"
<< "consumed by any other module in any of the Paths the ConditionalTask\n"
<< "was associated with. Perhaps they should be either removed from the\n"
<< "job, or moved to a Task to make it explicit they are unscheduled.\n";
for (auto const& modLabel : labelsToPrint) {
log.format(" {}\n", modLabel);
}
}
}
} // StreamSchedule::StreamSchedule

void StreamSchedule::initializeEarlyDelete(ModuleRegistry& modReg,
@@ -753,7 +787,8 @@ namespace edm {
bool ignoreFilters,
PathWorkers& out,
std::vector<std::string> const& endPathNames,
ConditionalTaskHelper const& conditionalTaskHelper) {
ConditionalTaskHelper const& conditionalTaskHelper,
std::unordered_set<std::string>& allConditionalModules) {
vstring modnames = proc_pset.getParameter<vstring>(pathName);
PathWorkers tmpworkers;

@@ -775,6 +810,9 @@ namespace edm {

conditionalModsBranches = conditionalTaskHelper.conditionalModuleBranches(conditionalmods);
modnames.erase(std::prev(condRange.first), modnames.end());

// Make a union of all conditional modules from all Paths
allConditionalModules.insert(conditionalmods.begin(), conditionalmods.end());
}

unsigned int placeInPath = 0;
@@ -856,10 +894,19 @@ namespace edm {
std::string const& name,
TrigResPtr trptr,
std::vector<std::string> const& endPathNames,
ConditionalTaskHelper const& conditionalTaskHelper) {
ConditionalTaskHelper const& conditionalTaskHelper,
std::unordered_set<std::string>& allConditionalModules) {
PathWorkers tmpworkers;
fillWorkers(
proc_pset, preg, prealloc, processConfiguration, name, false, tmpworkers, endPathNames, conditionalTaskHelper);
fillWorkers(proc_pset,
preg,
prealloc,
processConfiguration,
name,
false,
tmpworkers,
endPathNames,
conditionalTaskHelper,
allConditionalModules);

// an empty path will cause an extra bit that is not used
if (!tmpworkers.empty()) {
@@ -880,10 +927,19 @@ namespace edm {
int bitpos,
std::string const& name,
std::vector<std::string> const& endPathNames,
ConditionalTaskHelper const& conditionalTaskHelper) {
ConditionalTaskHelper const& conditionalTaskHelper,
std::unordered_set<std::string>& allConditionalModules) {
PathWorkers tmpworkers;
fillWorkers(
proc_pset, preg, prealloc, processConfiguration, name, true, tmpworkers, endPathNames, conditionalTaskHelper);
fillWorkers(proc_pset,
preg,
prealloc,
processConfiguration,
name,
true,
tmpworkers,
endPathNames,
conditionalTaskHelper,
allConditionalModules);

if (!tmpworkers.empty()) {
end_paths_.emplace_back(bitpos,
2 changes: 2 additions & 0 deletions FWCore/Framework/test/BuildFile.xml
Original file line number Diff line number Diff line change
@@ -379,6 +379,8 @@
<test name="testFWCoreFrameworkGetByType_getterOfProduct" command="cmsRun ${LOCALTOP}/src/FWCore/Framework/test/test_get_by_type_cfg.py"/>
<test name="testFWCoreFrameworkGetByType_getterOfProduct_alias" command="cmsRun ${LOCALTOP}/src/FWCore/Framework/test/test_get_by_type_cfg.py --useEDAlias"/>

<test name="testFWCoreFrameworkConditionalTaskNonConsumed" command="test_conditionaltasks_nonconsumed.sh"/>

<test name="testFWCoreFrameworkOptionsNumberOfThreadsType" command="cmsRun ${LOCALTOP}/src/FWCore/Framework/test/test_wrongOptionsType_cfg.py --name=numberOfThreads --value='cms.untracked.uint32(1)'"/>
<test name="testFWCoreFrameworkWrongOptionsNumberOfThreadsType" command="run_wrongOptionsType.sh numberOfThreads 'cms.untracked.int32(1)'"/>
<test name="testFWCoreFrameworkWrongOptionsNumberOfStreamsType" command="run_wrongOptionsType.sh numberOfStreams 'cms.untracked.int32(1)'"/>
16 changes: 16 additions & 0 deletions FWCore/Framework/test/test_conditionaltasks_nonconsumed.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/bin/bash

function die { echo Failure $1: status $2 ; exit $2 ; }

CONFIG=${LOCALTOP}/src/FWCore/Framework/test/test_conditionaltasks_nonconsumed_cfg.py
OUTPUT=conditionaltasks_nonconsumed.log
REFERENCE=${LOCALTOP}/src/FWCore/Framework/test/unit_test_outputs/$OUTPUT

function run {
cmsRun $CONFIG $@
tail -n +2 conditionaltasks_nonconsumed.log | diff - $REFERENCE || die "cmsRun $CONFIG $@ provides unexpected log"
}

run
run --filterSucceeds
run --testView
104 changes: 104 additions & 0 deletions FWCore/Framework/test/test_conditionaltasks_nonconsumed_cfg.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import FWCore.ParameterSet.Config as cms

import argparse
import sys

parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ConditionalTasks.')

parser.add_argument("--filterSucceeds", help="Have filter succeed", action="store_true")
parser.add_argument("--testView", help="Get data via a view", action="store_true")

args = parser.parse_args()

process = cms.Process("Test")

process.source = cms.Source("EmptySource")

# ensure the printout is done only once
process.maxEvents.input = 4
process.options.numberOfThreads = 4

process.MessageLogger.files.conditionaltasks_nonconsumed = dict()
process.MessageLogger.files.conditionaltasks_nonconsumed.default = dict(limit=0)
process.MessageLogger.files.conditionaltasks_nonconsumed.NonConsumedConditionalModules = dict(limit=100)

process.a = cms.EDProducer("IntProducer", ivalue = cms.int32(1))
process.b = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag(cms.InputTag("a")))

process.f1 = cms.EDFilter("IntProductFilter", label = cms.InputTag("b"))

process.c = cms.EDProducer("IntProducer", ivalue = cms.int32(2))
process.d = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag(cms.InputTag("c")))
process.e = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag(cms.InputTag("d")))

process.nonconsumed = process.a.clone()
process.nonconsumed2 = process.b.clone(
labels = ["nonconsumed"]
)
process.nonconsumedConditionalTask = cms.ConditionalTask(
process.nonconsumed,
process.nonconsumed2,
)

process.consumedInOnePath = process.a.clone()
process.nonconsumedConditionalTask2 = cms.ConditionalTask(
process.nonconsumed,
process.consumedInOnePath
)

process.explicitlyInDifferentPath = process.a.clone()
process.consumedInUnrelatedPath = process.a.clone()
process.nonconsumedConditionalTask3 = cms.ConditionalTask(
process.nonconsumed,
process.explicitlyInDifferentPath,
process.consumedInUnrelatedPath
)
process.nonconsumedTask3 = cms.Task(
process.consumedInUnrelatedPath
)

process.prodOnPath = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag(cms.InputTag("d"), cms.InputTag("e")))
process.prodOnPath2 = process.prodOnPath.clone(
labels = ["consumedInOnePath"]
)
process.prodOnPath3 = process.prodOnPath.clone(
labels = ["consumedInUnrelatedPath"]
)

if args.filterSucceeds:
threshold = 1
else:
threshold = 3

process.f2 = cms.EDFilter("IntProductFilter", label = cms.InputTag("e"), threshold = cms.int32(threshold))

if args.testView:
process.f3 = cms.EDAnalyzer("SimpleViewAnalyzer",
label = cms.untracked.InputTag("f"),
sizeMustMatch = cms.untracked.uint32(10),
checkSize = cms.untracked.bool(False)
)
process.f = cms.EDProducer("OVSimpleProducer", size = cms.int32(10))
producttype = "edmtestSimplesOwned"
else:
process.f= cms.EDProducer("IntProducer", ivalue = cms.int32(3))
process.f3 = cms.EDFilter("IntProductFilter", label = cms.InputTag("f"))
producttype = "edmtestIntProduct"

process.p = cms.Path(process.f1+process.prodOnPath+process.f2+process.f3, cms.ConditionalTask(process.a, process.b, process.c, process.d, process.e, process.f, process.nonconsumedConditionalTask, process.nonconsumedConditionalTask2))

process.p2 = cms.Path(process.prodOnPath2, process.nonconsumedConditionalTask2, process.nonconsumedConditionalTask3)

process.p3 = cms.Path(process.explicitlyInDifferentPath)

process.p4 = cms.Path(process.prodOnPath3, process.nonconsumedTask3)

process.tst = cms.EDAnalyzer("IntTestAnalyzer", moduleLabel = cms.untracked.InputTag("f"), valueMustMatch = cms.untracked.int32(3),
valueMustBeMissing = cms.untracked.bool(not args.filterSucceeds))

process.nonconsumedConsumer = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag(cms.InputTag("nonconsumed")))

process.endp = cms.EndPath(process.tst+process.nonconsumedConsumer)

#process.add_(cms.Service("Tracer"))
#process.options.wantSummary=True
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
The following modules were part of some ConditionalTask, but were not
consumed by any other module in any of the Paths the ConditionalTask
was associated with. Perhaps they should be either removed from the
job, or moved to a Task to make it explicit they are unscheduled.
consumedInUnrelatedPath
nonconsumed
nonconsumed2

%MSG

0 comments on commit e5c6296

Please sign in to comment.