Skip to content

Commit

Permalink
Create infrastructure to allow alternative module types to be loaded
Browse files Browse the repository at this point in the history
Provide API to allow alternative module implementation to be loaded based on the name of the type of the module given in the configuration.
  • Loading branch information
Dr15Jones committed Apr 28, 2022
1 parent a951722 commit c23d4d2
Show file tree
Hide file tree
Showing 7 changed files with 282 additions and 18 deletions.
4 changes: 4 additions & 0 deletions FWCore/Framework/interface/ModuleRegistry.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
// user include files
#include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
#include "FWCore/Utilities/interface/propagate_const.h"
#include "FWCore/Framework/interface/ModuleTypeResolverBase.h"

// forward declarations
namespace edm {
Expand All @@ -39,6 +40,8 @@ namespace edm {

class ModuleRegistry {
public:
ModuleRegistry() = default;
explicit ModuleRegistry(std::unique_ptr<ModuleTypeResolverBase>);
std::shared_ptr<maker::ModuleHolder> getModule(MakeModuleParams const& p,
std::string const& moduleLabel,
signalslot::Signal<void(ModuleDescription const&)>& iPre,
Expand All @@ -62,6 +65,7 @@ namespace edm {

private:
std::map<std::string, edm::propagate_const<std::shared_ptr<maker::ModuleHolder>>> labelToModule_;
std::unique_ptr<ModuleTypeResolverBase> typeResolver_;
};
} // namespace edm

Expand Down
43 changes: 43 additions & 0 deletions FWCore/Framework/interface/ModuleTypeResolverBase.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#ifndef FWCore_Framework_ModuleTypeResolverBase_h
#define FWCore_Framework_ModuleTypeResolverBase_h
// -*- C++ -*-
//
// Package: FWCore/Framework
// Class : ModuleTypeResolverBase
//
/**\class edm::ModuleTypeResolverBase ModuleTypeResolverBase.h "FWCore/Framework/interface/ModuleTypeResolverBase.h"
Description: Base class for deriving alternative module types to use when loading
Usage:
This is meant to be used as part of a do...while loop. The condition should be the rturned int is not kDone and the
type returned is not what you need.
*/
//
// Original Author: Chris Jones
// Created: Wed, 27 Apr 2022 16:21:10 GMT
//

// system include files
#include <string>

// user include files

// forward declarations
namespace edm {
class ModuleTypeResolverBase {
public:
static constexpr int kInitialIndex = 0;
static constexpr int kLastIndex = -1;
virtual ~ModuleTypeResolverBase() = default;

/**This function is meant to be called multiple times with different values for index. The first call should set index
to kInitialIndex. The int returned from the function is the new index to use on next call or is a value of kLastIndex which
means no further calls should be made. The returned string is the next concrete type to be used when making a call.
On subsequent call, the argument basename can be the same string as returned from the previous call to the function.
**/
virtual std::pair<std::string, int> resolveType(std::string basename, int index) const = 0;
};
} // namespace edm

#endif
68 changes: 52 additions & 16 deletions FWCore/Framework/src/Factory.cc
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@

#include "FWCore/Framework/src/Factory.h"
#include "FWCore/Framework/interface/maker/MakerPluginFactory.h"
#include "FWCore/Framework/interface/ModuleTypeResolverBase.h"
#include "FWCore/Utilities/interface/DebugMacros.h"
#include "FWCore/Utilities/interface/EDMException.h"
#include "FWCore/Utilities/interface/Algorithms.h"
Expand All @@ -23,30 +24,64 @@ namespace edm {

Factory const* Factory::get() { return &singleInstance_; }

Maker* Factory::findMaker(const MakeModuleParams& p) const {
static void annotateExceptionAndRethrow(cms::Exception& except, const MakeModuleParams& p, std::string const& modtype, ModuleTypeResolverBase const* resolver) {
if(not resolver) {
throw except;
}
//if needed, create list of alternative types that were tried
std::string alternativeTypes;
auto index = resolver->kInitialIndex;
auto newType = modtype;
int tries = 0;
do {
++tries;
if(not alternativeTypes.empty()) {
alternativeTypes.append(", ");
}
auto [ttype,tindex] = resolver->resolveType(std::move(newType), index);
newType = std::move(ttype);
index = tindex;
alternativeTypes.append(newType);
} while(index != resolver->kLastIndex);
if(tries==1 and alternativeTypes == modtype) {
throw except;
}
alternativeTypes.insert(0,"These alternative types were tried: ");
except.addAdditionalInfo(alternativeTypes);
throw except;
}

Maker* Factory::findMaker(const MakeModuleParams& p, ModuleTypeResolverBase const* resolver) const {
std::string modtype = p.pset_->getParameter<std::string>("@module_type");
FDEBUG(1) << "Factory: module_type = " << modtype << std::endl;
MakerMap::iterator it = makers_.find(modtype);

if (it == makers_.end()) {
std::unique_ptr<Maker> wm(MakerPluginFactory::get()->create(modtype));

if (wm.get() == nullptr)
throw edm::Exception(errors::Configuration, "UnknownModule")
<< "Module " << modtype << " with version " << p.processConfiguration_->releaseVersion()
<< " was not registered.\n"
<< "Perhaps your module type is misspelled or is not a "
<< "framework plugin.\n"
<< "Try running EdmPluginDump to obtain a list of "
<< "available Plugins.";

auto make = [](auto resolver, const auto& modtype, auto const& p) {
if(resolver) {
auto index = resolver->kInitialIndex;
auto newType = modtype;
do {
auto [ttype,tindex] = resolver->resolveType(std::move(newType), index);
newType = std::move(ttype);
index = tindex;
auto m = MakerPluginFactory::get()->tryToCreate(newType);
if(m) { return m;}
} while(index != resolver->kLastIndex);
try {
//failed to find a plugin
return MakerPluginFactory::get()->create(modtype);
} catch(cms::Exception& iExcept) {
annotateExceptionAndRethrow(iExcept, p, modtype, resolver);
}
}
return MakerPluginFactory::get()->create(modtype);
};
std::unique_ptr<Maker> wm = make(resolver, modtype, p);
FDEBUG(1) << "Factory: created worker of type " << modtype << std::endl;

std::pair<MakerMap::iterator, bool> ret = makers_.insert(std::pair<std::string, Maker*>(modtype, wm.get()));

// if(ret.second==false)
// throw runtime_error("Worker Factory map insert failed");

it = ret.first;
wm.release();
}
Expand All @@ -55,9 +90,10 @@ namespace edm {

std::shared_ptr<maker::ModuleHolder> Factory::makeModule(
const MakeModuleParams& p,
const ModuleTypeResolverBase* resolver,
signalslot::Signal<void(const ModuleDescription&)>& pre,
signalslot::Signal<void(const ModuleDescription&)>& post) const {
auto maker = findMaker(p);
auto maker = findMaker(p, resolver);
auto mod(maker->makeModule(p, pre, post));
return mod;
}
Expand Down
5 changes: 4 additions & 1 deletion FWCore/Framework/src/Factory.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
#include "FWCore/Utilities/interface/thread_safety_macros.h"

namespace edm {
class ModuleTypeResolverBase;

class Factory {
public:
typedef std::map<std::string, edm::propagate_const<Maker*>> MakerMap;
Expand All @@ -23,14 +25,15 @@ namespace edm {

//This function is not const-thread safe
std::shared_ptr<maker::ModuleHolder> makeModule(const MakeModuleParams&,
const ModuleTypeResolverBase*,
signalslot::Signal<void(const ModuleDescription&)>& pre,
signalslot::Signal<void(const ModuleDescription&)>& post) const;

std::shared_ptr<maker::ModuleHolder> makeReplacementModule(const edm::ParameterSet&) const;

private:
Factory();
Maker* findMaker(const MakeModuleParams& p) const;
Maker* findMaker(const MakeModuleParams& p, const ModuleTypeResolverBase*) const;
static Factory const singleInstance_;
//It is not safe to create modules across threads
CMS_SA_ALLOW mutable MakerMap makers_;
Expand Down
2 changes: 1 addition & 1 deletion FWCore/Framework/src/ModuleRegistry.cc
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ namespace edm {
signalslot::Signal<void(ModuleDescription const&)>& iPost) {
auto modItr = labelToModule_.find(moduleLabel);
if (modItr == labelToModule_.end()) {
auto modPtr = Factory::get()->makeModule(p, iPre, iPost);
auto modPtr = Factory::get()->makeModule(p, typeResolver_.get(), iPre, iPost);

// Transfer ownership of worker to the registry
labelToModule_[moduleLabel] = modPtr;
Expand Down
5 changes: 5 additions & 0 deletions FWCore/Framework/test/BuildFile.xml
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,11 @@
<use name="FWCore/Framework"/>
</bin>

<bin file="test_catch2_main.cc,test_catch2_Factory.cc" name="TestFWCoreFrameworkFactory">
<use name="catch2"/>
<use name="FWCore/Framework"/>
</bin>

<test name="testFWCoreFrameworkNonEventOrdering" command="test_non_event_ordering.sh"/>
<test name="testFWCoreFramework1ThreadESPrefetch" command="run_test_1_thread_es_prefetching.sh"/>
<test name="testFWCoreFrameworkModuleDeletion" command="run_module_delete_tests.sh"/>
Expand Down
173 changes: 173 additions & 0 deletions FWCore/Framework/test/test_catch2_Factory.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
#include "catch.hpp"

#include "FWCore/Framework/src/Factory.h"
#include "FWCore/Framework/interface/ModuleTypeResolverBase.h"
#include "FWCore/Framework/interface/global/EDProducer.h"
#include "FWCore/Framework/interface/MakerMacros.h"
#include "FWCore/Framework/interface/PreallocationConfiguration.h"
#include "FWCore/PluginManager/interface/PluginManager.h"
#include "FWCore/PluginManager/interface/standard.h"
#include "DataFormats/Provenance/interface/ProductRegistry.h"

#include <iostream>

namespace edm::test {
class SimpleTestTypeResolver : public edm::ModuleTypeResolverBase {
public:
SimpleTestTypeResolver() = default;
std::pair<std::string, int> resolveType(std::string basename, int index) const final {
return {basename,kLastIndex};
}
};

class ComplexTestTypeResolver : public edm::ModuleTypeResolverBase {
public:
ComplexTestTypeResolver() = default;
std::pair<std::string, int> resolveType(std::string basename, int index) const final {
constexpr auto kGeneric = "generic::";
constexpr auto kOther = "edm::test::other::";
constexpr auto kCPU = "edm::test::cpu::";
if(index != kInitialIndex and index != kLastIndex) {
basename.replace(basename.find(kOther), strlen(kOther), kCPU);
return {basename, kLastIndex};
}
if(index == kInitialIndex and basename.find(kGeneric) != std::string::npos) {
basename.replace(basename.find(kGeneric), strlen(kGeneric), kOther);
return {basename, kInitialIndex+1};
}
return {basename,kLastIndex};
}
};

class FactoryTestAProd : public edm::global::EDProducer<> {
public:
explicit FactoryTestAProd(edm::ParameterSet const&) {}
void produce(StreamID, edm::Event&, edm::EventSetup const&) const final {}
};

namespace cpu {
class FactoryTestAProd : public edm::global::EDProducer<> {
public:
explicit FactoryTestAProd(edm::ParameterSet const&) {}
void produce(StreamID, edm::Event&, edm::EventSetup const&) const final {}
};
}
namespace other {
class FactoryTestAProd : public edm::global::EDProducer<> {
public:
explicit FactoryTestAProd(edm::ParameterSet const&) {}
void produce(StreamID, edm::Event&, edm::EventSetup const&) const final {}
};
}
} // namespace edm::test

DEFINE_FWK_MODULE(edm::test::FactoryTestAProd);
namespace edm::test {
using FactoryTestBProd = FactoryTestAProd;
}
DEFINE_FWK_MODULE(edm::test::FactoryTestBProd);
DEFINE_FWK_MODULE(edm::test::cpu::FactoryTestAProd);
DEFINE_FWK_MODULE(edm::test::other::FactoryTestAProd);
namespace edm::test::cpu {
using FactoryTestCProd = FactoryTestAProd;
}
DEFINE_FWK_MODULE(edm::test::cpu::FactoryTestCProd);

static bool called = false;
using namespace edm;
TEST_CASE("test edm::Factory", "[Factory]") {
signalslot::Signal<void(const ModuleDescription&)> pre;
signalslot::Signal<void(const ModuleDescription&)> post;
ProductRegistry prodReg;
PreallocationConfiguration preallocConfig;
std::shared_ptr<ProcessConfiguration const> procConfig = std::make_shared<ProcessConfiguration>();
if(not called) {
edmplugin::PluginManager::configure(edmplugin::standard::config());
called = true;
}

SECTION("test missing plugin") {
auto factory = Factory::get();
ParameterSet pset;
pset.addParameter<std::string>("@module_type", "DoesNotExistModule");
pset.addParameter<std::string>("@module_edm_type", "EDProducer");

CHECK_THROWS(factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), nullptr, pre,post));
try {
factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), nullptr, pre,post);
}catch(cms::Exception const& iE) {
//std::cout <<iE.what()<<"\n";
REQUIRE(std::string(iE.what()).find("DoesNotExistModule") !=std::string::npos);
}
}
SECTION("test missing plugin with simple resolver") {
auto factory = Factory::get();
ParameterSet pset;
pset.addParameter<std::string>("@module_type", "DoesNotExistModule");
pset.addParameter<std::string>("@module_edm_type", "EDProducer");
edm::test::SimpleTestTypeResolver resolver;
CHECK_THROWS(factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), &resolver, pre,post));
try {
factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), &resolver, pre,post);
}catch(cms::Exception const& iE) {
//std::cout <<iE.what()<<"\n";
REQUIRE(std::string(iE.what()).find("DoesNotExistModule") !=std::string::npos);
}
}
SECTION("test missing plugin with complex resolver") {
auto factory = Factory::get();
ParameterSet pset;
pset.addParameter<std::string>("@module_type", "generic::DoesNotExistModule");
pset.addParameter<std::string>("@module_edm_type", "EDProducer");
edm::test::ComplexTestTypeResolver resolver;
CHECK_THROWS(factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), &resolver, pre,post));
try {
factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), &resolver, pre,post);
}catch(cms::Exception const& iE) {
REQUIRE(std::string(iE.what()).find("generic::DoesNotExistModule") !=std::string::npos);
REQUIRE(std::string(iE.what()).find("edm::test::other::DoesNotExistModule") !=std::string::npos);
REQUIRE(std::string(iE.what()).find("edm::test::cpu::DoesNotExistModule") !=std::string::npos);
}
}

SECTION("test found plugin") {
auto factory = Factory::get();
ParameterSet pset;
pset.addParameter<std::string>("@module_type", "edm::test::FactoryTestAProd");
pset.addParameter<std::string>("@module_label", "a");
pset.addParameter<std::string>("@module_edm_type", "EDProducer");

REQUIRE(factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), nullptr, pre,post));
}
SECTION("test found plugin with simple resolver") {
auto factory = Factory::get();
ParameterSet pset;
pset.addParameter<std::string>("@module_type", "edm::test::FactoryTestBProd");
pset.addParameter<std::string>("@module_label", "b");
pset.addParameter<std::string>("@module_edm_type", "EDProducer");
edm::test::SimpleTestTypeResolver resolver;
REQUIRE(factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), &resolver, pre,post));
}
SECTION("test found plugin with complex resolver") {
SECTION("find other") {
auto factory = Factory::get();
ParameterSet pset;
pset.addParameter<std::string>("@module_type", "generic::FactoryTestAProd");
pset.addParameter<std::string>("@module_label", "gen");
pset.addParameter<std::string>("@module_edm_type", "EDProducer");
edm::test::ComplexTestTypeResolver resolver;
REQUIRE(factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), &resolver, pre,post));
}
SECTION("find cpu") {
auto factory = Factory::get();
ParameterSet pset;
pset.addParameter<std::string>("@module_type", "generic::FactoryTestCProd");
pset.addParameter<std::string>("@module_label", "cgen");
pset.addParameter<std::string>("@module_edm_type", "EDProducer");
edm::test::ComplexTestTypeResolver resolver;
REQUIRE(factory->makeModule(MakeModuleParams(&pset,prodReg,&preallocConfig, procConfig), &resolver, pre,post));
}
}


}

0 comments on commit c23d4d2

Please sign in to comment.