From acefa3368343229f94d50271c2ec146f12994734 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Tue, 13 Feb 2024 14:43:20 -0600 Subject: [PATCH] Alternative python module configuration syntax Generates a new python file from the C++ configuration description which describes the default. This can be imported to allow a more simplified module configuration syntax. --- .../Integration/test/check_empty_event_cfg.py | 5 +- .../test/delayedreader_throw_cfg.py | 24 ++++++---- FWCore/ParameterSet/bin/edmWriteConfigs.cpp | 19 ++++++++ .../interface/ConfigurationDescriptions.h | 2 + FWCore/ParameterSet/python/ModulesProxy.py | 24 ++++++++++ .../src/ConfigurationDescriptions.cc | 48 +++++++++++++++++++ 6 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 FWCore/ParameterSet/python/ModulesProxy.py diff --git a/FWCore/Integration/test/check_empty_event_cfg.py b/FWCore/Integration/test/check_empty_event_cfg.py index bb0c69925e27a..1d9d5d31907ab 100644 --- a/FWCore/Integration/test/check_empty_event_cfg.py +++ b/FWCore/Integration/test/check_empty_event_cfg.py @@ -1,14 +1,15 @@ import FWCore.ParameterSet.Config as cms +from FWCore.Modules.modules import EventContentAnalyzer, EmptySource process = cms.Process("EMPTY") -process.test = cms.EDAnalyzer("EventContentAnalyzer", listPathStatus = cms.untracked.bool(True)) +process.test = EventContentAnalyzer(listPathStatus = True) process.e = cms.EndPath(process.test) #process.out = cms.OutputModule("AsciiOutputModule") #process.e2 = cms.EndPath(process.out) -process.source = cms.Source("EmptySource") +process.source = EmptySource() process.maxEvents.input = 1 diff --git a/FWCore/Integration/test/delayedreader_throw_cfg.py b/FWCore/Integration/test/delayedreader_throw_cfg.py index dc2be5da2056a..c1c7e80494da6 100644 --- a/FWCore/Integration/test/delayedreader_throw_cfg.py +++ b/FWCore/Integration/test/delayedreader_throw_cfg.py @@ -1,27 +1,31 @@ import FWCore.ParameterSet.Config as cms +from FWCore.Framework.modules import AddIntsProducer, IntProductFilter +from FWCore.Modules.modules import AsciiOutputModule +from FWCore.Integration.modules import DelayedReaderThrowingSource process = cms.Process("TEST") -process.source = cms.Source("DelayedReaderThrowingSource", labels = cms.untracked.vstring("test", "test2", "test3")) +process.source = DelayedReaderThrowingSource( labels = ["test", "test2", "test3"]) -process.getter = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag(cms.InputTag("test","","INPUTTEST"))) -process.onPath = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag(cms.InputTag("test2", "", "INPUTTEST"), cms.InputTag("getter", "other"))) -process.f1 = cms.EDFilter("IntProductFilter", label = cms.InputTag("onPath"), shouldProduce = cms.bool(True)) -process.f2 = cms.EDFilter("IntProductFilter", label = cms.InputTag("onPath"), shouldProduce = cms.bool(True)) -process.inFront = cms.EDFilter("IntProductFilter", label = cms.InputTag("test3")) +process.getter = AddIntsProducer(labels = [("test","","INPUTTEST")]) +process.onPath = AddIntsProducer(labels = [("test2", "", "INPUTTEST"), ("getter", "other")]) +process.f1 = IntProductFilter( label = "onPath", shouldProduce = True) +process.f2 = IntProductFilter( label = "onPath", shouldProduce = True) +process.inFront = IntProductFilter( label = "test3") process.p1 = cms.Path(process.inFront+process.onPath+process.f1+process.f2) process.p3 = cms.Path(process.onPath+process.f1, cms.Task(process.getter)) process.p2 = cms.Path(process.onPath+process.f2) - -#process.dump = cms.EDAnalyzer("EventContentAnalyzer") +#from FWCore.Modules.modules import EventContentAnalyzer import * +#process.dump = EventContentAnalyzer() #process.p = cms.Path(process.dump) -process.out = cms.OutputModule("AsciiOutputModule") +process.out = AsciiOutputModule() process.e = cms.EndPath(process.out, cms.Task(process.getter)) process.maxEvents.input = 1 -#process.add_(cms.Service("Tracer")) +#from FWCore.Services.modules import Tracer +#process.add_(Tracer()) diff --git a/FWCore/ParameterSet/bin/edmWriteConfigs.cpp b/FWCore/ParameterSet/bin/edmWriteConfigs.cpp index af3b08c778923..b056ed9074eb1 100644 --- a/FWCore/ParameterSet/bin/edmWriteConfigs.cpp +++ b/FWCore/ParameterSet/bin/edmWriteConfigs.cpp @@ -52,6 +52,8 @@ #include #include "FWCore/Utilities/interface/Signal.h" #include +#include +#include static char const* const kHelpOpt = "help"; static char const* const kHelpCommandOpt = "help,h"; @@ -133,6 +135,20 @@ namespace { pluginNames.push_back(nameAndType.first); } } + + void writeModulesFile() { + if (std::filesystem::exists(std::filesystem::current_path() / "modules.py")) + return; + std::array buffer; + std::tmpnam(buffer.data()); + std::ofstream file{buffer.data()}; + + file << "from FWCore.ParameterSet.ModulesProxy import _setupProxies\n" + "locals().update(_setupProxies(__file__))\n"; + file.close(); + std::filesystem::copy(buffer.data(), "modules.py"); + std::filesystem::remove(buffer.data()); + } } // namespace int main(int argc, char** argv) try { @@ -287,6 +303,9 @@ int main(int argc, char** argv) try { std::set usedCfiFileNames; edm::for_all(pluginNames, std::bind(&writeCfisForPlugin, _1, factory, std::ref(usedCfiFileNames))); + if (not pluginNames.empty()) { + writeModulesFile(); + } }); } catch (cms::Exception& iException) { if (!library.empty()) { diff --git a/FWCore/ParameterSet/interface/ConfigurationDescriptions.h b/FWCore/ParameterSet/interface/ConfigurationDescriptions.h index a69206eacd29e..c0925cd40bfce 100644 --- a/FWCore/ParameterSet/interface/ConfigurationDescriptions.h +++ b/FWCore/ParameterSet/interface/ConfigurationDescriptions.h @@ -87,6 +87,8 @@ namespace edm { std::string const& pluginName, std::set& usedCfiFileNames); + void writeClassFile(ParameterSetDescription const&) const; + void printForLabel(std::pair const& labelAndDesc, std::ostream& os, std::string const& moduleLabel, diff --git a/FWCore/ParameterSet/python/ModulesProxy.py b/FWCore/ParameterSet/python/ModulesProxy.py new file mode 100644 index 0000000000000..910c04236081b --- /dev/null +++ b/FWCore/ParameterSet/python/ModulesProxy.py @@ -0,0 +1,24 @@ +import os +from os.path import sep, join +import importlib + +class _ModuleProxy (object): + def __init__(self, package, name): + self._package = package + self._name = name + self._caller = None + def __call__(self,**kwargs): + if not self._caller: + self._caller = getattr(importlib.import_module(self._package+'.'+self._name),self._name) + return self._caller(**kwargs) + + +def _setupProxies(fullName): + _cmssw_package_name='.'.join(fullName.split(sep)[-3:-1]) + basename = fullName.split(sep)[-1] + pathname = fullName[:-1*len(basename)] + proxies = dict() + for filename in ( x for x in os.listdir(pathname) if (len(x) > 3 and x[-3:] == '.py' and x != basename and ((len(x) < 6) or (x[-6:] != 'cfi.py')))): + name = filename[:-3] + proxies[name] = _ModuleProxy(_cmssw_package_name, name) + return proxies diff --git a/FWCore/ParameterSet/src/ConfigurationDescriptions.cc b/FWCore/ParameterSet/src/ConfigurationDescriptions.cc index 8d9d5e98a0b8b..3ffa8ed2a44b9 100644 --- a/FWCore/ParameterSet/src/ConfigurationDescriptions.cc +++ b/FWCore/ParameterSet/src/ConfigurationDescriptions.cc @@ -147,6 +147,54 @@ namespace edm { std::cref(baseType_), std::cref(pluginName_), std::ref(usedCfiFileNames))); + if (defaultDescDefined_) { + writeClassFile(defaultDesc_); + } else if (descriptions_.size() == 1) { + writeClassFile(descriptions_.begin()->second); + } + } + + void ConfigurationDescriptions::writeClassFile(ParameterSetDescription const& iDesc) const { + std::string pluginName = pluginName_; + auto found = pluginName.find("::"); + while (found != std::string::npos) { + pluginName.replace(found, 2, "_"); + found = pluginName.find("::"); + } + //Symbols that can appear in our plugin names but can't in python function names + const std::string toReplace("@<>,"); + found = pluginName.find_first_of(toReplace); + while (found != std::string::npos) { + pluginName.replace(found, 1, "_"); + found = pluginName.find_first_of(toReplace, found); + } + + std::string fileName = pluginName + ".py"; + std::ofstream outFile(fileName.c_str()); + if (outFile.fail()) { + edm::Exception ex(edm::errors::LogicError, "Creating class file failed.\n"); + ex << "Opening a file '" << fileName << "' failed.\n"; + ex << "Error code from errno " << errno << ": " << std::strerror(errno) << "\n"; + + ex.addContext("Executing function ConfigurationDescriptions::writeDefault"); + throw ex; + } + outFile << "import FWCore.ParameterSet.Config as cms\n\n"; + outFile << "def " << pluginName + << "(**kwargs):\n" + " mod = cms." + << baseType_ << "('" << pluginName_ << "'"; + + bool startWithComma = true; + int indentation = 4; + iDesc.writeCfi(outFile, startWithComma, indentation); + + outFile << ")\n" + " for k,v in kwargs.items():\n" + " setattr(mod, k, v)\n" + " return mod\n"; + + outFile.close(); } void ConfigurationDescriptions::writeCfiForLabel(std::pair const& labelAndDesc,