diff --git a/DataFormats/TestObjects/interface/SchemaEvolutionTestObjects.h b/DataFormats/TestObjects/interface/SchemaEvolutionTestObjects.h new file mode 100644 index 0000000000000..3eb527df94ac3 --- /dev/null +++ b/DataFormats/TestObjects/interface/SchemaEvolutionTestObjects.h @@ -0,0 +1,337 @@ +#ifndef DataFormats_TestObjects_SchemaEvolutionTestObjects_h +#define DataFormats_TestObjects_SchemaEvolutionTestObjects_h + +#include + +// Don't delete the following comment line. +// This #define is required when generating data files +// using the old formats. These data files are saved and used +// as input in unit tests to verify that ROOT can use schema +// evolution to read the old formats with a release that has +// modified formats. When reading this #define should be commented +// out and it should be commented out in the code repository. +// Note that the data files are generated manually and this +// line and classes_def.xml must be manually modified and built +// when generating new data files. The data files are saved +// in this repository: https://github.com/cms-data/IOPool-Input. +//#define DataFormats_TestObjects_USE_OLD +#if defined DataFormats_TestObjects_USE_OLD + +#include + +#else + +#include +#include +#include +#include + +#endif + +namespace edmtest { + + class SchemaEvolutionChangeOrder { + public: +#if defined DataFormats_TestObjects_USE_OLD + SchemaEvolutionChangeOrder() : a_(0), b_(0) {} + SchemaEvolutionChangeOrder(int a, int b) : a_(a), b_(b) {} + int a_; + int b_; +#else + SchemaEvolutionChangeOrder() : b_(0), a_(0) {} + SchemaEvolutionChangeOrder(int a, int b) : b_(b), a_(a) {} + int b_; + int a_; +#endif + }; + + class SchemaEvolutionAddMember { + public: +#if defined DataFormats_TestObjects_USE_OLD + SchemaEvolutionAddMember() : a_(0), b_(0) {} + SchemaEvolutionAddMember(int a, int b, int) : a_(a), b_(b) {} + int a_; + int b_; +#else + SchemaEvolutionAddMember() : a_(0), b_(0), c_(0) {} + SchemaEvolutionAddMember(int a, int b, int c) : a_(a), b_(b), c_(c) {} + int a_; + int b_; + int c_; +#endif + }; + + class SchemaEvolutionRemoveMember { + public: +#if defined DataFormats_TestObjects_USE_OLD + SchemaEvolutionRemoveMember() : a_(0), b_(0) {} + SchemaEvolutionRemoveMember(int a, int b) : a_(a), b_(b) {} + int a_; + int b_; +#else + SchemaEvolutionRemoveMember() : a_(0) {} + SchemaEvolutionRemoveMember(int a, int) : a_(a) {} + int a_; +#endif + }; + +#if defined DataFormats_TestObjects_USE_OLD + class SchemaEvolutionBase { + public: + SchemaEvolutionBase() : d_(0) {} + SchemaEvolutionBase(int d) : d_(d) {} + int d_; + }; + + class SchemaEvolutionMoveToBase : public SchemaEvolutionBase { + public: + SchemaEvolutionMoveToBase() : a_(0), b_(0), c_(0) {} + SchemaEvolutionMoveToBase(int a, int b, int c, int d) : SchemaEvolutionBase(d), a_(a), b_(b), c_(c) {} + int a_; + int b_; + int c_; + }; +#else + class SchemaEvolutionBase { + public: + SchemaEvolutionBase() : c_(0), d_(0) {} + SchemaEvolutionBase(int c, int d) : c_(c), d_(d) {} + int c_; + int d_; + }; + + class SchemaEvolutionMoveToBase : public SchemaEvolutionBase { + public: + SchemaEvolutionMoveToBase() : a_(0), b_(0) {} + SchemaEvolutionMoveToBase(int a, int b, int c, int d) : SchemaEvolutionBase(c, d), a_(a), b_(b) {} + int a_; + int b_; + }; +#endif + + class SchemaEvolutionChangeType { + public: +#if defined DataFormats_TestObjects_USE_OLD + SchemaEvolutionChangeType() : a_(0), b_(0) {} + SchemaEvolutionChangeType(int a, int b) : a_(a), b_(b) {} + int a_; + int b_; +#else + SchemaEvolutionChangeType() : a_(0.0), b_(0LL) {} + SchemaEvolutionChangeType(int a, int b) : a_(a), b_(b) {} + double a_; + long long b_; +#endif + }; + + class SchemaEvolutionBaseA { + public: + SchemaEvolutionBaseA() : c_(0) {} + SchemaEvolutionBaseA(int c) : c_(c) {} + int c_; + }; + +#if defined DataFormats_TestObjects_USE_OLD + class SchemaEvolutionAddBase { + public: + SchemaEvolutionAddBase() : a_(0), b_(0) {} + SchemaEvolutionAddBase(int a, int b, int) : a_(a), b_(b) {} +#else + class SchemaEvolutionAddBase : public SchemaEvolutionBaseA { + public: + SchemaEvolutionAddBase() : a_(0), b_(0) {} + SchemaEvolutionAddBase(int a, int b, int c) : SchemaEvolutionBaseA(c), a_(a), b_(b) {} +#endif + int a_; + int b_; + }; + + class SchemaEvolutionContained { + public: + SchemaEvolutionContained() : c_(0) {} + SchemaEvolutionContained(int c) : c_(c) {} + int c_; + }; + + class SchemaEvolutionPointerToMember { + public: +#if defined DataFormats_TestObjects_USE_OLD + SchemaEvolutionPointerToMember() : a_(0), b_(0), contained_(nullptr) {} + SchemaEvolutionPointerToMember(SchemaEvolutionPointerToMember const& other) + : a_(other.a_), b_(other.b_), contained_(new SchemaEvolutionContained(other.contained_->c_)) {} + SchemaEvolutionPointerToMember(SchemaEvolutionPointerToMember&&) = delete; + SchemaEvolutionPointerToMember& operator=(SchemaEvolutionPointerToMember const&) = delete; + SchemaEvolutionPointerToMember& operator=(SchemaEvolutionPointerToMember&&) = delete; + + SchemaEvolutionPointerToMember(int a, int b, int c) : a_(a), b_(b), contained_(new SchemaEvolutionContained(c)) {} + ~SchemaEvolutionPointerToMember() { delete contained_; } + int c() const { return contained_->c_; } + int a_; + int b_; + SchemaEvolutionContained* contained_; +#else + SchemaEvolutionPointerToMember() : a_(0), b_(0) {} + SchemaEvolutionPointerToMember(int a, int b, int c) : a_(a), b_(b), contained_(c) {} + int c() const { return contained_.c_; } + int a_; + int b_; + SchemaEvolutionContained contained_; +#endif + }; + + class SchemaEvolutionPointerToUniquePtr { + public: + SchemaEvolutionPointerToUniquePtr(SchemaEvolutionPointerToUniquePtr&&) = delete; + SchemaEvolutionPointerToUniquePtr& operator=(SchemaEvolutionPointerToUniquePtr const&) = delete; + SchemaEvolutionPointerToUniquePtr& operator=(SchemaEvolutionPointerToUniquePtr&&) = delete; + +#if defined DataFormats_TestObjects_USE_OLD + SchemaEvolutionPointerToUniquePtr() : a_(0), b_(0), contained_(nullptr) {} + SchemaEvolutionPointerToUniquePtr(SchemaEvolutionPointerToUniquePtr const& other) + : a_(other.a_), b_(other.b_), contained_(new SchemaEvolutionContained(other.contained_->c_)) {} + + SchemaEvolutionPointerToUniquePtr(int a, int b, int c) + : a_(a), b_(b), contained_(new SchemaEvolutionContained(c)) {} + ~SchemaEvolutionPointerToUniquePtr() { delete contained_; } + int a_; + int b_; + SchemaEvolutionContained* contained_; +#else + SchemaEvolutionPointerToUniquePtr() : a_(0), b_(0) {} + SchemaEvolutionPointerToUniquePtr(int a, int b, int c) + : a_(a), b_(b), contained_(std::make_unique(c)) {} + SchemaEvolutionPointerToUniquePtr(SchemaEvolutionPointerToUniquePtr const& other) + : a_(other.a_), b_(other.b_), contained_(std::make_unique(other.contained_->c_)) {} + int a_; + int b_; + std::unique_ptr contained_; +#endif + }; + + class SchemaEvolutionCArrayToStdArray { + public: +#if defined DataFormats_TestObjects_USE_OLD + SchemaEvolutionCArrayToStdArray() : a_{0, 0, 0} {} + SchemaEvolutionCArrayToStdArray(int x, int y, int z) : a_{x, y, z} {} + int a_[3]; +#else + SchemaEvolutionCArrayToStdArray() : a_{{0, 0, 0}} {} + SchemaEvolutionCArrayToStdArray(int x, int y, int z) : a_{{x, y, z}} {} + std::array a_; +#endif + }; + + class SchemaEvolutionCArrayToStdVector { + public: +#if defined DataFormats_TestObjects_USE_OLD + SchemaEvolutionCArrayToStdVector() : a_{new int[fSize_]{0, 0, 0}} {} + SchemaEvolutionCArrayToStdVector(int x, int y, int z) : a_{new int[fSize_]{x, y, z}} {} + SchemaEvolutionCArrayToStdVector(SchemaEvolutionCArrayToStdVector const& other) + : a_(new int[fSize_]{other.a_[0], other.a_[1], other.a_[2]}) {} + SchemaEvolutionCArrayToStdVector(SchemaEvolutionCArrayToStdVector&&) = delete; + SchemaEvolutionCArrayToStdVector& operator=(SchemaEvolutionCArrayToStdVector const&) = delete; + SchemaEvolutionCArrayToStdVector& operator=(SchemaEvolutionCArrayToStdVector&&) = delete; + ~SchemaEvolutionCArrayToStdVector() { delete[] a_; } + + int fSize_ = 3; + int* a_; //[fSize_] +#else + SchemaEvolutionCArrayToStdVector() : a_{0, 0, 0} {} + SchemaEvolutionCArrayToStdVector(int x, int y, int z) : a_{x, y, z} {} + std::vector a_; +#endif + }; + + class SchemaEvolutionVectorToList { + public: + SchemaEvolutionVectorToList() : a_{0, 0, 0} {} + SchemaEvolutionVectorToList(int x, int y, int z) : a_{x, y, z} {} +#if defined DataFormats_TestObjects_USE_OLD + std::vector a_; +#else + std::list a_; +#endif + }; + + class SchemaEvolutionMapToUnorderedMap { + public: + SchemaEvolutionMapToUnorderedMap() { + a_.insert({0, 0}); + a_.insert({1, 0}); + a_.insert({2, 0}); + } + SchemaEvolutionMapToUnorderedMap(int keyX, int x, int keyY, int y, int keyZ, int z) { + a_.insert({keyX, x}); + a_.insert({keyY, y}); + a_.insert({keyZ, z}); + } +#if defined DataFormats_TestObjects_USE_OLD + std::map a_; +#else + std::unordered_map a_; +#endif + }; + + class VectorVectorElement { + public: + VectorVectorElement(); + VectorVectorElement(int a, + int b, + SchemaEvolutionChangeOrder const&, + SchemaEvolutionAddMember const&, + SchemaEvolutionRemoveMember const&, + SchemaEvolutionMoveToBase const&, + SchemaEvolutionChangeType const&, + SchemaEvolutionAddBase const&, + SchemaEvolutionPointerToMember const&, + SchemaEvolutionPointerToUniquePtr const&, + SchemaEvolutionCArrayToStdArray const&, + // SchemaEvolutionCArrayToStdVector const&, + SchemaEvolutionVectorToList const&, + SchemaEvolutionMapToUnorderedMap const&); +#if defined DataFormats_TestObjects_USE_OLD + int a_; + int b_; +#else + int a_; + int b_; + int c_ = 0; +#endif + SchemaEvolutionChangeOrder changeOrder_; + SchemaEvolutionAddMember addMember_; + SchemaEvolutionRemoveMember removeMember_; + SchemaEvolutionMoveToBase moveToBase_; + SchemaEvolutionChangeType changeType_; + SchemaEvolutionAddBase addBase_; + SchemaEvolutionPointerToMember pointerToMember_; + SchemaEvolutionPointerToUniquePtr pointerToUniquePtr_; + SchemaEvolutionCArrayToStdArray cArrayToStdArray_; + // This one is commented out because it fails reading an old format + // input file with an executable built with the modified format. + // If the issue in ROOT is ever fixed and this is added back, + // it also would need to be added into the constructor above. + // SchemaEvolutionCArrayToStdVector cArrayToStdVector_; + SchemaEvolutionVectorToList vectorToList_; + SchemaEvolutionMapToUnorderedMap mapToUnorderedMap_; + }; + + class VectorVectorElementNonSplit { + public: + VectorVectorElementNonSplit(); + VectorVectorElementNonSplit(int a, int b); +#if defined DataFormats_TestObjects_USE_OLD + // This version of the class is forced to be non-split because + // it has only one data member. The unit test this is used by + // was developed in response to a ROOT bug in the version of ROOT + // associated with CMSSW_13_0_0. This bug only affected non split + // classes and this class was necessary to reproduce it. + int a_; +#else + int a_; + int b_; +#endif + }; + +} // namespace edmtest + +#endif diff --git a/DataFormats/TestObjects/interface/VectorVectorTop.h b/DataFormats/TestObjects/interface/VectorVectorTop.h new file mode 100644 index 0000000000000..727a86daa8d0f --- /dev/null +++ b/DataFormats/TestObjects/interface/VectorVectorTop.h @@ -0,0 +1,41 @@ +#ifndef DataFormats_TestObjects_VectorVectorTop_h +#define DataFormats_TestObjects_VectorVectorTop_h + +/** \class VectorVectorTop + +\author W. David Dagenhart, created 21 July, 2023 + +*/ + +#include "DataFormats/TestObjects/interface/SchemaEvolutionTestObjects.h" + +#include + +namespace edmtest { + + class VectorVectorMiddle { + public: + VectorVectorMiddle(); + std::vector middleVector_; + }; + + class VectorVectorTop { + public: + VectorVectorTop(); + std::vector outerVector_; + }; + + class VectorVectorMiddleNonSplit { + public: + VectorVectorMiddleNonSplit(); + std::vector middleVector_; + }; + + class VectorVectorTopNonSplit { + public: + VectorVectorTopNonSplit(); + std::vector outerVector_; + }; + +} // namespace edmtest +#endif diff --git a/DataFormats/TestObjects/src/SchemaEvolutionTestObjects.cc b/DataFormats/TestObjects/src/SchemaEvolutionTestObjects.cc new file mode 100644 index 0000000000000..d79d7b663ab71 --- /dev/null +++ b/DataFormats/TestObjects/src/SchemaEvolutionTestObjects.cc @@ -0,0 +1,43 @@ + +#include "DataFormats/TestObjects/interface/SchemaEvolutionTestObjects.h" + +namespace edmtest { + VectorVectorElement::VectorVectorElement() : a_(0), b_(0) {} + VectorVectorElement::VectorVectorElement(int a, + int b, + SchemaEvolutionChangeOrder const& changeOrder, + SchemaEvolutionAddMember const& addMember, + SchemaEvolutionRemoveMember const& removeMember, + SchemaEvolutionMoveToBase const& moveToBase, + SchemaEvolutionChangeType const& changeType, + SchemaEvolutionAddBase const& addBase, + SchemaEvolutionPointerToMember const& pointerToMember, + SchemaEvolutionPointerToUniquePtr const& pointerToUniquePtr, + SchemaEvolutionCArrayToStdArray const& cArrayToStdArray, + // SchemaEvolutionCArrayToStdVector const& cArrayToStdVector, + SchemaEvolutionVectorToList const& vectorToList, + SchemaEvolutionMapToUnorderedMap const& mapToUnorderedMap) + : a_(a), + b_(b), + changeOrder_(changeOrder), + addMember_(addMember), + removeMember_(removeMember), + moveToBase_(moveToBase), + changeType_(changeType), + addBase_(addBase), + pointerToMember_(pointerToMember), + pointerToUniquePtr_(pointerToUniquePtr), + cArrayToStdArray_(cArrayToStdArray), + // cArrayToStdVector_(cArrayToStdVector), + vectorToList_(vectorToList), + mapToUnorderedMap_(mapToUnorderedMap) {} + +#if defined DataFormats_TestObjects_USE_OLD + VectorVectorElementNonSplit::VectorVectorElementNonSplit() : a_(0) {} + VectorVectorElementNonSplit::VectorVectorElementNonSplit(int a, int) : a_(a) {} +#else + VectorVectorElementNonSplit::VectorVectorElementNonSplit() : a_(0), b_(0) {} + VectorVectorElementNonSplit::VectorVectorElementNonSplit(int a, int b) : a_(a), b_(b) {} +#endif + +} // namespace edmtest diff --git a/DataFormats/TestObjects/src/VectorVectorTop.cc b/DataFormats/TestObjects/src/VectorVectorTop.cc new file mode 100644 index 0000000000000..8f5ccd58acd6c --- /dev/null +++ b/DataFormats/TestObjects/src/VectorVectorTop.cc @@ -0,0 +1,11 @@ +#include "DataFormats/TestObjects/interface/VectorVectorTop.h" + +namespace edmtest { + + VectorVectorMiddle::VectorVectorMiddle() {} + VectorVectorTop::VectorVectorTop() {} + + VectorVectorMiddleNonSplit::VectorVectorMiddleNonSplit() {} + VectorVectorTopNonSplit::VectorVectorTopNonSplit() {} + +} // namespace edmtest diff --git a/DataFormats/TestObjects/src/classes.h b/DataFormats/TestObjects/src/classes.h index c71629aa9b41f..10aee962eda9b 100644 --- a/DataFormats/TestObjects/src/classes.h +++ b/DataFormats/TestObjects/src/classes.h @@ -10,6 +10,7 @@ #include "DataFormats/Common/interface/AssociationVector.h" #include "DataFormats/Common/interface/Wrapper.h" +#include "DataFormats/TestObjects/interface/SchemaEvolutionTestObjects.h" #include "DataFormats/TestObjects/interface/MissingDictionaryTestObject.h" #include "DataFormats/TestObjects/interface/OtherThingCollection.h" #include "DataFormats/TestObjects/interface/ThingCollection.h" @@ -28,6 +29,8 @@ #include "DataFormats/TestObjects/interface/DeleteEarly.h" +#include "DataFormats/TestObjects/interface/VectorVectorTop.h" + #include "DataFormats/Common/interface/Holder.h" #include "DataFormats/Common/interface/RefProd.h" #include "DataFormats/Common/interface/RefToBaseProd.h" diff --git a/DataFormats/TestObjects/src/classes_def.xml b/DataFormats/TestObjects/src/classes_def.xml index f72dc5545321f..ab5e3287d766c 100644 --- a/DataFormats/TestObjects/src/classes_def.xml +++ b/DataFormats/TestObjects/src/classes_def.xml @@ -170,7 +170,7 @@ - + @@ -252,7 +252,149 @@ exception when running testMissingDictionaryChecking_cfg.py. --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/IOPool/Input/test/BuildFile.xml b/IOPool/Input/test/BuildFile.xml index b960208fe6a6b..aadb8a4b57436 100644 --- a/IOPool/Input/test/BuildFile.xml +++ b/IOPool/Input/test/BuildFile.xml @@ -13,7 +13,16 @@ - - - + + + + + + + + + + + + diff --git a/IOPool/Input/test/SchemaEvolutionTestRead.cc b/IOPool/Input/test/SchemaEvolutionTestRead.cc new file mode 100644 index 0000000000000..ae418c532f99a --- /dev/null +++ b/IOPool/Input/test/SchemaEvolutionTestRead.cc @@ -0,0 +1,292 @@ +// -*- C++ -*- +// +// Package: IOPool/Input +// Class: SchemaEvolutionTestRead +// +/**\class edmtest::SchemaEvolutionTestRead + Description: Used as part of tests of ROOT's schema evolution + features. These features allow reading a persistent object + whose data format has changed since it was written. +*/ +// Original Author: W. David Dagenhart +// Created: 28 July 2023 + +#include "DataFormats/TestObjects/interface/VectorVectorTop.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/global/EDAnalyzer.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "FWCore/Utilities/interface/InputTag.h" +#include "FWCore/Utilities/interface/StreamID.h" + +#include + +namespace edmtest { + + class SchemaEvolutionTestRead : public edm::global::EDAnalyzer<> { + public: + SchemaEvolutionTestRead(edm::ParameterSet const&); + void analyze(edm::StreamID, edm::Event const&, edm::EventSetup const&) const override; + static void fillDescriptions(edm::ConfigurationDescriptions&); + + private: + void analyzeVectorVector(edm::Event const&) const; + void analyzeVectorVectorNonSplit(edm::Event const&) const; + + void throwWithMessageFromConstructor(const char*) const; + void throwWithMessage(const char*) const; + + // These expected values are meaningless other than we use them + // to check that values read from persistent storage match the values + // we know were written. + + const std::vector expectedVectorVectorIntegralValues_; + const edm::EDGetTokenT vectorVectorToken_; + const edm::EDGetTokenT vectorVectorNonSplitToken_; + }; + + SchemaEvolutionTestRead::SchemaEvolutionTestRead(edm::ParameterSet const& iPSet) + : expectedVectorVectorIntegralValues_(iPSet.getParameter>("expectedVectorVectorIntegralValues")), + vectorVectorToken_(consumes(iPSet.getParameter("vectorVectorTag"))), + vectorVectorNonSplitToken_(consumes(iPSet.getParameter("vectorVectorTag"))) { + if (expectedVectorVectorIntegralValues_.size() != 15) { + throwWithMessageFromConstructor("test configuration error, expectedVectorVectorIntegralValues must have size 15"); + } + } + + void SchemaEvolutionTestRead::analyze(edm::StreamID, edm::Event const& iEvent, edm::EventSetup const&) const { + analyzeVectorVector(iEvent); + analyzeVectorVectorNonSplit(iEvent); + } + + void SchemaEvolutionTestRead::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.add>("expectedVectorVectorIntegralValues"); + desc.add("vectorVectorTag"); + descriptions.addDefault(desc); + } + + void SchemaEvolutionTestRead::analyzeVectorVector(edm::Event const& iEvent) const { + auto const& vectorVector = iEvent.get(vectorVectorToken_); + unsigned int vectorSize = 2 + iEvent.id().event() % 4; + if (vectorVector.outerVector_.size() != vectorSize) { + throwWithMessage("analyzeVectorVector, vectorVector does not have expected size"); + } + unsigned int i = 0; + for (auto const& middleVector : vectorVector.outerVector_) { + if (middleVector.middleVector_.size() != vectorSize) { + throwWithMessage("analyzeVectorVector, middleVector does not have expected size"); + } + int iOffset = static_cast(iEvent.id().event() + i); + + int j = 0; + for (auto const& element : middleVector.middleVector_) { + if (element.a_ != expectedVectorVectorIntegralValues_[0] + iOffset + j * 10) { + throwWithMessage("analyzeVectorVector, element a_ does not contain expected value"); + } + if (element.b_ != expectedVectorVectorIntegralValues_[0] + iOffset + j * 100) { + throwWithMessage("analyzeVectorVector, element b_ does not contain expected value"); + } + + SchemaEvolutionChangeOrder const& changeOrder = element.changeOrder_; + if (changeOrder.a_ != expectedVectorVectorIntegralValues_[1] + iOffset + j * 11) { + throwWithMessage("analyzeVectorVector, changeOrder a_ does not contain expected value"); + } + if (changeOrder.b_ != expectedVectorVectorIntegralValues_[1] + iOffset + j * 101) { + throwWithMessage("analyzeVectorVector, changeOrder b_ does not contain expected value"); + } + + SchemaEvolutionAddMember const& addMember = element.addMember_; + if (addMember.a_ != expectedVectorVectorIntegralValues_[2] + iOffset + j * 12) { + throwWithMessage("analyzeVectorVector, addMember a_ does not contain expected value"); + } + if (addMember.b_ != expectedVectorVectorIntegralValues_[2] + iOffset + j * 102) { + throwWithMessage("analyzeVectorVector, addMember b_ does not contain expected value"); + } + + SchemaEvolutionRemoveMember const& removeMember = element.removeMember_; + if (removeMember.a_ != expectedVectorVectorIntegralValues_[3] + iOffset + j * 13) { + throwWithMessage("analyzeVectorVector, removeMember a_ does not contain expected value"); + } + + SchemaEvolutionMoveToBase const& moveToBase = element.moveToBase_; + if (moveToBase.a_ != expectedVectorVectorIntegralValues_[4] + iOffset + j * 14) { + throwWithMessage("analyzeVectorVector, moveToBase a_ does not contain expected value"); + } + if (moveToBase.b_ != expectedVectorVectorIntegralValues_[4] + iOffset + j * 104) { + throwWithMessage("analyzeVectorVector, moveToBase b_ does not contain expected value"); + } + if (moveToBase.c_ != expectedVectorVectorIntegralValues_[4] + iOffset + j * 1004) { + throwWithMessage("analyzeVectorVector, moveToBase c_ does not contain expected value"); + } + if (moveToBase.d_ != expectedVectorVectorIntegralValues_[4] + iOffset + j * 10004) { + throwWithMessage("analyzeVectorVector, moveToBase d_ does not contain expected value"); + } + + SchemaEvolutionChangeType const& changeType = element.changeType_; + if (static_cast(changeType.a_) != expectedVectorVectorIntegralValues_[5] + iOffset + j * 15) { + throwWithMessage("analyzeVectorVector, changeType a_ does not contain expected value"); + } + if (static_cast(changeType.b_) != expectedVectorVectorIntegralValues_[5] + iOffset + j * 105) { + throwWithMessage("analyzeVectorVector, changeType b_ does not contain expected value"); + } + + SchemaEvolutionAddBase const& addBase = element.addBase_; + if (addBase.a_ != expectedVectorVectorIntegralValues_[6] + iOffset + j * 16) { + throwWithMessage("analyzeVectorVector, addToBase a_ does not contain expected value"); + } + if (addBase.b_ != expectedVectorVectorIntegralValues_[6] + iOffset + j * 106) { + throwWithMessage("analyzeVectorVector, addToBase b_ does not contain expected value"); + } + + SchemaEvolutionPointerToMember const& pointerToMember = element.pointerToMember_; + if (pointerToMember.a_ != expectedVectorVectorIntegralValues_[7] + iOffset + j * 17) { + throwWithMessage("analyzeVectorVector, pointerToMember a_ does not contain expected value"); + } + if (pointerToMember.b_ != expectedVectorVectorIntegralValues_[7] + iOffset + j * 107) { + throwWithMessage("analyzeVectorVector, pointerToMember b_ does not contain expected value"); + } + // This part is commented out because it fails. My conclusion is that ROOT + // does not properly support schema evolution in this case. CMS does not + // usually use pointers in persistent formats. So for now we are just ignoring + // this issue. + // if (pointerToMember.c() != expectedVectorVectorIntegralValues_[7] + iOffset + j * 1007) { + // throwWithMessage("analyzeVectorVector, pointerToMember c_ does not contain expected value"); + // } + + SchemaEvolutionPointerToUniquePtr const& pointerToUniquePtr = element.pointerToUniquePtr_; + if (pointerToUniquePtr.a_ != expectedVectorVectorIntegralValues_[8] + iOffset + j * 18) { + throwWithMessage("analyzeVectorVector, pointerToUniquePtr a_ does not contain expected value"); + } + if (pointerToUniquePtr.b_ != expectedVectorVectorIntegralValues_[8] + iOffset + j * 108) { + throwWithMessage("analyzeVectorVector, pointerToUniquePtr b_ does not contain expected value"); + } + if (pointerToUniquePtr.contained_->c_ != expectedVectorVectorIntegralValues_[8] + iOffset + j * 1008) { + throwWithMessage("analyzeVectorVector, pointerToUniquePtr c_ does not contain expected value"); + } + + SchemaEvolutionCArrayToStdArray const& cArrayToStdArray = element.cArrayToStdArray_; + if (cArrayToStdArray.a_[0] != expectedVectorVectorIntegralValues_[9] + iOffset + j * 19) { + throwWithMessage("analyzeVectorVector, cArrayToStdArray a_[0] does not contain expected value"); + } + if (cArrayToStdArray.a_[1] != expectedVectorVectorIntegralValues_[9] + iOffset + j * 109) { + throwWithMessage("analyzeVectorVector, cArrayToStdArray a_[1] does not contain expected value"); + } + if (cArrayToStdArray.a_[2] != expectedVectorVectorIntegralValues_[9] + iOffset + j * 1009) { + throwWithMessage("analyzeVectorVector, cArrayToStdArray a_[2] does not contain expected value"); + } + + // This part is commented out because it fails. My conclusion is that ROOT + // does not properly support schema evolution in this case. CMS does not + // usually use pointers in persistent formats. So for now we are just ignoring + // this issue. Note that pointerToMember fails because the values are incorrect. + // This one is also commented out of the format, the write function and here + // because simply reading the object causes a fatal exception even without + // checking the values. + // SchemaEvolutionCArrayToStdVector const& cArrayToStdVector = element.cArrayToStdVector_; + // if (cArrayToStdVector.a_[0] != expectedVectorVectorIntegralValues_[10] + iOffset + j * 20) { + // throwWithMessage("analyzeVectorVector, cArrayToStdVector a_[0] does not contain expected value"); + // } + // if (cArrayToStdVector.a_[1] != expectedVectorVectorIntegralValues_[10] + iOffset + j * 110) { + // throwWithMessage("analyzeVectorVector, cArrayToStdVector a_[1] does not contain expected value"); + // } + // if (cArrayToStdVector.a_[2] != expectedVectorVectorIntegralValues_[10] + iOffset + j * 1010) { + // throwWithMessage("analyzeVectorVector, cArrayToStdVector a_[2] does not contain expected value"); + // } + + { + SchemaEvolutionVectorToList const& vectorToList = element.vectorToList_; + auto iter = vectorToList.a_.cbegin(); + auto iter0 = iter; + auto iter1 = ++iter; + auto iter2 = ++iter; + if (*iter0 != expectedVectorVectorIntegralValues_[11] + iOffset + j * 21) { + throwWithMessage("vectorToList, element 0 does not contain expected value"); + } + if (*iter1 != expectedVectorVectorIntegralValues_[11] + iOffset + j * 111) { + throwWithMessage("vectorToList, element 1 does not contain expected value"); + } + if (*iter2 != expectedVectorVectorIntegralValues_[11] + iOffset + j * 1011) { + throwWithMessage("vectorToList, element 2 does not contain expected value"); + } + } + + { + SchemaEvolutionMapToUnorderedMap const& mapToUnorderedMap = element.mapToUnorderedMap_; + if (mapToUnorderedMap.a_.size() != 3) { + throwWithMessage("mapToUnorderedMap, map has unexpected size"); + } + auto iter = mapToUnorderedMap.a_.cbegin(); + + // Easier to check values if we sort them first so sort them in a regular map + std::map orderedMap; + orderedMap.insert(*iter); + ++iter; + orderedMap.insert(*iter); + ++iter; + orderedMap.insert(*iter); + + auto orderedIter = orderedMap.cbegin(); + auto iter0 = orderedIter; + auto iter1 = ++orderedIter; + auto iter2 = ++orderedIter; + if (iter0->first != expectedVectorVectorIntegralValues_[12] + iOffset + j * 22) { + throwWithMessage("mapToUnorderedMap, element 0 key does not contain expected value"); + } + if (iter0->second != expectedVectorVectorIntegralValues_[12] + iOffset + j * 112) { + throwWithMessage("mapToUnorderedMap, element 0 does not contain expected value"); + } + if (iter1->first != expectedVectorVectorIntegralValues_[12] + iOffset + j * 1012 + 1012) { + throwWithMessage("mapToUnorderedMap, element 1 key does not contain expected value"); + } + if (iter1->second != expectedVectorVectorIntegralValues_[12] + iOffset + j * 10012) { + throwWithMessage("mapToUnorderedMap, element 1 does not contain expected value"); + } + if (iter2->first != expectedVectorVectorIntegralValues_[12] + iOffset + j * 100012 + 100012) { + throwWithMessage("mapToUnorderedMap, element 2 key does not contain expected value"); + } + if (iter2->second != expectedVectorVectorIntegralValues_[12] + iOffset + j * 1000012) { + throwWithMessage("mapToUnorderedMap, element 2 does not contain expected value"); + } + } + ++j; + } + ++i; + } + } + + void SchemaEvolutionTestRead::analyzeVectorVectorNonSplit(edm::Event const& iEvent) const { + auto const& vectorVectorNonSplit = iEvent.get(vectorVectorNonSplitToken_); + unsigned int vectorSize = 1; + if (vectorVectorNonSplit.outerVector_.size() != vectorSize) { + throwWithMessage("analyzeVectorVectorNonSplit, outerVector does not have expected size"); + } + for (auto const& middleVector : vectorVectorNonSplit.outerVector_) { + if (middleVector.middleVector_.size() != vectorSize) { + throwWithMessage("analyzeVectorVectorNonSplit, middleVector does not have expected size"); + } + for (auto const& element : middleVector.middleVector_) { + if (element.a_ != expectedVectorVectorIntegralValues_[13]) { + throwWithMessage("analyzeVectorVectorNonSplit, element a_ does not contain expected value"); + } + } + } + } + + void SchemaEvolutionTestRead::throwWithMessageFromConstructor(const char* msg) const { + throw cms::Exception("TestFailure") << "SchemaEvolutionTestRead constructor, " << msg; + } + + void SchemaEvolutionTestRead::throwWithMessage(const char* msg) const { + throw cms::Exception("TestFailure") << "SchemaEvolutionTestRead::analyze, " << msg; + } + +} // namespace edmtest + +using edmtest::SchemaEvolutionTestRead; +DEFINE_FWK_MODULE(SchemaEvolutionTestRead); diff --git a/IOPool/Input/test/SchemaEvolutionTestWrite.cc b/IOPool/Input/test/SchemaEvolutionTestWrite.cc new file mode 100644 index 0000000000000..a951bb1870b00 --- /dev/null +++ b/IOPool/Input/test/SchemaEvolutionTestWrite.cc @@ -0,0 +1,156 @@ +// -*- C++ -*- +// +// Package: IOPool/Input +// Class: SchemaEvolutionTestWrite +// +/**\class edmtest::SchemaEvolutionTestWrite + Description: Used as part of tests of ROOT's schema evolution + features. These features allow reading a persistent object + whose data format has changed since it was written. +*/ +// Original Author: W. David Dagenhart +// Created: 24 July 2023 + +#include "DataFormats/TestObjects/interface/VectorVectorTop.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/global/EDProducer.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/Utilities/interface/EDPutToken.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "FWCore/Utilities/interface/StreamID.h" + +#include +#include +#include + +namespace edmtest { + + class SchemaEvolutionTestWrite : public edm::global::EDProducer<> { + public: + SchemaEvolutionTestWrite(edm::ParameterSet const&); + void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override; + static void fillDescriptions(edm::ConfigurationDescriptions&); + + private: + void produceVectorVector(edm::Event&) const; + void produceVectorVectorNonSplit(edm::Event&) const; + + void throwWithMessage(const char*) const; + + const std::vector testIntegralValues_; + const edm::EDPutTokenT vectorVectorPutToken_; + const edm::EDPutTokenT vectorVectorNonSplitPutToken_; + }; + + SchemaEvolutionTestWrite::SchemaEvolutionTestWrite(edm::ParameterSet const& iPSet) + : testIntegralValues_(iPSet.getParameter>("testIntegralValues")), + vectorVectorPutToken_(produces()), + vectorVectorNonSplitPutToken_(produces()) { + if (testIntegralValues_.size() != 15) { + throwWithMessage("testIntegralValues must have 15 elements and it does not"); + } + } + + void SchemaEvolutionTestWrite::produce(edm::StreamID, edm::Event& iEvent, edm::EventSetup const&) const { + // Fill test objects. Make sure all the containers inside + // of them have something in them (not empty). The values are meaningless. + // We will later check that after writing these objects to persistent storage + // and then reading them in a later process we obtain matching values for + // all this content. + + produceVectorVector(iEvent); + produceVectorVectorNonSplit(iEvent); + } + + void SchemaEvolutionTestWrite::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.add>("testIntegralValues"); + descriptions.addDefault(desc); + } + + void SchemaEvolutionTestWrite::produceVectorVector(edm::Event& iEvent) const { + auto vectorVector = std::make_unique(); + unsigned int vectorSize = 2 + iEvent.id().event() % 4; + vectorVector->outerVector_.reserve(vectorSize); + for (unsigned int i = 0; i < vectorSize; ++i) { + int iOffset = static_cast(iEvent.id().event() + i); + VectorVectorMiddle middleVector; + middleVector.middleVector_.reserve(vectorSize); + for (int j = 0; j < static_cast(vectorSize); ++j) { + SchemaEvolutionChangeOrder changeOrder(testIntegralValues_[1] + iOffset + j * 11, + testIntegralValues_[1] + iOffset + j * 101); + SchemaEvolutionAddMember addMember(testIntegralValues_[2] + iOffset + j * 12, + testIntegralValues_[2] + iOffset + j * 102, + testIntegralValues_[2] + iOffset + j * 1002); + SchemaEvolutionRemoveMember removeMember(testIntegralValues_[3] + iOffset + j * 13, + testIntegralValues_[3] + iOffset + j * 103); + SchemaEvolutionMoveToBase moveToBase(testIntegralValues_[4] + iOffset + j * 14, + testIntegralValues_[4] + iOffset + j * 104, + testIntegralValues_[4] + iOffset + j * 1004, + testIntegralValues_[4] + iOffset + j * 10004); + SchemaEvolutionChangeType changeType(testIntegralValues_[5] + iOffset + j * 15, + testIntegralValues_[5] + iOffset + j * 105); + SchemaEvolutionAddBase addBase(testIntegralValues_[6] + iOffset + j * 16, + testIntegralValues_[6] + iOffset + j * 106, + testIntegralValues_[6] + iOffset + j * 1006); + SchemaEvolutionPointerToMember pointerToMember(testIntegralValues_[7] + iOffset + j * 17, + testIntegralValues_[7] + iOffset + j * 107, + testIntegralValues_[7] + iOffset + j * 1007); + SchemaEvolutionPointerToUniquePtr pointerToUniquePtr(testIntegralValues_[8] + iOffset + j * 18, + testIntegralValues_[8] + iOffset + j * 108, + testIntegralValues_[8] + iOffset + j * 1008); + SchemaEvolutionCArrayToStdArray cArrayToStdArray(testIntegralValues_[9] + iOffset + j * 19, + testIntegralValues_[9] + iOffset + j * 109, + testIntegralValues_[9] + iOffset + j * 1009); + // This is commented out because schema evolution fails for this case + // SchemaEvolutionCArrayToStdVector cArrayToStdVector(testIntegralValues_[10] + iOffset + j * 20, testIntegralValues_[10] + iOffset + j * 110, testIntegralValues_[10] + iOffset + j * 1010); + SchemaEvolutionVectorToList vectorToList(testIntegralValues_[11] + iOffset + j * 21, + testIntegralValues_[11] + iOffset + j * 111, + testIntegralValues_[11] + iOffset + j * 1011); + SchemaEvolutionMapToUnorderedMap mapToUnorderedMap(testIntegralValues_[12] + iOffset + j * 22, + testIntegralValues_[12] + iOffset + j * 112, + testIntegralValues_[12] + iOffset + j * 1012 + 1012, + testIntegralValues_[12] + iOffset + j * 10012, + testIntegralValues_[12] + iOffset + j * 100012 + 100012, + testIntegralValues_[12] + iOffset + j * 1000012); + + middleVector.middleVector_.emplace_back(testIntegralValues_[0] + iOffset + j * 10, + testIntegralValues_[0] + iOffset + j * 100, + changeOrder, + addMember, + removeMember, + moveToBase, + changeType, + addBase, + pointerToMember, + pointerToUniquePtr, + cArrayToStdArray, + // cArrayToStdVector, + vectorToList, + mapToUnorderedMap); + } + vectorVector->outerVector_.push_back(std::move(middleVector)); + } + iEvent.put(vectorVectorPutToken_, std::move(vectorVector)); + } + + void SchemaEvolutionTestWrite::produceVectorVectorNonSplit(edm::Event& iEvent) const { + auto vectorVector = std::make_unique(); + VectorVectorMiddleNonSplit middleVector; + middleVector.middleVector_.emplace_back(testIntegralValues_[13], testIntegralValues_[14]); + vectorVector->outerVector_.push_back(middleVector); + iEvent.put(vectorVectorNonSplitPutToken_, std::move(vectorVector)); + } + + void SchemaEvolutionTestWrite::throwWithMessage(const char* msg) const { + throw cms::Exception("TestFailure") << "SchemaEvolutionTestWrite constructor, test configuration error, " << msg; + } + +} // namespace edmtest + +using edmtest::SchemaEvolutionTestWrite; +DEFINE_FWK_MODULE(SchemaEvolutionTestWrite); diff --git a/IOPool/Input/test/SchemaEvolution_create_test_file_cfg.py b/IOPool/Input/test/SchemaEvolution_create_test_file_cfg.py new file mode 100644 index 0000000000000..eb290cf5a5a86 --- /dev/null +++ b/IOPool/Input/test/SchemaEvolution_create_test_file_cfg.py @@ -0,0 +1,25 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") + +process.source = cms.Source("EmptySource") +process.maxEvents.input = 10 + +process.writeSchemaEvolutionTest = cms.EDProducer("SchemaEvolutionTestWrite", + # Test values below are meaningless. We just make sure when we read + # we get the same values. Note only values exactly convertible to + # float are used to avoid precision and rounding issues in + # in comparisons. + testIntegralValues = cms.vint32( + 11, 21, 31, 41, 51, 61, 71, 81, 91, 101, 111, 121, 131, 141, 151 + ) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('SchemaEvolutionTest.root') +) + +process.path = cms.Path(process.writeSchemaEvolutionTest) +process.endPath = cms.EndPath(process.out) diff --git a/IOPool/Input/test/SchemaEvolution_test_read_cfg.py b/IOPool/Input/test/SchemaEvolution_test_read_cfg.py new file mode 100644 index 0000000000000..e45194b2d5ce2 --- /dev/null +++ b/IOPool/Input/test/SchemaEvolution_test_read_cfg.py @@ -0,0 +1,30 @@ +import FWCore.ParameterSet.Config as cms +import sys +import argparse + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ROOT Schema Evolution') + +parser.add_argument("--inputFile", type=str, help="Input file name (default: SchemaEvolutionTest.root)", default="SchemaEvolutionTest.root") +parser.add_argument("--outputFileName", type=str, help="Output file name (default: SchemaEvolutionTest2.root)", default="SchemaEvolutionTest2.root") +args = parser.parse_args() + +process = cms.Process("READ") + +process.source = cms.Source("PoolSource", fileNames = cms.untracked.vstring("file:"+args.inputFile)) + +process.schemaEvolutionTestRead = cms.EDAnalyzer("SchemaEvolutionTestRead", + # I stick to values exactly convertable to float + # to avoid potential rounding issues in the test. + expectedVectorVectorIntegralValues = cms.vint32( + 11, 21, 31, 41, 51, 61, 71, 81, 91, 101, 111, 121, 131, 141, 151 + ), + vectorVectorTag = cms.InputTag("writeSchemaEvolutionTest", "", "PROD") +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string(args.outputFileName) +) + +process.path = cms.Path(process.schemaEvolutionTestRead) + +process.endPath = cms.EndPath(process.out) diff --git a/IOPool/Input/test/testForStreamerInfo.C b/IOPool/Input/test/testForStreamerInfo.C new file mode 100644 index 0000000000000..caa4474333012 --- /dev/null +++ b/IOPool/Input/test/testForStreamerInfo.C @@ -0,0 +1,111 @@ +// Use something like the following to run this file: +// +// root.exe -b -l -q SchemaEvolutionTest.root 'IOPool/Input/test/testForStreamerInfo.C(gFile)' | sort -u +// +// In the output, lines with "Missing" indicate problems. + +// Note the script ignores classes whose name start with "T" +// to ignore ROOT classes. In the context this is used, none +// of the interesting classes start with "T" (although if used +// generally use I can imagine this could cause some confusion...). + +// This is a modified version of some temporary code Philippe Canal +// provided us while debugging a problem. + +#include "TFile.h" +#include "TStreamerInfo.h" +#include "TList.h" +#include "TVirtualCollectionProxy.h" +#include "TStreamerElement.h" +#include + +void check(TClass &cl, TList &streamerInfoList) { + std::string name(cl.GetName()); + if (name == "string") + return; + if (0 == name.compare(0, strlen("pair<"), "pair<")) + return; + std::cout << "check TClass: " << name << std::endl; + bool found = streamerInfoList.FindObject(cl.GetName()) != 0; + if (!found) + std::cout << "Missing: " << cl.GetName() << '\n'; +} + +void check(TVirtualCollectionProxy &proxy, TList &streamerInfoList) { + auto inner = proxy.GetValueClass(); + if (inner) { + auto subproxy = inner->GetCollectionProxy(); + if (subproxy) { + check(*subproxy, streamerInfoList); + } else { + check(*inner, streamerInfoList); + } + } +} + +void check(TStreamerElement &element, TList &streamerInfoList) { + auto cl = element.GetClass(); + if (cl == nullptr) { + return; + } + // Ignore all classes that start with a T with the intent + // to ignore all internal ROOT classes + // (In general this might ignore other interesting classes, + // but this is intended to be used in a specific test where the + // the interesting classes don't start with T). + if (*(cl->GetName()) == 'T') { + return; + } + if (cl->GetCollectionProxy()) { + check(*cl->GetCollectionProxy(), streamerInfoList); + } else { + check(*cl, streamerInfoList); + } +} + +// This is called once for each TStreamerInfo in +// streamerInfoList. The info is the first argument +// and the list of all the infos is the second argument. +void scan(TStreamerInfo &info, TList &streamerInfoList) { + auto cl = TClass::GetClass(info.GetName()); + // print error message and do skip the info if there + // is not a TClass available. + if (!cl) { + //std::cerr << "Error no TClass for " << info.GetName() << '\n'; + return; + } + auto proxy = cl->GetCollectionProxy(); + if (proxy) + check(*proxy, streamerInfoList); + for (auto e : TRangeDynCast(*info.GetElements())) { + if (!e) + continue; + check(*e, streamerInfoList); + } +} + +void scan(TList *streamerInfoList) { + if (!streamerInfoList) + return; + for (auto l : TRangeDynCast(*streamerInfoList)) { + if (!l) + continue; + // Ignore all classes that start with a T with the intent + // to ignore all internal ROOT classes + // (In general this might ignore other interesting classes, + // but this is intended to be used in a specific test where the + // the interesting classes don't start with T). + if (*(l->GetName()) == 'T') { + continue; + } + //std::cout << "Seeing: " << l->GetName() << " " << l->GetClassVersion() << '\n'; + scan(*l, *streamerInfoList); + } + delete streamerInfoList; +} + +void testForStreamerInfo(TFile *file) { + if (!file) + return; + scan(file->GetStreamerInfoList()); +} diff --git a/IOPool/Input/test/testSchemaEvolution.sh b/IOPool/Input/test/testSchemaEvolution.sh new file mode 100755 index 0000000000000..1738174685efc --- /dev/null +++ b/IOPool/Input/test/testSchemaEvolution.sh @@ -0,0 +1,105 @@ +#!/bin/sh -x + +function die { echo $1: status $2 ; exit $2; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + +# The purpose of this is to test schema evolution in ROOT. + +# In the first two cmsRun processes, there is no schema evolution +# going on. The first one writes a data file containing a test +# product. The second one reads the test products and checks +# that the values read from it match values that the first process +# should have written. The purpose here is to validate that the +# test code is actually working properly and if we see failures +# in the later cmsRun processes, they are more likely to be +# caused by a failure in ROOT schema evolution. +cmsRun ${LOCAL_TEST_DIR}/SchemaEvolution_create_test_file_cfg.py || die 'Failure using SchemaEvolution_create_test_file_cfg.py' $? +cmsRun ${LOCAL_TEST_DIR}/SchemaEvolution_test_read_cfg.py || die 'Failure using SchemaEvolution_create_test_file_cfg.py' $? + +# For each StreamerInfo in the input file, test for existence of StreamerInfo for +# nested classes (members, base, elements of containers). +root.exe -b -l -q file:SchemaEvolutionTest.root "${LOCAL_TEST_DIR}/testForStreamerInfo.C(gFile)" | sort -u > testForStreamerInfo1.log +grep "Missing" testForStreamerInfo1.log && die "Missing nested streamer info" 1 +grep "SchemaEvolutionChangeOrder" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionChangeOrder in testForStreamerInfo1.log' $? +grep "SchemaEvolutionAddMember" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionAddMember in testForStreamerInfo1.log' $? +grep "SchemaEvolutionRemoveMember" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionRemoveMember in testForStreamerInfo1.log' $? +grep "SchemaEvolutionMoveToBase" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionMoveToBase" in testForStreamerInfo1.log' $? +grep "SchemaEvolutionChangeType" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionChangeType in testForStreamerInfo1.log' $? +grep "SchemaEvolutionAddBase" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionAddBase in testForStreamerInfo1.log' $? +grep "SchemaEvolutionPointerToMember" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionPointerToMember in testForStreamerInfo1.log' $? +grep "SchemaEvolutionPointerToUniquePtr" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionPointerToUniquePtr in testForStreamerInfo1.log' $? +grep "SchemaEvolutionCArrayToStdArray" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionCArrayToStdArray in testForStreamerInfo1.log' $? +grep "SchemaEvolutionVectorToList" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionVectorToList in testForStreamerInfo1.log' $? +grep "SchemaEvolutionMapToUnorderedMap" testForStreamerInfo1.log || die 'Failure cannot find SchemaEvolutionMapToUnorderedMap in testForStreamerInfo1.log' $? +grep "VectorVectorElementNonSplit" testForStreamerInfo1.log || die 'Failure cannot find VectorVectorElementNonSplit in testForStreamerInfo1.log' $? + +# Then we read permanently saved data files from the cms-data +# repository. When these data files were written, the working area +# was built with different class definitions used for the contents +# of the test product. This forces ROOT to perform schema evolution +# as it reads the test product. We check both that the job +# completes successfully and that the values contained in +# the test product after schema evolution are what we expect. +# The plan is to generate new data files each time there is a major +# revision to ROOT. + +# The old files read below were generated as follows. +# +# Check out the release you want to use to generate the file +# and build a working area. You will at least need to checkout +# IOPool/Input and DataFormats/TestObjects. +# +# If the release is before the first release that includes this +# file (testSchemaEvolution.sh), then you will need to manually +# merge in the commit that added the file. Most of the files are +# completely new files, although BuildFile.xml, classes.h, and +# classes_def.xml already existed may require manually merging a +# few lines of code. +# +# Manually edit 2 files. +# +# 1. Modify the following line in the file: +# DataFormats/TestObjects/interface/SchemaEvolutionTestObjects.h +# //#define DataFormats_TestObjects_USE_OLD +# To generate an input file with the old format remove the "//" +# at the beginning so the macro is defined. +# +# 2. In file: DataFormats/TestObjects/src/classes_def.xml +# There is a section of code starting with SchemaEvolutionChangeOrder +# continuing to SchemaEvolutionMapToUnorderedMap. This section appears +# twice. The first section defines old formats that should be used +# to generate old format data files (These are all ClassVersion= "3") +# When reading, use the new version (the section with ClassVersion="4" +# which should be the one enabled in the CMSSW repository). +# Of these two sections, exactly one should be commented out. +# To generate the input data files, you will need to manually +# enable the ClassVersion 3 section. +# +# Then rebuild the working area. +# +# Then run the configuration IOPool/Input/test/SchemaEvolution_create_test_file_cfg.py +# This will generate an output file named SchemaEvolutionTest.root. +# Rename this file appropriately to include the release used and use +# it as an input for this test by adding additional cases below. +# The new data file will need to added to the cms-data repository +# named IOPool-Input. + +file=SchemaEvolutionTestOLD13_2_3.root +inputfile=$(edmFileInPath IOPool/Input/data/$file) || die "Failure edmFileInPath IOPool/Input/data/$file" $? +cmsRun ${LOCAL_TEST_DIR}/SchemaEvolution_test_read_cfg.py --inputFile "$inputfile" || die "Failed to read old file $file" $? + +file=SchemaEvolutionTestOLD13_0_0.root +inputfile=$(edmFileInPath IOPool/Input/data/$file) || die "Failure edmFileInPath IOPool/Input/data/$file" $? +# These fail because there was a bug in the version of ROOT associated with CMSSW_13_0_0 +# The bug caused StreamerInfo objects to missing from the ROOT file. In this case, +# schema evolution fails and also the testForStreamerInfo.C script will find +# missing StreamerInfo objects. +# Lets keep this code around because it may be useful if we need to +# do additional work related to data files written using an executable +# built from code having the bug. +#cmsRun ${LOCAL_TEST_DIR}/SchemaEvolution_test_read_cfg.py --inputFile "$inputfile" || die "Failed to read old file $file" $? +#root.exe -b -l -q file:$inputfile "${LOCAL_TEST_DIR}/testForStreamerInfo.C(gFile)" | sort -u | grep Missing > testForStreamerInfo2.log +#grep "Missing" testForStreamerInfo2.log && die "Missing nested streamer info" 1 + +exit 0