From f614add8384476b9c1e1401cfdbf8f190c03d890 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Wed, 7 Jul 2021 13:00:42 +0200 Subject: [PATCH 01/17] Check functionality after merging with latest cmsdist --- cmssw-tool-conf.spec | 196 +++-------------------------------------- scram-tools.spec | 16 ++++ scramv1-tool-conf.file | 68 ++++++++++---- 3 files changed, 80 insertions(+), 200 deletions(-) create mode 100644 scram-tools.spec diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index ae77bd0e288..07f9dfad29e 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -5,190 +5,22 @@ ## INSTALL_DEPENDENCIES cmsLHEtoEOSManager gcc-fixincludes cmssw-osenv cms-git-tools ## UPLOAD_DEPENDENCIES dqmgui -Requires: crab -Requires: cmssw-wm-tools -Requires: google-benchmark-toolfile -Requires: catch2-toolfile -Requires: starlight-toolfile -Requires: alpgen-toolfile -Requires: boost-toolfile -Requires: bz2lib-toolfile -Requires: charybdis-toolfile -Requires: classlib-toolfile -Requires: clhep-toolfile -Requires: coral-toolfile -Requires: cppunit-toolfile -Requires: curl-toolfile -Requires: das_client-toolfile -Requires: db6-toolfile -Requires: davix-toolfile -Requires: evtgen-toolfile -Requires: expat-toolfile -Requires: fakesystem -Requires: fastjet-toolfile -Requires: flatbuffers-toolfile -Requires: fmt-toolfile -Requires: gbl-toolfile -Requires: gcc-toolfile -Requires: gdbm-toolfile -Requires: geant4-toolfile -Requires: geant4data-toolfile -Requires: vecgeom-toolfile -Requires: glimpse-toolfile -Requires: gmake-toolfile -Requires: gsl-toolfile -Requires: highfive-toolfile -Requires: hector-toolfile -Requires: hepmc-toolfile -Requires: heppdt-toolfile -Requires: herwig-toolfile -Requires: herwig7-toolfile -Requires: hydjet-toolfile -Requires: ittnotify-toolfile -Requires: jemalloc-toolfile -Requires: jemalloc-debug-toolfile -Requires: jimmy-toolfile -Requires: json-toolfile -Requires: ktjet-toolfile -Requires: lhapdf-toolfile -Requires: libjpeg-turbo-toolfile -Requires: libpng-toolfile -Requires: libtiff-toolfile -Requires: libungif-toolfile -Requires: libxml2-toolfile -Requires: lwtnn-toolfile -Requires: meschach-toolfile -Requires: pcre-toolfile -Requires: photos-toolfile -Requires: photospp-toolfile -Requires: pyquen-toolfile -Requires: pythia6-toolfile -Requires: pythia8-toolfile -Requires: python-toolfile -Requires: python3-toolfile -Requires: root-toolfile -Requires: sherpa-toolfile -Requires: libpciaccess-toolfile -Requires: numactl-toolfile -Requires: hwloc-toolfile -%ifnarch aarch64 -Requires: gdrcopy-toolfile -%endif -Requires: ucx-toolfile -Requires: openmpi-toolfile -Requires: sigcpp-toolfile -Requires: sqlite-toolfile -Requires: systemtools -Requires: tauola-toolfile -Requires: tauolapp-toolfile -Requires: thepeg-toolfile -Requires: toprex-toolfile -Requires: libuuid-toolfile -Requires: xerces-c-toolfile -Requires: zlib-toolfile -Requires: dcap-toolfile -Requires: frontier_client-toolfile -Requires: xrootd-toolfile -Requires: dd4hep-toolfile -Requires: graphviz-toolfile -Requires: valgrind-toolfile -Requires: cmsswdata-toolfile -Requires: zstd-toolfile -Requires: hls-toolfile -Requires: opencv-toolfile -Requires: grpc-toolfile -Requires: onnxruntime-toolfile -Requires: triton-inference-server-toolfile -Requires: hdf5-toolfile -Requires: rivet-toolfile -Requires: cascade-toolfile -Requires: yoda-toolfile -Requires: fftw3-toolfile -Requires: fftjet-toolfile -Requires: professor2-toolfile -Requires: xz-toolfile -Requires: lz4-toolfile -Requires: protobuf-toolfile -Requires: lcov-toolfile -Requires: llvm-gcc-toolfile -Requires: tbb-toolfile -Requires: mctester-toolfile -Requires: vdt-toolfile -Requires: icc-gcc-toolfile -Requires: gnuplot-toolfile -Requires: sloccount-toolfile -Requires: millepede-toolfile -Requires: pacparser-toolfile -Requires: git-toolfile -Requires: cgal-toolfile -Requires: doxygen-toolfile -Requires: yaml-cpp-toolfile -Requires: gmp-static-toolfile -Requires: mpfr-static-toolfile -Requires: fastjet-contrib-toolfile -Requires: opencl-toolfile -Requires: opencl-cpp-toolfile -Requires: qd-toolfile -Requires: blackhat-toolfile -Requires: sherpa-toolfile -Requires: geant4-parfullcms-toolfile -Requires: fasthadd -Requires: eigen-toolfile -Requires: gdb-toolfile -Requires: libxslt-toolfile -Requires: giflib-toolfile -Requires: freetype-toolfile -Requires: utm-toolfile -Requires: libffi-toolfile -Requires: CSCTrackFinderEmulation-toolfile -Requires: tinyxml2-toolfile -Requires: md5-toolfile -Requires: gosamcontrib-toolfile -Requires: gosam-toolfile -Requires: madgraph5amcatnlo-toolfile -Requires: python_tools -Requires: dasgoclient -Requires: OpenBLAS-toolfile -Requires: mxnet-predict-toolfile -Requires: mkfit-toolfile -Requires: dablooms-toolfile - -# Only for Linux platform. -%ifos linux -Requires: openldap-toolfile -Requires: gperftools-toolfile -Requires: cuda-toolfile -Requires: alpaka-toolfile -Requires: cupla-toolfile - -%if "%{cmsos}" != "slc7_aarch64" -Requires: cudnn-toolfile -%endif - -Requires: libunwind-toolfile -%ifnarch ppc64le -Requires: igprof-toolfile -Requires: openloops-toolfile -%endif - -%ifarch x86_64 -Requires: dmtcp-toolfile -Requires: tkonlinesw-toolfile -Requires: oracle-toolfile +BuildRequires: scram-tools +Requires: gcc +Requires: herwig +Requires: libpng +Requires: OpenBLAS +Requires: root +Requires: tbb +Requires: zlib +Requires: zstd Requires: intel-vtune -Requires: cmsmon-tools -Requires: dip-toolfile -%else -Requires: tkonlinesw-fake-toolfile -Requires: oracle-fake-toolfile -%endif -%endif - -Requires: tensorflow-toolfile -Requires: xtensor-toolfile -Requires: xtl-toolfile -Requires: xgboost-toolfile +Requires: das_client +Requires: llvm +Requires: boost %define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug +# the vectorization flags/macroses will be available + ## IMPORT scramv1-tool-conf diff --git a/scram-tools.spec b/scram-tools.spec new file mode 100644 index 00000000000..b5b70f5fbf1 --- /dev/null +++ b/scram-tools.spec @@ -0,0 +1,16 @@ +### RPM cms scram-tools 1.0 + +%define branch main +%define tag 33e20671985a2b7774681adcdfce655c0186b650 +%define github_user cmss-sw +%define github_repo %{n} + +Source: git+https://github.com/%{github_user}/%{github_repo}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz + +%prep +%setup -n %{n}-%{realversion} + +%build + +%install +cp -r %{_builddir}/%{n}-%{realversion} %{i}/%{n} diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index a8bf551476a..a93c49f07e2 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -8,18 +8,22 @@ Source: none %define CheckScramTools \ uctool=`echo $tool | tr '[a-z-]' '[A-Z_]'` \ toolbase=`eval echo \\\\$${uctool}_ROOT` \ - [ -d $toolbase/etc/scram.d ] || continue + [ -d ${SCRAM_TOOLS_ROOT}/scram-tools/$tool ] || continue %define CopyScramTools \ echo ">> Copying tool files from: $tool" \ - for xml in $(find $toolbase/etc/scram.d -type f) ; do \ + for xml in $(find ${SCRAM_TOOLS_ROOT}/scram-tools/$tool -type f -name "*.xml") ; do \ bxml=$(basename $xml | tr '[A-Z]' '[a-z]') \ [ -f %i/tools/selected/$bxml ] && continue \ [ -f %i/tools/available/$bxml ] && continue \ cp $xml %i/tools/selected/${bxml} \ + T_VERSION=`eval echo \\\\$${uctool}_VERSION` \ + sed -i -e 's|@TOOL_ROOT@|'"$toolbase"'|g' %i/tools/selected/${bxml} \ + sed -i -e 's|@TOOL_VERSION@|'"$T_VERSION"'|g' %i/tools/selected/${bxml} \ echo " Copied $bxml" \ - done - + done \ + tool_script=${SCRAM_TOOLS_ROOT}/scram-tools/$tool/$(echo $tool | tr '[A-Z]' '[a-z]').sh \ + if [ -f $tool_script ]; then ($tool_script %i/tools/selected); fi %prep %build @@ -42,6 +46,23 @@ mkdir -p %i/tools/selected %i/tools/available %define skipreqtools %{nil} %endif +%if "%{?package_vectorization}" != "" +PKG_VECTORIZATION=`echo %package_vectorization` +export PKG_VECTORIZATION +%endif + +COMP_ARCH_SPECIFIC_FLAGS="" +%ifarch x86_64 +COMP_ARCH_SPECIFIC_FLAGS=$(for v in %package_vectorization ; do %{cmsdist_directory}/vectorization/cmsdist_packages.py ${v} ; done) +%endif +%ifarch aarch64 +COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields" +%endif +%ifarch ppc64le +COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields %ppc64le_build_flags" +%endif +export COMP_ARCH_SPECIFIC_FLAGS + DoneTools=" " SkipTools= for tool in %requiredtools; do @@ -69,23 +90,11 @@ for tool in $SkipTools ; do %{CopyScramTools} done -# Fixes logic in above loop in case of online release: -# the descriptions of explicitly defined systemtools should -# take precedence over those from cms-rebuilt rpms -%if "%online" == "true" -for tool in %systemtools ${PKGTOOLS_SYSTEM_TOOLS} ; do - lc_tool=$(echo $tool | tr '[A-Z]' '[a-z]') - rm -f %i/tools/selected/${lc_tool} %i/tools/selected/${lc_tool}.xml - cp -rf %{onlinesystemtoolsroot}/etc/scram.d/${tool}.xml %i/tools/selected/${lc_tool}.xml -done -# For now copy all systemtools files. If needed, this can be done more selectively. -%endif - for stool in $(echo %skipreqtools | tr '[A-Z]' '[a-z]') ; do [ -f %i/tools/selected/${stool}.xml ] || continue mv %i/tools/selected/${stool}.xml %i/tools/available done - + if [ -e $SCRAMV1_ROOT/bin/chktool ] ; then find %i/tools -name '*.xml' -type f | (xargs $SCRAMV1_ROOT/bin/chktool > %i/errors.log 2>&1 || true) ToolErrors=`cat %i/errors.log | wc -l | awk '{print $1}'` @@ -141,6 +150,10 @@ for pkg in $(echo %{allpkgreqs} | tr ' ' '\n' | grep -v '\-toolfile/' | grep '/ echo "" >> %{i}/tools/selected/${pk_name}.xml done DUP_BIN=$(echo "${ALL_PY_BIN}" | tr ' ' '\n' | sort | uniq -c | sed 's|^\s*||' | grep -v '^1 ' | sed 's|^.* ||') + +echo "echoing the install path" +echo %i + set +x if [ "${DUP_BIN}" != "" ] ; then for p in ${DUP_BIN} ; do @@ -150,7 +163,26 @@ if [ "${DUP_BIN}" != "" ] ; then exit 1 fi set -x + +#vectorized packages +%if "%{?vectorized_packages}" != "" +for vect_package in %vectorized_packages; do + echo $vect_package + package_uc=$(echo $vect_package | tr '[a-z-]' '[A-Z_]') + r="" + #echo $package_uc + for v in $(echo %{package_vectorization} | tr '[a-z-]' '[A-Z_]'); do + r=$r" \n" + #echo $v + done + r=$r'' + perl -p -i -e "s||${r}|g" %i/tools/selected/$(echo $vect_package | tr '[A-Z]' '[a-z]').xml +done +%endif + %post + %{relocateCmsFiles} $(find $RPM_INSTALL_PREFIX/%{pkgrel}/tools -type f) %{relocateCmsFiles} $(find $RPM_INSTALL_PREFIX/%{pkgrel}/lib -type f) - From 489f32a5fb00bc3dc634bb5cbf45d6041ea57f34 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Wed, 7 Jul 2021 14:47:52 +0200 Subject: [PATCH 02/17] typo --- scram-tools.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scram-tools.spec b/scram-tools.spec index b5b70f5fbf1..d54a26411d8 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ %define branch main %define tag 33e20671985a2b7774681adcdfce655c0186b650 -%define github_user cmss-sw +%define github_user cms-sw %define github_repo %{n} Source: git+https://github.com/%{github_user}/%{github_repo}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz From 00ed1e3f181dca6eb59c07d062273e145ca167e2 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Thu, 8 Jul 2021 00:46:19 +0200 Subject: [PATCH 03/17] add pkgreqs in init files for cmsswdata --- cmssw-tool-conf.spec | 2 ++ cmsswdata.spec | 4 +++- geant4data.spec | 1 + scram-tools.spec | 5 +++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index 07f9dfad29e..ae3fc7a5d03 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -18,6 +18,8 @@ Requires: intel-vtune Requires: das_client Requires: llvm Requires: boost +Requires: cmsswdata +Requires: geant4 %define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug diff --git a/cmsswdata.spec b/cmsswdata.spec index 15bff051bb2..987525f3fda 100644 --- a/cmsswdata.spec +++ b/cmsswdata.spec @@ -93,6 +93,8 @@ Requires: data-GeneratorInterface-ReggeGribovPartonMCInterface %post echo "%{BaseTool}_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh echo "set %{BaseTool}_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh +echo "%{BaseTool}_PKGREQUIRED='%{pkgreqs}'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh +echo "set %{BaseTool}_PKGREQUIRED='%{pkgreqs}'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh for DATA_PATH in %directpkgreqs; do PKG_DIR=$(echo $DATA_PATH | cut -d/ -f2) @@ -113,4 +115,4 @@ for DATA_PATH in %directpkgreqs; do if [ ! -L $SOURCE/$PKG_DATA ] ; then rm -rf $SOURCE/$PKG_DATA && ln -fs ../../../../share/$DATA_PATH/$PKG_DATA $SOURCE/$PKG_DATA fi -done \ No newline at end of file +done diff --git a/geant4data.spec b/geant4data.spec index f5e179abed9..bc9e8c9350c 100644 --- a/geant4data.spec +++ b/geant4data.spec @@ -21,3 +21,4 @@ echo "GEANT4DATA_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{p echo "set GEANT4DATA_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh echo "GEANT4DATA_VERSION='%v'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh echo "set GEANT4DATA_VERSION='%v'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh +echo "GEANT4DATA_PKGREQUIRED='%v'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh diff --git a/scram-tools.spec b/scram-tools.spec index d54a26411d8..0502d3859b9 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -1,7 +1,8 @@ ### RPM cms scram-tools 1.0 +## NOCOMPILER -%define branch main -%define tag 33e20671985a2b7774681adcdfce655c0186b650 +%define branch IB/CMSSW_12_0_X/master +%define tag HEAD %define github_user cms-sw %define github_repo %{n} From 8c53d252e116955cfd5f2c1f6e5f773d79df2240 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Thu, 8 Jul 2021 12:47:58 +0200 Subject: [PATCH 04/17] add get_tools script in scramv1-tool-conf --- cmsswdata.spec | 2 +- scram-tools.spec | 2 +- scramv1-tool-conf.file | 29 +++++++++++++++++------------ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/cmsswdata.spec b/cmsswdata.spec index 987525f3fda..828d3c67372 100644 --- a/cmsswdata.spec +++ b/cmsswdata.spec @@ -93,7 +93,7 @@ Requires: data-GeneratorInterface-ReggeGribovPartonMCInterface %post echo "%{BaseTool}_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh echo "set %{BaseTool}_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh -echo "%{BaseTool}_PKGREQUIRED='%{pkgreqs}'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh +echo "export %{BaseTool}_PKGREQUIRED='%{pkgreqs}'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh echo "set %{BaseTool}_PKGREQUIRED='%{pkgreqs}'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh for DATA_PATH in %directpkgreqs; do diff --git a/scram-tools.spec b/scram-tools.spec index 0502d3859b9..f3340d5f95e 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch IB/CMSSW_12_0_X/master -%define tag HEAD +%define tag f042f65d7b292e942af0cb547c4a12e5987026bf %define github_user cms-sw %define github_repo %{n} diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index a93c49f07e2..bfeb2a914d0 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -12,18 +12,23 @@ Source: none %define CopyScramTools \ echo ">> Copying tool files from: $tool" \ - for xml in $(find ${SCRAM_TOOLS_ROOT}/scram-tools/$tool -type f -name "*.xml") ; do \ - bxml=$(basename $xml | tr '[A-Z]' '[a-z]') \ - [ -f %i/tools/selected/$bxml ] && continue \ - [ -f %i/tools/available/$bxml ] && continue \ - cp $xml %i/tools/selected/${bxml} \ - T_VERSION=`eval echo \\\\$${uctool}_VERSION` \ - sed -i -e 's|@TOOL_ROOT@|'"$toolbase"'|g' %i/tools/selected/${bxml} \ - sed -i -e 's|@TOOL_VERSION@|'"$T_VERSION"'|g' %i/tools/selected/${bxml} \ - echo " Copied $bxml" \ - done \ - tool_script=${SCRAM_TOOLS_ROOT}/scram-tools/$tool/$(echo $tool | tr '[A-Z]' '[a-z]').sh \ - if [ -f $tool_script ]; then ($tool_script %i/tools/selected); fi + T_VERSION=`eval echo \\\\$${uctool}_VERSION` \ + PYTV=`eval echo $PYTHON_VERSION | cut -f1,2 -d.` + PYTV3=`eval echo $PYTHON3_VERSION | cut -f1,2 -d.` + ${SCRAM_TOOLS_ROOT}/get_tools "$toolbase" "$T_VERSION" %i "${tool}" "${PYTV}" "${PYTV3}"\ + + #for xml in $(find ${SCRAM_TOOLS_ROOT}/scram-tools/$tool -type f -name "*.xml") ; do \ + # bxml=$(basename $xml | tr '[A-Z]' '[a-z]') \ + # [ -f %i/tools/selected/$bxml ] && continue \ + # [ -f %i/tools/available/$bxml ] && continue \ + # cp $xml %i/tools/selected/${bxml} \ + # T_VERSION=`eval echo \\\\$${uctool}_VERSION` \ + # sed -i -e 's|@TOOL_ROOT@|'"$toolbase"'|g' %i/tools/selected/${bxml} \ + # sed -i -e 's|@TOOL_VERSION@|'"$T_VERSION"'|g' %i/tools/selected/${bxml} \ + # echo " Copied $bxml" \ + #done \ + #tool_script=${SCRAM_TOOLS_ROOT}/scram-tools/$tool/$(echo $tool | tr '[A-Z]' '[a-z]').sh \ + #if [ -f $tool_script ]; then ($tool_script %i/tools/selected); fi %prep %build From 53ccd7f57ab3b1f9ec97f4538e41d31e06e210ed Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Thu, 8 Jul 2021 16:20:25 +0200 Subject: [PATCH 05/17] remove old lines and use last scram-tools commit --- scram-tools.spec | 2 +- scramv1-tool-conf.file | 20 ++++---------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/scram-tools.spec b/scram-tools.spec index f3340d5f95e..86e301f5377 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch IB/CMSSW_12_0_X/master -%define tag f042f65d7b292e942af0cb547c4a12e5987026bf +%define tag f32993e0e9d084f0ec4b0bae9756f705efb0827d %define github_user cms-sw %define github_repo %{n} diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index bfeb2a914d0..025e2613bab 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -13,22 +13,10 @@ Source: none %define CopyScramTools \ echo ">> Copying tool files from: $tool" \ T_VERSION=`eval echo \\\\$${uctool}_VERSION` \ - PYTV=`eval echo $PYTHON_VERSION | cut -f1,2 -d.` - PYTV3=`eval echo $PYTHON3_VERSION | cut -f1,2 -d.` - ${SCRAM_TOOLS_ROOT}/get_tools "$toolbase" "$T_VERSION" %i "${tool}" "${PYTV}" "${PYTV3}"\ - - #for xml in $(find ${SCRAM_TOOLS_ROOT}/scram-tools/$tool -type f -name "*.xml") ; do \ - # bxml=$(basename $xml | tr '[A-Z]' '[a-z]') \ - # [ -f %i/tools/selected/$bxml ] && continue \ - # [ -f %i/tools/available/$bxml ] && continue \ - # cp $xml %i/tools/selected/${bxml} \ - # T_VERSION=`eval echo \\\\$${uctool}_VERSION` \ - # sed -i -e 's|@TOOL_ROOT@|'"$toolbase"'|g' %i/tools/selected/${bxml} \ - # sed -i -e 's|@TOOL_VERSION@|'"$T_VERSION"'|g' %i/tools/selected/${bxml} \ - # echo " Copied $bxml" \ - #done \ - #tool_script=${SCRAM_TOOLS_ROOT}/scram-tools/$tool/$(echo $tool | tr '[A-Z]' '[a-z]').sh \ - #if [ -f $tool_script ]; then ($tool_script %i/tools/selected); fi + PYTV=`eval echo $PYTHON_VERSION | cut -f1,2 -d.` \ + PYTV3=`eval echo $PYTHON3_VERSION | cut -f1,2 -d.` \ + ${SCRAM_TOOLS_ROOT}/scram-tools/get_tools "$toolbase" "$T_VERSION" %i "${tool}" "${PYTV}" "${PYTV3}" + %prep %build From f23cc00c3622634ae7e22c5545d1983d43e86f4b Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Thu, 8 Jul 2021 17:13:22 +0200 Subject: [PATCH 06/17] add an export to cmsswdata and remove toolfiles --- cmssw-tool-conf.spec | 186 ++++++++++++++++++++++++++++++++++++++++--- cmsswdata.spec | 4 +- 2 files changed, 178 insertions(+), 12 deletions(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index ae3fc7a5d03..a1eb5a0ee9d 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -5,24 +5,190 @@ ## INSTALL_DEPENDENCIES cmsLHEtoEOSManager gcc-fixincludes cmssw-osenv cms-git-tools ## UPLOAD_DEPENDENCIES dqmgui -BuildRequires: scram-tools +Requires: crab +Requires: cmssw-wm-tools +Requires: google-benchmark +Requires: catch2 +Requires: starlight +Requires: alpgen +Requires: boost +Requires: bz2lib +Requires: charybdis +Requires: classlib +Requires: clhep +Requires: coral +Requires: cppunit +Requires: curl +Requires: das_client +Requires: db6 +Requires: davix +Requires: evtgen +Requires: expat +Requires: fakesystem +Requires: fastjet +Requires: flatbuffers +Requires: fmt +Requires: gbl Requires: gcc +Requires: gdbm +Requires: geant4 +Requires: geant4data +Requires: vecgeom +Requires: glimpse +Requires: gmake +Requires: gsl +Requires: highfive +Requires: hector +Requires: hepmc +Requires: heppdt Requires: herwig +Requires: herwig7 +Requires: hydjet +Requires: ittnotify +Requires: jemalloc +Requires: jemalloc-debug +Requires: jimmy +Requires: json +Requires: ktjet +Requires: lhapdf +Requires: libjpeg-turbo Requires: libpng -Requires: OpenBLAS +Requires: libtiff +Requires: libungif +Requires: libxml2 +Requires: lwtnn +Requires: meschach +Requires: pcre +Requires: photos +Requires: photospp +Requires: pyquen +Requires: pythia6 +Requires: pythia8 +Requires: python +Requires: python3 Requires: root -Requires: tbb +Requires: sherpa +Requires: libpciaccess +Requires: numactl +Requires: hwloc +%ifnarch aarch64 +Requires: gdrcopy +%endif +Requires: ucx +Requires: openmpi +Requires: sigcpp +Requires: sqlite +Requires: systemtools +Requires: tauola +Requires: tauolapp +Requires: thepeg +Requires: toprex +Requires: libuuid +Requires: xerces-c Requires: zlib +Requires: dcap +Requires: frontier_client +Requires: xrootd +Requires: dd4hep +Requires: graphviz +Requires: valgrind +Requires: cmsswdata Requires: zstd -Requires: intel-vtune -Requires: das_client +Requires: hls +Requires: opencv +Requires: grpc +Requires: onnxruntime +Requires: triton-inference-server +Requires: hdf5 +Requires: rivet +Requires: cascade +Requires: yoda +Requires: fftw3 +Requires: fftjet +Requires: professor2 +Requires: xz +Requires: lz4 +Requires: protobuf +Requires: lcov Requires: llvm -Requires: boost -Requires: cmsswdata -Requires: geant4 +Requires: tbb +Requires: mctester +Requires: vdt +Requires: icc +Requires: gnuplot +Requires: sloccount +Requires: millepede +Requires: pacparser +Requires: git +Requires: cgal +Requires: doxygen +Requires: yaml-cpp +Requires: gmp-static +Requires: mpfr-static +Requires: fastjet-contrib +Requires: opencl +Requires: opencl-cpp +Requires: qd +Requires: blackhat +Requires: sherpa +Requires: geant4-parfullcms +Requires: fasthadd +Requires: eigen +Requires: gdb +Requires: libxslt +Requires: giflib +Requires: freetype +Requires: utm +Requires: libffi +Requires: CSCTrackFinderEmulation +Requires: tinyxml2 +Requires: md5 +Requires: gosamcontrib +Requires: gosam +Requires: madgraph5amcatnlo +Requires: python_tools +Requires: dasgoclient +Requires: OpenBLAS +Requires: mxnet-predict +Requires: mkfit +Requires: dablooms -%define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug +# Only for Linux platform. +%ifos linux +Requires: openldap +Requires: gperftools +Requires: cuda +Requires: alpaka +Requires: cupla + +%if "%{cmsos}" != "slc7_aarch64" +Requires: cudnn +%endif -# the vectorization flags/macroses will be available +Requires: libunwind +%ifnarch ppc64le +Requires: igprof +Requires: openloops +%endif + +%ifarch x86_64 +Requires: dmtcp +Requires: tkonlinesw +Requires: oracle +Requires: intel-vtune +Requires: cmsmon-tools +Requires: dip +%else +Requires: tkonlinesw-fake +Requires: oracle-fake +%endif +%endif + +Requires: tensorflow +Requires: xtensor +Requires: xtl +Requires: xgboost + +%define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug ## IMPORT scramv1-tool-conf diff --git a/cmsswdata.spec b/cmsswdata.spec index 828d3c67372..19c9652414e 100644 --- a/cmsswdata.spec +++ b/cmsswdata.spec @@ -93,8 +93,6 @@ Requires: data-GeneratorInterface-ReggeGribovPartonMCInterface %post echo "%{BaseTool}_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh echo "set %{BaseTool}_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh -echo "export %{BaseTool}_PKGREQUIRED='%{pkgreqs}'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh -echo "set %{BaseTool}_PKGREQUIRED='%{pkgreqs}'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh for DATA_PATH in %directpkgreqs; do PKG_DIR=$(echo $DATA_PATH | cut -d/ -f2) @@ -116,3 +114,5 @@ for DATA_PATH in %directpkgreqs; do rm -rf $SOURCE/$PKG_DATA && ln -fs ../../../../share/$DATA_PATH/$PKG_DATA $SOURCE/$PKG_DATA fi done + +export %{BaseTool}_PKGREQUIRED='%{pkgreqs}' From 89588d1ca7d5450250244035cb9133297f567212 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Thu, 8 Jul 2021 18:30:16 +0200 Subject: [PATCH 07/17] remove toolfile string for fwlite and coral --- cmssw-tool-conf.spec | 2 +- coral-tool-conf.spec | 26 +++++++------- fwlite-tool-conf.spec | 84 +++++++++++++++++++++---------------------- scram-tools.spec | 2 +- 4 files changed, 57 insertions(+), 57 deletions(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index a1eb5a0ee9d..98a80ecc2d7 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -16,7 +16,7 @@ Requires: bz2lib Requires: charybdis Requires: classlib Requires: clhep -Requires: coral +#Requires: coral Requires: cppunit Requires: curl Requires: das_client diff --git a/coral-tool-conf.spec b/coral-tool-conf.spec index a1164f5db60..ea97109d28f 100644 --- a/coral-tool-conf.spec +++ b/coral-tool-conf.spec @@ -1,22 +1,22 @@ ### RPM cms coral-tool-conf 2.1 ## NOCOMPILER -Requires: pcre-toolfile -Requires: python-toolfile -Requires: expat-toolfile -Requires: boost-toolfile -Requires: frontier_client-toolfile -Requires: gcc-toolfile +Requires: pcre +Requires: python +Requires: expat +Requires: boost +Requires: frontier_client +Requires: gcc -Requires: sqlite-toolfile -Requires: libuuid-toolfile -Requires: zlib-toolfile -Requires: bz2lib-toolfile -Requires: cppunit-toolfile -Requires: xerces-c-toolfile +Requires: sqlite +Requires: libuuid +Requires: zlib +Requires: bz2lib +Requires: cppunit +Requires: xerces-c Requires: systemtools %ifarch x86_64 -Requires: oracle-toolfile +Requires: oracle %endif %define skipreqtools jcompiler diff --git a/fwlite-tool-conf.spec b/fwlite-tool-conf.spec index 14c579b6435..bab330e720b 100644 --- a/fwlite-tool-conf.spec +++ b/fwlite-tool-conf.spec @@ -4,54 +4,54 @@ # tool is added ## INITENV SET CMSSW_TOOL_CONF_ROOT $FWLITE_TOOL_CONF_ROOT -Requires: eigen-toolfile -Requires: fmt-toolfile -Requires: tbb-toolfile -Requires: boost-toolfile -Requires: bz2lib-toolfile -Requires: clhep-toolfile -Requires: db6-toolfile -Requires: dcap-toolfile -Requires: expat-toolfile +Requires: eigen +Requires: fmt +Requires: tbb +Requires: boost +Requires: bz2lib +Requires: clhep +Requires: db6 +Requires: dcap +Requires: expat Requires: fakesystem -Requires: fftw3-toolfile -Requires: fwlitedata-toolfile -Requires: gcc-toolfile -Requires: gdbm-toolfile -Requires: gmake-toolfile -Requires: gsl-toolfile -Requires: hepmc-toolfile -Requires: libjpeg-turbo-toolfile -Requires: libpng-toolfile -Requires: libtiff-toolfile -Requires: libungif-toolfile -Requires: pcre-toolfile -Requires: python-toolfile -Requires: python3-toolfile -Requires: root-toolfile -Requires: sigcpp-toolfile -Requires: sqlite-toolfile +Requires: fftw3 +Requires: fwlitedata +Requires: gcc +Requires: gdbm +Requires: gmake +Requires: gsl +Requires: hepmc +Requires: libjpeg-turbo +Requires: libpng +Requires: libtiff +Requires: libungif +Requires: pcre +Requires: python +Requires: python3 +Requires: root +Requires: sigcpp +Requires: sqlite Requires: systemtools -Requires: libuuid-toolfile -Requires: xerces-c-toolfile -Requires: xrootd-toolfile -Requires: xz-toolfile -Requires: zlib-toolfile -Requires: libxml2-toolfile -Requires: llvm-gcc-toolfile -Requires: vdt-toolfile -Requires: tinyxml2-toolfile -Requires: md5-toolfile -Requires: davix-toolfile -Requires: py3-numpy-toolfile -Requires: OpenBLAS-toolfile -Requires: py3-pybind11-toolfile +Requires: libuuid +Requires: xerces-c +Requires: xrootd +Requires: xz +Requires: zlib +Requires: libxml2 +Requires: llvm +Requires: vdt +Requires: tinyxml2 +Requires: md5 +Requires: davix +Requires: py3-numpy +Requires: OpenBLAS +Requires: py3-pybind11 Requires: fwlite_python_tools -Requires: zstd-toolfile +Requires: zstd # Only for Darwin platform. %ifarch darwin -Requires: freetype-toolfile +Requires: freetype %endif %define skipreqtools jcompiler db6 expat fftw3 sqlite diff --git a/scram-tools.spec b/scram-tools.spec index 86e301f5377..1f980aef753 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch IB/CMSSW_12_0_X/master -%define tag f32993e0e9d084f0ec4b0bae9756f705efb0827d +%define tag e56e5c726a3de02de67c3e9231e0ce8f760bd04f %define github_user cms-sw %define github_repo %{n} From 3625f68554ec11146d858d1b0b43a60c6c32286e Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Mon, 12 Jul 2021 23:42:36 +0200 Subject: [PATCH 08/17] save fixes and changes for geant4data cmsswdata systemtools --- cmsswdata.spec | 27 ++++++++++++++- coral-tool-conf.spec | 1 + geant4data.spec | 21 +++++++++++ scram-tools.spec | 4 +-- scramv1-tool-conf.file | 9 +++-- systemtools.spec | 79 ------------------------------------------ 6 files changed, 54 insertions(+), 87 deletions(-) diff --git a/cmsswdata.spec b/cmsswdata.spec index 19c9652414e..f1c0a5192ff 100644 --- a/cmsswdata.spec +++ b/cmsswdata.spec @@ -90,9 +90,35 @@ Requires: data-GeneratorInterface-ReggeGribovPartonMCInterface %install +mkdir -p %i/etc/scram.d +cat << \EOF_TOOLFILE >%i/etc/scram.d/cmsswdata.xml + + + + +EOF_TOOLFILE + +cat << \EOF_TOOLFILE > %i/searchpath.xml + + +EOF_TOOLFILE + +for toolbase in `echo %pkgreqs | tr ' ' '\n' | grep 'cms/data-'` ; do + toolver=`basename $toolbase` + pack=`echo $toolbase | cut -d/ -f2 | sed 's|data-||;s|-|/|'` + echo " " >> %i/etc/scram.d/cmsswdata.xml + echo " " >> %i/searchpath.xml +done + +cat %i/searchpath.xml >> %i/etc/scram.d/cmsswdata.xml +echo " " >> %i/etc/scram.d/cmsswdata.xml +rm -f %i/searchpath.xml + %post echo "%{BaseTool}_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh echo "set %{BaseTool}_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh +echo "%{BaseTool}_PKGREQUIRED='%pkgreqs'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh +echo "set %{BaseTool}_PKGREQUIRED='%pkgreqs'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh for DATA_PATH in %directpkgreqs; do PKG_DIR=$(echo $DATA_PATH | cut -d/ -f2) @@ -115,4 +141,3 @@ for DATA_PATH in %directpkgreqs; do fi done -export %{BaseTool}_PKGREQUIRED='%{pkgreqs}' diff --git a/coral-tool-conf.spec b/coral-tool-conf.spec index ea97109d28f..e0c9c4a6c4e 100644 --- a/coral-tool-conf.spec +++ b/coral-tool-conf.spec @@ -1,6 +1,7 @@ ### RPM cms coral-tool-conf 2.1 ## NOCOMPILER +BuildRequires: scram-tools Requires: pcre Requires: python Requires: expat diff --git a/geant4data.spec b/geant4data.spec index bc9e8c9350c..a48025336a2 100644 --- a/geant4data.spec +++ b/geant4data.spec @@ -15,6 +15,27 @@ Requires: geant4-G4INCL %prep %build %install + +mkdir -p %i/etc/scram.d +cat << \EOF_TOOLFILE >%i/etc/scram.d/geant4data.xml + + + + +EOF_TOOLFILE + +for tool in `echo %requiredtools | tr ' ' '\n' | grep 'geant4-'` ; do + uctool=`echo $tool | tr '-' '_' | tr '[a-z]' '[A-Z]'` + toolbase=`eval echo \\$${uctool}_ROOT` + toolenv=`eval echo \\$${uctool}_RUNTIME` + echo "$uctool = $toolbase, $toolenv" + if [ "X$toolbase" = X -o "X$toolenv" = X -o ! -d $toolbase/data ] ; then continue ; fi + tooldata=`ls -d $toolbase/data/* | tail -1` + if [ "X$tooldata" = X ] ; then continue ; fi + echo " " >> %i/etc/scram.d/geant4data.xml +done +echo "" >> %i/etc/scram.d/geant4data.xml + %post if [ "X$CMS_INSTALL_PREFIX" = "X" ] ; then CMS_INSTALL_PREFIX=$RPM_INSTALL_PREFIX; export CMS_INSTALL_PREFIX; fi echo "GEANT4DATA_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh diff --git a/scram-tools.spec b/scram-tools.spec index 1f980aef753..8d01fda9c30 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -1,8 +1,8 @@ ### RPM cms scram-tools 1.0 ## NOCOMPILER -%define branch IB/CMSSW_12_0_X/master -%define tag e56e5c726a3de02de67c3e9231e0ce8f760bd04f +%define branch main +%define tag 90649b9063b5bef81619d96651100448f46645a7 %define github_user cms-sw %define github_repo %{n} diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index 025e2613bab..de1afc3fc76 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -8,14 +8,12 @@ Source: none %define CheckScramTools \ uctool=`echo $tool | tr '[a-z-]' '[A-Z_]'` \ toolbase=`eval echo \\\\$${uctool}_ROOT` \ - [ -d ${SCRAM_TOOLS_ROOT}/scram-tools/$tool ] || continue + [ -d ${SCRAM_TOOLS_ROOT}/scram-tools/tools/$tool ] || continue %define CopyScramTools \ echo ">> Copying tool files from: $tool" \ T_VERSION=`eval echo \\\\$${uctool}_VERSION` \ - PYTV=`eval echo $PYTHON_VERSION | cut -f1,2 -d.` \ - PYTV3=`eval echo $PYTHON3_VERSION | cut -f1,2 -d.` \ - ${SCRAM_TOOLS_ROOT}/scram-tools/get_tools "$toolbase" "$T_VERSION" %i "${tool}" "${PYTV}" "${PYTV3}" + ${SCRAM_TOOLS_ROOT}/scram-tools/bin/get_tools "$toolbase" "$T_VERSION" %i "${tool}" %prep %build @@ -55,6 +53,7 @@ COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields" COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields %ppc64le_build_flags" %endif export COMP_ARCH_SPECIFIC_FLAGS +export ORACLE_ENV_ROOT DoneTools=" " SkipTools= @@ -167,7 +166,7 @@ for vect_package in %vectorized_packages; do for v in $(echo %{package_vectorization} | tr '[a-z-]' '[A-Z_]'); do r=$r" \n" + r=$r${package_uc}_${v}_ROOT/lib"\"/>\n" #echo $v done r=$r'' diff --git a/systemtools.spec b/systemtools.spec index a213d6ddcb8..d43d22aac1f 100644 --- a/systemtools.spec +++ b/systemtools.spec @@ -2,7 +2,6 @@ ## NOCOMPILER Source: none - %if "%{?use_system_gcc:set}" == "set" %define compilertools ccompiler cxxcompiler f77compiler %else @@ -22,82 +21,4 @@ Source: none %prep %build %install -# SCRAM ToolBox toolfile -mkdir -p %i/etc/scram.d -#OpenSSL -cat << \EOF_TOOLFILE >%i/etc/scram.d/openssl.xml - - - - -EOF_TOOLFILE - -# Sockets -cat << \EOF_TOOLFILE >%i/etc/scram.d/sockets.xml - -EOF_TOOLFILE -%ifos linux -cat << \EOF_TOOLFILE >>%i/etc/scram.d/sockets.xml - - - - -EOF_TOOLFILE -%endif -%ifarch darwin -cat << \EOF_TOOLFILE >>%i/etc/scram.d/sockets.xml - -EOF_TOOLFILE -%endif -echo " " >>%i/etc/scram.d/sockets.xml - -# OpenGL -cat << \EOF_TOOLFILE >%i/etc/scram.d/opengl.xml - - - - - -EOF_TOOLFILE -%ifarch darwin -cat << \EOF_TOOLFILE >>%i/etc/scram.d/opengl.xml - - - - - -EOF_TOOLFILE -%endif -echo " " >>%i/etc/scram.d/opengl.xml - -# X11 -cat << \EOF_TOOLFILE >%i/etc/scram.d/x11.xml - -EOF_TOOLFILE -%ifarch darwin -cat << \EOF_TOOLFILE >>%i/etc/scram.d/x11.xml - - - - - - - - - - - - - -EOF_TOOLFILE -%endif -cat << \EOF_TOOLFILE >>%i/etc/scram.d/x11.xml - - -EOF_TOOLFILE - -export ORACLE_ENV_ROOT -perl -p -i -e 's|\@([^@]*)\@|$ENV{$1}|g' %i/etc/scram.d/*.xml -%post -%{relocateConfig}etc/scram.d/*.xml From 5df88b7f49648fffb6bcdaa7fafd9f5860157e27 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Tue, 13 Jul 2021 10:03:55 +0200 Subject: [PATCH 09/17] bring back coral. coral builds now --- cmssw-tool-conf.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index 98a80ecc2d7..a1eb5a0ee9d 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -16,7 +16,7 @@ Requires: bz2lib Requires: charybdis Requires: classlib Requires: clhep -#Requires: coral +Requires: coral Requires: cppunit Requires: curl Requires: das_client From 424d7b7137e20d63e633569b99a15d7febf0baf7 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Tue, 13 Jul 2021 19:13:45 +0200 Subject: [PATCH 10/17] numpy changes --- cmssw-tool-conf.spec | 2 ++ fwlite-tool-conf.spec | 1 + python_tools.spec | 7 +------ scram-tools.spec | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index a1eb5a0ee9d..68a6274d829 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -5,6 +5,7 @@ ## INSTALL_DEPENDENCIES cmsLHEtoEOSManager gcc-fixincludes cmssw-osenv cms-git-tools ## UPLOAD_DEPENDENCIES dqmgui +BuildRequires: scram-tools Requires: crab Requires: cmssw-wm-tools Requires: google-benchmark @@ -152,6 +153,7 @@ Requires: OpenBLAS Requires: mxnet-predict Requires: mkfit Requires: dablooms +Requires: py3-numpy # Only for Linux platform. %ifos linux diff --git a/fwlite-tool-conf.spec b/fwlite-tool-conf.spec index bab330e720b..e0c974b25f1 100644 --- a/fwlite-tool-conf.spec +++ b/fwlite-tool-conf.spec @@ -4,6 +4,7 @@ # tool is added ## INITENV SET CMSSW_TOOL_CONF_ROOT $FWLITE_TOOL_CONF_ROOT +BuildRequires: scram-tools Requires: eigen Requires: fmt Requires: tbb diff --git a/python_tools.spec b/python_tools.spec index d4fc7f0b7a4..3a8e211e899 100644 --- a/python_tools.spec +++ b/python_tools.spec @@ -132,7 +132,7 @@ Requires: py3-python-ldap Requires: py3-plac Requires: py3-matplotlib -Requires: py3-numpy-toolfile +Requires: py3-numpy Requires: py2-sqlalchemy Requires: py3-sqlalchemy Requires: py3-pygithub @@ -231,9 +231,4 @@ Requires: py3-correctionlib %build %install -mkdir -p %{i}/etc/scram.d -cat << \EOF_TOOLFILE >%i/etc/scram.d/python_tools.xml - - -EOF_TOOLFILE diff --git a/scram-tools.spec b/scram-tools.spec index 8d01fda9c30..2104b1c1787 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch main -%define tag 90649b9063b5bef81619d96651100448f46645a7 +%define tag 3df77f0a29f4d9b3912e4dbcafe4f22d99487af7 %define github_user cms-sw %define github_repo %{n} From 3ee88808e8043ea9822cfdf9a0f7c702a1bb3ab1 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Tue, 13 Jul 2021 19:49:09 +0200 Subject: [PATCH 11/17] all toolfiles deployed in cmssw-tool-conf, coral and fwlite building --- scram-tools.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scram-tools.spec b/scram-tools.spec index 2104b1c1787..03da843ee92 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch main -%define tag 3df77f0a29f4d9b3912e4dbcafe4f22d99487af7 +%define tag 689a50952ad5a35535f76a7dae89355916fce490 %define github_user cms-sw %define github_repo %{n} From 4dbbf72a8a153251b6e7bbfe6ecd7cde2f9c5b00 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Tue, 13 Jul 2021 20:44:18 +0200 Subject: [PATCH 12/17] keep python_tools as they were and remove redundant lines --- cmssw-tool-conf.spec | 1 - geant4data.spec | 2 +- python_tools.spec | 4 ++-- scramv1-tool-conf.file | 3 --- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index 68a6274d829..f9c33fadda3 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -153,7 +153,6 @@ Requires: OpenBLAS Requires: mxnet-predict Requires: mkfit Requires: dablooms -Requires: py3-numpy # Only for Linux platform. %ifos linux diff --git a/geant4data.spec b/geant4data.spec index a48025336a2..e14cc386828 100644 --- a/geant4data.spec +++ b/geant4data.spec @@ -42,4 +42,4 @@ echo "GEANT4DATA_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{p echo "set GEANT4DATA_ROOT='$CMS_INSTALL_PREFIX/%{pkgrel}'" > $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh echo "GEANT4DATA_VERSION='%v'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh echo "set GEANT4DATA_VERSION='%v'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.csh -echo "GEANT4DATA_PKGREQUIRED='%v'" >> $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/init.sh + diff --git a/python_tools.spec b/python_tools.spec index 3a8e211e899..5650dbb7227 100644 --- a/python_tools.spec +++ b/python_tools.spec @@ -136,7 +136,7 @@ Requires: py3-numpy Requires: py2-sqlalchemy Requires: py3-sqlalchemy Requires: py3-pygithub -Requires: py3-dxr-toolfile +Requires: py3-dxr Requires: py2-PyYAML Requires: py3-PyYAML Requires: py3-pylint @@ -147,7 +147,7 @@ Requires: py3-cx-Oracle %endif Requires: py3-cython Requires: py2-future -Requires: py3-pybind11-toolfile +Requires: py3-pybind11 Requires: py3-histbook Requires: py3-flake8 Requires: py3-autopep8 diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index de1afc3fc76..4118a5acd60 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -143,9 +143,6 @@ for pkg in $(echo %{allpkgreqs} | tr ' ' '\n' | grep -v '\-toolfile/' | grep '/ done DUP_BIN=$(echo "${ALL_PY_BIN}" | tr ' ' '\n' | sort | uniq -c | sed 's|^\s*||' | grep -v '^1 ' | sed 's|^.* ||') -echo "echoing the install path" -echo %i - set +x if [ "${DUP_BIN}" != "" ] ; then for p in ${DUP_BIN} ; do From e51a811c47c9abf6a0b5e95c971490769a761f3e Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Wed, 14 Jul 2021 01:23:40 +0200 Subject: [PATCH 13/17] fix cmsswdata and g4data paths and gmake-toolfile --- cmsswdata.spec | 2 +- geant4data.spec | 2 +- scram-tools.spec | 2 +- scramv1-tool-conf.file | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmsswdata.spec b/cmsswdata.spec index f1c0a5192ff..80ee1a13056 100644 --- a/cmsswdata.spec +++ b/cmsswdata.spec @@ -94,7 +94,7 @@ mkdir -p %i/etc/scram.d cat << \EOF_TOOLFILE >%i/etc/scram.d/cmsswdata.xml - + EOF_TOOLFILE diff --git a/geant4data.spec b/geant4data.spec index e14cc386828..74c5d0c680e 100644 --- a/geant4data.spec +++ b/geant4data.spec @@ -20,7 +20,7 @@ mkdir -p %i/etc/scram.d cat << \EOF_TOOLFILE >%i/etc/scram.d/geant4data.xml - + EOF_TOOLFILE diff --git a/scram-tools.spec b/scram-tools.spec index 03da843ee92..06f0aaa0ee6 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch main -%define tag 689a50952ad5a35535f76a7dae89355916fce490 +%define tag 588309281b221cdf0f4e36e69aa8bbc3a017336c %define github_user cms-sw %define github_repo %{n} diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index 4118a5acd60..a47e327c769 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -1,7 +1,7 @@ ### FILE scramv1-tool-conf ## NOCOMPILER ## BUILDREQUIRE-TOOLFILE -Requires: gmake-toolfile +Requires: gmake BuildRequires: SCRAMV1 Source: none %define online %(case %cmsplatf in (*onl_*_*) echo true;; (*) echo false;; esac) From ed4f9136b3830185a3638a8b084102f548540335 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Wed, 14 Jul 2021 17:20:13 +0200 Subject: [PATCH 14/17] fix vectorization variables for gcc flags and update tool-conf versions --- cmssw-tool-conf.spec | 2 +- coral-tool-conf.spec | 2 +- fwlite-tool-conf.spec | 2 +- scram-tools.spec | 2 +- scramv1-tool-conf.file | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index f9c33fadda3..405072eee3c 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -1,4 +1,4 @@ -### RPM cms cmssw-tool-conf 46.0 +### RPM cms cmssw-tool-conf 50.0 # With cmsBuild, change the above version only when a new tool is added ## NOCOMPILER diff --git a/coral-tool-conf.spec b/coral-tool-conf.spec index 89a59bb9f53..f654a98e8d7 100644 --- a/coral-tool-conf.spec +++ b/coral-tool-conf.spec @@ -1,4 +1,4 @@ -### RPM cms coral-tool-conf 2.1 +### RPM cms coral-tool-conf 10.0 ## NOCOMPILER BuildRequires: scram-tools diff --git a/fwlite-tool-conf.spec b/fwlite-tool-conf.spec index e0c974b25f1..449811e5a4c 100644 --- a/fwlite-tool-conf.spec +++ b/fwlite-tool-conf.spec @@ -1,4 +1,4 @@ -### RPM cms fwlite-tool-conf 11.0 +### RPM cms fwlite-tool-conf 20.0 ## NOCOMPILER # with cmsBuild, change the above version only when a new # tool is added diff --git a/scram-tools.spec b/scram-tools.spec index 06f0aaa0ee6..b72274b5c4a 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch main -%define tag 588309281b221cdf0f4e36e69aa8bbc3a017336c +%define tag 8c5f3b616bf4b52d1d7d857e6b179d21239ee022 %define github_user cms-sw %define github_repo %{n} diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index a47e327c769..2677d2335c9 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -38,13 +38,13 @@ mkdir -p %i/tools/selected %i/tools/available %endif %if "%{?package_vectorization}" != "" -PKG_VECTORIZATION=`echo %package_vectorization` -export PKG_VECTORIZATION +export PKG_VECTORIZATION=`echo %package_vectorization` +export CMSDIST_DIR=%cmsdist_directory %endif COMP_ARCH_SPECIFIC_FLAGS="" %ifarch x86_64 -COMP_ARCH_SPECIFIC_FLAGS=$(for v in %package_vectorization ; do %{cmsdist_directory}/vectorization/cmsdist_packages.py ${v} ; done) +COMP_ARCH_SPECIFIC_FLAGS=$(%{cmsdist_directory}/vectorization/cmsdist_packages.py) %endif %ifarch aarch64 COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields" From e31c9fda6e55b3efe5801ebccd294a3b05e2a776 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Thu, 15 Jul 2021 16:35:41 +0200 Subject: [PATCH 15/17] add the ppc flags in gcc/env.sh instead --- compilation_flags.file | 2 +- scram-tools.spec | 2 +- scramv1-tool-conf.file | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compilation_flags.file b/compilation_flags.file index dac80c9f392..f198b1851d4 100644 --- a/compilation_flags.file +++ b/compilation_flags.file @@ -1,4 +1,4 @@ %if "%{?ppc64le_build_flags:set}" != "set" -%define ppc64le_build_flags -mcpu=power8 -mtune=power8 --param=l1-cache-size=64 --param=l1-cache-line-size=128 --param=l2-cache-size=512 +%define ppc64le_build_flags -mcpu=power8 -mtune=power8 --param=l1-cache-size=64 --param=l1-cache-line-size=128 --param=l2-cache-size=512 %endif diff --git a/scram-tools.spec b/scram-tools.spec index b72274b5c4a..83a50b316a7 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch main -%define tag 8c5f3b616bf4b52d1d7d857e6b179d21239ee022 +%define tag 8af690460dcfe869aa9505f3e4ecbdbccea66f96 %define github_user cms-sw %define github_repo %{n} diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index 2677d2335c9..fd83fb2f18c 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -39,8 +39,8 @@ mkdir -p %i/tools/selected %i/tools/available %if "%{?package_vectorization}" != "" export PKG_VECTORIZATION=`echo %package_vectorization` -export CMSDIST_DIR=%cmsdist_directory %endif +export CMSDIST_DIR=%cmsdist_directory COMP_ARCH_SPECIFIC_FLAGS="" %ifarch x86_64 @@ -50,7 +50,7 @@ COMP_ARCH_SPECIFIC_FLAGS=$(%{cmsdist_directory}/vectorization/cmsdist_packages.p COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields" %endif %ifarch ppc64le -COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields %ppc64le_build_flags" +COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields" %endif export COMP_ARCH_SPECIFIC_FLAGS export ORACLE_ENV_ROOT From e50000c7c5a516cb40c2d89b9dec57cd832d7047 Mon Sep 17 00:00:00 2001 From: Mircho Rodozov Date: Fri, 16 Jul 2021 12:47:55 +0200 Subject: [PATCH 16/17] change IMPORT to INCLUDE for scramv1 --- cmssw-patch-tool-conf.spec | 2 +- cmssw-tool-conf.spec | 2 +- coral-tool-conf.spec | 2 +- fwlite-tool-conf.spec | 2 +- scram-tools.spec | 2 +- scramv1-tool-conf.file | 3 ++- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmssw-patch-tool-conf.spec b/cmssw-patch-tool-conf.spec index b9f8de03083..dccd6a77801 100644 --- a/cmssw-patch-tool-conf.spec +++ b/cmssw-patch-tool-conf.spec @@ -10,4 +10,4 @@ Requires: cmssw-toolfile # still need this (from the non-patch tool-conf spec ... %define skipreqtools jcompiler -## IMPORT scramv1-tool-conf +## INCLUDE scramv1-tool-conf diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index 405072eee3c..7a947e6242d 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -192,4 +192,4 @@ Requires: xgboost %define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug -## IMPORT scramv1-tool-conf +## INCLUDE scramv1-tool-conf diff --git a/coral-tool-conf.spec b/coral-tool-conf.spec index f654a98e8d7..e8d67a7b378 100644 --- a/coral-tool-conf.spec +++ b/coral-tool-conf.spec @@ -21,4 +21,4 @@ Requires: oracle %define skipreqtools jcompiler -## IMPORT scramv1-tool-conf +## INCLUDE scramv1-tool-conf diff --git a/fwlite-tool-conf.spec b/fwlite-tool-conf.spec index 449811e5a4c..e29603e4e50 100644 --- a/fwlite-tool-conf.spec +++ b/fwlite-tool-conf.spec @@ -57,4 +57,4 @@ Requires: freetype %define skipreqtools jcompiler db6 expat fftw3 sqlite -## IMPORT scramv1-tool-conf +## INCLUDE scramv1-tool-conf diff --git a/scram-tools.spec b/scram-tools.spec index 83a50b316a7..e1c117f2afc 100644 --- a/scram-tools.spec +++ b/scram-tools.spec @@ -2,7 +2,7 @@ ## NOCOMPILER %define branch main -%define tag 8af690460dcfe869aa9505f3e4ecbdbccea66f96 +%define tag c29ad2e62574733b9bb9bf7d5b995b9a4df2dbe8 %define github_user cms-sw %define github_repo %{n} diff --git a/scramv1-tool-conf.file b/scramv1-tool-conf.file index fd83fb2f18c..b2fea382231 100644 --- a/scramv1-tool-conf.file +++ b/scramv1-tool-conf.file @@ -3,6 +3,7 @@ ## BUILDREQUIRE-TOOLFILE Requires: gmake BuildRequires: SCRAMV1 +## INCLUDE compilation_flags Source: none %define online %(case %cmsplatf in (*onl_*_*) echo true;; (*) echo false;; esac) %define CheckScramTools \ @@ -50,7 +51,7 @@ COMP_ARCH_SPECIFIC_FLAGS=$(%{cmsdist_directory}/vectorization/cmsdist_packages.p COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields" %endif %ifarch ppc64le -COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields" +COMP_ARCH_SPECIFIC_FLAGS="-fsigned-char -fsigned-bitfields %ppc64le_build_flags" %endif export COMP_ARCH_SPECIFIC_FLAGS export ORACLE_ENV_ROOT From 5d461cf2b61bac1782b31e4ae435af52a5377d1f Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Mon, 19 Jul 2021 12:42:37 +0200 Subject: [PATCH 17/17] revert to avoid rebuild of packages --- compilation_flags.file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compilation_flags.file b/compilation_flags.file index f198b1851d4..dac80c9f392 100644 --- a/compilation_flags.file +++ b/compilation_flags.file @@ -1,4 +1,4 @@ %if "%{?ppc64le_build_flags:set}" != "set" -%define ppc64le_build_flags -mcpu=power8 -mtune=power8 --param=l1-cache-size=64 --param=l1-cache-line-size=128 --param=l2-cache-size=512 +%define ppc64le_build_flags -mcpu=power8 -mtune=power8 --param=l1-cache-size=64 --param=l1-cache-line-size=128 --param=l2-cache-size=512 %endif