Merge IB/CMSSW_11_1_X/master into IB/CMSSW_11_1_X/rootmaster.

_crab-startup.file

@@ -0,0 +1,67 @@
+#!/bin/bash -e
+#####################################################
+# **************** IMPORTANT NOTE ***************** #
+# Increament Crab Startup Revision for every change #
+#####################################################
+CRAB_STARTUP_REVISION=1
+
+#Check if we have not already found CRAB
+if [ -z "${CMS_CRAB_PATH}" ] ; then
+  cms_basedir="$(dirname $(cd $(dirname $0); /bin/pwd -P))"
+
+  #Use crab for crab-prob command
+  crab_pkg=$(basename $0)
+  if [ "${crab_pkg}" = "crab-prod" ] ; then crab_pkg="crab" ; fi
+
+  #Search for latest crab version for cmsos 
+  cms_os=$(${cms_basedir}/common/cmsos)
+  crab_dir=$(ls -d ${cms_basedir}/${cms_os}_gcc*/cms/${crab_pkg}/*/etc/profile.d/init.sh 2>/dev/null | sed 's|/etc/profile.d/init.sh$||' | sort | tail -1)
+  if [ -z "${crab_dir}" ] ; then
+    >&2 echo "Error: Unable to find '${crab_pkg}' installation for '${cms_os}' architecture"
+    exit 1
+  fi
+  export CMS_CRAB_PATH="${crab_dir}"
+  export CRAB_VERSION=$(basename ${crab_dir})
+
+  #Save CMSSW_BASE in case we have to set its env later
+  export ORIGINAL_CMSSW_BASE="${CMSSW_BASE}"
+
+  #Unset cmssw env
+  eval `scram unset -sh`
+
+  #set crab env
+  source ${CMS_CRAB_PATH}/etc/profile.d/init.sh
+
+  #If cjson not available then add crab/lib/extras
+  if ! python -c 'import cjson' >/dev/null  2>&1 ; then
+    export PYTHONPATH="${CMS_CRAB_PATH}/lib/extras${PYTHONPATH:+:$PYTHONPATH}"
+  fi
+
+  ## If it is a submit command run the bootstrap script
+  if [ $(echo "$@" | tr ' ' '\n' | grep 'sub\|submit' | wc -l) -gt 0 ] ; then
+    if [ -z "${ORIGINAL_CMSSW_BASE}" ] ; then
+      >&2 echo "Error: Please set SCRAM/CMSSW runtime environment before running crab"
+      exit 1
+    fi
+    ERR=0
+    BOOTSTRAP_OUT=$(eval `scram runtime -sh ${ORIGINAL_CMSSW_BASE}`; ${CMS_CRAB_PATH}/bin/crab3bootstrap $@) || ERR=$?
+    if [ $ERR -ne 0 ]; then
+      echo "$BOOTSTRAP_OUT"
+      exit $ERR
+    fi
+    ## The submit command will know what to do if this variable is set!
+    ## See TODO: add link to an hypothetical twiki
+    export CRAB3_BOOTSTRAP_DIR="$BOOTSTRAP_OUT"
+  fi
+else
+  CRAB3_BOOTSTRAP_DIR=""
+fi
+
+## Execute the client command in a pure COMP environment
+ERR=0
+${CMS_CRAB_PATH}/bin/crab "$@" || ERR=$?
+
+## Remove the temporary direcotry, but only if we run the bootstrap script
+[ -z "${CRAB3_BOOTSTRAP_DIR}" ] || rm -rf "${CRAB3_BOOTSTRAP_DIR}"
+
+exit $ERR

cjson.py.file

@@ -0,0 +1,10 @@
+import json
+
+class DecodeError(ValueError): pass
+
+def decode(data):
+  try: return json.loads(data)
+  except ValueError as e: raise DecodeError(e)
+
+def encode(data):
+  return json.dumps(data)

cms-common.spec

@@ -1,8 +1,8 @@
 ### RPM cms cms-common 1.0
-## REVISION 1207
+## REVISION 1208
 ## NOCOMPILER
 
-%define tag 5da960d0e56fc5315fac016d417ed37b159486ff
+%define tag cf4dd43612a2feb26f8a06b566bea485f888a010
 
 Source:  git+https://github.com/cms-sw/cms-common.git?obj=master/%{tag}&export=%{n}-%{realversion}-%{tag}&output=/%{n}-%{realversion}-%{tag}.tgz
 

crab-build.file

@@ -1,5 +1,6 @@
-## INITENV +PATH PYTHON27PATH %{i}/${PYTHON_LIB_SITE_PACKAGES}
+## INITENV +PATH PYTHONPATH %{i}/lib
 
+%define crab_startup _crab-startup
 %if "%{?wmcore_packages:set}" != "set"
   %define wmcore_packages PSetTweaks Utils WMCore
 %endif
@@ -10,12 +11,14 @@
   %define dbs_packages Client/src/python/dbs PycurlClient/src/python/RestClient
 %endif
 
+Requires: curl
+
 Source0: git://github.com/dmwm/CRABClient.git?obj=master/%{realversion}&export=CRABClient&output=/CRABClient-%{realversion}.tar.gz
 Source1: git://github.com/dmwm/WMCore.git?obj=master/%{wmcore_version}&export=WMCore&output=/WMCore-%{wmcore_version}.tar.gz
 Source2: git://github.com/dmwm/CRABServer.git?obj=master/%{crabserver_version}&export=CRABServer&output=/CRABServer-%{crabserver_version}.tar.gz
 Source3: git://github.com/dmwm/DBS.git?obj=master/%{dbs_version}&export=DBS&output=/DBS-%{dbs_version}.tar.gz
-
-Requires: py2-pycurl py2-python-cjson
+Source98: cjson.py
+Source99: %{crab_startup}
 
 %prep
 %setup -D -T -b 0 -n CRABClient
@@ -26,45 +29,61 @@ Requires: py2-pycurl py2-python-cjson
 %build
 
 %install
-mkdir -p %{i}/bin %{i}/${PYTHON_LIB_SITE_PACKAGES}
+
+mkdir -p %{i}/bin %{i}/lib/extras %{i}/etc/profile.d
 
 #Copy CRABClient
-rsync -a %{_builddir}/CRABClient/src/python/ %{i}/${PYTHON_LIB_SITE_PACKAGES}
-sed -i -e 's|"development"|"%{realversion}"|' %{i}/${PYTHON_LIB_SITE_PACKAGES}/CRABClient/__init__.py
+rsync -a %{_builddir}/CRABClient/src/python/ %{i}/lib
+sed -i -e 's|"development"|"%{realversion}"|' %{i}/lib/CRABClient/__init__.py
 rsync -a %{_builddir}/CRABClient/bin/  %{i}/bin/
 
-
 #Copy WMCore
 for pkg in %{wmcore_packages} ; do
   if [ -d %{_builddir}/WMCore/src/python/${pkg} ] ; then
-    rsync -a %{_builddir}/WMCore/src/python/${pkg} %{i}/${PYTHON_LIB_SITE_PACKAGES}/
+    rsync -a %{_builddir}/WMCore/src/python/${pkg} %{i}/lib/
   else
-    cp %{_builddir}/WMCore/src/python/${pkg} %{i}/${PYTHON_LIB_SITE_PACKAGES}/${pkg} 
+    cp %{_builddir}/WMCore/src/python/${pkg} %{i}/lib/${pkg}
   fi
 done
 
 #Copy CRABServer
 for pkg in %{crabserver_packages} ; do
   if [ -d %{_builddir}/CRABServer/src/python/${pkg} ] ; then
-    rsync -a %{_builddir}/CRABServer/src/python/${pkg} %{i}/${PYTHON_LIB_SITE_PACKAGES}/
+    rsync -a %{_builddir}/CRABServer/src/python/${pkg} %{i}/lib/
   else
-    cp %{_builddir}/CRABServer/src/python/${pkg} %{i}/${PYTHON_LIB_SITE_PACKAGES}/${pkg}
+    cp %{_builddir}/CRABServer/src/python/${pkg} %{i}/lib/${pkg}
   fi
 done
 
 #Copy DBS
 for pkg in %{dbs_packages} ; do
   if [ -d %{_builddir}/DBS/${pkg} ] ; then
-    rsync -a %{_builddir}/DBS/${pkg} %{i}/${PYTHON_LIB_SITE_PACKAGES}/
+    rsync -a %{_builddir}/DBS/${pkg} %{i}/lib/
   else
-    cp %{_builddir}/DBS/${pkg} %{i}/${PYTHON_LIB_SITE_PACKAGES}/${pkg}
+    cp %{_builddir}/DBS/${pkg} %{i}/lib/${pkg}
   fi
 done
-
+
+#Copy crab-startup script
+cp %{_sourcedir}/%{crab_startup} %{i}/bin/%{crab_startup}
+chmod +x %{i}/bin/%{crab_startup}
+
+cp %{_sourcedir}/cjson.py %{i}/lib/extras
+
 %post
-if [ "%{n}" != "crab" ] ; then
-  rm -f ${RPM_INSTALL_PREFIX}/common/%{n}
-  symlink="crab-prod"
-  if [ -f ${RPM_INSTALL_PREFIX}/common/crab ] ; then symlink="crab" ; fi
-  ln -s ${symlink} ${RPM_INSTALL_PREFIX}/common/%{n}
+%{relocateConfig}/bin/%{crab_startup}
+cd  ${RPM_INSTALL_PREFIX}/common
+CRAB_OLD_REV=0
+if [ -f %{crab_startup} ] ; then
+  CRAB_OLD_REV=$(grep '^\s*CRAB_STARTUP_REVISION\s*=' %{crab_startup} | sed 's|.*=||;s| ||g')
 fi
+CRAB_NEW_REV=$(grep '^\s*CRAB_STARTUP_REVISION\s*='  ${RPM_INSTALL_PREFIX}/%{pkgrel}/bin/%{crab_startup} | sed 's|.*=||;s| ||g')
+if [ ${CRAB_OLD_REV} -lt ${CRAB_NEW_REV} ] ; then
+  rm -f %{crab_startup}
+  cp ${RPM_INSTALL_PREFIX}/%{pkgrel}/bin/%{crab_startup} %{crab_startup}
+fi
+
+symlink="%n"
+if [ "%{n}" = "crab" ] ; then symlink="%{n}-prod" ; fi
+rm -f ${symlink}
+ln -s %{crab_startup} ${symlink}