Merge pull request #35825 from smuzaffar/12_1-py3subprocess-ALCA_RECO…

…NSTRUCTION

[ALCA_RECONSTRUCTION] [PY3] Fix needed for python3: Use of subprocess instead of commands

RecoVertex/BeamSpotProducer/scripts/BeamSpotWorkflow.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #____________________________________________________________
 #
 #  BeamSpotWorkflow
@@ -35,7 +35,7 @@
 
 from builtins import range
 import sys,os
-import commands, re, time
+import subprocess, re, time
 import datetime
 import configparser as ConfigParser
 import xmlrpclib
@@ -58,7 +58,7 @@
 def getLastUploadedIOV(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"):
     #return 582088327592295
     listIOVCommand = "cmscond_list_iov -c " + destDB + " -P /afs/cern.ch/cms/DB/conddb -t " + tagName 
-    dbError = commands.getstatusoutput( listIOVCommand )
+    dbError = subprocess.getstatusoutput( listIOVCommand )
     if dbError[0] != 0 :
         if dbError[1].find("metadata entry \"" + tagName + "\" does not exist") != -1:
             print("Creating a new tag because I got the following error contacting the DB")
@@ -70,7 +70,7 @@ def getLastUploadedIOV(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEA
 
 
     aCommand = listIOVCommand + " | grep DB= | tail -1 | awk \'{print $1}\'"
-    output = commands.getstatusoutput( aCommand )
+    output = subprocess.getstatusoutput( aCommand )
 
     #WARNING when we pass to lumi IOV this should be long long
     if output[1] == '':
@@ -85,14 +85,14 @@ def getListOfFilesToProcess(dataSet,lastRun=-1):
         queryCommand = queryCommand + " and run > " + str(lastRun)
     queryCommand = queryCommand + "\" | grep .root"    
 #    print " >> " + queryCommand
-    output = commands.getstatusoutput( queryCommand )
+    output = subprocess.getstatusoutput( queryCommand )
     return output[1].split('\n')
 
 ########################################################################
 def getNumberOfFilesToProcessForRun(dataSet,run):
     queryCommand = "dbs --search --query \"find file where dataset=" + dataSet + " and run = " + str(run) + "\" | grep .root"
     #print " >> " + queryCommand
-    output = commands.getstatusoutput( queryCommand )
+    output = subprocess.getstatusoutput( queryCommand )
     if output[0] != 0:
         return 0
     else:
@@ -110,7 +110,7 @@ def getListOfRunsAndLumiFromDBS(dataSet,lastRun=-1):
         print(" >> " + queryCommand)
         output = []
         for i in range(0,3):
-            output = commands.getstatusoutput( queryCommand )
+            output = subprocess.getstatusoutput( queryCommand )
             if output[0] == 0 and not (output[1].find("ERROR") != -1 or output[1].find("Error") != -1) :
                 break
         if output[0] != 0:
@@ -654,7 +654,7 @@ def main():
         if len(copiedFiles) == len(selectedFilesToProcess):
             break;
         else:
-            commands.getstatusoutput("rm -rf " + workingDir)
+            subprocess.getstatusoutput("rm -rf " + workingDir)
     if len(copiedFiles) != len(selectedFilesToProcess):
         error = "ERROR: I can't copy more than " + str(len(copiedFiles)) + " files out of " + str(len(selectedFilesToProcess)) + " from " + archiveDir + " to " + workingDir 
         sendEmail(mailList,error)
@@ -693,7 +693,7 @@ def main():
     iovTillLast   = '0';
 
     #Creating the final name for the combined sqlite file
-    uuid = commands.getstatusoutput('uuidgen -t')[1]
+    uuid = subprocess.getstatusoutput('uuidgen -t')[1]
     final_sqlite_file_name = databaseTag + '@' + uuid
     sqlite_file     = workingDir + final_sqlite_file_name + ".db"
     metadata_file   = workingDir + final_sqlite_file_name + ".txt"
@@ -774,9 +774,9 @@ def main():
     archive_results_file_name = "Payloads_" + iovSinceFirst + "_" + iovTillLast + "_" + databaseTag + ".txt"
     if not os.path.isdir(archiveDir + 'payloads'):
         os.mkdir(archiveDir + 'payloads')
-    commands.getstatusoutput('mv ' + sqlite_file   + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.db')
-    commands.getstatusoutput('mv ' + metadata_file + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.txt')
-    commands.getstatusoutput('cp ' + workingDir + payloadFileName + ' ' + archiveDir + 'payloads/' + archive_results_file_name)
+    subprocess.getstatusoutput('mv ' + sqlite_file   + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.db')
+    subprocess.getstatusoutput('mv ' + metadata_file + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.txt')
+    subprocess.getstatusoutput('cp ' + workingDir + payloadFileName + ' ' + archiveDir + 'payloads/' + archive_results_file_name)
 
     print(archiveDir + "payloads/" + archive_sqlite_file_name + '.db')
     print(archiveDir + "payloads/" + archive_sqlite_file_name + '.txt')

RecoVertex/BeamSpotProducer/scripts/CommonMethods.py

@@ -1,6 +1,6 @@
 from __future__ import print_function
 from builtins import range
-import math, re, optparse, commands, os, sys, time, datetime
+import math, re, optparse, subprocess, os, sys, time, datetime
 from BeamSpotObj import BeamSpot
 from IOVObj import IOV
 
@@ -56,7 +56,7 @@ def timeoutManager(type,timeout=-1,fileName=".timeout"):
 
     if not fileExist or not alreadyThere or isTimeout or (reset and alreadyThere):
         if fileExist:
-            commands.getstatusoutput("rm -rf " + fileName)
+            subprocess.getstatusoutput("rm -rf " + fileName)
         file = open(fileName,'w')
         file.write(text)
         file.close()
@@ -80,13 +80,13 @@ def checkLock():
 ###########################################################################################
 def lock():
     global lockFile
-    commands.getstatusoutput( "touch " + lockFile)
+    subprocess.getstatusoutput( "touch " + lockFile)
 
 ###########################################################################################
 def rmLock():
     global lockFile
     if checkLock():
-        commands.getstatusoutput( "rm " + lockFile)
+        subprocess.getstatusoutput( "rm " + lockFile)
 
 ###########################################################################################
 def exit(msg=""):
@@ -159,7 +159,7 @@ def sendEmail(mailList,error):
 def dirExists(dir):
     if dir.find("castor") != -1:
         lsCommand = "nsls " + dir
-        output = commands.getstatusoutput( lsCommand )
+        output = subprocess.getstatusoutput( lsCommand )
         return not output[0]
     else:
         return os.path.exists(dir)
@@ -179,7 +179,7 @@ def ls(dir,filter=""):
     if filter != "":
         aCommand  += " | grep " + filter 
 
-    tmpStatus = commands.getstatusoutput( aCommand )
+    tmpStatus = subprocess.getstatusoutput( aCommand )
     listOfFiles = tmpStatus[1].split('\n')
     if len(listOfFiles) == 1:
         if listOfFiles[0].find('No such file or directory') != -1:
@@ -213,7 +213,7 @@ def cp(fromDir,toDir,listOfFiles,overwrite=False,smallList=False):
         # copy to local disk
         aCommand = cpCommand + 'cp '+ fromDir + file + " " + toDir
         print(" >> " + aCommand)
-        tmpStatus = commands.getstatusoutput( aCommand )
+        tmpStatus = subprocess.getstatusoutput( aCommand )
         if tmpStatus[0] == 0:
             copiedFiles.append(file)
         else:
@@ -897,7 +897,7 @@ def writeSqliteFile(sqliteFileName,tagName,timeType,beamSpotFile,sqliteTemplateF
 
     wNewFile.close()
     print("writing sqlite file ...")
-    status_wDB = commands.getstatusoutput('cmsRun '+ writeDBOut)
+    status_wDB = subprocess.getstatusoutput('cmsRun '+ writeDBOut)
     print(status_wDB[1])
 
     os.system("rm -f " + writeDBOut)
@@ -919,7 +919,7 @@ def readSqliteFile(sqliteFileName,tagName,sqliteTemplateFile,tmpDir="/tmp/"):
         rNewFile.write(line)
 
     rNewFile.close()
-    status_rDB = commands.getstatusoutput('cmsRun '+ readDBOut)
+    status_rDB = subprocess.getstatusoutput('cmsRun '+ readDBOut)
 
     outtext = status_rDB[1]
     print(outtext)
@@ -930,15 +930,15 @@ def readSqliteFile(sqliteFileName,tagName,sqliteTemplateFile,tmpDir="/tmp/"):
 def appendSqliteFile(combinedSqliteFileName, sqliteFileName, tagName, IOVSince, IOVTill ,tmpDir="/tmp/"):
     aCommand = "conddb_import -c sqlite_file:" + tmpDir + combinedSqliteFileName + " -f sqlite_file:" + sqliteFileName + " -i " + tagName + " -t " + tagName + " -b " + IOVSince + " -e " + IOVTill
     print(aCommand)
-    std = commands.getstatusoutput(aCommand)
+    std = subprocess.getstatusoutput(aCommand)
     print(std[1])
     return not std[0]
 
 ###########################################################################################
 def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
     # Changing permissions to metadata
     acmd = "chmod a+w " + sqliteFileDirName + sqliteFileName + ".txt"
-    outcmd = commands.getstatusoutput(acmd)
+    outcmd = subprocess.getstatusoutput(acmd)
     print(acmd)
 #    print outcmd[1]
     if outcmd[0]:
@@ -947,22 +947,22 @@ def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
 
     acmd = "cp " + sqliteFileDirName + sqliteFileName + ".db " + sqliteFileDirName + sqliteFileName + ".txt ." 
     print(acmd)
-    outcmd = commands.getstatusoutput(acmd)
+    outcmd = subprocess.getstatusoutput(acmd)
     print(outcmd[1])
     if outcmd[0]:
         print("Couldn't cd to " + sqliteFileDirName)
         return False
 
     acmd = "tar -cvjf " + sqliteFileName + ".tar.bz2 " + sqliteFileName + ".db " + sqliteFileName + ".txt"
     print(acmd)
-    outcmd = commands.getstatusoutput(acmd)
+    outcmd = subprocess.getstatusoutput(acmd)
     print(outcmd[1])
     if outcmd[0]:
         print("Couldn't zip the files!")
         return False
 
     acmd = "chmod a+w " + sqliteFileName + ".tar.bz2"
-    outcmd = commands.getstatusoutput(acmd)
+    outcmd = subprocess.getstatusoutput(acmd)
     print(acmd)
 #    print outcmd[1]
     if outcmd[0]:
@@ -971,7 +971,7 @@ def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
 
     acmd = "scp -p " + sqliteFileName + ".tar.bz2" + " webcondvm.cern.ch:" + dropbox
     print(acmd)
-    outcmd = commands.getstatusoutput(acmd)
+    outcmd = subprocess.getstatusoutput(acmd)
     print(outcmd[1])
     if outcmd[0]:
         print("Couldn't scp the files to DropBox!")
@@ -980,22 +980,22 @@ def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
 
     acmd = "mv " + sqliteFileName + ".tar.bz2 " + sqliteFileDirName
     print(acmd)
-    outcmd = commands.getstatusoutput(acmd)
+    outcmd = subprocess.getstatusoutput(acmd)
     print(outcmd[1])
     if outcmd[0]:
         print("Couldn't mv the file to " + sqliteFileDirName)
         return False
 
     acmd = "rm " + sqliteFileName + ".db " + sqliteFileName + ".txt"
     print(acmd)
-    outcmd = commands.getstatusoutput(acmd)
+    outcmd = subprocess.getstatusoutput(acmd)
     print(outcmd[1])
     if outcmd[0]:
         print("Couldn't rm the db and txt files")
         return False
 
 #    acmd = "scp -p " + sqliteFileDirName + sqliteFileName + ".txt webcondvm.cern.ch:/tmp"
-#    outcmd = commands.getstatusoutput(acmd)
+#    outcmd = subprocess.getstatusoutput(acmd)
 #    print acmd
 #    print outcmd[1]
 #    if outcmd[0]:
@@ -1004,14 +1004,14 @@ def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
 
 #    acmd = "ssh webcondvm.cern.ch \"mv /tmp/" + sqliteFileName + ".db /tmp/" + sqliteFileName + ".txt " + dropbox +"\""
 #    print acmd
-#    outcmd = commands.getstatusoutput(acmd)
+#    outcmd = subprocess.getstatusoutput(acmd)
 #    print outcmd[1]
 #    if outcmd[0]:
 #        print "Can't move files from tmp to dropbox!"
         return False
 
 #    acmd = "ssh webcondvm.cern.ch \"mv /tmp/" + final_sqlite_file_name + ".txt "+dropbox +"\""
-#    outcmd = commands.getstatusoutput(acmd)
+#    outcmd = subprocess.getstatusoutput(acmd)
 #    print acmd
 #    print outcmd[1]
 #    if outcmd[0]:

RecoVertex/BeamSpotProducer/scripts/beamvalidation.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #____________________________________________________________
 #
 #
@@ -30,7 +30,7 @@
 
 from builtins import range
 import os, string, re, sys, math
-import commands, time
+import subprocess, time
 
 #_______________OPTIONS________________
 import optparse
@@ -187,7 +187,7 @@ def get_listoftags(dest, auth,):
 
     queryTags_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -a | grep BeamSpotObjects"
     print(queryTags_cmd)
-    outcmd = commands.getstatusoutput( queryTags_cmd )
+    outcmd = subprocess.getstatusoutput( queryTags_cmd )
     print(outcmd[1])
 
     listtags = outcmd[1].split()
@@ -242,7 +242,7 @@ def get_lastIOVs( listoftags, dest, auth ):
         queryIOVs_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -t "+ lasttag
         print(queryIOVs_cmd)
 
-        outcmd = commands.getstatusoutput( queryIOVs_cmd )
+        outcmd = subprocess.getstatusoutput( queryIOVs_cmd )
 
         tmparr = outcmd[1].split('\n')
 
@@ -303,19 +303,19 @@ def get_plots(path,output, iovs, tag):
     initial = str(int(initial) -100 )
     cmd = path+"/plotBeamSpotDB.py -b -P -t "+tag+" -i "+initial +" -f "+final
     print(cmd)
-    outcmd = commands.getstatusoutput( cmd )
+    outcmd = subprocess.getstatusoutput( cmd )
     print(outcmd[1])
 
     cmd = "ls *.png"
-    outcmd = commands.getstatusoutput( cmd )
+    outcmd = subprocess.getstatusoutput( cmd )
 
     pngfiles = outcmd[1].split('\n')
     print(pngfiles)
 
     cmd = "cp *.png "+os.path.dirname(output)
-    outcmd = commands.getstatusoutput( cmd )
+    outcmd = subprocess.getstatusoutput( cmd )
     cmd = "rm *.png"
-    outcmd = commands.getstatusoutput( cmd )
+    outcmd = subprocess.getstatusoutput( cmd )
 
     pngfiles.sort()
     return pngfiles
@@ -345,7 +345,7 @@ def write_plots(lines, plots,web):
 #________________________________
 def get_productionFiles( directory ):
 
-    list = commands.getstatusoutput('ls -t '+directory)
+    list = subprocess.getstatusoutput('ls -t '+directory)
     list = list[1].split()
     newlist = []
     for i in list:

RecoVertex/BeamSpotProducer/scripts/checkPayloads.py

@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import print_function
-import sys,os,commands,re
+import sys,os,subprocess,re
 import xmlrpclib
 from CommonMethods import *
 
@@ -16,7 +16,7 @@
 #####################################################################################
 def getUploadedIOVs(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"):
     listIOVCommand = "cmscond_list_iov -c " + destDB + " -P /afs/cern.ch/cms/DB/conddb -t " + tagName
-    dbError = commands.getstatusoutput( listIOVCommand )
+    dbError = subprocess.getstatusoutput( listIOVCommand )
     if dbError[0] != 0 :
         if dbError[1].find("metadata entry \"" + tagName + "\" does not exist") != -1:
             exit(dbError[1])
@@ -26,7 +26,7 @@ def getUploadedIOVs(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSP
 
     aCommand = listIOVCommand + " | grep DB= | awk \'{print $1}\'"
     #print aCommand
-    output = commands.getstatusoutput( aCommand )
+    output = subprocess.getstatusoutput( aCommand )
 
     #WARNING when we pass to lumi IOV this should be long long
     if output[1] == '':

RecoVertex/BeamSpotProducer/scripts/checkRuns.py

@@ -1,7 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import print_function
 from builtins import range
-import sys,os,commands,re
+import sys,os,re
 import xmlrpclib
 from CommonMethods import *
 try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson

RecoVertex/BeamSpotProducer/scripts/copyAndRename.py

@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import print_function
-import sys,os,commands
+import sys,os,subprocess
 from CommonMethods import *
 def main():
     if len(sys.argv) < 3:
@@ -36,7 +36,7 @@ def main():
         if fileName != newFileName:
             aCmd = "mv " + destDir + fileName + " " + destDir + newFileName
             print(aCmd)
-            output =  commands.getstatusoutput(aCmd)
+            output =  subprocess.getstatusoutput(aCmd)
             if output[0] != 0:
                 print(output[1])
         else:

RecoVertex/BeamSpotProducer/scripts/copyFiles.py

@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import print_function
-import sys,os,commands
+import sys,os
 from CommonMethods import *
 
 def main():