Skip to content

Commit

Permalink
Merge pull request #34373 from ggovi/uploadconditions-py3-1-120X
Browse files Browse the repository at this point in the history 
Moving uploadCondition.py to python3
  • Loading branch information
@cmsbuild
cmsbuild authored Jul 7, 2021
2 parents 3744d7c + e9bcc04 commit 9a544f7
Showing 1 changed file with 31 additions and 45 deletions.
76 changes: 31 additions & 45 deletions CondCore/Utilities/scripts/uploadConditions.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
'''Script that uploads to the new CMS conditions uploader.
Adapted to the new infrastructure from v6 of the upload.py script for the DropBox from Miguel Ojeda.
'''
Expand Down Expand Up @@ -43,17 +43,17 @@
# common/http.py start (plus the "# Try to extract..." section bit)
import time
import logging
import cStringIO
import io

import pycurl
import socket
import copy

def getInput(default, prompt = ''):
'''Like raw_input() but with a default and automatic strip().
'''Like input() but with a default and automatic strip().
'''

answer = raw_input(prompt)
answer = input(prompt)
if answer:
return answer.strip()

Expand Down Expand Up @@ -89,11 +89,11 @@ def getInputChoose(optionsList, default, prompt = ''):


def getInputRepeat(prompt = ''):
'''Like raw_input() but repeats if nothing is provided and automatic strip().
'''Like input() but repeats if nothing is provided and automatic strip().
'''

while True:
answer = raw_input(prompt)
answer = input(prompt)
if answer:
return answer.strip()

Expand All @@ -107,32 +107,17 @@ def runWizard(basename, dataFilename, metadataFilename):
I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename)

# Try to get the available inputTags
try:
dataConnection = sqlite3.connect(dataFilename)
dataCursor = dataConnection.cursor()
dataCursor.execute('select name from sqlite_master where type == "table"')
tables = set(zip(*dataCursor.fetchall())[0])

# only conddb V2 supported...
if 'TAG' in tables:
dataCursor.execute('select NAME from TAG')
# In any other case, do not try to get the inputTags
else:
raise Exception()
dataConnection = sqlite3.connect(dataFilename)
dataCursor = dataConnection.cursor()

inputTags = dataCursor.fetchall()
if len(inputTags) == 0:
raise Exception()
inputTags = zip(*inputTags)[0]

except Exception:
inputTags = []
dataCursor.execute('select NAME from TAG')
records = dataCursor.fetchall()
inputTags = []
for rec in records:
inputTags.append(rec[0])

if len(inputTags) == 0:
print('\nI could not find any input tag in your data file, but you can still specify one manually.')

inputTag = getInputRepeat(
'\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
raise Exception("Could not find any input tag in the data file.")

else:
print('\nI found the following input tags in your SQLite data file:')
Expand All @@ -147,14 +132,18 @@ def runWizard(basename, dataFilename, metadataFilename):
while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
if ntry==0:
inputMessage = \
'\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: '
'\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (or prod); oracle://cms_orcoff_prep/CMS_CONDITIONS (or prep) \ndestinationDatabase: '
elif ntry==1:
inputMessage = \
'\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
\ndestinationDatabase: '
else:
raise Exception('No valid destination chosen. Bailing out...')
destinationDatabase = getInputRepeat(inputMessage)
if destinationDatabase == 'prod':
destinationDatabase = 'oracle://cms_orcon_prod/CMS_CONDITIONS'
if destinationDatabase == 'prep':
destinationDatabase = 'oracle://cms_orcoff_prep/CMS_CONDITIONS'
ntry += 1

while True:
Expand Down Expand Up @@ -205,7 +194,7 @@ def runWizard(basename, dataFilename, metadataFilename):
'\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
break
logging.info('Saving generated metadata in %s...', metadataFilename)
with open(metadataFilename, 'wb') as metadataFile:
with open(metadataFilename, 'w') as metadataFile:
metadataFile.write(metadata)

class HTTPError(Exception):
Expand Down Expand Up @@ -314,7 +303,7 @@ def getToken(self, username, password):
# self.curl.setopt( self.curl.POST, {})
self.curl.setopt(self.curl.HTTPGET, 0)

response = cStringIO.StringIO()
response = io.BytesIO()
self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
logging.debug('going to connect to server at: %s' % url )
Expand All @@ -325,12 +314,12 @@ def getToken(self, username, password):
if code in ( 502,503,504 ):
logging.debug('Trying again after %d seconds...', waitForRetry)
time.sleep( waitForRetry )
response = cStringIO.StringIO()
response = io.StringIO()
self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
self.curl.perform()
code = self.curl.getinfo(pycurl.RESPONSE_CODE)
resp = response.getvalue()
code = self.curl.getinfo(pycurl.RESPONSE_CODE)
resp = response.getvalue().decode('UTF-8')
errorMsg = None
if code==500 and not resp.find("INVALID_CREDENTIALS")==-1:
logging.error("Invalid credentials provided.")
Expand All @@ -355,9 +344,9 @@ def getToken(self, username, password):
raise Exception(errorMsg)

logging.debug('token: %s', self.token)
logging.debug('returning: %s', response.getvalue())
logging.debug('returning: %s', response.getvalue().decode('UTF-8'))

return resp
return response.getvalue()

def query(self, url, data = None, files = None, keepCookies = True):
'''Queries a URL, optionally with some data (dictionary).
Expand Down Expand Up @@ -410,11 +399,11 @@ def query(self, url, data = None, files = None, keepCookies = True):
if files is not None:
for (key, fileName) in files.items():
finalData[key] = (self.curl.FORM_FILE, fileName)
self.curl.setopt( self.curl.HTTPPOST, finalData.items() )
self.curl.setopt( self.curl.HTTPPOST, list(finalData.items()) )

self.curl.setopt(pycurl.VERBOSE, 0)

response = cStringIO.StringIO()
response = io.BytesIO()
self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
self.curl.perform()

Expand Down Expand Up @@ -471,7 +460,6 @@ def signIn(self, username, password ):
self.http.setBaseUrl(self.urlTemplate % self.hostname)
'''Signs in the server.
'''

logging.info('%s: Signing in user %s ...', self.hostname, username)
try:
self.token = self.http.getToken(username, password)
Expand All @@ -487,7 +475,6 @@ def signIn(self, username, password ):
if not self.token:
logging.error("could not get token for user %s from %s" % (username, self.hostname) )
return -2

logging.debug( "got: '%s'", str(self.token) )
self.userName = username
self.password = password
Expand Down Expand Up @@ -553,9 +540,9 @@ def uploadFile(self, filename, backend = defaultBackend, temporaryFile = default
logging.error(msg)
raise Exception(msg)

with tempfile.NamedTemporaryFile() as metadata:
with open('%s.txt' % basepath, 'rb') as originalMetadata:
json.dump(json.load(originalMetadata), metadata, sort_keys = True, indent = 4)
with tempfile.NamedTemporaryFile(mode='rb+') as metadata:
with open('%s.txt' % basepath, 'r') as originalMetadata:
metadata.write(json.dumps(json.load(originalMetadata), sort_keys = True, indent = 4).encode())

metadata.seek(0)
addToTarFile(tarFile, metadata, 'metadata.txt')
Expand Down Expand Up @@ -821,7 +808,6 @@ def re_upload( options ):
netrcPath = os.path.join( options.authPath,'.netrc' )
try:
netrcKey = '%s/%s' %(logDbSrv,logDbSchema)
print('#netrc key=%s' %netrcKey)
# Try to find the netrc entry
(username, account, password) = netrc.netrc( netrcPath ).authenticators( netrcKey )
except IOError as e:
Expand Down

0 comments on commit 9a544f7

Please sign in to comment.