From df05e87cce1b0d4e06325313d7f7fa2a7b09c49e Mon Sep 17 00:00:00 2001 From: Sam Harper Date: Fri, 6 Jan 2023 12:27:55 +0000 Subject: [PATCH 1/3] expanding proxy support for other databases and adding tunnel support --- .../Configuration/python/Tools/confdb.py | 2 +- .../python/Tools/confdbOfflineConverter.py | 43 ++++++++++++++++--- .../Configuration/python/Tools/options.py | 4 +- .../Configuration/scripts/hltGetConfiguration | 15 ++++++- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/HLTrigger/Configuration/python/Tools/confdb.py b/HLTrigger/Configuration/python/Tools/confdb.py index d8ef59d7da5f1..f88875b40c9eb 100644 --- a/HLTrigger/Configuration/python/Tools/confdb.py +++ b/HLTrigger/Configuration/python/Tools/confdb.py @@ -50,7 +50,7 @@ def __init__(self, configuration): # get the configuration from ConfdB from .confdbOfflineConverter import OfflineConverter - self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port) + self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port, tunnel = self.config.tunnel, tunnelPort = self.config.tunnel_port) self.buildPathList() self.buildOptions() self.getSetupConfigurationFromDB() diff --git a/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py b/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py index e6271d7e612c2..2b28e7e2624b6 100644 --- a/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py +++ b/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py @@ -46,7 +46,9 @@ class OfflineConverter: ips_for_proxy = { 'cmsr1-s.cern.ch' : '10.116.96.89', 'cmsr2-s.cern.ch' : '10.116.96.139', - 'cmsr3-s.cern.ch' : '10.116.96.105' + 'cmsr3-s.cern.ch' : '10.116.96.105', + 'cmsonr1-adg1-s.cern.ch' : '10.116.96.109', + 'cmsonr1-s.cms' : '10.176.84.78' } databases['v3-beta'] = dict(databases['v3']) @@ -70,7 +72,8 @@ def CheckTempDirectory(dir): def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False, - proxy = False, proxyHost = 'localhost', proxyPort = '8080'): + proxy = False, proxyHost = 'localhost', proxyPort = '8080', + tunnel = False, tunnelPort = '10121'): self.verbose = verbose self.version = version self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version @@ -83,6 +86,12 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals self.proxy = proxy self.proxyHost = proxyHost self.proxyPort = proxyPort + self.tunnel = tunnel + self.tunnelPort = tunnelPort + + if self.proxy and self.tunnel: + sys.stderr.write( "ERROR: proxy and tunnel options can not both be true" ) + sys.exit(1) # check the schema version if version not in self.databases: @@ -109,6 +118,16 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals else: self.proxy_connect_args = () + # this sets the host to localhost + if self.tunnel: + temp_connect = list(self.connect) + host_index = temp_connect.index('-h') + temp_connect[host_index+1] = "localhost" + self.connect = tuple(temp_connect) + self.tunnel_connect_args = ('--dbport', self.tunnelPort) + else: + self.tunnel_connect_args = () + # check for a custom base URL if url is not None: self.baseUrl = url @@ -165,7 +184,7 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals def query(self, *args): - args = self.javaCmd + self.connect + self.proxy_connect_args + args + args = self.javaCmd + self.connect + self.proxy_connect_args + self.tunnel_connect_args + args if self.verbose: sys.stderr.write("\n" + ' '.join(args) + "\n\n" ) sub = subprocess.Popen( @@ -288,7 +307,20 @@ def main(): if '--dbproxyport' in args: proxy_port = args.pop(args.index('--dbproxyport')+1) args.remove('--dbproxyport') - + + tunnel = False + tunnel_port = "10121" + if '--dbtunnel' in args: + tunnel = True + args.remove('--dbtunnel') + + if '--dbtunnelport' in args: + tunnel_port = args.pop(args.index('--dbtunnelport')+1) + args.remove('--dbtunnelport') + + if tunnel and proxy: + sys.stderr.write( 'ERROR: conflicting connection specifications, "--dbtunnel" and "--dbproxy" are mutually exclusive options\n' ) + sys.exit(1) _dbs = {} _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ] @@ -315,7 +347,8 @@ def main(): sys.exit(1) converter = OfflineConverter(version = version, database = db, verbose = verbose, - proxy = proxy, proxyHost = proxy_host, proxyPort=proxy_port) + proxy = proxy, proxyHost = proxy_host, proxyPort=proxy_port, + tunnel = tunnel, tunnelPort = tunnel_port) out, err = converter.query( * args ) if 'ERROR' in err: sys.stderr.write( "%s: error while retriving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) ) diff --git a/HLTrigger/Configuration/python/Tools/options.py b/HLTrigger/Configuration/python/Tools/options.py index 9fed7bdea76aa..c36c4810ad806 100644 --- a/HLTrigger/Configuration/python/Tools/options.py +++ b/HLTrigger/Configuration/python/Tools/options.py @@ -131,7 +131,9 @@ def __init__(self): self.setup = None # if set, downlad the setup_cff from the specified configuration and load it. self.proxy = False # use a socks proxy to connect self.proxy_host = 'localhost' # host of the proxy server - self.proxy_port = '8080' # port of the proxy server + self.proxy_port = '8080' # port of the proxy server + self.tunnel = False # use a direct tunnel on localhost to connect + self.tunnel_port = '10121' # port to connect to on localhost when tunneling # convert HLT and L1 menus to a dedicated object representation on the fly def __setattr__(self, name, value): diff --git a/HLTrigger/Configuration/scripts/hltGetConfiguration b/HLTrigger/Configuration/scripts/hltGetConfiguration index 6157019b9f761..587cfe25cbdfd 100755 --- a/HLTrigger/Configuration/scripts/hltGetConfiguration +++ b/HLTrigger/Configuration/scripts/hltGetConfiguration @@ -75,11 +75,18 @@ parser.add_argument('--l1-emulator', const = 'Full', help = 'Run the Full stage-2 L1T emulator.' ) -parser.add_argument('--dbproxy', +group = parser.add_mutually_exclusive_group() +group.add_argument('--dbproxy', dest = 'proxy', action = 'store_true', default = defaults.proxy, help = 'Use a socks proxy to connect outside CERN network (default: False)' ) +group.add_argument('--dbtunnel', + dest = 'tunnel', + action = 'store_true', + default = defaults.tunnel, + help = 'Use direct tunnel to connect outside CERN network (default: False)' ) + parser.add_argument('--dbproxyport', dest = 'proxy_port', action = 'store', @@ -92,6 +99,12 @@ parser.add_argument('--dbproxyhost', metavar = 'PROXYHOST', default = defaults.proxy_host, help = 'Host of the socks proxy (default: "localhost")' ) +parser.add_argument('--dbtunnelport', + dest = 'tunnel_port', + action = 'store', + metavar = 'TUNNELPORT', + default = defaults.tunnel_port, + help = 'Port when using a direct tunnel on localhost (default: 10121)' ) group = parser.add_mutually_exclusive_group() group.add_argument('--prescale', From bad4349e962239bb142c1d7617572a027346ba50 Mon Sep 17 00:00:00 2001 From: Sam Harper Date: Mon, 9 Jan 2023 10:49:36 +0000 Subject: [PATCH 2/3] code review --- .../python/Tools/confdbOfflineConverter.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py b/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py index 2b28e7e2624b6..c0ad7083e6d24 100644 --- a/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py +++ b/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py @@ -242,6 +242,14 @@ def help(): --modules (include modules, referenced or not!) --blocks (generate parameter blocks) + Options to connect to target db via SOCKS proxy, or direct tunnel: + [the options --dbproxy and --dbtunnel are mutually exclusive] + --dbproxy (use a SOCKS proxy to connect outside CERN network [default: False]) + --dbproxyhost (host of the SOCKS proxy [default: "localhost"]) + --dbproxyport (port of the SOCKS proxy [default: 8080]) + --dbtunnel (use direct tunnel to connect outside CERN network [default: False]) + --dbtunnelport (port when using a direct tunnel on localhost [default: 10121]) + --verbose (print additional details) """) @@ -294,8 +302,8 @@ def main(): version = 'v3-test' db = 'dev' args.remove('--v3-test') - - proxy=False + + proxy = False proxy_host = "localhost" proxy_port = "8080" if '--dbproxy' in args: @@ -347,11 +355,11 @@ def main(): sys.exit(1) converter = OfflineConverter(version = version, database = db, verbose = verbose, - proxy = proxy, proxyHost = proxy_host, proxyPort=proxy_port, + proxy = proxy, proxyHost = proxy_host, proxyPort = proxy_port, tunnel = tunnel, tunnelPort = tunnel_port) out, err = converter.query( * args ) if 'ERROR' in err: - sys.stderr.write( "%s: error while retriving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) ) + sys.stderr.write( "%s: error while retrieving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) ) sys.exit(1) else: sys.stdout.write( out ) From 2281f649a10354861a7423d17fa697b71b50dd36 Mon Sep 17 00:00:00 2001 From: Sam Harper Date: Mon, 9 Jan 2023 10:52:44 +0000 Subject: [PATCH 3/3] updating to keep tunnel options out of the config --- HLTrigger/Configuration/scripts/hltGetConfiguration | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/HLTrigger/Configuration/scripts/hltGetConfiguration b/HLTrigger/Configuration/scripts/hltGetConfiguration index 587cfe25cbdfd..1a0a6aa338b37 100755 --- a/HLTrigger/Configuration/scripts/hltGetConfiguration +++ b/HLTrigger/Configuration/scripts/hltGetConfiguration @@ -255,14 +255,14 @@ parser.add_argument('--help', # parse command line arguments and options config = parser.parse_args(namespace = options.HLTProcessOptions()) -# do not include db-proxy options in 1st-line comment +# do not include db-proxy/tunnel options in 1st-line comment cmdArgs, skipNext = [], False for cmdArg in sys.argv: if skipNext: skipNext = False continue - if cmdArg.startswith('--dbproxy'): - if cmdArg.startswith('--dbproxyh') or cmdArg.startswith('--dbproxyp'): + if cmdArg.startswith('--dbproxy') or cmdArg.startswith('--dbtunnel'): + if cmdArg.startswith('--dbproxyh') or cmdArg.startswith('--dbproxyp') or cmdArg.startswith('--dbtunnelp'): skipNext = '=' not in cmdArg continue cmdArgs += [cmdArg]