Merge pull request #40454 from Sam-Harper/ConfdbProxyTunnels_1300pre2

Expanding HLT Configuration Proxy Support and Adding Tunnel Support : 13_X

HLTrigger/Configuration/python/Tools/confdb.py

@@ -50,7 +50,7 @@ def __init__(self, configuration):
 
     # get the configuration from ConfdB
     from .confdbOfflineConverter import OfflineConverter
-    self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port)
+    self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port, tunnel = self.config.tunnel, tunnelPort = self.config.tunnel_port)
     self.buildPathList()
     self.buildOptions()
     self.getSetupConfigurationFromDB()

HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py

@@ -46,7 +46,9 @@ class OfflineConverter:
     ips_for_proxy = {
         'cmsr1-s.cern.ch' : '10.116.96.89',
         'cmsr2-s.cern.ch' : '10.116.96.139',
-        'cmsr3-s.cern.ch' : '10.116.96.105'
+        'cmsr3-s.cern.ch' : '10.116.96.105',
+        'cmsonr1-adg1-s.cern.ch' : '10.116.96.109',
+        'cmsonr1-s.cms' : '10.176.84.78'
     }
 
     databases['v3-beta'] = dict(databases['v3'])
@@ -70,7 +72,8 @@ def CheckTempDirectory(dir):
 
 
     def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False,
-                 proxy = False, proxyHost = 'localhost', proxyPort = '8080'): 
+                 proxy = False, proxyHost = 'localhost', proxyPort = '8080',
+                 tunnel = False, tunnelPort = '10121'):
         self.verbose = verbose
         self.version = version
         self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
@@ -83,6 +86,12 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals
         self.proxy = proxy
         self.proxyHost = proxyHost
         self.proxyPort = proxyPort
+        self.tunnel = tunnel
+        self.tunnelPort = tunnelPort
+
+        if self.proxy and self.tunnel:
+            sys.stderr.write( "ERROR: proxy and tunnel options can not both be true" )
+            sys.exit(1)
 
         # check the schema version
         if version not in self.databases:
@@ -109,6 +118,16 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals
         else:
             self.proxy_connect_args = ()
 
+        # this sets the host to localhost
+        if self.tunnel:
+            temp_connect = list(self.connect)
+            host_index = temp_connect.index('-h')
+            temp_connect[host_index+1] = "localhost"
+            self.connect = tuple(temp_connect)
+            self.tunnel_connect_args = ('--dbport', self.tunnelPort)
+        else:
+            self.tunnel_connect_args = ()
+
         # check for a custom base URL
         if url is not None:
             self.baseUrl = url
@@ -165,7 +184,7 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals
 
 
     def query(self, *args):
-        args = self.javaCmd + self.connect + self.proxy_connect_args + args 
+        args = self.javaCmd + self.connect + self.proxy_connect_args + self.tunnel_connect_args + args 
         if self.verbose:
             sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
         sub = subprocess.Popen(
@@ -223,6 +242,14 @@ def help():
         --modules <p1[,p2]>         (include modules, referenced or not!)
         --blocks <m1::p1[,p2][,m2]> (generate parameter blocks)
 
+        Options to connect to target db via SOCKS proxy, or direct tunnel:
+          [the options --dbproxy and --dbtunnel are mutually exclusive]
+        --dbproxy                   (use a SOCKS proxy to connect outside CERN network [default: False])
+        --dbproxyhost <hostname>    (host of the SOCKS proxy [default: "localhost"])
+        --dbproxyport <port>        (port of the SOCKS proxy [default: 8080])
+        --dbtunnel                  (use direct tunnel to connect outside CERN network [default: False])
+        --dbtunnelport <port>       (port when using a direct tunnel on localhost [default: 10121])
+
         --verbose                   (print additional details)
 """)
 
@@ -275,8 +302,8 @@ def main():
         version = 'v3-test'
         db      = 'dev'
         args.remove('--v3-test')
-    
-    proxy=False
+
+    proxy = False
     proxy_host = "localhost"
     proxy_port = "8080"
     if '--dbproxy' in args:
@@ -288,7 +315,20 @@ def main():
     if '--dbproxyport' in args:
         proxy_port = args.pop(args.index('--dbproxyport')+1)
         args.remove('--dbproxyport')
-
+
+    tunnel = False
+    tunnel_port = "10121"
+    if '--dbtunnel' in args:
+        tunnel = True
+        args.remove('--dbtunnel')
+
+    if '--dbtunnelport' in args:
+        tunnel_port = args.pop(args.index('--dbtunnelport')+1)
+        args.remove('--dbtunnelport')
+
+    if tunnel and proxy:
+        sys.stderr.write( 'ERROR: conflicting connection specifications, "--dbtunnel" and "--dbproxy" are mutually exclusive options\n' )
+        sys.exit(1)
 
     _dbs = {}
     _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ]
@@ -315,10 +355,11 @@ def main():
             sys.exit(1)
 
     converter = OfflineConverter(version = version, database = db, verbose = verbose,
-                                 proxy = proxy, proxyHost = proxy_host, proxyPort=proxy_port)
+                                 proxy = proxy, proxyHost = proxy_host, proxyPort = proxy_port,
+                                 tunnel = tunnel, tunnelPort = tunnel_port)
     out, err = converter.query( * args )
     if 'ERROR' in err:
-        sys.stderr.write( "%s: error while retriving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) )
+        sys.stderr.write( "%s: error while retrieving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) )
         sys.exit(1)
     else:
         sys.stdout.write( out )

HLTrigger/Configuration/python/Tools/options.py

@@ -131,7 +131,9 @@ def __init__(self):
     self.setup      = None        #     if set, downlad the setup_cff from the specified configuration and load it.
     self.proxy      = False       #     use a socks proxy to connect
     self.proxy_host = 'localhost' #     host of the proxy server
-    self.proxy_port = '8080'        #     port of the proxy server
+    self.proxy_port = '8080'      #     port of the proxy server
+    self.tunnel     = False       #     use a direct tunnel on localhost to connect
+    self.tunnel_port = '10121'    #     port to connect to on localhost when tunneling
 
   # convert HLT and L1 menus to a dedicated object representation on the fly
   def __setattr__(self, name, value):

HLTrigger/Configuration/scripts/hltGetConfiguration

@@ -75,11 +75,18 @@ parser.add_argument('--l1-emulator',
                     const   = 'Full',
                     help    = 'Run the Full stage-2 L1T emulator.' )
 
-parser.add_argument('--dbproxy',
+group = parser.add_mutually_exclusive_group()
+group.add_argument('--dbproxy',
                     dest    = 'proxy',
                     action  = 'store_true',
                     default = defaults.proxy,
                     help    = 'Use a socks proxy to connect outside CERN network (default: False)' )
+group.add_argument('--dbtunnel',
+                    dest    = 'tunnel',
+                    action  = 'store_true',
+                    default = defaults.tunnel,
+                    help    = 'Use direct tunnel to connect outside CERN network (default: False)' )
+
 parser.add_argument('--dbproxyport',
                     dest    = 'proxy_port',
                     action  = 'store',
@@ -92,6 +99,12 @@ parser.add_argument('--dbproxyhost',
                     metavar = 'PROXYHOST',
                     default = defaults.proxy_host,
                     help    = 'Host of the socks proxy (default: "localhost")' )
+parser.add_argument('--dbtunnelport',
+                    dest    = 'tunnel_port',
+                    action  = 'store',
+                    metavar = 'TUNNELPORT',
+                    default = defaults.tunnel_port,
+                    help    = 'Port when using a direct tunnel on localhost (default: 10121)' )
 
 group = parser.add_mutually_exclusive_group()
 group.add_argument('--prescale',
@@ -242,14 +255,14 @@ parser.add_argument('--help',
 # parse command line arguments and options
 config = parser.parse_args(namespace = options.HLTProcessOptions())
 
-# do not include db-proxy options in 1st-line comment
+# do not include db-proxy/tunnel options in 1st-line comment
 cmdArgs, skipNext = [], False
 for cmdArg in sys.argv:
   if skipNext:
     skipNext = False
     continue
-  if cmdArg.startswith('--dbproxy'):
-    if cmdArg.startswith('--dbproxyh') or cmdArg.startswith('--dbproxyp'):
+  if cmdArg.startswith('--dbproxy') or cmdArg.startswith('--dbtunnel'):
+    if cmdArg.startswith('--dbproxyh') or cmdArg.startswith('--dbproxyp') or cmdArg.startswith('--dbtunnelp'):
       skipNext = '=' not in cmdArg
     continue
   cmdArgs += [cmdArg]