[Core] Proxy, fix the way we set the no proxy environment setting

Fix #1594

Also:
* simplify the proxy parsing
* Move the code in a new utils module

config.py

@@ -23,6 +23,7 @@
 # project
 from util import get_os, yLoader
 from utils.platform import Platform
+from utils.proxy import get_proxy
 
 # 3rd party
 import yaml
@@ -607,59 +608,6 @@ def set_win32_cert_path():
     log.info("Windows certificate path: %s" % crt_path)
     tornado.simple_httpclient._DEFAULT_CA_CERTS = crt_path
 
-def get_proxy(agentConfig, use_system_settings=False):
-    proxy_settings = {}
-
-    # First we read the proxy configuration from datadog.conf
-    proxy_host = agentConfig.get('proxy_host', None)
-    if proxy_host is not None and not use_system_settings:
-        proxy_settings['host'] = proxy_host
-        try:
-            proxy_settings['port'] = int(agentConfig.get('proxy_port', 3128))
-        except ValueError:
-            log.error('Proxy port must be an Integer. Defaulting it to 3128')
-            proxy_settings['port'] = 3128
-
-        proxy_settings['user'] = agentConfig.get('proxy_user', None)
-        proxy_settings['password'] = agentConfig.get('proxy_password', None)
-        proxy_settings['system_settings'] = False
-        log.debug("Proxy Settings: %s:%s@%s:%s" % (proxy_settings['user'], "*****", proxy_settings['host'], proxy_settings['port']))
-        return proxy_settings
-
-    # If no proxy configuration was specified in datadog.conf
-    # We try to read it from the system settings
-    try:
-        import urllib
-        proxies = urllib.getproxies()
-        proxy = proxies.get('https', None)
-        if proxy is not None:
-            try:
-                proxy = proxy.split('://')[1]
-            except Exception:
-                pass
-            px = proxy.split(':')
-            proxy_settings['host'] = px[0]
-            proxy_settings['port'] = int(px[1])
-            proxy_settings['user'] = None
-            proxy_settings['password'] = None
-            proxy_settings['system_settings'] = True
-            if '@' in proxy_settings['host']:
-                creds = proxy_settings['host'].split('@')[0].split(':')
-                proxy_settings['user'] = creds[0]
-                if len(creds) == 2:
-                    proxy_settings['password'] = creds[1]
-
-            log.debug("Proxy Settings: %s:%s@%s:%s" % (proxy_settings['user'], "*****", proxy_settings['host'], proxy_settings['port']))
-            return proxy_settings
-
-    except Exception, e:
-        log.debug("Error while trying to fetch proxy settings using urllib %s. Proxy is probably not set" % str(e))
-
-    log.debug("No proxy configured")
-
-    return None
-
-
 def get_confd_path(osname=None):
     if not osname:
         osname = get_os()

dogstatsd.py

@@ -10,11 +10,8 @@
 import os
 os.umask(022)
 
-# Starting with Agent 5.0.0, there should always be a local forwarder
-# running and all payloads should go through it. So we should make sure
-# that we pass the no_proxy environment variable that will be used by requests
-# See: https://github.com/kennethreitz/requests/pull/945
-os.environ['no_proxy'] = '127.0.0.1,localhost'
+from utils.proxy import set_no_proxy_settings
+set_no_proxy_settings()
 
 # stdlib
 import logging

emitter.py

@@ -1,7 +1,6 @@
 # stdlib
 from hashlib import md5
 import logging
-import os
 import re
 import zlib
 
@@ -12,11 +11,8 @@
 # project
 from config import get_version
 
-# Starting with Agent 5.0.0, there should always be a local forwarder
-# running and all payloads should go through it. So we should make sure
-# that we pass the no_proxy environment variable that will be used by requests
-# See: https://github.com/kennethreitz/requests/pull/945
-os.environ['no_proxy'] = '127.0.0.1,localhost'
+from utils.proxy import set_no_proxy_settings
+set_no_proxy_settings()
 
 # urllib3 logs a bunch of stuff at the info level
 requests_log = logging.getLogger("requests.packages.urllib3")

tests/core/test_common.py

@@ -20,6 +20,7 @@
 from tests.checks.common import load_check
 from util import get_hostname
 from utils.ntp import get_ntp_datadog_host
+from utils.proxy import get_proxy
 
 logger = logging.getLogger()
 
@@ -233,14 +234,14 @@ def test_no_proxy(self):
 
         self.assertTrue("no_proxy" in env)
 
-        self.assertEquals(env["no_proxy"], "127.0.0.1,localhost")
+        self.assertEquals(env["no_proxy"], "127.0.0.1,localhost,169.254.169.254")
         self.assertEquals({}, get_environ_proxies(
             "http://localhost:17123/intake"))
 
         expected_proxies = {
             'http': 'http://localhost:3128',
             'https': 'http://localhost:3128',
-            'no': '127.0.0.1,localhost'
+            'no': '127.0.0.1,localhost,169.254.169.254'
         }
         environ_proxies = get_environ_proxies("https://www.google.com")
         self.assertEquals(expected_proxies, environ_proxies,
@@ -252,6 +253,36 @@ def test_no_proxy(self):
         del env["HTTP_PROXY"]
         del env["HTTPS_PROXY"]
 
+    def test_get_proxy(self):
+
+        agentConfig = {
+            "proxy_host": "localhost",
+            "proxy_port": 4242,
+            "proxy_user": "foo",
+            "proxy_password": "bar"
+        }
+        proxy_from_config = get_proxy(agentConfig)
+
+        self.assertEqual(proxy_from_config,
+            {
+                "host": "localhost",
+                "port": 4242,
+                "user": "foo",
+                "password": "bar",
+            })
+
+        os.environ["HTTPS_PROXY"] = "https://fooenv:barenv@google.com:4444"
+        proxy_from_env = get_proxy({})
+        self.assertEqual(proxy_from_env,
+            {
+                "host": "google.com",
+                "port": 4444,
+                "user": "fooenv",
+                "password": "barenv"
+            })
+
+
+
     def test_min_collection_interval(self):
         if os.environ.get('TRAVIS', False):
             raise SkipTest('ntp server times out too often on Travis')

tests/core/test_dogstatsd.py

@@ -848,14 +848,14 @@ def test_no_proxy(self):
 
         self.assertTrue("no_proxy" in env)
 
-        self.assertEquals(env["no_proxy"], "127.0.0.1,localhost")
+        self.assertEquals(env["no_proxy"], "127.0.0.1,localhost,169.254.169.254")
         self.assertEquals({}, get_environ_proxies(
             "http://localhost:17123/api/v1/series"))
 
         expected_proxies = {
             'http': 'http://localhost:3128',
             'https': 'http://localhost:3128',
-            'no': '127.0.0.1,localhost'
+            'no': '127.0.0.1,localhost,169.254.169.254'
         }
         environ_proxies = get_environ_proxies("https://www.google.com")
         self.assertEquals(expected_proxies, environ_proxies,

utils/proxy.py

@@ -0,0 +1,66 @@
+import logging
+import os
+from urllib import getproxies
+from urlparse import urlparse
+
+log = logging.getLogger(__name__)
+
+def set_no_proxy_settings():
+    # Starting with Agent 5.0.0, there should always be a local forwarder
+    # running and all payloads should go through it. So we should make sure
+    # that we pass the no_proxy environment variable that will be used by requests
+    # See: https://github.com/kennethreitz/requests/pull/945
+    to_add = ["127.0.0.1", "localhost", "169.254.169.254"]
+    no_proxy = os.environ.get("no_proxy", "")
+    if not no_proxy.strip():
+        no_proxy = []
+    else:
+        no_proxy = no_proxy.split(',')
+
+    for host in to_add:
+        if host not in no_proxy:
+            no_proxy.append(host)
+
+    os.environ['no_proxy'] = ','.join(no_proxy)
+
+def get_proxy(agentConfig):
+    proxy_settings = {}
+
+    # First we read the proxy configuration from datadog.conf
+    proxy_host = agentConfig.get('proxy_host')
+    if proxy_host is not None:
+        proxy_settings['host'] = proxy_host
+        try:
+            proxy_settings['port'] = int(agentConfig.get('proxy_port', 3128))
+        except ValueError:
+            log.error('Proxy port must be an Integer. Defaulting it to 3128')
+            proxy_settings['port'] = 3128
+
+        proxy_settings['user'] = agentConfig.get('proxy_user')
+        proxy_settings['password'] = agentConfig.get('proxy_password')
+        log.debug("Proxy Settings: %s:*****@%s:%s", proxy_settings['user'],
+            proxy_settings['host'], proxy_settings['port'])
+        return proxy_settings
+
+    # If no proxy configuration was specified in datadog.conf
+    # We try to read it from the system settings
+    try:
+        proxy = getproxies().get('https')
+        if proxy is not None:
+            parse = urlparse(proxy)
+            proxy_settings['host'] = parse.hostname
+            proxy_settings['port'] = int(parse.port)
+            proxy_settings['user'] = parse.username
+            proxy_settings['password'] = parse.password
+
+            log.debug("Proxy Settings: %s:*****@%s:%s", proxy_settings['user'],
+                proxy_settings['host'], proxy_settings['port'])
+            return proxy_settings
+
+    except Exception, e:
+        log.debug("Error while trying to fetch proxy settings using urllib %s."
+            "Proxy is probably not set", str(e))
+
+    log.debug("No proxy configured")
+
+    return None