From 1c7d4419fb64b64c87ef066dba722cd575585338 Mon Sep 17 00:00:00 2001 From: Remi Hakim Date: Fri, 29 May 2015 16:56:06 -0400 Subject: [PATCH] [Core] Proxy, fix the way we set the no proxy environment setting Fix #1594 Also: * simplify the proxy parsing * Move the code in a new utils module --- config.py | 54 +---------------------------- dogstatsd.py | 7 ++-- emitter.py | 8 ++--- tests/core/test_common.py | 35 +++++++++++++++++-- tests/core/test_dogstatsd.py | 4 +-- utils/proxy.py | 66 ++++++++++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 68 deletions(-) create mode 100644 utils/proxy.py diff --git a/config.py b/config.py index fef84ad09b..9dd231e681 100644 --- a/config.py +++ b/config.py @@ -23,6 +23,7 @@ # project from util import get_os, yLoader from utils.platform import Platform +from utils.proxy import get_proxy # 3rd party import yaml @@ -607,59 +608,6 @@ def set_win32_cert_path(): log.info("Windows certificate path: %s" % crt_path) tornado.simple_httpclient._DEFAULT_CA_CERTS = crt_path -def get_proxy(agentConfig, use_system_settings=False): - proxy_settings = {} - - # First we read the proxy configuration from datadog.conf - proxy_host = agentConfig.get('proxy_host', None) - if proxy_host is not None and not use_system_settings: - proxy_settings['host'] = proxy_host - try: - proxy_settings['port'] = int(agentConfig.get('proxy_port', 3128)) - except ValueError: - log.error('Proxy port must be an Integer. Defaulting it to 3128') - proxy_settings['port'] = 3128 - - proxy_settings['user'] = agentConfig.get('proxy_user', None) - proxy_settings['password'] = agentConfig.get('proxy_password', None) - proxy_settings['system_settings'] = False - log.debug("Proxy Settings: %s:%s@%s:%s" % (proxy_settings['user'], "*****", proxy_settings['host'], proxy_settings['port'])) - return proxy_settings - - # If no proxy configuration was specified in datadog.conf - # We try to read it from the system settings - try: - import urllib - proxies = urllib.getproxies() - proxy = proxies.get('https', None) - if proxy is not None: - try: - proxy = proxy.split('://')[1] - except Exception: - pass - px = proxy.split(':') - proxy_settings['host'] = px[0] - proxy_settings['port'] = int(px[1]) - proxy_settings['user'] = None - proxy_settings['password'] = None - proxy_settings['system_settings'] = True - if '@' in proxy_settings['host']: - creds = proxy_settings['host'].split('@')[0].split(':') - proxy_settings['user'] = creds[0] - if len(creds) == 2: - proxy_settings['password'] = creds[1] - - log.debug("Proxy Settings: %s:%s@%s:%s" % (proxy_settings['user'], "*****", proxy_settings['host'], proxy_settings['port'])) - return proxy_settings - - except Exception, e: - log.debug("Error while trying to fetch proxy settings using urllib %s. Proxy is probably not set" % str(e)) - - log.debug("No proxy configured") - - return None - - def get_confd_path(osname=None): if not osname: osname = get_os() diff --git a/dogstatsd.py b/dogstatsd.py index 3d1d087844..5e724c24db 100755 --- a/dogstatsd.py +++ b/dogstatsd.py @@ -10,11 +10,8 @@ import os os.umask(022) -# Starting with Agent 5.0.0, there should always be a local forwarder -# running and all payloads should go through it. So we should make sure -# that we pass the no_proxy environment variable that will be used by requests -# See: https://github.com/kennethreitz/requests/pull/945 -os.environ['no_proxy'] = '127.0.0.1,localhost' +from utils.proxy import set_no_proxy_settings +set_no_proxy_settings() # stdlib import logging diff --git a/emitter.py b/emitter.py index 8f831f798c..4a845b08fe 100644 --- a/emitter.py +++ b/emitter.py @@ -1,7 +1,6 @@ # stdlib from hashlib import md5 import logging -import os import re import zlib @@ -12,11 +11,8 @@ # project from config import get_version -# Starting with Agent 5.0.0, there should always be a local forwarder -# running and all payloads should go through it. So we should make sure -# that we pass the no_proxy environment variable that will be used by requests -# See: https://github.com/kennethreitz/requests/pull/945 -os.environ['no_proxy'] = '127.0.0.1,localhost' +from utils.proxy import set_no_proxy_settings +set_no_proxy_settings() # urllib3 logs a bunch of stuff at the info level requests_log = logging.getLogger("requests.packages.urllib3") diff --git a/tests/core/test_common.py b/tests/core/test_common.py index e6f4259c71..84945cb3e2 100644 --- a/tests/core/test_common.py +++ b/tests/core/test_common.py @@ -20,6 +20,7 @@ from tests.checks.common import load_check from util import get_hostname from utils.ntp import get_ntp_datadog_host +from utils.proxy import get_proxy logger = logging.getLogger() @@ -233,14 +234,14 @@ def test_no_proxy(self): self.assertTrue("no_proxy" in env) - self.assertEquals(env["no_proxy"], "127.0.0.1,localhost") + self.assertEquals(env["no_proxy"], "127.0.0.1,localhost,169.254.169.254") self.assertEquals({}, get_environ_proxies( "http://localhost:17123/intake")) expected_proxies = { 'http': 'http://localhost:3128', 'https': 'http://localhost:3128', - 'no': '127.0.0.1,localhost' + 'no': '127.0.0.1,localhost,169.254.169.254' } environ_proxies = get_environ_proxies("https://www.google.com") self.assertEquals(expected_proxies, environ_proxies, @@ -252,6 +253,36 @@ def test_no_proxy(self): del env["HTTP_PROXY"] del env["HTTPS_PROXY"] + def test_get_proxy(self): + + agentConfig = { + "proxy_host": "localhost", + "proxy_port": 4242, + "proxy_user": "foo", + "proxy_password": "bar" + } + proxy_from_config = get_proxy(agentConfig) + + self.assertEqual(proxy_from_config, + { + "host": "localhost", + "port": 4242, + "user": "foo", + "password": "bar", + }) + + os.environ["HTTPS_PROXY"] = "https://fooenv:barenv@google.com:4444" + proxy_from_env = get_proxy({}) + self.assertEqual(proxy_from_env, + { + "host": "google.com", + "port": 4444, + "user": "fooenv", + "password": "barenv" + }) + + + def test_min_collection_interval(self): if os.environ.get('TRAVIS', False): raise SkipTest('ntp server times out too often on Travis') diff --git a/tests/core/test_dogstatsd.py b/tests/core/test_dogstatsd.py index 9be255a207..7303859221 100644 --- a/tests/core/test_dogstatsd.py +++ b/tests/core/test_dogstatsd.py @@ -848,14 +848,14 @@ def test_no_proxy(self): self.assertTrue("no_proxy" in env) - self.assertEquals(env["no_proxy"], "127.0.0.1,localhost") + self.assertEquals(env["no_proxy"], "127.0.0.1,localhost,169.254.169.254") self.assertEquals({}, get_environ_proxies( "http://localhost:17123/api/v1/series")) expected_proxies = { 'http': 'http://localhost:3128', 'https': 'http://localhost:3128', - 'no': '127.0.0.1,localhost' + 'no': '127.0.0.1,localhost,169.254.169.254' } environ_proxies = get_environ_proxies("https://www.google.com") self.assertEquals(expected_proxies, environ_proxies, diff --git a/utils/proxy.py b/utils/proxy.py new file mode 100644 index 0000000000..0db41c1b54 --- /dev/null +++ b/utils/proxy.py @@ -0,0 +1,66 @@ +import logging +import os +from urllib import getproxies +from urlparse import urlparse + +log = logging.getLogger(__name__) + +def set_no_proxy_settings(): + # Starting with Agent 5.0.0, there should always be a local forwarder + # running and all payloads should go through it. So we should make sure + # that we pass the no_proxy environment variable that will be used by requests + # See: https://github.com/kennethreitz/requests/pull/945 + to_add = ["127.0.0.1", "localhost", "169.254.169.254"] + no_proxy = os.environ.get("no_proxy", "") + if not no_proxy.strip(): + no_proxy = [] + else: + no_proxy = no_proxy.split(',') + + for host in to_add: + if host not in no_proxy: + no_proxy.append(host) + + os.environ['no_proxy'] = ','.join(no_proxy) + +def get_proxy(agentConfig): + proxy_settings = {} + + # First we read the proxy configuration from datadog.conf + proxy_host = agentConfig.get('proxy_host') + if proxy_host is not None: + proxy_settings['host'] = proxy_host + try: + proxy_settings['port'] = int(agentConfig.get('proxy_port', 3128)) + except ValueError: + log.error('Proxy port must be an Integer. Defaulting it to 3128') + proxy_settings['port'] = 3128 + + proxy_settings['user'] = agentConfig.get('proxy_user') + proxy_settings['password'] = agentConfig.get('proxy_password') + log.debug("Proxy Settings: %s:*****@%s:%s", proxy_settings['user'], + proxy_settings['host'], proxy_settings['port']) + return proxy_settings + + # If no proxy configuration was specified in datadog.conf + # We try to read it from the system settings + try: + proxy = getproxies().get('https') + if proxy is not None: + parse = urlparse(proxy) + proxy_settings['host'] = parse.hostname + proxy_settings['port'] = int(parse.port) + proxy_settings['user'] = parse.username + proxy_settings['password'] = parse.password + + log.debug("Proxy Settings: %s:*****@%s:%s", proxy_settings['user'], + proxy_settings['host'], proxy_settings['port']) + return proxy_settings + + except Exception, e: + log.debug("Error while trying to fetch proxy settings using urllib %s." + "Proxy is probably not set", str(e)) + + log.debug("No proxy configured") + + return None