From 038e51b39cd9d896afeb26f7ee973578f0ed0553 Mon Sep 17 00:00:00 2001 From: Remi Hakim Date: Wed, 16 Jul 2014 10:39:07 -0400 Subject: [PATCH 1/4] Tell curl not to change http method when hitting a 301 when using a proxy --- ddagent.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ddagent.py b/ddagent.py index 47214b3b13..1bacfcab60 100755 --- a/ddagent.py +++ b/ddagent.py @@ -43,6 +43,9 @@ from transaction import Transaction, TransactionManager import modules +# 3rd party +import pycurl + log = logging.getLogger('forwarder') log.setLevel(get_logging_config()['log_level'] or logging.INFO) @@ -216,6 +219,10 @@ def flush(self): tornado_client_params['proxy_port'] = proxy_settings['port'] tornado_client_params['proxy_username'] = proxy_settings['user'] tornado_client_params['proxy_password'] = proxy_settings['password'] + + # See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get + tornado_client_params['prepare_curl_callback'] = lambda curl: curl.setopt(pycurl.POSTREDIR, pycurl.REDIR_POST_ALL) + force_use_curl = True if not self._application.use_simple_http_client or force_use_curl: From 393488983db020525f389d4635c6dd0961a340cb Mon Sep 17 00:00:00 2001 From: Remi Hakim Date: Tue, 22 Jul 2014 13:41:08 -0400 Subject: [PATCH 2/4] Add pycurl in the requirements in order for travis to pass --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index fa04020434..0489fc4746 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ psutil gearman pylint boto +pycurl pysnmp pysnmp-mibs PyMySQL From d881636c31511565b1683e645d69d42c0eb5dd74 Mon Sep 17 00:00:00 2001 From: Remi Hakim Date: Tue, 22 Jul 2014 18:38:18 -0400 Subject: [PATCH 3/4] Do not set options by default --- config.py | 4 ++++ datadog.conf.example | 3 +++ ddagent.py | 5 +++-- .../datadog-agent/win32/install_files/datadog_win32.conf | 3 +++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 43768dec5c..4e5f65fb69 100644 --- a/config.py +++ b/config.py @@ -434,6 +434,10 @@ def get_config(parse_args=True, cfg_path=None, options=None): if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative(config.get("Main", "collect_instance_metadata")) + agentConfig["proxy_forbid_method_switch"] = False + if config.has_option("Main", "proxy_forbid_method_switch"): + agentConfig["proxy_forbid_method_switch"] = _is_affirmative(config.get("Main", "proxy_forbid_method_switch")) + agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags")) diff --git a/datadog.conf.example b/datadog.conf.example index 96dda688d9..6950f02865 100644 --- a/datadog.conf.example +++ b/datadog.conf.example @@ -8,6 +8,9 @@ dd_url: https://app.datadoghq.com # proxy_port: 3128 # proxy_user: user # proxy_password: password +# To be used with some proxys that return a 302 which make curl switch from POST to GET +# See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get +# proxy_forbid_method_switch: no # If you run the agent behind haproxy, you might want to set this to yes # skip_ssl_validation: no diff --git a/ddagent.py b/ddagent.py index 1bacfcab60..1489bd0a66 100755 --- a/ddagent.py +++ b/ddagent.py @@ -220,8 +220,9 @@ def flush(self): tornado_client_params['proxy_username'] = proxy_settings['user'] tornado_client_params['proxy_password'] = proxy_settings['password'] - # See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get - tornado_client_params['prepare_curl_callback'] = lambda curl: curl.setopt(pycurl.POSTREDIR, pycurl.REDIR_POST_ALL) + if self._application._agentConfig.get('proxy_forbid_method_switch'): + # See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get + tornado_client_params['prepare_curl_callback'] = lambda curl: curl.setopt(pycurl.POSTREDIR, pycurl.REDIR_POST_ALL) force_use_curl = True diff --git a/packaging/datadog-agent/win32/install_files/datadog_win32.conf b/packaging/datadog-agent/win32/install_files/datadog_win32.conf index c4c1128e2e..8c81399b28 100644 --- a/packaging/datadog-agent/win32/install_files/datadog_win32.conf +++ b/packaging/datadog-agent/win32/install_files/datadog_win32.conf @@ -13,6 +13,9 @@ api_key: APIKEYHERE # proxy_port: 3128 # proxy_user: user # proxy_password: password +# To be used with some proxys that return a 302 which make curl switch from POST to GET +# See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get +# proxy_forbid_method_switch: no # If you run the agent behind haproxy, you might want to set this to yes # skip_ssl_validation: no From 38fe83fb580446d9c959a7fd5f5a9b7a038d0c07 Mon Sep 17 00:00:00 2001 From: Remi Hakim Date: Wed, 23 Jul 2014 11:21:09 -0400 Subject: [PATCH 4/4] pycurl might not be available on source install --- ddagent.py | 53 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/ddagent.py b/ddagent.py index 1489bd0a66..a267d24812 100755 --- a/ddagent.py +++ b/ddagent.py @@ -44,7 +44,11 @@ import modules # 3rd party -import pycurl +try: + import pycurl +except ImportError: + # For the source install, pycurl might not be installed + pycurl = None log = logging.getLogger('forwarder') log.setLevel(get_logging_config()['log_level'] or logging.INFO) @@ -206,35 +210,38 @@ def flush(self): force_use_curl = False if proxy_settings is not None and endpoint != PUP_ENDPOINT: - - # When using a proxy we do a CONNECT request why shouldn't include Content-Length - # This is pretty hacky though as it should be done in pycurl or curl or tornado - if 'Content-Length' in tornado_client_params['headers']: - del tornado_client_params['headers']['Content-Length'] - log.debug("Removing Content-Length header.") - - log.debug("Configuring tornado to use proxy settings: %s:****@%s:%s" % (proxy_settings['user'], - proxy_settings['host'], proxy_settings['port'])) - tornado_client_params['proxy_host'] = proxy_settings['host'] - tornado_client_params['proxy_port'] = proxy_settings['port'] - tornado_client_params['proxy_username'] = proxy_settings['user'] - tornado_client_params['proxy_password'] = proxy_settings['password'] - - if self._application._agentConfig.get('proxy_forbid_method_switch'): - # See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get - tornado_client_params['prepare_curl_callback'] = lambda curl: curl.setopt(pycurl.POSTREDIR, pycurl.REDIR_POST_ALL) - force_use_curl = True - - if not self._application.use_simple_http_client or force_use_curl: + if pycurl is not None: + # When using a proxy we do a CONNECT request why shouldn't include Content-Length + # This is pretty hacky though as it should be done in pycurl or curl or tornado + if 'Content-Length' in tornado_client_params['headers']: + del tornado_client_params['headers']['Content-Length'] + log.debug("Removing Content-Length header.") + + log.debug("Configuring tornado to use proxy settings: %s:****@%s:%s" % (proxy_settings['user'], + proxy_settings['host'], proxy_settings['port'])) + tornado_client_params['proxy_host'] = proxy_settings['host'] + tornado_client_params['proxy_port'] = proxy_settings['port'] + tornado_client_params['proxy_username'] = proxy_settings['user'] + tornado_client_params['proxy_password'] = proxy_settings['password'] + + if self._application._agentConfig.get('proxy_forbid_method_switch'): + # See http://stackoverflow.com/questions/8156073/curl-violate-rfc-2616-10-3-2-and-switch-from-post-to-get + tornado_client_params['prepare_curl_callback'] = lambda curl: curl.setopt(pycurl.POSTREDIR, pycurl.REDIR_POST_ALL) + + if (not self._application.use_simple_http_client or force_use_curl) and pycurl is not None: ssl_certificate = self._application._agentConfig.get('ssl_certificate', None) tornado_client_params['ca_certs'] = ssl_certificate req = tornado.httpclient.HTTPRequest(**tornado_client_params) use_curl = force_use_curl or self._application._agentConfig.get("use_curl_http_client") and not self._application.use_simple_http_client + if use_curl: - log.debug("Using CurlAsyncHTTPClient") - tornado.httpclient.AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") + if pycurl is None: + log.error("dd-agent is configured to use the Curl HTTP Client, but pycurl is not available on this system.") + else: + log.debug("Using CurlAsyncHTTPClient") + tornado.httpclient.AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") else: log.debug("Using SimpleHTTPClient") http = tornado.httpclient.AsyncHTTPClient()