Auto merge of #6166 - alexcrichton:debug-curl, r=ehuss

Add convenience debugging for HTTP requests in Cargo

This is something we probably should have added long long ago given the
number of network issues that arise over time. This adds a new
configuration setting for Cargo, `http.debug`, which activates curl's
`verbose` interface which prints out information like redirects and
headers flying back and forth. This is by default routed through Cargo's
normal debug logging facilities, meaning one way to use this could be:

    CARGO_HTTP_DEBUG=true \
      RUST_LOG=cargo::ops::registry \
      cargo build

and you should get lots of nice verbose logs as a result! This should
hopefully make it much easier to remotely debug HTTP issues as we can in
theory do a lot less guessing and a lot more manual inspection.

src/cargo/ops/registry.rs

@@ -1,10 +1,12 @@
 use std::collections::BTreeMap;
 use std::fs::{self, File};
 use std::iter::repeat;
+use std::str;
 use std::time::Duration;
 use std::{cmp, env};
 
-use curl::easy::{Easy, SslOpt};
+use log::Level;
+use curl::easy::{Easy, SslOpt, InfoType};
 use git2;
 use registry::{NewCrate, NewCrateDependency, Registry};
 
@@ -388,6 +390,32 @@ pub fn configure_http_handle(config: &Config, handle: &mut Easy) -> CargoResult<
     } else {
         handle.useragent(&version().to_string())?;
     }
+
+    if let Some(true) = config.get::<Option<bool>>("http.debug")? {
+        handle.verbose(true)?;
+        handle.debug_function(|kind, data| {
+            let (prefix, level) = match kind {
+                InfoType::Text => ("*", Level::Debug),
+                InfoType::HeaderIn => ("<", Level::Debug),
+                InfoType::HeaderOut => (">", Level::Debug),
+                InfoType::DataIn => ("{", Level::Trace),
+                InfoType::DataOut => ("}", Level::Trace),
+                InfoType::SslDataIn |
+                InfoType::SslDataOut => return,
+                _ => return,
+            };
+            match str::from_utf8(data) {
+                Ok(s) => {
+                    for line in s.lines() {
+                        log!(level, "http-debug: {} {}", prefix, line);
+                    }
+                }
+                Err(_) => {
+                    log!(level, "http-debug: {} ({} bytes of data)", prefix, data.len());
+                }
+            }
+        })?;
+    }
     Ok(())
 }
 

src/doc/src/reference/config.md

@@ -104,6 +104,16 @@ check-revoke = true # Indicates whether SSL certs are checked for revocation
 low-speed-limit = 5 # Lower threshold for bytes/sec (10 = default, 0 = disabled)
 multiplexing = false  # whether or not to use HTTP/2 multiplexing where possible
 
+# This setting can be used to help debug what's going on with HTTP requests made
+# by Cargo. When set to `true` then Cargo's normal debug logging will be filled
+# in with HTTP information, which you can extract with
+# `RUST_LOG=cargo::ops::registry=debug` (and `trace` may print more).
+#
+# Be wary when posting these logs elsewhere though, it may be the case that a
+# header has an authentication token in it you don't want leaked! Be sure to
+# briefly review logs before posting them.
+debug = false
+
 [build]
 jobs = 1                  # number of parallel jobs, defaults to # of CPUs
 rustc = "rustc"           # the rust compiler tool