Implement experimental registry HTTP API from RFC

crates/cargo-test-support/src/lib.rs

@@ -1525,6 +1525,7 @@ fn substitute_macros(input: &str) -> String {
         ("[DOCUMENTING]", " Documenting"),
         ("[FRESH]", "       Fresh"),
         ("[UPDATING]", "    Updating"),
+        ("[PREFETCHING]", " Prefetching"),
         ("[ADDING]", "      Adding"),
         ("[REMOVING]", "    Removing"),
         ("[DOCTEST]", "   Doc-tests"),

crates/cargo-test-support/src/registry.rs

@@ -7,7 +7,12 @@ use flate2::Compression;
 use std::collections::HashMap;
 use std::fs::{self, File};
 use std::io::prelude::*;
+use std::io::BufReader;
+use std::net::{SocketAddr, TcpListener};
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread;
 use tar::{Builder, Header};
 use url::Url;
 
@@ -213,6 +218,231 @@ pub fn init() {
     );
 }
 
+#[derive(Debug, Copy, Clone)]
+pub enum RegistryServerConfiguration {
+    NoChangelog,
+    WithChangelog,
+    ChangelogNoRange,
+}
+
+pub struct RegistryServer {
+    done: Arc<AtomicBool>,
+    server: Option<thread::JoinHandle<()>>,
+    addr: SocketAddr,
+}
+
+impl RegistryServer {
+    pub fn addr(&self) -> SocketAddr {
+        self.addr
+    }
+}
+
+impl Drop for RegistryServer {
+    fn drop(&mut self) {
+        self.done.store(true, Ordering::SeqCst);
+        // NOTE: we can't actually await the server since it's blocked in accept()
+        let _ = self.server.take().unwrap();
+    }
+}
+
+#[must_use]
+pub fn serve_registry(
+    registry_path: PathBuf,
+    config: RegistryServerConfiguration,
+) -> RegistryServer {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let addr = listener.local_addr().unwrap();
+    let done = Arc::new(AtomicBool::new(false));
+    let done2 = done.clone();
+
+    let t = thread::spawn(move || {
+        let support_range = !matches!(config, RegistryServerConfiguration::ChangelogNoRange);
+
+        let mut line = String::new();
+        'server: while !done2.load(Ordering::SeqCst) {
+            let (socket, _) = listener.accept().unwrap();
+            // Let's implement a very naive static file HTTP server.
+            let mut buf = BufReader::new(socket);
+
+            // First, the request line:
+            // GET /path HTTPVERSION
+            line.clear();
+            if buf.read_line(&mut line).unwrap() == 0 {
+                // Connection terminated.
+                continue;
+            }
+
+            assert!(line.starts_with("GET "), "got non-GET request: {}", line);
+            let path = PathBuf::from(
+                line.split_whitespace()
+                    .skip(1)
+                    .next()
+                    .unwrap()
+                    .trim_start_matches('/'),
+            );
+
+            let file = registry_path.join(path);
+            let mut exists = file.exists();
+            if file.ends_with("changelog")
+                && matches!(config, RegistryServerConfiguration::NoChangelog)
+            {
+                exists = false;
+            }
+
+            if exists {
+                // Grab some other headers we may care about.
+                let mut range = None;
+                let mut if_modified_since = None;
+                let mut if_none_match = None;
+                loop {
+                    line.clear();
+                    if buf.read_line(&mut line).unwrap() == 0 {
+                        continue 'server;
+                    }
+
+                    if line == "\r\n" {
+                        // End of headers.
+                        line.clear();
+                        break;
+                    }
+
+                    let value = line
+                        .splitn(2, ':')
+                        .skip(1)
+                        .next()
+                        .map(|v| v.trim())
+                        .unwrap();
+
+                    if line.starts_with("Range:") {
+                        let value = value.strip_prefix("bytes=").unwrap_or(value);
+                        if !value.is_empty() {
+                            let mut parts = value.split('-');
+                            let start = parts.next().unwrap().parse::<usize>().unwrap();
+                            let end = parts.next().unwrap();
+                            let end = if end.is_empty() {
+                                None
+                            } else {
+                                Some(end.parse::<usize>().unwrap())
+                            };
+                            range = Some((start, end));
+                        }
+                    } else if line.starts_with("If-Modified-Since:") {
+                        if_modified_since = Some(value.to_owned());
+                    } else if line.starts_with("If-None-Match:") {
+                        if_none_match = Some(value.trim_matches('"').to_owned());
+                    }
+                }
+
+                // Now grab info about the file.
+                let data = fs::read(&file).unwrap();
+                let etag = Sha256::new().update(&data).finish_hex();
+                let last_modified = format!("{:?}", file.metadata().unwrap().modified().unwrap());
+
+                // Start to construct our response:
+                let mut any_match = false;
+                let mut all_match = true;
+                if let Some(expected) = if_none_match {
+                    if etag != expected {
+                        all_match = false;
+                    } else {
+                        any_match = true;
+                    }
+                }
+                if let Some(expected) = if_modified_since {
+                    // NOTE: Equality comparison is good enough for tests.
+                    if last_modified != expected {
+                        all_match = false;
+                    } else {
+                        any_match = true;
+                    }
+                }
+                if any_match {
+                    assert!(range.is_none());
+                }
+
+                // Write out the main response line.
+                let data_len = data.len();
+                let mut data = &data[..];
+                if any_match && all_match {
+                    buf.get_mut()
+                        .write_all(b"HTTP/1.1 304 Not Modified\r\n")
+                        .unwrap();
+                } else if range.is_none() || !support_range {
+                    buf.get_mut().write_all(b"HTTP/1.1 200 OK\r\n").unwrap();
+                } else if let Some((start, end)) = range {
+                    if start >= data.len()
+                        || end.unwrap_or(0) >= data.len()
+                        || end.unwrap_or(start) <= start
+                    {
+                        buf.get_mut()
+                            .write_all(b"HTTP/1.1 416 Range Not Satisfiable\r\n")
+                            .unwrap();
+                    } else {
+                        buf.get_mut()
+                            .write_all(b"HTTP/1.1 206 Partial Content\r\n")
+                            .unwrap();
+
+                        // Slice the data as requested and include a header indicating that.
+                        // Note that start and end are both inclusive!
+                        data = &data[start..=end.unwrap_or(data_len - 1)];
+                        buf.get_mut()
+                            .write_all(
+                                format!(
+                                    "Content-Range: bytes {}-{}/{}\r\n",
+                                    start,
+                                    end.unwrap_or(data_len - 1),
+                                    data_len
+                                )
+                                .as_bytes(),
+                            )
+                            .unwrap();
+                    }
+                }
+                // TODO: Support 451 for crate index deletions.
+
+                // Write out other headers.
+                buf.get_mut()
+                    .write_all(format!("Content-Length: {}\r\n", data.len()).as_bytes())
+                    .unwrap();
+                buf.get_mut()
+                    .write_all(format!("ETag: \"{}\"\r\n", etag).as_bytes())
+                    .unwrap();
+                buf.get_mut()
+                    .write_all(format!("Last-Modified: {}\r\n", last_modified).as_bytes())
+                    .unwrap();
+
+                // And finally, write out the body.
+                buf.get_mut().write_all(b"\r\n").unwrap();
+                buf.get_mut().write_all(data).unwrap();
+            } else {
+                loop {
+                    line.clear();
+                    if buf.read_line(&mut line).unwrap() == 0 {
+                        // Connection terminated.
+                        continue 'server;
+                    }
+
+                    if line == "\r\n" {
+                        break;
+                    }
+                }
+
+                buf.get_mut()
+                    .write_all(b"HTTP/1.1 404 Not Found\r\n\r\n")
+                    .unwrap();
+                buf.get_mut().write_all(b"\r\n").unwrap();
+            }
+            buf.get_mut().flush().unwrap();
+        }
+    });
+
+    RegistryServer {
+        addr,
+        server: Some(t),
+        done,
+    }
+}
+
 pub fn init_registry(registry_path: PathBuf, dl_url: String, api_url: Url, api_path: PathBuf) {
     // Initialize a new registry.
     repo(&registry_path)
@@ -454,6 +684,26 @@ impl Package {
         t!(fs::create_dir_all(dst.parent().unwrap()));
         t!(fs::write(&dst, prev + &line[..] + "\n"));
 
+        // Update changelog.
+        let dst = registry_path.join("changelog");
+        t!(fs::create_dir_all(dst.parent().unwrap()));
+        let mut epoch = 1;
+        if dst.exists() {
+            // Fish out the current epoch.
+            let prev = fs::read_to_string(&dst).unwrap_or_default();
+            let e = prev.split_whitespace().next().unwrap();
+            if !e.is_empty() {
+                epoch = e.parse::<usize>().unwrap();
+            }
+        }
+        let mut changelog = t!(fs::OpenOptions::new().append(true).create(true).open(dst));
+        t!(writeln!(
+            changelog,
+            "{} 2020-11-20 16:54:07 {}",
+            epoch, self.name
+        ));
+        t!(changelog.flush());
+
         // Add the new file to the index.
         if !self.local {
             let repo = t!(git2::Repository::open(&registry_path));

crates/resolver-tests/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::many_single_char_names)]
 #![allow(clippy::needless_range_loop)] // false positives
 
+use std::borrow::Cow;
 use std::cell::RefCell;
 use std::cmp::PartialEq;
 use std::cmp::{max, min};
@@ -125,6 +126,14 @@ pub fn resolve_with_config_raw(
         used: HashSet<PackageId>,
     };
     impl<'a> Registry for MyRegistry<'a> {
+        fn prefetch(
+            &mut self,
+            _deps: &mut dyn Iterator<Item = Cow<'_, Dependency>>,
+        ) -> CargoResult<()> {
+            // Doing nothing is a valid way to prefetch.
+            Ok(())
+        }
+
         fn query(
             &mut self,
             dep: &Dependency,

src/bin/cargo/cli.rs

@@ -44,6 +44,7 @@ Available unstable (nightly-only) flags:
     -Z terminal-width      -- Provide a terminal width to rustc for error truncation
     -Z namespaced-features -- Allow features with `dep:` prefix
     -Z weak-dep-features   -- Allow `dep_name?/feature` feature syntax
+    -Z http-registries     -- Support HTTP-based crate registries
 
 Run with 'cargo -Z [FLAG] [SUBCOMMAND]'"
         );

src/cargo/core/dependency.rs

@@ -60,7 +60,7 @@ struct SerializedDependency<'a> {
     target: Option<&'a Platform>,
     /// The registry URL this dependency is from.
     /// If None, then it comes from the default registry (crates.io).
-    registry: Option<&'a str>,
+    registry: Option<String>,
 }
 
 impl ser::Serialize for Dependency {
@@ -79,7 +79,7 @@ impl ser::Serialize for Dependency {
             features: self.features(),
             target: self.platform(),
             rename: self.explicit_name_in_toml().map(|s| s.as_str()),
-            registry: registry_id.as_ref().map(|sid| sid.url().as_str()),
+            registry: registry_id.as_ref().map(|sid| sid.as_url().to_string()),
         }
         .serialize(s)
     }

src/cargo/core/features.rs

@@ -360,6 +360,7 @@ pub struct CliUnstable {
     pub namespaced_features: bool,
     pub weak_dep_features: bool,
     pub extra_link_arg: bool,
+    pub http_registry: bool,
 }
 
 fn deserialize_build_std<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>
@@ -468,6 +469,7 @@ impl CliUnstable {
             "namespaced-features" => self.namespaced_features = parse_empty(k, v)?,
             "weak-dep-features" => self.weak_dep_features = parse_empty(k, v)?,
             "extra-link-arg" => self.extra_link_arg = parse_empty(k, v)?,
+            "http-registry" => self.http_registry = parse_empty(k, v)?,
             _ => bail!("unknown `-Z` flag specified: {}", k),
         }
 

src/cargo/core/package.rs

@@ -8,12 +8,11 @@ use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use std::time::{Duration, Instant};
 
-use anyhow::Context;
 use bytesize::ByteSize;
 use curl::easy::{Easy, HttpVersion};
 use curl::multi::{EasyHandle, Multi};
 use lazycell::LazyCell;
-use log::{debug, warn};
+use log::debug;
 use semver::Version;
 use serde::Serialize;
 
@@ -579,23 +578,6 @@ impl<'cfg> PackageSet<'cfg> {
     }
 }
 
-// When dynamically linked against libcurl, we want to ignore some failures
-// when using old versions that don't support certain features.
-macro_rules! try_old_curl {
-    ($e:expr, $msg:expr) => {
-        let result = $e;
-        if cfg!(target_os = "macos") {
-            if let Err(e) = result {
-                warn!("ignoring libcurl {} error: {}", $msg, e);
-            }
-        } else {
-            result.with_context(|| {
-                anyhow::format_err!("failed to enable {}, is curl not built right?", $msg)
-            })?;
-        }
-    };
-}
-
 impl<'a, 'cfg> Downloads<'a, 'cfg> {
     /// Starts to download the package for the `id` specified.
     ///