diff --git a/CHANGELOG.md b/CHANGELOG.md index a61dbdf1b..444f623a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #795 - added images for additional toolchains maintained by cross-rs. - #792 - added `CROSS_CONTAINER_IN_CONTAINER` environment variable to replace `CROSS_DOCKER_IN_DOCKER`. - #782 - added `build-std` config option, which builds the rust standard library from source if enabled. -- #775 - forward Cargo exit code to host +- #678 - Add optional `target.{target}.dockerfile[.file]`, `target.{target}.dockerfile.context` and `target.{target}.dockerfile.build-args` to invoke docker/podman build before using an image. +- #678 - Add `target.{target}.pre-build` config for running commands before building the image. - #772 - added `CROSS_CONTAINER_OPTS` environment variable to replace `DOCKER_OPTS`. - #767, #788 - added the `cross-util` and `xtask` commands. - #745 - added `thumbv7neon-*` targets. @@ -27,6 +28,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Changed +- #775 - forward Cargo exit code to host - #762 - re-enabled `x86_64-unknown-dragonfly` target. - #747 - reduced android image sizes. - #746 - limit image permissions for android images. diff --git a/Cargo.lock b/Cargo.lock index f15812f40..91b2c1371 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,6 +160,7 @@ dependencies = [ "serde", "serde_ignored", "serde_json", + "sha1_smol", "shell-escape", "shell-words", "thiserror", @@ -473,6 +474,12 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sharded-slab" version = "0.1.4" diff --git a/Cargo.toml b/Cargo.toml index fe254fb55..62a3c62d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_ignored = "0.1.2" shell-words = "1.1.0" +sha1_smol = "1.0.0" [target.'cfg(not(windows))'.dependencies] nix = { version = "0.24", default-features = false, features = ["user"] } diff --git a/README.md b/README.md index 1ffb6358c..56a4da455 100644 --- a/README.md +++ b/README.md @@ -112,10 +112,22 @@ the default one. Normal Docker behavior applies, so: - If only `tag` is omitted, then Docker will use the `latest` tag. +#### Dockerfiles + +If you're using a custom Dockerfile, you can use `target.{{TARGET}}.dockerfile` to automatically build it + +``` toml +[target.aarch64-unknown-linux-gnu.dockerfile] +dockerfile = "./path/to/where/the/Dockerfile/resides" +``` + +`cross` will build and use the image that was built instead of the default image. + It's recommended to base your custom image on the default Docker image that cross uses: `ghcr.io/cross-rs/{{TARGET}}:{{VERSION}}` (where `{{VERSION}}` is cross's version). This way you won't have to figure out how to install a cross C toolchain in your -custom image. Example below: +custom image. + ``` Dockerfile FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:latest @@ -125,8 +137,23 @@ RUN dpkg --add-architecture arm64 && \ apt-get install --assume-yes libfoo:arm64 ``` +If you want cross to provide the `FROM` instruction, you can do the following + +``` Dockerfile +ARG CROSS_BASE_IMAGE +FROM $CROSS_BASE_IMAGE + +RUN ... ``` -$ docker build -t my/image:tag path/to/where/the/Dockerfile/resides + +#### Pre-build hook + +`cross` enables you to add dependencies and run other necessary commands in the image before using it. +This action will be added to the used image, so it won't be ran/built every time you use `cross`. + +``` toml +[target.x86_64-unknown-linux-gnu] +pre-build = ["dpkg --add-architecture arm64 && apt-get update && apt-get install --assume-yes libfoo:arm64"] ``` ### Docker in Docker diff --git a/docs/cross_toml.md b/docs/cross_toml.md index 16f306886..bedfc49d9 100644 --- a/docs/cross_toml.md +++ b/docs/cross_toml.md @@ -1,6 +1,7 @@ The `cross` configuration in the `Cross.toml` file, can contain the following elements: # `build` + The `build` key allows you to set global variables, e.g.: ```toml @@ -11,6 +12,7 @@ default-target = "x86_64-unknown-linux-gnu" ``` # `build.env` + With the `build.env` key you can globally set volumes that should be mounted in the Docker container or environment variables that should be passed through. For example: @@ -22,6 +24,7 @@ passthrough = ["IMPORTANT_ENV_VARIABLES"] ``` # `target.TARGET` + The `target` key allows you to specify parameters for specific compilation targets. ```toml @@ -29,10 +32,12 @@ The `target` key allows you to specify parameters for specific compilation targe xargo = false build-std = false image = "test-image" +pre-build = ["apt-get update"] runner = "custom-runner" ``` # `target.TARGET.env` + The `target` key allows you to specify environment variables that should be used for a specific compilation target. This is similar to `build.env`, but allows you to be more specific per target. @@ -41,3 +46,19 @@ This is similar to `build.env`, but allows you to be more specific per target. volumes = ["VOL1_ARG", "VOL2_ARG"] passthrough = ["IMPORTANT_ENV_VARIABLES"] ``` + +# `target.TARGET.dockerfile` + +```toml +[target.x86_64-unknown-linux-gnu.dockerfile] +file = "./Dockerfile" # The dockerfile to use relative to the `Cargo.toml` +context = "." # What folder to run the build script in +build-args = { ARG1 = "foo" } # https://docs.docker.com/engine/reference/builder/#arg +``` + +also supports + +```toml +[target.x86_64-unknown-linux-gnu] +dockerfile = "./Dockerfile" +``` diff --git a/src/bin/commands/images.rs b/src/bin/commands/images.rs index 955e3d018..83466c87c 100644 --- a/src/bin/commands/images.rs +++ b/src/bin/commands/images.rs @@ -186,7 +186,7 @@ fn remove_images( } command.args(images); if execute { - command.run(verbose).map_err(Into::into) + command.run(verbose, false).map_err(Into::into) } else { println!("{:?}", command); Ok(()) diff --git a/src/cargo.rs b/src/cargo.rs index 133f7bbe0..9146025e5 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -148,7 +148,9 @@ pub fn cargo_metadata_with_args( /// Pass-through mode pub fn run(args: &[String], verbose: bool) -> Result { - Command::new("cargo").args(args).run_and_get_status(verbose) + Command::new("cargo") + .args(args) + .run_and_get_status(verbose, false) } /// run cargo and get the output, does not check the exit status diff --git a/src/config.rs b/src/config.rs index e5b73c966..319cd0b68 100644 --- a/src/config.rs +++ b/src/config.rs @@ -68,6 +68,23 @@ impl Environment { self.get_target_var(target, "IMAGE") } + fn dockerfile(&self, target: &Target) -> Option { + let res = self.get_values_for("DOCKERFILE", target, |s| s.to_string()); + res.0.or(res.1) + } + + fn dockerfile_context(&self, target: &Target) -> Option { + let res = self.get_values_for("DOCKERFILE_CONTEXT", target, |s| s.to_string()); + res.0.or(res.1) + } + + fn pre_build(&self, target: &Target) -> Option> { + let res = self.get_values_for("PRE_BUILD", target, |v| { + v.split('\n').map(String::from).collect() + }); + res.0.or(res.1) + } + fn runner(&self, target: &Target) -> Option { self.get_target_var(target, "RUNNER") } @@ -178,16 +195,34 @@ impl Config { fn vec_from_config( &self, target: &Target, - env: impl Fn(&Environment, &Target) -> (Option>, Option>), - config_build: impl for<'a> Fn(&'a CrossToml) -> Option<&'a [String]>, - config_target: impl for<'a> Fn(&'a CrossToml, &Target) -> Option<&'a [String]>, - ) -> Result> { + env: impl for<'a> Fn(&'a Environment, &Target) -> (Option>, Option>), + config: impl for<'a> Fn(&'a CrossToml, &Target) -> (Option<&'a [String]>, Option<&'a [String]>), + sum: bool, + ) -> Result>> { let (env_build, env_target) = env(&self.env, target); - let mut collected = self.sum_of_env_toml_values(env_build, config_build)?; - collected.extend(self.sum_of_env_toml_values(env_target, |t| config_target(t, target))?); + if sum { + return self.sum_of_env_toml_values(env_target, |t| config(t, target)); + } else if let Some(env_target) = env_target { + return Ok(Some(env_target)); + } + + let (build, target) = self + .toml + .as_ref() + .map(|t| config(t, target)) + .unwrap_or_default(); + + // FIXME: let expression + if target.is_none() && env_build.is_some() { + return Ok(env_build); + } - Ok(collected) + if target.is_none() { + Ok(build.map(ToOwned::to_owned)) + } else { + Ok(target.map(ToOwned::to_owned)) + } } #[cfg(test)] @@ -211,22 +246,17 @@ impl Config { self.string_from_config(target, Environment::runner, CrossToml::runner) } - pub fn env_passthrough(&self, target: &Target) -> Result> { + pub fn env_passthrough(&self, target: &Target) -> Result>> { self.vec_from_config( target, Environment::passthrough, - CrossToml::env_passthrough_build, - CrossToml::env_passthrough_target, + CrossToml::env_passthrough, + true, ) } - pub fn env_volumes(&self, target: &Target) -> Result> { - self.vec_from_config( - target, - Environment::volumes, - CrossToml::env_volumes_build, - CrossToml::env_volumes_target, - ) + pub fn env_volumes(&self, target: &Target) -> Result>> { + self.vec_from_config(target, Environment::volumes, CrossToml::env_volumes, false) } pub fn target(&self, target_list: &TargetList) -> Option { @@ -238,19 +268,78 @@ impl Config { .and_then(|t| t.default_target(target_list)) } + pub fn dockerfile(&self, target: &Target) -> Result> { + self.string_from_config(target, Environment::dockerfile, CrossToml::dockerfile) + } + + pub fn dockerfile_context(&self, target: &Target) -> Result> { + self.string_from_config( + target, + Environment::dockerfile_context, + CrossToml::dockerfile_context, + ) + } + + pub fn dockerfile_build_args( + &self, + target: &Target, + ) -> Result>> { + // This value does not support env variables + self.toml + .as_ref() + .map_or(Ok(None), |t| Ok(t.dockerfile_build_args(target))) + } + + pub fn pre_build(&self, target: &Target) -> Result>> { + self.vec_from_config( + target, + |e, t| (None, e.pre_build(t)), + CrossToml::pre_build, + false, + ) + } + fn sum_of_env_toml_values<'a>( &'a self, - env_values: Option>, - toml_getter: impl FnOnce(&'a CrossToml) -> Option<&'a [String]>, - ) -> Result> { + env_values: Option>, + toml_getter: impl FnOnce(&'a CrossToml) -> (Option<&'a [String]>, Option<&'a [String]>), + ) -> Result>> { + let mut defined = false; let mut collect = vec![]; - if let Some(mut vars) = env_values { - collect.append(&mut vars); - } else if let Some(toml_values) = self.toml.as_ref().and_then(toml_getter) { - collect.extend(toml_values.iter().cloned()); + if let Some(vars) = env_values { + collect.extend(vars.as_ref().iter().cloned()); + defined = true; + } else if let Some((build, target)) = self.toml.as_ref().map(toml_getter) { + if let Some(build) = build { + collect.extend(build.iter().cloned()); + defined = true; + } + if let Some(target) = target { + collect.extend(target.iter().cloned()); + defined = true; + } } + if !defined { + Ok(None) + } else { + Ok(Some(collect)) + } + } +} - Ok(collect) +pub fn opt_merge + IntoIterator>( + opt1: Option, + opt2: Option, +) -> Option { + match (opt1, opt2) { + (None, None) => None, + (None, Some(opt2)) => Some(opt2), + (Some(opt1), None) => Some(opt1), + (Some(opt1), Some(opt2)) => { + let mut res = opt2; + res.extend(opt1); + Some(res) + } } } @@ -389,7 +478,8 @@ mod tests { let config = Config::new_with(Some(toml(TOML_BUILD_VOLUMES)?), env); let expected = vec!["VOLUME1".to_string(), "VOLUME2".into()]; - let result = config.env_volumes(&target()).unwrap(); + let result = config.env_volumes(&target()).unwrap().unwrap_or_default(); + dbg!(&result); assert!(result.len() == 2); assert!(result.contains(&expected[0])); assert!(result.contains(&expected[1])); @@ -404,7 +494,8 @@ mod tests { let config = Config::new_with(Some(toml(TOML_BUILD_VOLUMES)?), env); let expected = vec!["VOLUME3".to_string(), "VOLUME4".into()]; - let result = config.env_volumes(&target()).unwrap(); + let result = config.env_volumes(&target()).unwrap().unwrap_or_default(); + dbg!(&result); assert!(result.len() == 2); assert!(result.contains(&expected[0])); assert!(result.contains(&expected[1])); diff --git a/src/cross_toml.rs b/src/cross_toml.rs index 683d475ee..2d073002f 100644 --- a/src/cross_toml.rs +++ b/src/cross_toml.rs @@ -1,9 +1,10 @@ #![doc = include_str!("../docs/cross_toml.md")] -use crate::errors::*; +use crate::{config, errors::*}; use crate::{Target, TargetList}; use serde::Deserialize; use std::collections::{BTreeSet, HashMap}; +use std::str::FromStr; /// Environment configuration #[derive(Debug, Deserialize, PartialEq, Eq, Default)] @@ -21,6 +22,9 @@ pub struct CrossBuildConfig { xargo: Option, build_std: Option, default_target: Option, + pre_build: Option>, + #[serde(default, deserialize_with = "opt_string_or_struct")] + dockerfile: Option, } /// Target configuration @@ -30,11 +34,35 @@ pub struct CrossTargetConfig { xargo: Option, build_std: Option, image: Option, + #[serde(default, deserialize_with = "opt_string_or_struct")] + dockerfile: Option, + pre_build: Option>, runner: Option, #[serde(default)] env: CrossEnvConfig, } +/// Dockerfile configuration +#[derive(Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct CrossTargetDockerfileConfig { + file: String, + context: Option, + build_args: Option>, +} + +impl FromStr for CrossTargetDockerfileConfig { + type Err = std::convert::Infallible; + + fn from_str(s: &str) -> Result { + Ok(CrossTargetDockerfileConfig { + file: s.to_string(), + context: None, + build_args: None, + }) + } +} + /// Cross configuration #[derive(Debug, Deserialize, PartialEq, Eq)] pub struct CrossToml { @@ -67,12 +95,55 @@ impl CrossToml { /// Returns the `target.{}.image` part of `Cross.toml` pub fn image(&self, target: &Target) -> Option { - self.get_string(target, |t| &t.image) + self.get_string(target, |_| None, |t| t.image.as_ref()) + } + + /// Returns the `{}.dockerfile` or `{}.dockerfile.file` part of `Cross.toml` + pub fn dockerfile(&self, target: &Target) -> Option { + self.get_string( + target, + |b| b.dockerfile.as_ref().map(|c| &c.file), + |t| t.dockerfile.as_ref().map(|c| &c.file), + ) + } + + /// Returns the `target.{}.dockerfile.context` part of `Cross.toml` + pub fn dockerfile_context(&self, target: &Target) -> Option { + self.get_string( + target, + |b| b.dockerfile.as_ref().and_then(|c| c.context.as_ref()), + |t| t.dockerfile.as_ref().and_then(|c| c.context.as_ref()), + ) + } + + /// Returns the `target.{}.dockerfile.build_args` part of `Cross.toml` + pub fn dockerfile_build_args(&self, target: &Target) -> Option> { + let target = self + .get_target(target) + .and_then(|t| t.dockerfile.as_ref()) + .and_then(|d| d.build_args.as_ref()); + + let build = self + .build + .dockerfile + .as_ref() + .and_then(|d| d.build_args.as_ref()); + + config::opt_merge(target.cloned(), build.cloned()) + } + + /// Returns the `build.dockerfile.pre-build` and `target.{}.dockerfile.pre-build` part of `Cross.toml` + pub fn pre_build(&self, target: &Target) -> (Option<&[String]>, Option<&[String]>) { + self.get_vec( + target, + |b| b.pre_build.as_deref(), + |t| t.pre_build.as_deref(), + ) } /// Returns the `target.{}.runner` part of `Cross.toml` pub fn runner(&self, target: &Target) -> Option { - self.get_string(target, |t| &t.runner) + self.get_string(target, |_| None, |t| t.runner.as_ref()) } /// Returns the `build.xargo` or the `target.{}.xargo` part of `Cross.toml` @@ -85,24 +156,18 @@ impl CrossToml { self.get_bool(target, |b| b.build_std, |t| t.build_std) } - /// Returns the list of environment variables to pass through for `build`, - pub fn env_passthrough_build(&self) -> Option<&[String]> { - self.build.env.passthrough.as_deref() - } - - /// Returns the list of environment variables to pass through for `target`, - pub fn env_passthrough_target(&self, target: &Target) -> Option<&[String]> { - self.get_vec(target, |e| e.passthrough.as_deref()) + /// Returns the list of environment variables to pass through for `build` and `target` + pub fn env_passthrough(&self, target: &Target) -> (Option<&[String]>, Option<&[String]>) { + self.get_vec(target, |_| None, |t| t.env.passthrough.as_deref()) } - /// Returns the list of environment variables to pass through for `build`, - pub fn env_volumes_build(&self) -> Option<&[String]> { - self.build.env.volumes.as_deref() - } - - /// Returns the list of environment variables to pass through for `target`, - pub fn env_volumes_target(&self, target: &Target) -> Option<&[String]> { - self.get_vec(target, |e| e.volumes.as_deref()) + /// Returns the list of environment variables to pass through for `build` and `target` + pub fn env_volumes(&self, target: &Target) -> (Option<&[String]>, Option<&[String]>) { + self.get_vec( + target, + |build| build.env.volumes.as_deref(), + |t| t.env.volumes.as_deref(), + ) } /// Returns the default target to build, @@ -118,12 +183,16 @@ impl CrossToml { self.targets.get(target) } - fn get_string( - &self, + fn get_string<'a>( + &'a self, target: &Target, - get: impl Fn(&CrossTargetConfig) -> &Option, + get_build: impl Fn(&'a CrossBuildConfig) -> Option<&'a String>, + get_target: impl Fn(&'a CrossTargetConfig) -> Option<&'a String>, ) -> Option { - self.get_target(target).and_then(|t| get(t).clone()) + self.get_target(target) + .and_then(get_target) + .or_else(|| get_build(&self.build)) + .map(ToOwned::to_owned) } fn get_bool( @@ -140,13 +209,60 @@ impl CrossToml { fn get_vec( &self, - target: &Target, - get: impl Fn(&CrossEnvConfig) -> Option<&[String]>, - ) -> Option<&[String]> { - self.get_target(target).and_then(|t| get(&t.env)) + target_triple: &Target, + build: impl Fn(&CrossBuildConfig) -> Option<&[String]>, + target: impl Fn(&CrossTargetConfig) -> Option<&[String]>, + ) -> (Option<&[String]>, Option<&[String]>) { + let target = if let Some(t) = self.get_target(target_triple) { + target(t) + } else { + None + }; + (build(&self.build), target) } } +fn opt_string_or_struct<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: Deserialize<'de> + std::str::FromStr, + D: serde::Deserializer<'de>, +{ + use std::{fmt, marker::PhantomData}; + + use serde::de::{self, MapAccess, Visitor}; + + struct StringOrStruct(PhantomData T>); + + impl<'de, T> Visitor<'de> for StringOrStruct + where + T: Deserialize<'de> + FromStr, + { + type Value = Option; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("string or map") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(FromStr::from_str(value).ok()) + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + let t: Result = + Deserialize::deserialize(de::value::MapAccessDeserializer::new(map)); + t.map(Some) + } + } + + deserializer.deserialize_any(StringOrStruct(PhantomData)) +} + #[cfg(test)] mod tests { use super::*; @@ -177,12 +293,15 @@ mod tests { xargo: Some(true), build_std: None, default_target: None, + pre_build: Some(vec!["echo 'Hello World!'".to_string()]), + dockerfile: None, }, }; let test_str = r#" [build] xargo = true + pre-build = ["echo 'Hello World!'"] [build.env] volumes = ["VOL1_ARG", "VOL2_ARG"] @@ -212,6 +331,8 @@ mod tests { build_std: Some(true), image: Some("test-image".to_string()), runner: None, + dockerfile: None, + pre_build: Some(vec![]), }, ); @@ -228,6 +349,7 @@ mod tests { xargo = false build-std = true image = "test-image" + pre-build = [] "#; let (parsed_cfg, unused) = CrossToml::parse(test_str)?; @@ -245,14 +367,20 @@ mod tests { triple: "aarch64-unknown-linux-gnu".to_string(), }, CrossTargetConfig { - env: CrossEnvConfig { - passthrough: None, - volumes: Some(vec!["VOL".to_string()]), - }, xargo: Some(false), build_std: None, image: None, + dockerfile: Some(CrossTargetDockerfileConfig { + file: "Dockerfile.test".to_string(), + context: None, + build_args: None, + }), + pre_build: Some(vec!["echo 'Hello'".to_string()]), runner: None, + env: CrossEnvConfig { + passthrough: None, + volumes: Some(vec!["VOL".to_string()]), + }, }, ); @@ -266,18 +394,23 @@ mod tests { xargo: Some(true), build_std: None, default_target: None, + pre_build: Some(vec![]), + dockerfile: None, }, }; let test_str = r#" [build] xargo = true + pre-build = [] [build.env] passthrough = [] [target.aarch64-unknown-linux-gnu] xargo = false + dockerfile = "Dockerfile.test" + pre-build = ["echo 'Hello'"] [target.aarch64-unknown-linux-gnu.env] volumes = ["VOL"] diff --git a/src/docker/custom.rs b/src/docker/custom.rs new file mode 100644 index 000000000..7cb73821d --- /dev/null +++ b/src/docker/custom.rs @@ -0,0 +1,141 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::docker::Engine; +use crate::{config::Config, docker, CargoMetadata, Target}; +use crate::{errors::*, file, CommandExt}; + +use super::{image_name, parse_docker_opts}; + +#[derive(Debug, PartialEq, Eq)] +pub enum Dockerfile<'a> { + File { + path: &'a str, + context: Option<&'a str>, + name: Option<&'a str>, + }, + Custom { + content: String, + }, +} + +impl<'a> Dockerfile<'a> { + #[allow(clippy::too_many_arguments)] + pub fn build( + &self, + config: &Config, + metadata: &CargoMetadata, + engine: &Engine, + host_root: &Path, + build_args: impl IntoIterator, impl AsRef)>, + target_triple: &Target, + verbose: bool, + ) -> Result { + let mut docker_build = docker::subcommand(engine, "build"); + docker_build.current_dir(host_root); + docker_build.env("DOCKER_SCAN_SUGGEST", "false"); + docker_build.args([ + "--label", + &format!( + "{}.for-cross-target={target_triple}", + crate::CROSS_LABEL_DOMAIN + ), + ]); + + docker_build.args([ + "--label", + &format!( + "{}.workspace_root={}", + crate::CROSS_LABEL_DOMAIN, + metadata.workspace_root.display() + ), + ]); + + let image_name = self.image_name(target_triple, metadata); + docker_build.args(["--tag", &image_name]); + + for (key, arg) in build_args.into_iter() { + docker_build.args(["--build-arg", &format!("{}={}", key.as_ref(), arg.as_ref())]); + } + + if let Some(arch) = target_triple.deb_arch() { + docker_build.args(["--build-arg", &format!("CROSS_DEB_ARCH={arch}")]); + } + + let path = match self { + Dockerfile::File { path, .. } => PathBuf::from(path), + Dockerfile::Custom { content } => { + let path = metadata + .target_directory + .join(target_triple.to_string()) + .join(format!("Dockerfile.{}-custom", target_triple,)); + { + let mut file = file::write_file(&path, true)?; + file.write_all(content.as_bytes())?; + } + path + } + }; + + if matches!(self, Dockerfile::File { .. }) { + if let Ok(cross_base_image) = self::image_name(config, target_triple) { + docker_build.args([ + "--build-arg", + &format!("CROSS_BASE_IMAGE={cross_base_image}"), + ]); + } + } + + docker_build.args(["--file".into(), path]); + + if let Ok(build_opts) = std::env::var("CROSS_BUILD_OPTS") { + // FIXME: Use shellwords + docker_build.args(parse_docker_opts(&build_opts)?); + } + if let Some(context) = self.context() { + docker_build.arg(&context); + } else { + docker_build.arg("."); + } + + docker_build.run(verbose, true)?; + Ok(image_name) + } + + pub fn image_name(&self, target_triple: &Target, metadata: &CargoMetadata) -> String { + match self { + Dockerfile::File { + name: Some(name), .. + } => name.to_string(), + _ => format!( + "cross-custom-{package_name}:{target_triple}-{path_hash}{custom}", + package_name = metadata + .workspace_root + .file_name() + .expect("workspace_root can't end in `..`") + .to_string_lossy(), + path_hash = + sha1_smol::Sha1::from(&metadata.workspace_root.to_string_lossy().as_bytes()) + .digest() + .to_string() + .get(..5) + .expect("sha1 is expected to be at least 5 characters long"), + custom = if matches!(self, Self::File { .. }) { + "" + } else { + "-pre-build" + } + ), + } + } + + fn context(&self) -> Option<&'a str> { + match self { + Dockerfile::File { + context: Some(context), + .. + } => Some(context), + _ => None, + } + } +} diff --git a/src/docker/local.rs b/src/docker/local.rs index 6ea91b93d..7f2d35767 100644 --- a/src/docker/local.rs +++ b/src/docker/local.rs @@ -8,6 +8,7 @@ use crate::errors::Result; use crate::extensions::CommandExt; use crate::{Config, Target}; use atty::Stream; +use eyre::Context; #[allow(clippy::too_many_arguments)] // TODO: refactor pub(crate) fn run( @@ -44,7 +45,7 @@ pub(crate) fn run( docker.arg("--rm"); - docker_seccomp(&mut docker, engine.kind, target, verbose)?; + docker_seccomp(&mut docker, engine.kind, target, metadata, verbose)?; docker_user_id(&mut docker, engine.kind); docker @@ -84,10 +85,15 @@ pub(crate) fn run( docker.arg("-t"); } } + let mut image = image_name(config, target)?; + if needs_custom_image(target, config) { + image = custom_image_build(target, config, metadata, dirs, &engine, verbose) + .wrap_err("when building custom image")? + } docker - .arg(&container_name(config, target)?) + .arg(&image) .args(&["sh", "-c", &format!("PATH=$PATH:/rust/bin {:?}", cmd)]) - .run_and_get_status(verbose) + .run_and_get_status(verbose, false) .map_err(Into::into) } diff --git a/src/docker/mod.rs b/src/docker/mod.rs index ea075e7a0..ab431d41c 100644 --- a/src/docker/mod.rs +++ b/src/docker/mod.rs @@ -1,3 +1,4 @@ +mod custom; mod engine; mod local; mod shared; diff --git a/src/docker/shared.rs b/src/docker/shared.rs index 861283049..52a997f62 100644 --- a/src/docker/shared.rs +++ b/src/docker/shared.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{env, fs}; +use super::custom::Dockerfile; use super::engine::*; use crate::cargo::CargoMetadata; use crate::config::Config; @@ -138,7 +139,7 @@ pub(crate) fn register(target: &Target, verbose: bool) -> Result<()> { .arg("--rm") .arg("ubuntu:16.04") .args(&["sh", "-c", cmd]) - .run(verbose) + .run(verbose, false) .map_err(Into::into) } @@ -157,7 +158,7 @@ fn validate_env_var(var: &str) -> Result<(&str, Option<&str>)> { Ok((key, value)) } -fn parse_docker_opts(value: &str) -> Result> { +pub fn parse_docker_opts(value: &str) -> Result> { shell_words::split(value).wrap_err_with(|| format!("could not parse docker opts of {}", value)) } @@ -185,7 +186,7 @@ pub(crate) fn mount( } pub(crate) fn docker_envvars(docker: &mut Command, config: &Config, target: &Target) -> Result<()> { - for ref var in config.env_passthrough(target)? { + for ref var in config.env_passthrough(target)?.unwrap_or_default() { validate_env_var(var)?; // Only specifying the environment variable name in the "-e" @@ -272,7 +273,7 @@ pub(crate) fn docker_mount( mount_volumes = true; } - for ref var in config.env_volumes(target)? { + for ref var in config.env_volumes(target)?.unwrap_or_default() { let (var, value) = validate_env_var(var)?; let value = match value { Some(v) => Ok(v.to_string()), @@ -351,6 +352,7 @@ pub(crate) fn docker_seccomp( docker: &mut Command, engine_type: EngineType, target: &Target, + metadata: &CargoMetadata, verbose: bool, ) -> Result<()> { // docker uses seccomp now on all installations @@ -361,11 +363,8 @@ pub(crate) fn docker_seccomp( "unconfined".to_string() } else { #[allow(unused_mut)] // target_os = "windows" - let mut path = env::current_dir() - .wrap_err("couldn't get current directory")? - .canonicalize() - .wrap_err_with(|| "when canonicalizing current_dir".to_string())? - .join("target") + let mut path = metadata + .target_directory .join(target.triple()) .join("seccomp.json"); if !path.exists() { @@ -385,7 +384,79 @@ pub(crate) fn docker_seccomp( Ok(()) } -pub(crate) fn container_name(config: &Config, target: &Target) -> Result { +pub fn needs_custom_image(target: &Target, config: &Config) -> bool { + config.dockerfile(target).unwrap_or_default().is_some() + || !config + .pre_build(target) + .unwrap_or_default() + .unwrap_or_default() + .is_empty() +} + +pub(crate) fn custom_image_build( + target: &Target, + config: &Config, + metadata: &CargoMetadata, + Directories { host_root, .. }: Directories, + engine: &Engine, + verbose: bool, +) -> Result { + let mut image = image_name(config, target)?; + + if let Some(path) = config.dockerfile(target)? { + let context = config.dockerfile_context(target)?; + let name = config.image(target)?; + + let build = Dockerfile::File { + path: &path, + context: context.as_deref(), + name: name.as_deref(), + }; + + image = build + .build( + config, + metadata, + engine, + &host_root, + config.dockerfile_build_args(target)?.unwrap_or_default(), + target, + verbose, + ) + .wrap_err("when building dockerfile")?; + } + let pre_build = config.pre_build(target)?; + + if let Some(pre_build) = pre_build { + if !pre_build.is_empty() { + let custom = Dockerfile::Custom { + content: format!( + r#" + FROM {image} + ARG CROSS_DEB_ARCH= + ARG CROSS_CMD + RUN eval "${{CROSS_CMD}}""# + ), + }; + custom + .build( + config, + metadata, + engine, + &host_root, + Some(("CROSS_CMD", pre_build.join("\n"))), + target, + verbose, + ) + .wrap_err("when pre-building") + .with_note(|| format!("CROSS_CMD={}", pre_build.join("\n")))?; + image = custom.image_name(target, metadata); + } + } + Ok(image) +} + +pub(crate) fn image_name(config: &Config, target: &Target) -> Result { if let Some(image) = config.image(target)? { return Ok(image); } diff --git a/src/extensions.rs b/src/extensions.rs index 0dfeafb17..2302f4b75 100644 --- a/src/extensions.rs +++ b/src/extensions.rs @@ -7,8 +7,12 @@ use crate::errors::*; pub trait CommandExt { fn print_verbose(&self, verbose: bool); fn status_result(&self, status: ExitStatus) -> Result<(), CommandError>; - fn run(&mut self, verbose: bool) -> Result<(), CommandError>; - fn run_and_get_status(&mut self, verbose: bool) -> Result; + fn run(&mut self, verbose: bool, silence_stdout: bool) -> Result<(), CommandError>; + fn run_and_get_status( + &mut self, + verbose: bool, + silence_stdout: bool, + ) -> Result; fn run_and_get_stdout(&mut self, verbose: bool) -> Result; fn run_and_get_output(&mut self, verbose: bool) -> Result; } @@ -16,7 +20,11 @@ pub trait CommandExt { impl CommandExt for Command { fn print_verbose(&self, verbose: bool) { if verbose { - println!("+ {:?}", self); + if let Some(cwd) = self.get_current_dir() { + println!("+ {:?} {:?}", cwd, self); + } else { + println!("+ {:?}", self); + } } } @@ -29,16 +37,24 @@ impl CommandExt for Command { } /// Runs the command to completion - fn run(&mut self, verbose: bool) -> Result<(), CommandError> { - let status = self.run_and_get_status(verbose)?; + fn run(&mut self, verbose: bool, silence_stdout: bool) -> Result<(), CommandError> { + let status = self.run_and_get_status(verbose, silence_stdout)?; self.status_result(status) } /// Runs the command to completion - fn run_and_get_status(&mut self, verbose: bool) -> Result { + fn run_and_get_status( + &mut self, + verbose: bool, + silence_stdout: bool, + ) -> Result { self.print_verbose(verbose); + if silence_stdout && !verbose { + self.stdout(std::process::Stdio::null()); + } self.status() .map_err(|e| CommandError::CouldNotExecute(Box::new(e), format!("{self:?}"))) + .map_err(Into::into) } /// Runs the command to completion and returns its stdout diff --git a/src/file.rs b/src/file.rs index 55468df81..445201a3c 100644 --- a/src/file.rs +++ b/src/file.rs @@ -44,9 +44,16 @@ pub fn write_file(path: impl AsRef, overwrite: bool) -> Result { })?, ) .wrap_err_with(|| format!("couldn't create directory `{}`", path.display()))?; - fs::OpenOptions::new() - .write(true) - .create_new(!overwrite) - .open(&path) - .wrap_err(format!("could't write to file `{}`", path.display())) + + let mut open = fs::OpenOptions::new(); + open.write(true); + + if overwrite { + open.truncate(true).create(true); + } else { + open.create_new(true); + } + + open.open(&path) + .wrap_err(format!("couldn't write to file `{}`", path.display())) } diff --git a/src/lib.rs b/src/lib.rs index 536aac821..f1a9d82d1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,6 +47,8 @@ use self::rustc::{TargetList, VersionMetaExt}; pub use self::errors::{install_panic_hook, Result}; pub use self::extensions::{CommandExt, OutputExt}; +pub const CROSS_LABEL_DOMAIN: &str = "org.cross-rs"; + #[allow(non_camel_case_types)] #[derive(Debug, Clone, PartialEq, Eq)] pub enum Host { @@ -235,6 +237,73 @@ impl Target { arch_32bit && self.is_android() } + + /// Returns the architecture name according to `dpkg` naming convention + /// + /// # Notes + /// + /// Some of these make no sense to use in our standard images + pub fn deb_arch(&self) -> Option<&'static str> { + match self.triple() { + "aarch64-unknown-linux-gnu" => Some("arm64"), + "aarch64-unknown-linux-musl" => Some("musl-linux-arm64"), + "aarch64-linux-android" => None, + "x86_64-unknown-linux-gnu" => Some("amd64"), + "x86_64-apple-darwin" => Some("darwin-amd64"), + "x86_64-unknown-linux-musl" => Some("musl-linux-amd64"), + + "x86_64-pc-windows-msvc" => None, + "arm-unknown-linux-gnueabi" => Some("armel"), + "arm-unknown-linux-gnueabihf" => Some("armhf"), + "armv7-unknown-linux-gnueabi" => Some("armel"), + "armv7-unknown-linux-gnueabihf" => Some("armhf"), + "thumbv7neon-unknown-linux-gnueabihf" => Some("armhf"), + "i586-unknown-linux-gnu" => Some("i386"), + "i686-unknown-linux-gnu" => Some("i386"), + "mips-unknown-linux-gnu" => Some("mips"), + "mipsel-unknown-linux-gnu" => Some("mipsel"), + "mips64-unknown-linux-gnuabi64" => Some("mips64"), + "mips64el-unknown-linux-gnuabi64" => Some("mips64el"), + "mips64-unknown-linux-muslabi64" => Some("musl-linux-mips64"), + "mips64el-unknown-linux-muslabi64" => Some("musl-linux-mips64el"), + "powerpc-unknown-linux-gnu" => Some("powerpc"), + "powerpc64-unknown-linux-gnu" => Some("ppc64"), + "powerpc64le-unknown-linux-gnu" => Some("ppc64el"), + "riscv64gc-unknown-linux-gnu" => Some("riscv64"), + "s390x-unknown-linux-gnu" => Some("s390x"), + "sparc64-unknown-linux-gnu" => Some("sparc64"), + "arm-unknown-linux-musleabihf" => Some("musl-linux-armhf"), + "arm-unknown-linux-musleabi" => Some("musl-linux-arm"), + "armv5te-unknown-linux-gnueabi" => None, + "armv5te-unknown-linux-musleabi" => None, + "armv7-unknown-linux-musleabi" => Some("musl-linux-arm"), + "armv7-unknown-linux-musleabihf" => Some("musl-linux-armhf"), + "i586-unknown-linux-musl" => Some("musl-linux-i386"), + "i686-unknown-linux-musl" => Some("musl-linux-i386"), + "mips-unknown-linux-musl" => Some("musl-linux-mips"), + "mipsel-unknown-linux-musl" => Some("musl-linux-mipsel"), + "arm-linux-androideabi" => None, + "armv7-linux-androideabi" => None, + "thumbv7neon-linux-androideabi" => None, + "i686-linux-android" => None, + "x86_64-linux-android" => None, + "x86_64-pc-windows-gnu" => None, + "i686-pc-windows-gnu" => None, + "asmjs-unknown-emscripten" => None, + "wasm32-unknown-emscripten" => None, + "x86_64-unknown-dragonfly" => Some("dragonflybsd-amd64"), + "i686-unknown-freebsd" => Some("freebsd-i386"), + "x86_64-unknown-freebsd" => Some("freebsd-amd64"), + "x86_64-unknown-netbsd" => Some("netbsd-amd64"), + "sparcv9-sun-solaris" => Some("solaris-sparc"), + "x86_64-sun-solaris" => Some("solaris-amd64"), + "thumbv6m-none-eabi" => Some("arm"), + "thumbv7em-none-eabi" => Some("arm"), + "thumbv7em-none-eabihf" => Some("armhf"), + "thumbv7m-none-eabi" => Some("arm"), + _ => None, + } + } } impl std::fmt::Display for Target { @@ -304,7 +373,7 @@ pub fn run() -> Result { .unwrap_or_else(|| Target::from(host.triple(), &target_list)); config.confusable_target(&target); - let image_exists = match docker::container_name(&config, &target) { + let image_exists = match docker::image_name(&config, &target) { Ok(_) => true, Err(err) => { eprintln!("Warning: {}", err); diff --git a/src/rustup.rs b/src/rustup.rs index 5b78006af..35cc41f17 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -73,7 +73,7 @@ pub fn available_targets(toolchain: &str, verbose: bool) -> Result Result<()> { Command::new("rustup") .args(&["toolchain", "add", toolchain, "--profile", "minimal"]) - .run(verbose) + .run(verbose, false) .wrap_err_with(|| format!("couldn't install toolchain `{toolchain}`")) } @@ -82,14 +82,14 @@ pub fn install(target: &Target, toolchain: &str, verbose: bool) -> Result<()> { Command::new("rustup") .args(&["target", "add", target, "--toolchain", toolchain]) - .run(verbose) + .run(verbose, false) .wrap_err_with(|| format!("couldn't install `std` for {target}")) } pub fn install_component(component: &str, toolchain: &str, verbose: bool) -> Result<()> { Command::new("rustup") .args(&["component", "add", component, "--toolchain", toolchain]) - .run(verbose) + .run(verbose, false) .wrap_err_with(|| format!("couldn't install the `{component}` component")) } diff --git a/xtask/src/build_docker_image.rs b/xtask/src/build_docker_image.rs index 4a0d37770..3c171ec30 100644 --- a/xtask/src/build_docker_image.rs +++ b/xtask/src/build_docker_image.rs @@ -214,6 +214,11 @@ pub fn build_docker_image( docker_build.args(&["--label", label]); } + docker_build.args([ + "--label", + &format!("{}.for-cross-target={target}", cross::CROSS_LABEL_DOMAIN), + ]); + docker_build.args(&["-f", dockerfile]); if gha || progress == "plain" { @@ -225,7 +230,7 @@ pub fn build_docker_image( docker_build.arg("."); if !dry_run && (force || !push || gha) { - let result = docker_build.run(verbose); + let result = docker_build.run(verbose, false); if gha && targets.len() > 1 { if let Err(e) = &result { // TODO: Determine what instruction errorred, and place warning on that line with appropriate warning diff --git a/xtask/src/target_info.rs b/xtask/src/target_info.rs index 249828705..9b77df2ef 100644 --- a/xtask/src/target_info.rs +++ b/xtask/src/target_info.rs @@ -58,7 +58,7 @@ fn pull_image(engine: &Path, image: &str, verbose: bool) -> cross::Result<()> { command.stdout(Stdio::null()); command.stderr(Stdio::null()); } - command.run(verbose).map_err(Into::into) + command.run(verbose, false).map_err(Into::into) } fn image_info( @@ -88,7 +88,7 @@ fn image_info( // capture stderr to avoid polluting table command.stderr(Stdio::null()); } - command.run(verbose).map_err(Into::into) + command.run(verbose, false).map_err(Into::into) } pub fn target_info(