From e63acb4136aad6c9d4541da7eb6eedbb28f77f0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Mon, 4 Jul 2022 22:01:29 +0200 Subject: [PATCH] add way to run script with pre-build --- CHANGELOG.md | 3 +- docs/cross_toml.md | 20 ++++++++- src/config.rs | 27 +++++++++--- src/cross_toml.rs | 99 ++++++++++++++++++++++++++++++++++++++------ src/docker/custom.rs | 40 +++++++++++++++++- src/docker/mod.rs | 2 +- src/docker/shared.rs | 88 ++++++++++++++++++++++++++++----------- 7 files changed, 233 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71057813e..364d8eec2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,11 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Added - #913 - added the `x86_64-unknown-illumos` target. +- #910 - `pre-build` can now take a string pointing to a script file to run. - #905 - added `qemu-runner` for musl images, allowing use of native or emulated runners. - #905 - added qemu emulation to `i586-unknown-linux-gnu`, `i686-unknown-linux-musl`, and `i586-unknown-linux-gnu`, so they can run on an `x86` CPU, rather than an `x86_64` CPU. - #900 - add the option to skip copying build artifacts back to host when using remote cross via `CROSS_REMOTE_SKIP_BUILD_ARTIFACTS`. -- #891 - support custom user namespace overrides by setting the `CROSS_CONTAINER_USER_NAMESPACE` environment variable. +- #891 - support custom user namespace overrides by setting the `CROSS_CONTAINER_USER_NAMESPACE` environment variable. - #890 - support rootless docker via the `CROSS_ROOTLESS_CONTAINER_ENGINE` environment variable. - #878 - added an image `ghcr.io/cross-rs/cross` containing cross. diff --git a/docs/cross_toml.md b/docs/cross_toml.md index 51f878302..91e3f36f3 100644 --- a/docs/cross_toml.md +++ b/docs/cross_toml.md @@ -34,10 +34,28 @@ The `target` key allows you to specify parameters for specific compilation targe xargo = false build-std = false image = "test-image" -pre-build = ["apt-get update"] +pre-build = ["apt-get update"] # can also be the path to a file to run runner = "custom-runner" ``` +# `target.TARGET.pre-build` + +The `pre-build` field can also reference a file to copy and run. This file is relative to the container context, which would be the workspace root, or the current directory if `--manifest-path` is used. For more involved scripts, consider using `target.TARGET.dockerfile` instead to directly control the execution. + +This script will be invoked as `RUN ./pre-build-script $CROSS_TARGET` where `$CROSS_TARGET` is the target triple. + +```toml +[target.aarch64-unknown-linux-gnu] +pre-build = "./scripts/my-script.sh" +``` + +```sh +$ cat ./scripts/my-script.sh +#!/usr/bin/env bash + +apt-get install libssl-dev -y +``` + # `target.TARGET.env` The `target` key allows you to specify environment variables that should be used for a specific compilation target. diff --git a/src/config.rs b/src/config.rs index 4e9fbd2f2..a5d583016 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,3 +1,4 @@ +use crate::docker::custom::PreBuild; use crate::shell::MessageInfo; use crate::{CrossToml, Result, Target, TargetList}; @@ -77,9 +78,17 @@ impl Environment { self.get_values_for("DOCKERFILE_CONTEXT", target, |s| s.to_owned()) } - fn pre_build(&self, target: &Target) -> (Option>, Option>) { + fn pre_build(&self, target: &Target) -> (Option, Option) { self.get_values_for("PRE_BUILD", target, |v| { - v.split('\n').map(String::from).collect() + let v: Vec<_> = v.split('\n').map(String::from).collect(); + if v.len() == 1 { + PreBuild::Single { + line: v.into_iter().next().expect("should contain one item"), + env: true, + } + } else { + PreBuild::Lines(v) + } }) } @@ -326,7 +335,7 @@ impl Config { .map_or(Ok(None), |t| Ok(t.dockerfile_build_args(target))) } - pub fn pre_build(&self, target: &Target) -> Result>> { + pub fn pre_build(&self, target: &Target) -> Result> { self.get_from_ref(target, Environment::pre_build, CrossToml::pre_build) } @@ -494,7 +503,10 @@ mod tests { assert_eq!(config.build_std(&target()), None); assert_eq!( config.pre_build(&target())?, - Some(vec![s!("apt-get update"), s!("apt-get install zlib-dev")]) + Some(PreBuild::Lines(vec![ + s!("apt-get update"), + s!("apt-get install zlib-dev") + ])) ); Ok(()) @@ -530,7 +542,7 @@ mod tests { } #[test] - pub fn env_target_and_toml_build_pre_build_then_use_toml() -> Result<()> { + pub fn env_target_and_toml_build_pre_build_then_use_env() -> Result<()> { let mut map = HashMap::new(); map.insert( "CROSS_TARGET_AARCH64_UNKNOWN_LINUX_GNU_PRE_BUILD", @@ -541,7 +553,10 @@ mod tests { let config = Config::new_with(Some(toml(TOML_BUILD_PRE_BUILD)?), env); assert_eq!( config.pre_build(&target())?, - Some(vec![s!("dpkg --add-architecture arm64")]) + Some(PreBuild::Single { + line: s!("dpkg --add-architecture arm64"), + env: true + }) ); Ok(()) diff --git a/src/cross_toml.rs b/src/cross_toml.rs index 2768cf629..819143f05 100644 --- a/src/cross_toml.rs +++ b/src/cross_toml.rs @@ -1,5 +1,6 @@ #![doc = include_str!("../docs/cross_toml.md")] +use crate::docker::custom::PreBuild; use crate::shell::MessageInfo; use crate::{config, errors::*}; use crate::{Target, TargetList}; @@ -24,7 +25,8 @@ pub struct CrossBuildConfig { xargo: Option, build_std: Option, default_target: Option, - pre_build: Option>, + #[serde(default, deserialize_with = "opt_string_or_string_vec")] + pre_build: Option, #[serde(default, deserialize_with = "opt_string_or_struct")] dockerfile: Option, } @@ -38,7 +40,8 @@ pub struct CrossTargetConfig { image: Option, #[serde(default, deserialize_with = "opt_string_or_struct")] dockerfile: Option, - pre_build: Option>, + #[serde(default, deserialize_with = "opt_string_or_string_vec")] + pre_build: Option, runner: Option, #[serde(default)] env: CrossEnvConfig, @@ -251,12 +254,8 @@ impl CrossToml { } /// Returns the `build.dockerfile.pre-build` and `target.{}.dockerfile.pre-build` part of `Cross.toml` - pub fn pre_build(&self, target: &Target) -> (Option<&[String]>, Option<&[String]>) { - self.get_ref( - target, - |b| b.pre_build.as_deref(), - |t| t.pre_build.as_deref(), - ) + pub fn pre_build(&self, target: &Target) -> (Option<&PreBuild>, Option<&PreBuild>) { + self.get_ref(target, |b| b.pre_build.as_ref(), |t| t.pre_build.as_ref()) } /// Returns the `target.{}.runner` part of `Cross.toml` @@ -395,6 +394,64 @@ where deserializer.deserialize_any(StringOrStruct(PhantomData)) } +fn opt_string_or_string_vec<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: Deserialize<'de> + std::str::FromStr + From>, + D: serde::Deserializer<'de>, +{ + use std::{fmt, marker::PhantomData}; + + use serde::de::{self, SeqAccess, Visitor}; + + struct StringOrStringVec(PhantomData T>); + + impl<'de, T> Visitor<'de> for StringOrStringVec + where + T: Deserialize<'de> + FromStr + From>, + { + type Value = Option; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("string or seq") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(FromStr::from_str(value).ok()) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut vec: Vec = vec![]; + + while let Some(inner) = seq.next_element::()? { + vec.push(inner); + } + Ok(Some(vec.into())) + } + + fn visit_none(self) -> Result + where + E: de::Error, + { + Ok(None) + } + + fn visit_unit(self) -> Result + where + E: de::Error, + { + Ok(None) + } + } + + deserializer.deserialize_any(StringOrStringVec(PhantomData)) +} + #[cfg(test)] mod tests { use super::*; @@ -438,7 +495,7 @@ mod tests { xargo: Some(true), build_std: None, default_target: None, - pre_build: Some(vec![s!("echo 'Hello World!'")]), + pre_build: Some(PreBuild::Lines(vec![s!("echo 'Hello World!'")])), dockerfile: None, }, }; @@ -477,7 +534,7 @@ mod tests { image: Some(s!("test-image")), runner: None, dockerfile: None, - pre_build: Some(vec![]), + pre_build: Some(PreBuild::Lines(vec![])), }, ); @@ -520,7 +577,7 @@ mod tests { context: None, build_args: None, }), - pre_build: Some(vec![s!("echo 'Hello'")]), + pre_build: Some(PreBuild::Lines(vec![s!("echo 'Hello'")])), runner: None, env: CrossEnvConfig { passthrough: None, @@ -539,7 +596,7 @@ mod tests { xargo: Some(true), build_std: None, default_target: None, - pre_build: Some(vec![]), + pre_build: Some(PreBuild::Lines(vec![])), dockerfile: None, }, }; @@ -771,4 +828,22 @@ mod tests { Ok(()) } + + #[test] + fn pre_build_script() -> Result<()> { + let toml_str = r#" + [target.aarch64-unknown-linux-gnu] + pre-build = "./my-script.sh" + + [build] + pre-build = ["echo Hello World"] + "#; + let (toml, unused) = CrossToml::parse_from_cross(toml_str, &mut m!())?; + assert!(unused.is_empty()); + assert!(matches!( + toml.pre_build(&Target::new_built_in("aarch64-unknown-linux-gnu")), + (Some(&PreBuild::Lines(_)), Some(&PreBuild::Single { .. })) + )); + Ok(()) + } } diff --git a/src/docker/custom.rs b/src/docker/custom.rs index 0c950ada7..31e2df6db 100644 --- a/src/docker/custom.rs +++ b/src/docker/custom.rs @@ -1,5 +1,6 @@ use std::io::Write; use std::path::PathBuf; +use std::str::FromStr; use crate::docker::{DockerOptions, DockerPaths}; use crate::shell::MessageInfo; @@ -22,6 +23,43 @@ pub enum Dockerfile<'a> { }, } +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum PreBuild { + /// A path to a file to copy or a single line to `RUN` if line comes from env + Single { line: String, env: bool }, + /// Lines to execute in a single `RUN` + Lines(Vec), +} + +impl FromStr for PreBuild { + type Err = std::convert::Infallible; + + fn from_str(s: &str) -> Result { + Ok(PreBuild::Single { + line: s.to_owned(), + env: false, + }) + } +} + +impl From> for PreBuild { + fn from(vec: Vec) -> Self { + PreBuild::Lines(vec) + } +} + +impl PreBuild { + #[must_use] + pub fn is_single(&self) -> bool { + matches!(self, Self::Single { .. }) + } + + #[must_use] + pub fn is_lines(&self) -> bool { + matches!(self, Self::Lines(..)) + } +} + impl<'a> Dockerfile<'a> { pub fn build( &self, @@ -96,7 +134,7 @@ impl<'a> Dockerfile<'a> { if let Some(context) = self.context() { docker_build.arg(&context); } else { - docker_build.arg("."); + docker_build.arg(paths.host_root()); } docker_build.run(msg_info, true)?; diff --git a/src/docker/mod.rs b/src/docker/mod.rs index b0b3de413..aececd69b 100644 --- a/src/docker/mod.rs +++ b/src/docker/mod.rs @@ -1,4 +1,4 @@ -mod custom; +pub mod custom; mod engine; mod local; pub mod remote; diff --git a/src/docker/shared.rs b/src/docker/shared.rs index 696817f74..6d26c34a9 100644 --- a/src/docker/shared.rs +++ b/src/docker/shared.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{env, fs}; -use super::custom::Dockerfile; +use super::custom::{Dockerfile, PreBuild}; use super::engine::*; use crate::cargo::{cargo_metadata_with_args, CargoMetadata}; use crate::config::{bool_from_envvar, Config}; @@ -62,12 +62,11 @@ impl DockerOptions { .dockerfile(&self.target) .unwrap_or_default() .is_some() - || !self + || self .config .pre_build(&self.target) .unwrap_or_default() - .unwrap_or_default() - .is_empty() + .is_some() } pub(crate) fn custom_image_build( @@ -101,26 +100,67 @@ impl DockerOptions { let pre_build = self.config.pre_build(&self.target)?; if let Some(pre_build) = pre_build { - if !pre_build.is_empty() { - let custom = Dockerfile::Custom { - content: format!( - r#" - FROM {image} - ARG CROSS_DEB_ARCH= - ARG CROSS_CMD - RUN eval "${{CROSS_CMD}}""# - ), - }; - custom - .build( - self, - paths, - Some(("CROSS_CMD", pre_build.join("\n"))), - msg_info, - ) - .wrap_err("when pre-building") - .with_note(|| format!("CROSS_CMD={}", pre_build.join("\n")))?; - image = custom.image_name(&self.target, &paths.metadata)?; + match pre_build { + super::custom::PreBuild::Single { + line: pre_build_script, + env, + } if !env + || !pre_build_script.contains('\n') + && paths.host_root().join(&pre_build_script).is_file() => + { + let custom = Dockerfile::Custom { + content: format!( + r#" + FROM {image} + ARG CROSS_DEB_ARCH= + ARG CROSS_SCRIPT + ARG CROSS_TARGET + COPY $CROSS_SCRIPT /pre-build-script + RUN chmod +x /pre-build-script + RUN ./pre-build-script $CROSS_TARGET"# + ), + }; + + image = custom + .build( + self, + paths, + vec![ + ("CROSS_SCRIPT", &*pre_build_script), + ("CROSS_TARGET", self.target.triple()), + ], + msg_info, + ) + .wrap_err("when pre-building") + .with_note(|| format!("CROSS_SCRIPT={pre_build_script}")) + .with_note(|| format!("CROSS_TARGET={}", self.target))?; + } + this => { + let pre_build = match this { + PreBuild::Single { line, .. } => vec![line], + PreBuild::Lines(lines) => lines, + }; + if !pre_build.is_empty() { + let custom = Dockerfile::Custom { + content: format!( + r#" + FROM {image} + ARG CROSS_DEB_ARCH= + ARG CROSS_CMD + RUN eval "${{CROSS_CMD}}""# + ), + }; + image = custom + .build( + self, + paths, + Some(("CROSS_CMD", pre_build.join("\n"))), + msg_info, + ) + .wrap_err("when pre-building") + .with_note(|| format!("CROSS_CMD={}", pre_build.join("\n")))?; + } + } } } Ok(image)