Skip to content

Commit

Permalink
Add support for use of prebuilt test Daphne container image.
Browse files Browse the repository at this point in the history 
This is controlled by a new compile-time DAPHNE_INTEROP_CONTAINER
environment variable (matching the JANUS_INTEROP_CONTAINER variable used
to control building of the Janus interop test containers). See the
updates to README.md for more details.

Also:
* Update Daphne to the latest commit. This is important as it picks up
  cloudflare/daphne#99 which de-flakes
  Janus/Daphne integration tests again after they were accidentally
  re-flaked by switching to DAP_DEPLOYMENT=prod.
* Add ability to skip building the Daphne container, and do so for our
  CI lints.
  • Loading branch information
@branlwyd
branlwyd committed Aug 26, 2022
1 parent e14fd45 commit 7c96da6
Show file tree
Hide file tree
Showing 6 changed files with 228 additions and 79 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ci-build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ jobs:
env:
CARGO_INCREMENTAL: 0
CARGO_TERM_COLOR: always
DAPHNE_INTEROP_CONTAINER: skip
JANUS_INTEROP_CONTAINER: skip
RUSTDOCFLAGS: "-D warnings"
RUSTFLAGS: "-D warnings"
Expand Down
2 changes: 1 addition & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion monolithic_integration_test/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ kube-openssl = ["kube/openssl-tls"]

[dependencies]
backoff = { version = "0.4", features = ["tokio"] }
daphne = { git = "https://github.com/cloudflare/daphne", rev = "6228556c7b87a7fe85e414a3186a2511407896f0", optional = true } # Match this to the version referenced in README.md.
daphne = { git = "https://github.com/cloudflare/daphne", rev = "e1b503eb2aefadfe2717abb0a359892848175534", optional = true } # Match this to the version referenced in README.md.
futures = "0.3.23"
hex = { version = "0.4", optional = true }
interop_binaries = { path = "../interop_binaries", features = ["testcontainer"] }
Expand Down Expand Up @@ -50,4 +50,5 @@ prio = "0.8.2"

[build-dependencies]
build_script_utils = { path = "../build_script_utils", optional = true }
serde_json = "1"
tempfile = "3"
19 changes: 18 additions & 1 deletion monolithic_integration_test/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ compiled version of Daphne inside a container. The test container is built by th
script based on the test Dockerfile in the Daphne repository; the test container is included in
necessary test binaries which know how to load the image into Docker themselves.

Daphne is compiled from commit [`6228556c7b87a7fe85e414a3186a2511407896f0`](
Daphne is compiled from commit [`e1b503eb2aefadfe2717abb0a359892848175534`](
https://github.com/cloudflare/daphne/commit/6228556c7b87a7fe85e414a3186a2511407896f0).

### Running Daphne integration tests
Expand All @@ -23,3 +23,20 @@ To update the version of Daphne in use:
1. Update `Cargo.toml` in this directory to reference the new commit of Daphne.
1. Update `build.rs` in this directory to reference the new commit of Daphne.
1. Update this README to note the new commit of Daphne.

### Using a prebuilt Daphne

It is possible to instruct Janus' tests to use a prebuilt Docker image for the Janus/Daphne
integration tests, rather than building an image as part of the build process. To do this, set the
`DAPHNE_INTEROP_CONTAINER` environment variable to `prebuilt=${IMAGE_NAME}:${IMAGE_TAG}`. For
example, to use an image named `test_daphne` with the `latest` tag, set
`DAPHNE_INTEROP_CONTAINER=prebuilt=test_daphne:latest`.

To build a new Daphne image suitable for use as a prebuilt image, clone the [Daphne repository](
https://github.com/cloudflare/daphne), check out the commit of interest, and then run a command like
`docker build --file=daphne_worker_test/docker/miniflare.Dockerfile --tag=test_daphne:$(git rev-parse HEAD | head -c8) .`.
This command will generate an image named `test_daphne` tagged with a prefix of the git commit hash
used to generate it.

Note that even when using prebuilt container images, it is still important to match the image
version with the version of Daphne referenced by this package in `Cargo.toml`.
210 changes: 149 additions & 61 deletions monolithic_integration_test/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,75 +2,163 @@ fn main() {
#[cfg(feature = "daphne")]
{
use build_script_utils::save_zstd_compressed_docker_image;
use serde_json::json;
use std::{env, fs::File, process::Command};
use tempfile::tempdir;

// This build script is self-contained, so we only need to rebuild if the build script
// itself changes.
// itself or one of its inputs changes.
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-env-changed=DAPHNE_INTEROP_CONTAINER");

// Check out Daphne repository at a fixed hash, then build & save off a container image for
// the test Daphne instance.
const DAPHNE_COMMIT_HASH: &str = "6228556c7b87a7fe85e414a3186a2511407896f0";
let daphne_checkout_dir = tempdir().unwrap();
let clone_output = Command::new("git")
.args(["clone", "-n", "https://github.com/cloudflare/daphne", "."])
.current_dir(&daphne_checkout_dir)
.output()
.expect("Failed to execute `git clone` for Daphne repository");
assert!(
clone_output.status.success(),
"Git clone of Daphne repository failed:\n{}",
String::from_utf8_lossy(&clone_output.stderr)
);
let checkout_output = Command::new("git")
.args(["checkout", DAPHNE_COMMIT_HASH])
.current_dir(&daphne_checkout_dir)
.output()
.expect("Failed to execute `git checkout` for Daphne repository");
assert!(
checkout_output.status.success(),
"Git checkout of Daphne repository failed:\n{}",
String::from_utf8_lossy(&checkout_output.stderr)
);
let container_strategy = env::var("DAPHNE_INTEROP_CONTAINER")
.ok()
.unwrap_or_else(|| "build".to_string());

// Note: `docker build` has an `--output` flag which writes the output to somewhere, which
// may be a tarfile. But `docker build --output` only exports the image filesystem, and not
// any other image metadata (such as exposed ports, the entrypoint, etc), so we can't easily
// use it.
let build_output = Command::new("docker")
.args([
"build",
"--file=daphne_worker_test/docker/miniflare.Dockerfile",
"--quiet",
".",
])
.current_dir(&daphne_checkout_dir)
.output()
.expect("Failed to execute `docker build` for test Daphne");
assert!(
build_output.status.success(),
"Docker build of test Daphne failed:\n{}",
String::from_utf8_lossy(&build_output.stderr)
);
let image_id = String::from_utf8(build_output.stdout).unwrap();
let image_id = image_id.trim();
if container_strategy == "build" {
// Check out Daphne repository at a fixed hash, then build & save off a container image for
// the test Daphne instance.
const DAPHNE_COMMIT_HASH: &str = "e1b503eb2aefadfe2717abb0a359892848175534";
let daphne_checkout_dir = tempdir().unwrap();
let clone_output = Command::new("git")
.args(["clone", "-n", "https://github.com/cloudflare/daphne", "."])
.current_dir(&daphne_checkout_dir)
.output()
.expect("Failed to execute `git clone` for Daphne repository");
assert!(
clone_output.status.success(),
"Git clone of Daphne repository failed:\n{}",
String::from_utf8_lossy(&clone_output.stderr)
);
let checkout_output = Command::new("git")
.args(["checkout", DAPHNE_COMMIT_HASH])
.current_dir(&daphne_checkout_dir)
.output()
.expect("Failed to execute `git checkout` for Daphne repository");
assert!(
checkout_output.status.success(),
"Git checkout of Daphne repository failed:\n{}",
String::from_utf8_lossy(&checkout_output.stderr)
);

let image_file = File::create(format!(
"{}/test_daphne.tar.zst",
env::var("OUT_DIR").unwrap()
))
.expect("Couldn't create test Daphne image file");
save_zstd_compressed_docker_image(image_id, &image_file);
image_file
.sync_all()
.expect("Couldn't write compressed image file");
drop(image_file);
// Note: `docker build` has an `--output` flag which writes the output to somewhere, which
// may be a tarfile. But `docker build --output` only exports the image filesystem, and not
// any other image metadata (such as exposed ports, the entrypoint, etc), so we can't easily
// use it.
let build_output = Command::new("docker")
.args([
"build",
"--file=daphne_worker_test/docker/miniflare.Dockerfile",
"--quiet",
".",
])
.current_dir(&daphne_checkout_dir)
.output()
.expect("Failed to execute `docker build` for test Daphne");
assert!(
build_output.status.success(),
"Docker build of test Daphne failed:\n{}",
String::from_utf8_lossy(&build_output.stderr)
);
let image_id = String::from_utf8(build_output.stdout).unwrap();
let image_id = image_id.trim();

// Make a best-effort attempt to clean up after ourselves.
Command::new("docker")
.args(["rmi", image_id])
.status()
.expect("Failed to execute `docker rmi` for test Daphne");
let image_file = File::create(format!(
"{}/test_daphne.tar.zst",
env::var("OUT_DIR").unwrap()
))
.expect("Couldn't create test Daphne image file");
save_zstd_compressed_docker_image(image_id, &image_file);
image_file
.sync_all()
.expect("Couldn't write compressed image file");
drop(image_file);

// Write metadata file instructing runtime to reference the container we built.
let metadata_file = File::create(format!(
"{}/test_daphne.metadata",
env::var("OUT_DIR").unwrap()
))
.expect("Couldn't create test Daphne metadata file");
serde_json::to_writer(&metadata_file, &json!({"strategy": "build"}))
.expect("Couldn't write metadata file");
metadata_file
.sync_all()
.expect("Couldn't write metadata file");
drop(metadata_file);

// Make a best-effort attempt to clean up after ourselves.
Command::new("docker")
.args(["image", "remove", image_id])
.status()
.expect("Failed to execute `docker image remove` for test Daphne");
} else if container_strategy.starts_with("prebuilt=") {
let (image_name, image_tag) = container_strategy
.strip_prefix("prebuilt=")
.unwrap()
.split_once(':')
.unwrap();

// Write empty image file (required for compilation to succeed).
let image_file = File::create(format!(
"{}/test_daphne.tar.zst",
env::var("OUT_DIR").unwrap()
))
.expect("Couldn't create test Daphne image file");
image_file
.sync_all()
.expect("Couldn't write compressed image file");
drop(image_file);

// Write metadata file instructing runtime to reference our prebuilt container.
let metadata_file = File::create(format!(
"{}/test_daphne.metadata",
env::var("OUT_DIR").unwrap()
))
.expect("Couldn't create test Daphne metadata file");
serde_json::to_writer(
&metadata_file,
&json!({
"strategy": "prebuilt",
"image_name": image_name,
"image_tag": image_tag,
}),
)
.expect("Couldn't write metadata file");
metadata_file
.sync_all()
.expect("Couldn't write metadata file");
drop(metadata_file);
} else if container_strategy == "skip" {
// The "skip" strategy causes us to skip building a container at all. Tests which
// depend on having Daphne interop test images available will fail.

// Write empty image file (required for compilation to succeed).
let image_file = File::create(format!(
"{}/test_daphne.tar.zst",
env::var("OUT_DIR").unwrap()
))
.expect("Couldn't create test Daphne image file");
image_file
.sync_all()
.expect("Couldn't write compressed image file");
drop(image_file);

// Write metadata file instructing runtime that we skipped the container build (which
// will cause anyone attempting to instantiate the container to panic).
let metadata_file = File::create(format!(
"{}/test_daphne.metadata",
env::var("OUT_DIR").unwrap()
))
.expect("Couldn't create test Daphne metadata file");
serde_json::to_writer(&metadata_file, &json!({"strategy": "skip"}))
.expect("Couldn't write metadata file");
metadata_file
.sync_all()
.expect("Couldn't write metadata file");
drop(metadata_file);
} else {
panic!("Unexpected DAPHNE_INTEROP_CONTAINER value {container_strategy:?} (valid values are \"build\", \"prebuilt=image_name:image_tag\", & \"skip\")")
}
}
}
72 changes: 57 additions & 15 deletions monolithic_integration_test/src/daphne.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ use testcontainers::{core::Port, images::generic::GenericImage, Container, Runna
use tokio::{select, sync::oneshot, task, time::interval};

// test_daphne.tar.zst is generated by this package's build.rs.
const TEST_DAPHNE_METADATA_BYTES: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/test_daphne.metadata"));
const TEST_DAPHNE_IMAGE_BYTES: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/test_daphne.tar.zst"));
static TEST_DAPHNE_IMAGE_HASH: Mutex<Option<String>> = Mutex::new(None);
static TEST_DAPHNE_IMAGE_NAME_AND_TAG: Mutex<Option<(String, String)>> = Mutex::new(None);

/// Represents a running Daphne test instance.
pub struct Daphne {
Expand Down Expand Up @@ -101,14 +103,47 @@ impl Daphne {
String::new()
};

// Get the test Daphne docker image hash; if necessary, do one-time setup to write the image
// to Docker.
let image_hash = {
let mut image_hash = TEST_DAPHNE_IMAGE_HASH.lock().unwrap();
if image_hash.is_none() {
*image_hash = Some(load_zstd_compressed_docker_image(TEST_DAPHNE_IMAGE_BYTES));
// Get the test Daphne docker image hash; if necessary, do one-time setup to determine the
// image name & tag (which may involve writing the image to docker depending on compilation
// settings).
let (image_name, image_tag) = {
let mut image_name_and_tag = TEST_DAPHNE_IMAGE_NAME_AND_TAG.lock().unwrap();
if image_name_and_tag.is_none() {
let metadata: serde_json::Value =
serde_json::from_slice(TEST_DAPHNE_METADATA_BYTES)
.expect("Couldn't parse Daphne test image metadata");
let strategy = metadata["strategy"].as_str().expect(&format!(
"Metadata strategy field was not a string: {}",
metadata["strategy"]
));

*image_name_and_tag = Some(match strategy {
"build" => (
"sha256".to_string(),
load_zstd_compressed_docker_image(TEST_DAPHNE_IMAGE_BYTES),
),

"prebuilt" => (
metadata["image_name"]
.as_str()
.expect(&format!(
"Daphne test image metadata image_name field was not a string: {}",
metadata["image_name"]
))
.to_string(),
metadata["image_tag"]
.as_str()
.expect(&format!(
"Daphne test image metadata image_tag field was not a string: {}",
metadata["image_tag"]
))
.to_string(),
),

_ => panic!("Unknown Daphne test image build strategy: {strategy:?}"),
});
}
image_hash.as_ref().unwrap().clone()
image_name_and_tag.as_ref().unwrap().clone()
};

// Start the Daphne test container running.
Expand All @@ -124,6 +159,12 @@ impl Daphne {
"DAP_DEPLOYMENT".to_string(),
"prod".to_string(),
),
(
// Works around https://github.com/cloudflare/daphne/issues/73. Remove once
// that issue is closed & the version of Daphne under test has picked up the fix.
"DAP_ISSUE73_DISABLE_AGG_JOB_QUEUE_GARBAGE_COLLECTION".to_string(),
"true".to_string(),
),
(
"DAP_AGGREGATOR_ROLE".to_string(),
task.role.as_str().to_string(),
Expand Down Expand Up @@ -158,13 +199,14 @@ impl Daphne {
.into_iter()
.chain(args)
.collect();
let runnable_image = RunnableImage::from((GenericImage::new("sha256", &image_hash), args))
.with_network(network)
.with_container_name(endpoint.host_str().unwrap())
.with_mapped_port(Port {
local: port,
internal: 8080,
});
let runnable_image =
RunnableImage::from((GenericImage::new(&image_name, &image_tag), args))
.with_network(network)
.with_container_name(endpoint.host_str().unwrap())
.with_mapped_port(Port {
local: port,
internal: 8080,
});
let daphne_container = CONTAINER_CLIENT.run(runnable_image);

// Wait for Daphne container to begin listening on the port.
Expand Down

0 comments on commit 7c96da6

Please sign in to comment.