Skip to content

Commit

Permalink
Improve package caching (#32)
Browse files Browse the repository at this point in the history 
* refactor: improve source preparation and validation

- Commented out unused `std::env` import in `vorpal.rs`.
- Added validation for empty source hash, name, and signature in `run_prepare.rs`.
- Refactored `run_prepare.rs` to remove redundant checks and improve readability.
- Introduced `validate_hashes` and `send_source` functions in `package/mod.rs` for better separation of concerns.
- Updated `prepare` function in `package/mod.rs` to use the new helper functions.
- Changed file permissions for package tar files to `0o440` in `package/mod.rs`.
- Renamed variables for clarity and consistency in `archives.rs`.

* refactor: uncomment and update package request logic

- Uncommented the `use std::env` statement.
- Renamed the `request` variable to `coreutils` and added logic to process its stream.
- Uncommented and updated the `example` package request logic.
- Ensured both `coreutils` and `example` package requests are processed correctly.

* fix(proxy/package): remove redundant references and improve string handling

- Removed redundant references to `workdir_path` and `name` in function calls.
- Replaced string comparison with `is_empty()` for better readability and performance.
  • Loading branch information
@erikreinert
erikreinert authored Jun 16, 2024
1 parent 645801d commit d3e53f4
Show file tree
Hide file tree
Showing 4 changed files with 238 additions and 159 deletions.
45 changes: 42 additions & 3 deletions example/rust/vorpal.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use anyhow::Result;
use std::env;
use tokio_stream::StreamExt;
use vorpal::api::command_service_client::CommandServiceClient;
use vorpal::api::{PackageRequest, PackageSource, PackageSourceKind};
Expand All @@ -7,7 +8,7 @@ use vorpal::api::{PackageRequest, PackageSource, PackageSourceKind};
pub async fn main() -> Result<(), anyhow::Error> {
let mut client = CommandServiceClient::connect("http://[::1]:15323").await?;

let request = client
let coreutils = client
.package(PackageRequest {
build_deps: Vec::new(),
build_phase: r#"
Expand All @@ -21,15 +22,53 @@ pub async fn main() -> Result<(), anyhow::Error> {
install_deps: Vec::new(),
name: "coreutils".to_string(),
source: Some(PackageSource {
hash: Some("af6d643afd6241ec35c7781b7f999b97a66c84bea4710ad2bb15e75a5caf11b4".to_string()),
hash: Some(
"af6d643afd6241ec35c7781b7f999b97a66c84bea4710ad2bb15e75a5caf11b4".to_string(),
),
ignore_paths: vec![],
kind: PackageSourceKind::Http.into(),
uri: "https://ftp.gnu.org/gnu/coreutils/coreutils-9.5.tar.gz".to_string(),
}),
})
.await?;

let mut stream = request.into_inner();
let mut stream = coreutils.into_inner();
while let Some(package_response) = stream.next().await {
let response = package_response?;
if !response.package_log.is_empty() {
println!("{}", response.package_log);
}
}

let example = client
.package(PackageRequest {
build_deps: Vec::new(),
build_phase: r#"
mkdir -p $OUTPUT/bin
touch $OUTPUT/bin/example.txt
"#
.to_string(),
install_phase: r#"
echo "Hello, World!" >> $OUTPUT/bin/example.txt
cat $OUTPUT/bin/example.txt
"#
.to_string(),
install_deps: Vec::new(),
name: "example".to_string(),
source: Some(PackageSource {
hash: None,
ignore_paths: vec![
".git".to_string(),
".gitignore".to_string(),
"target".to_string(),
],
kind: PackageSourceKind::Local.into(),
uri: env::current_dir()?.to_string_lossy().to_string(),
}),
})
.await?;

let mut stream = example.into_inner();
while let Some(package_response) = stream.next().await {
let response = package_response?;
if !response.package_log.is_empty() {
Expand Down
119 changes: 61 additions & 58 deletions src/service/build/run_prepare.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,26 +42,31 @@ pub async fn run(mut stream: Streaming<PrepareRequest>) -> Result<Response<Prepa
source_data.extend_from_slice(&chunk.source_data);
}

if source_hash.is_empty() {
return Err(Status::internal("source hash is empty"));
}

if source_name.is_empty() {
return Err(Status::internal("source name is empty"));
}

if source_signature.is_empty() {
return Err(Status::internal("source signature is empty"));
}

tx.send(Ok(PrepareResponse {
source_id: 0,
source_log: format!("source chunks received: {}", source_chunks),
}))
.await
.unwrap();

let db = database::connect(paths::get_database_path())
.map_err(|_| Status::internal("failed to connect to database"))?;

let public_key = notary::get_public_key()
.await
.map_err(|_| Status::internal("failed to get public key"))?;

let verifying_key = VerifyingKey::<Sha256>::new(public_key);

if source_signature.is_empty() {
return Err(Status::internal("source signature is empty"));
}

let signature_decode = hex::decode(source_signature)
.map_err(|_| Status::internal("hex decode of signature failed"))?;

Expand All @@ -72,14 +77,6 @@ pub async fn run(mut stream: Streaming<PrepareRequest>) -> Result<Response<Prepa
.verify(&source_data, &signature)
.map_err(|_| Status::internal("failed to verify signature"))?;

if source_hash.is_empty() {
return Err(Status::internal("source hash is empty"));
}

if source_name.is_empty() {
return Err(Status::internal("source name is empty"));
}

let package_source_path = paths::get_package_source_path(&source_name, &source_hash);
let package_source_tar_path =
paths::get_package_source_tar_path(&source_name, &source_hash);
Expand All @@ -104,64 +101,70 @@ pub async fn run(mut stream: Streaming<PrepareRequest>) -> Result<Response<Prepa
.await
.unwrap();
}
}

fs::create_dir_all(&package_source_path).await?;
fs::create_dir_all(&package_source_path).await?;

if let Err(err) =
archives::unpack_tar_gz(&package_source_path, &package_source_tar_path).await
{
return Err(Status::internal(format!(
"Failed to unpack source tar: {}",
err
)));
}
if let Err(err) =
archives::unpack_tar_gz(&package_source_path, &package_source_tar_path).await
{
return Err(Status::internal(format!(
"Failed to unpack source tar: {}",
err
)));
}

let source_file_paths =
paths::get_file_paths(&package_source_path, &Vec::<&str>::new()).map_err(|e| {
Status::internal(format!("Failed to get source files: {:?}", e))
})?;
let source_file_paths = paths::get_file_paths(&package_source_path, &Vec::<&str>::new())
.map_err(|e| Status::internal(format!("Failed to get source files: {:?}", e)))?;

tx.send(Ok(PrepareResponse {
source_id: 0,
source_log: format!("source files: {:?}", source_file_paths.len()),
}))
.await
.unwrap();
tx.send(Ok(PrepareResponse {
source_id: 0,
source_log: format!("source files: {:?}", source_file_paths.len()),
}))
.await
.unwrap();

let source_files_hashes = hashes::get_files(&source_file_paths).map_err(|e| {
Status::internal(format!("Failed to get source file hashes: {:?}", e))
})?;
let source_files_hashes = hashes::get_files(&source_file_paths)
.map_err(|e| Status::internal(format!("Failed to get source file hashes: {:?}", e)))?;

let source_hash_computed = hashes::get_source(&source_files_hashes)
.map_err(|e| Status::internal(format!("Failed to get source hash: {:?}", e)))?;
let source_hash_computed = hashes::get_source(&source_files_hashes)
.map_err(|e| Status::internal(format!("Failed to get source hash: {:?}", e)))?;

tx.send(Ok(PrepareResponse {
source_id: 0,
source_log: format!("source hash: {}", source_hash),
}))
.await
.unwrap();
tx.send(Ok(PrepareResponse {
source_id: 0,
source_log: format!("source hash: {}", source_hash),
}))
.await
.unwrap();

tx.send(Ok(PrepareResponse {
source_id: 0,
source_log: format!("source hash expected: {}", source_hash_computed),
}))
.await
.unwrap();
tx.send(Ok(PrepareResponse {
source_id: 0,
source_log: format!("source hash expected: {}", source_hash_computed),
}))
.await
.unwrap();

if source_hash != source_hash_computed {
return Err(Status::internal("source hash mismatch"));
}
if source_hash != source_hash_computed {
return Err(Status::internal("source hash mismatch"));
}

let db = database::connect(paths::get_database_path())
.map_err(|_| Status::internal("failed to connect to database"))?;

let mut source_id = database::find_source(&db, &source_hash, &source_name)
.map(|source| source.id)
.unwrap_or(0);

if source_id == 0 {
database::insert_source(&db, &source_hash, &source_name)
.map_err(|e| Status::internal(format!("Failed to insert source: {:?}", e)))?;

fs::remove_dir_all(package_source_path).await?;
source_id = database::find_source(&db, &source_hash, &source_name)
.map(|source| source.id)
.map_err(|e| Status::internal(format!("Failed to find source: {:?}", e)))?;
}

let source_id = database::find_source(&db, &source_hash, &source_name)
.map(|source| source.id)
.map_err(|e| Status::internal(format!("Failed to find source: {:?}", e.to_string())))?;
fs::remove_dir_all(package_source_path).await?;

tx.send(Ok(PrepareResponse {
source_id,
Expand Down
Loading

0 comments on commit d3e53f4

Please sign in to comment.