Implements store paths for builds (#34)

* refactor: simplify services and build steps

* refactor: improve caching for services and structure

* chore(deps): update dependencies in Cargo.lock and flake.nix

- Update `sync_wrapper` to version `0.1.2` in Cargo.lock
- Update `httparse` to version `1.9.4` in Cargo.lock
- Add `hyper-rustls` version `0.27.2` to Cargo.lock
- Update `spin` to version `0.5.2` in Cargo.lock
- Update `miniz_oxide` to version `0.7.4` in Cargo.lock
- Update `reqwest` to version `0.12.5` in Cargo.lock
- Add `ring` version `0.17.8` to Cargo.lock
- Add `rustls` version `0.23.10` to Cargo.lock
- Add `rustls-webpki` version `0.102.4` to Cargo.lock
- Add `spin` version `0.9.8` to Cargo.lock
- Add `sync_wrapper` version `1.0.1` to Cargo.lock
- Add `tokio-rustls` version `0.26.0` to Cargo.lock
- Add `untrusted` version `0.9.0` to Cargo.lock
- Update `zstd-sys` to version `2.0.11+zstd.1.5.6` in Cargo.lock
- Update `cargoSha256` in flake.nix to `sha256-kDe3EEc2InW39X+VXyUUwNYPY+mlUh5A3KOpc3ITqNc=`

* feat: adding package outputs to sandbox PATH environment variable

* feat(proto): add environment map to ConfigPackageBuild and PackageBuildRequest

- Updated `config.proto` to include `environment` map in `ConfigPackageBuild`.
- Updated `package.proto` to include `build_environment` map in `PackageBuildRequest`.

feat(clean): add clean-cache target to justfile

- Added `clean-cache` target to `justfile` to remove store cache.

refactor(agent): update build package handling in ConfigService

- Modified `ConfigService` implementation to handle `build_environment` and populate `build_packages`.

refactor(worker): enhance logging and environment handling in PackageService

- Updated `PackageService` to log sandbox paths and build packages.
- Added handling for `build_environment` in sandbox command.

* feat(proto): add sandbox option to config and package build messages

- Added `sandbox` field to `ConfigPackageBuild` message in `config.proto`.
- Added `build_sandbox` field to `PackageBuildRequest` message in `package.proto`.
- Updated `PackagePrepareResponse` and `PackageBuildResponse` messages to use `log_output` as the first field.

refactor(service): enhance package handling and sandbox execution

- Updated `service.rs` to handle new `sandbox` and `build_sandbox` fields.
- Improved package output handling by including `ConfigPackageOutput` in responses.
- Enhanced sandbox execution by adding support for Gzip and Xz decoders.
- Refined environment variable handling for sandbox execution.

fix(worker): correct source hash mismatch error message

- Updated error message for source hash mismatch in `service.rs`.

refactor(archives): improve tar archive handling

- Replaced `Archive` with `ArchiveBuilder` for better control over tar archive extraction.
- Removed unnecessary logging in `compress_tar_gz` function.

* feat: added support for zip sources

* fix: lint

* fix: update cargoSha256 for vorpal package in flake.nix

The cargoSha256 hash for the vorpal package has been updated to ensure the integrity and correctness of the build process.

* refactor: migrate all logs to send as bytes

* refactor: update source preparation and file copying logic

- Refactored `source_prepare` function to use a temporary source path for unpacking archives and copying files.
- Added `source_hash` parameter to `source_prepare` function.
- Moved `copy_files` function from `service/worker/service.rs` to `store/paths.rs`.
- Updated imports in `service/worker/service.rs` to use consolidated imports.
- Updated calls to `copy_files` to use the new location in `paths.rs`.
- Improved logging messages for better clarity during source preparation and file operations.

* fix(package): remove unnecessary references in function calls

Removed unnecessary references in the function calls to `get_package_source_path` and `compress_tar_gz` to align with the expected parameter types. This change improves code readability and correctness.

* chore: update flake.lock dependencies

- Remove flake-utils and systems dependencies
- Update nixpkgs to revision d603719ec6e294f034936c0d0dc06f689d91b6c3
- Update rust-overlay to revision c9a793a5278f711a59fe77b9bf54b215667022c6

Cargo.toml

@@ -10,25 +10,24 @@ path = "src/bin/main.rs"
 [dependencies]
 anyhow = "1"
 async-compression = { version = "0", features = ["all"] }
+async_zip = { version = "0", features = ["deflate", "tokio"] }
 clap = { version = "4", features = ["derive"] }
 dirs = "5"
-flate2 = "1"
-futures = "0"
-futures-util = "0"
+futures-lite = { version = "2", default-features = false, features = ["std"] }
 git2 = "0"
 hex = "0"
 infer = "0"
 process-stream = "0"
 prost = "0"
 rand = "0"
 rsa = { version = "0", features = ["sha2"] }
-rusqlite = { version = "0", features = ["bundled"] }
 sha256 = "1"
 reqwest = { version = "0", features = ["json"] }
 tera = { version = "1", default-features = false }
 tokio = { version = "1", features = ["full"] }
 tokio-stream = "0"
 tokio-tar = "0"
+tokio-util = { version = "0", features = ["compat"] }
 tonic = "0"
 url = "2"
 uuid = { version = "1", features = ["v7"] }

api/v0/config/config.proto

@@ -0,0 +1,45 @@
+syntax = "proto3";
+
+package vorpal.config.v0;
+
+service ConfigService {
+    rpc Package (ConfigPackageRequest) returns (stream ConfigPackageResponse);
+    // rpc Secret (ConfigSecretRequest) returns (stream ConfigSecretResponse);
+}
+
+enum ConfigPackageSourceKind {
+    UNKNOWN = 0;
+    LOCAL = 1;
+    HTTP = 2;
+    GIT = 3;
+}
+
+message ConfigPackageOutput {
+    string name = 1;
+    string hash = 2;
+}
+
+message ConfigPackageBuild {
+    bool sandbox = 1;
+    map<string, string> environment = 2;
+    repeated ConfigPackageOutput packages = 3;
+    string script = 4;
+}
+
+message ConfigPackageSource {
+    ConfigPackageSourceKind kind = 1;
+    optional string hash = 2;
+    repeated string ignore_paths = 3;
+    string uri = 4;
+}
+
+message ConfigPackageRequest {
+    ConfigPackageBuild build = 1;
+    ConfigPackageSource source = 2;
+    string name = 3;
+}
+
+message ConfigPackageResponse {
+    optional ConfigPackageOutput package_output = 1;
+    bytes log_output = 2;
+}

api/v0/package/package.proto

@@ -3,38 +3,35 @@ syntax = "proto3";
 package vorpal.package.v0;
 
 service PackageService {
-    rpc Prepare (stream PrepareRequest) returns (stream PrepareResponse);
-    rpc Build (BuildRequest) returns (stream BuildResponse);
+    rpc Prepare (stream PackagePrepareRequest) returns (stream PackagePrepareResponse);
+    rpc Build (PackageBuildRequest) returns (stream PackageBuildResponse);
 }
 
-enum Status {
-    CREATED = 0;
-    COMPLETED = 1;
-    FAILED = 2;
-}
-
-message PrepareRequest {
+message PackagePrepareRequest {
     bytes source_data = 1;
     string source_hash = 2;
     string source_name = 3;
     string source_signature = 4;
 }
 
-message PrepareResponse {
-    string source_log = 1;
-    int32 source_id = 2;
+message PackagePrepareResponse {
+    bytes log_output = 1;
+}
+
+message PrepareBuildPackage {
+    string hash = 1;
+    string name = 2;
 }
 
-message BuildRequest {
-    repeated string build_deps = 1;
-    repeated string install_deps = 2;
-    string build_phase = 3;
-    string install_phase = 4;
-    int32 source_id = 5;
+message PackageBuildRequest {
+    bool build_sandbox = 1;
+    map<string, string> build_environment = 2;
+    repeated PrepareBuildPackage build_packages = 3;
+    string build_script = 4;
+    string source_hash = 5;
+    string source_name = 6;
 }
 
-message BuildResponse {
-    bool is_archive = 1;
-    bytes package_data = 2;
-    string package_log = 3;
+message PackageBuildResponse {
+    bytes log_output = 1;
 }

api/v0/store/store.proto

@@ -0,0 +1,28 @@
+syntax = "proto3";
+
+package vorpal.store.v0;
+
+service StoreService {
+    rpc Fetch (StorePath) returns (stream StoreFetchResponse);
+    rpc Path (StorePath) returns (StorePathResponse);
+}
+
+enum StorePathKind {
+    UNKNOWN = 0;
+    SOURCE = 1;
+    PACKAGE = 2;
+}
+
+message StorePath {
+    StorePathKind kind = 1;
+    string hash = 2;
+    string name = 3;
+}
+
+message StoreFetchResponse {
+    bytes data = 1;
+}
+
+message StorePathResponse {
+    string uri = 1;
+}

build.rs

@@ -1,6 +1,10 @@
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     tonic_build::configure().compile(
-        &["command/command.proto", "package/package.proto"],
+        &[
+            "config/config.proto",
+            "package/package.proto",
+            "store/store.proto",
+        ],
         &["api/v0"],
     )?;
     Ok(())