Added service client authentication and improved modularity

TraceMachina · Jan 11, 2025 · 70f93cf · 70f93cf
1 parent 0b456d3
commit 70f93cf
Show file tree

Hide file tree

Showing 13 changed files with 1,026 additions and 898 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/nativelink-config/examples/README.md b/nativelink-config/examples/README.md
@@ -41,7 +41,7 @@ The value of `stores` includes top-level keys, which are user supplied names sto
 ### Store Type
 
 Once the store has been named and its object exists,
-the next key is the type of store. The options are `filesystem`, `memory`, `compression`, `dedup`, `fast_slow`, `verify`, and `experimental_s3_store`.
+the next key is the type of store. The options are `filesystem`, `memory`, `compression`, `dedup`, `fast_slow`, `verify`, `experimental_s3_store` and `experimental_gcs_store`.
 
 ```json5
 {

diff --git a/nativelink-config/examples/gcs_backend.json5 b/nativelink-config/examples/gcs_backend.json5
diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs
@@ -85,7 +85,7 @@ pub enum StoreSpec {
     /// **Example JSON Config:**
     /// ```json
     /// "experimental_gcs_store": {
-    ///   "project_id": "sample-project",
+    ///   "service_email": "email@domain.com",
     ///   "bucket": "test-bucket",
     ///   "key_prefix": "test-prefix-index/",
     ///   "retry": {
@@ -813,7 +813,7 @@ pub struct S3Spec {
 pub struct GcsSpec {
     /// Project ID for the GCS service
     #[serde(default, deserialize_with = "convert_string_with_shellexpand")]
-    pub project_id: String,
+    pub service_email: String,
 
     /// Bucket name to use as the backend
     #[serde(default, deserialize_with = "convert_string_with_shellexpand")]

diff --git a/nativelink-store/BUILD.bazel b/nativelink-store/BUILD.bazel
@@ -18,6 +18,10 @@ rust_library(
         "src/existence_cache_store.rs",
         "src/fast_slow_store.rs",
         "src/filesystem_store.rs",
+        "src/gcs_client/auth.rs",
+        "src/gcs_client/client.rs",
+        "src/gcs_client/grpc_client.rs",
+        "src/gcs_client/mod.rs",
         "src/gcs_store.rs",
         "src/grpc_store.rs",
         "src/lib.rs",
@@ -62,6 +66,7 @@ rust_library(
         "@crates//:http-body",
         "@crates//:hyper-0.14.31",
         "@crates//:hyper-rustls",
+        "@crates//:jsonwebtoken",
         "@crates//:lz4_flex",
         "@crates//:parking_lot",
         "@crates//:patricia_tree",

diff --git a/nativelink-store/Cargo.toml b/nativelink-store/Cargo.toml
@@ -40,6 +40,7 @@ fred = { version = "10.0.1", default-features = false, features = [
   "subscriber-client",
 ] }
 googleapis-tonic-google-storage-v2 = "0.17.0"
+jsonwebtoken = "9.3.0"
 patricia_tree = { version = "0.8.0", default-features = false }
 futures = { version = "0.3.31", default-features = false }
 hex = { version = "0.4.3", default-features = false }