diff --git a/storage/internal/benchmarks/README.md b/storage/internal/benchmarks/README.md
index 1ab328be0bbb..08d2477fdffb 100644
--- a/storage/internal/benchmarks/README.md
+++ b/storage/internal/benchmarks/README.md
@@ -31,6 +31,7 @@ This runs 1000 iterations on 512kib to 2Gib files in the background, sending out
| -connection_pool_size | GRPC connection pool size | any positive integer | 4 |
| -force_garbage_collection | whether to force garbage collection
before every write or read benchmark | `true` or `false` (present/not present) | `false` |
| -timeout | timeout (maximum time running benchmarks)
the program may run for longer while it finishes running processes | any [time.Duration](https://pkg.go.dev/time#Duration) | `1h` |
+| -timeout_per_op | timeout on a single upload or download | any [time.Duration](https://pkg.go.dev/time#Duration) | `5m` |
| -workload | `1` will run a w1r3 (write 1 read 3) benchmark
`6` will run a benchmark uploading and downloading (once each)
a single directory with `-directory_num_objects` number of files (no subdirectories) | `1` or `6` | `1` |
| -directory_num_objects | total number of objects in a directory (directory will only contain files,
no subdirectories); only applies to workload 6 | any positive integer | `1000` |
diff --git a/storage/internal/benchmarks/directory_benchmark.go b/storage/internal/benchmarks/directory_benchmark.go
index 8681bd9f6c35..6f7f967532e3 100644
--- a/storage/internal/benchmarks/directory_benchmark.go
+++ b/storage/internal/benchmarks/directory_benchmark.go
@@ -205,6 +205,7 @@ func (r *directoryBenchmark) uploadDirectory(ctx context.Context, numWorkers int
object: objectName,
useDefaultChunkSize: opts.minChunkSize == useDefault || opts.maxChunkSize == useDefault,
objectPath: filePath,
+ timeout: r.opts.timeoutPerOp,
})
return err
})
@@ -278,6 +279,7 @@ func (r *directoryBenchmark) downloadDirectory(ctx context.Context, numWorkers i
downloadToDirectory: r.downloadDirectoryPath,
rangeStart: rangeStart,
rangeLength: rangeLength,
+ timeout: r.opts.timeoutPerOp,
})
return err
})
diff --git a/storage/internal/benchmarks/download_benchmark.go b/storage/internal/benchmarks/download_benchmark.go
index 77fa4b76fe92..ce0cc3670911 100644
--- a/storage/internal/benchmarks/download_benchmark.go
+++ b/storage/internal/benchmarks/download_benchmark.go
@@ -33,6 +33,7 @@ type downloadOpts struct {
rangeStart int64
rangeLength int64
downloadToDirectory string
+ timeout time.Duration
}
func downloadBenchmark(ctx context.Context, dopts downloadOpts) (elapsedTime time.Duration, rerr error) {
@@ -45,7 +46,7 @@ func downloadBenchmark(ctx context.Context, dopts downloadOpts) (elapsedTime tim
defer func() { elapsedTime = time.Since(start) }()
// Set additional timeout
- ctx, cancel := context.WithTimeout(ctx, time.Minute*2)
+ ctx, cancel := context.WithTimeout(ctx, dopts.timeout)
defer cancel()
o := dopts.client.Bucket(dopts.bucket).Object(dopts.object)
diff --git a/storage/internal/benchmarks/main.go b/storage/internal/benchmarks/main.go
index 3ec08b57d838..1ca1b63e26ad 100644
--- a/storage/internal/benchmarks/main.go
+++ b/storage/internal/benchmarks/main.go
@@ -73,7 +73,8 @@ type benchmarkOptions struct {
forceGC bool
connPoolSize int
- timeout time.Duration
+ timeout time.Duration
+ timeoutPerOp time.Duration
continueOnFail bool
@@ -166,6 +167,7 @@ func parseFlags() {
flag.BoolVar(&opts.forceGC, "force_garbage_collection", false, "force garbage collection at the beginning of each upload")
flag.DurationVar(&opts.timeout, "timeout", time.Hour, "timeout")
+ flag.DurationVar(&opts.timeoutPerOp, "timeout_per_op", time.Minute*5, "timeout per upload/download")
flag.StringVar(&outputFile, "o", "", "file to output results to - if empty, will output to stdout")
flag.BoolVar(&opts.continueOnFail, "continue_on_fail", false, "continue even if a run fails")
@@ -209,6 +211,12 @@ func main() {
ctx, cancel := context.WithDeadline(context.Background(), start.Add(opts.timeout))
defer cancel()
+ // Print a message once deadline is exceeded
+ go func() {
+ <-ctx.Done()
+ log.Printf("total configured timeout exceeded")
+ }()
+
// Create bucket if necessary
if len(opts.bucket) < 1 {
opts.bucket = randomName(bucketPrefix)
diff --git a/storage/internal/benchmarks/upload_benchmark.go b/storage/internal/benchmarks/upload_benchmark.go
index 8024c2726e3c..0c5347237e4a 100644
--- a/storage/internal/benchmarks/upload_benchmark.go
+++ b/storage/internal/benchmarks/upload_benchmark.go
@@ -38,6 +38,7 @@ type uploadOpts struct {
object string
useDefaultChunkSize bool
objectPath string
+ timeout time.Duration
}
func uploadBenchmark(ctx context.Context, uopts uploadOpts) (elapsedTime time.Duration, rerr error) {
@@ -50,7 +51,7 @@ func uploadBenchmark(ctx context.Context, uopts uploadOpts) (elapsedTime time.Du
defer func() { elapsedTime = time.Since(start) }()
// Set additional timeout
- ctx, cancel := context.WithTimeout(ctx, time.Minute*2)
+ ctx, cancel := context.WithTimeout(ctx, uopts.timeout)
defer cancel()
// Open file
diff --git a/storage/internal/benchmarks/w1r3.go b/storage/internal/benchmarks/w1r3.go
index bc1f289541d7..18d1bd9fc57c 100644
--- a/storage/internal/benchmarks/w1r3.go
+++ b/storage/internal/benchmarks/w1r3.go
@@ -157,6 +157,7 @@ func (r *w1r3) run(ctx context.Context) error {
object: r.objectName,
useDefaultChunkSize: opts.minChunkSize == useDefault || opts.maxChunkSize == useDefault,
objectPath: r.objectPath,
+ timeout: r.opts.timeoutPerOp,
})
})
@@ -186,6 +187,7 @@ func (r *w1r3) run(ctx context.Context) error {
rangeStart: rangeStart,
rangeLength: rangeLength,
downloadToDirectory: r.directoryPath,
+ timeout: r.opts.timeoutPerOp,
})
})
if err != nil {