create continous benchmark doc (#2352) (#2395)

* improve Backport Pull Request Title & Description Quality (#2373) * 📝 create continous benchmark doc * style: format code with Gofumpt and Prettier This commit fixes the style issues introduced in ace001a according to the output from Gofumpt and Prettier. Details: #2352 * ♻️ rafactor default params * style: format code with Gofumpt and Prettier This commit fixes the style issues introduced in 9880dfc according to the output from Gofumpt and Prettier. Details: #2352 * ♻️ Fix format * Revert "improve Backport Pull Request Title & Description Quality (#2373)" This reverts commit 4246def. * ♻️ remove dimension from con-bench * 📝 Add sample benchmark manifest * Update docs/performance/continuos-benchmatk.md * Update docs/performance/continuos-benchmatk.md --------- Signed-off-by: kpango <kpango@vdaas.org> Signed-off-by: vankichi <kyukawa315@gmail.com> Signed-off-by: Kiichiro YUKAWA <kyukawa315@gmail.com> Co-authored-by: Kiichiro YUKAWA <kyukawa315@gmail.com> Co-authored-by: Yusuke Kato <kpango@vdaas.org> Co-authored-by: Kosuke Morimoto <ksk@vdaas.org> Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com> Co-authored-by: Hiroto Funakoshi <hiroto.funakoshi.hiroto@gmail.com>
vdaas · Feb 21, 2024 · f0dc950 · f0dc950
1 parent b8dfb45
commit f0dc950
Show file tree

Hide file tree

Showing 14 changed files with 290 additions and 45 deletions.
diff --git a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml
@@ -277,9 +277,6 @@ spec:
                     - indexes
                     - group
                     - range
-                dimension:
-                  type: integer
-                  minimum: 1
                 global_config:
                   type: object
                   properties:

diff --git a/charts/vald-benchmark-operator/schemas/job-values.yaml b/charts/vald-benchmark-operator/schemas/job-values.yaml
@@ -47,9 +47,6 @@ dataset:
   # @schema {"name": "dataset.url", "type": "string"}
   # dataset.url -- the dataset url which is used for executing benchmark job with user defined hdf5 file
   url: ""
-# @schema {"name": "dimension", "type": "integer", "minimum": 1}
-# dimension -- vector dimension
-dimension: 784
 # @schema {"name": "replica", "type": "integer", "minimum": 1}
 # replica -- the number of running concurrency job
 replica: 1

diff --git a/charts/vald-benchmark-operator/schemas/scenario-values.yaml b/charts/vald-benchmark-operator/schemas/scenario-values.yaml
@@ -59,7 +59,6 @@ jobs:
       range:
         start: 1
         end: 1000
-    dimension: 784
     replica: 1
     repetition: 1
     job_type: "search"

diff --git a/charts/vald-benchmark-operator/values/benchmark-job.yaml b/charts/vald-benchmark-operator/values/benchmark-job.yaml
@@ -27,7 +27,6 @@ spec:
       start: 1
       end: 1000
   job_type: "search"
-  dimension: 784
   repetition: 1
   replica: 1
   rules: []

diff --git a/charts/vald-benchmark-operator/values/benchmark-scenario.yaml b/charts/vald-benchmark-operator/values/benchmark-scenario.yaml
@@ -45,7 +45,6 @@ spec:
   jobs:
     # @schema {"name": "jobs.items.dataset", "type": "object"}
     - job_type: "insert"
-      dimension: 784
       repetition: 1
       replica: 1
       rules: []
@@ -62,7 +61,6 @@ spec:
         health_check_duration: "10s"
       rps: 500
     - job_type: "update"
-      dimension: 784
       repetition: 1
       replica: 1
       rules: []
@@ -79,7 +77,6 @@ spec:
         health_check_duration: "10s"
       rps: 500
     - job_type: "search"
-      dimension: 784
       repetition: 1
       replica: 1
       rules: []
@@ -94,7 +91,6 @@ spec:
         health_check_duration: "10s"
       rps: 2000
     - job_type: "upsert"
-      dimension: 784
       repetition: 1
       replica: 1
       rules: []
@@ -111,7 +107,6 @@ spec:
         health_check_duration: "10s"
       rps: 1000
     - job_type: "search"
-      dimension: 784
       repetition: 2
       replica: 1
       rules: []
@@ -133,7 +128,6 @@ spec:
         health_check_duration: "10s"
       rps: 4000
     - job_type: "exists"
-      dimension: 784
       repetition: 1
       replica: 1
       rules: []
@@ -148,7 +142,6 @@ spec:
         health_check_duration: "10s"
       rps: 1000
     - job_type: "getobject"
-      dimension: 784
       repetition: 1
       replica: 1
       rules: []
@@ -163,7 +156,6 @@ spec:
         health_check_duration: "10s"
       rps: 1000
     - job_type: "remove"
-      dimension: 784
       repetition: 1
       replica: 1
       rules: []

diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md
diff --git a/internal/config/benchmark.go b/internal/config/benchmark.go
@@ -21,7 +21,6 @@ package config
 type BenchmarkJob struct {
 	Target             *BenchmarkTarget    `json:"target,omitempty"               yaml:"target"`
 	Dataset            *BenchmarkDataset   `json:"dataset,omitempty"              yaml:"dataset"`
-	Dimension          int                 `json:"dimension,omitempty"            yaml:"dimension"`
 	Replica            int                 `json:"replica,omitempty"              yaml:"replica"`
 	Repetition         int                 `json:"repetition,omitempty"           yaml:"repetition"`
 	JobType            string              `json:"job_type,omitempty"             yaml:"job_type"`

diff --git a/internal/k8s/vald/benchmark/api/v1/job_types.go b/internal/k8s/vald/benchmark/api/v1/job_types.go
@@ -28,7 +28,6 @@ type BenchmarkJobSpec struct {
 	ServerConfig            *config.Servers            `json:"server_config,omitempty"              yaml:"server_config"`
 	Target                  *BenchmarkTarget           `json:"target,omitempty"                     yaml:"target"`
 	Dataset                 *BenchmarkDataset          `json:"dataset,omitempty"                    yaml:"dataset"`
-	Dimension               int                        `json:"dimension,omitempty"                  yaml:"dimension"`
 	Replica                 int                        `json:"replica,omitempty"                    yaml:"replica"`
 	Repetition              int                        `json:"repetition,omitempty"                 yaml:"repetition"`
 	JobType                 string                     `json:"job_type,omitempty"                   yaml:"job_type"`

diff --git a/internal/k8s/vald/benchmark/job/job_template_option.go b/internal/k8s/vald/benchmark/job/job_template_option.go
@@ -64,13 +64,21 @@ func WithImagePullPolicy(p ImagePullPolicy) BenchmarkJobTplOption {
 // BenchmarkJobOption represents the option for create benchmark job template.
 type BenchmarkJobOption func(b *jobs.Job) error
 
-// defaultTTLSeconds represents the default TTLSecondsAfterFinished for benchmark job template.
-const defaultTTLSeconds int32 = 600
+const (
+	// defaultTTLSeconds represents the default TTLSecondsAfterFinished for benchmark job template.
+	defaultTTLSeconds int32 = 600
+	// defaultCompletions represents the default completions for benchmark job template.
+	defaultCompletions int32 = 1
+	// defaultParallelism represents the default parallelism for benchmark job template.
+	defaultParallelism int32 = 1
+)
 
 var defaultBenchmarkJobOpts = []BenchmarkJobOption{
 	WithSvcAccountName(svcAccount),
 	WithRestartPolicy(RestartPolicyNever),
 	WithTTLSecondsAfterFinished(defaultTTLSeconds),
+	WithCompletions(defaultCompletions),
+	WithParallelism(defaultParallelism),
 }
 
 // WithSvcAccountName sets the service account name for benchmark job.

diff --git a/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml b/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml
@@ -277,9 +277,6 @@ spec:
                     - indexes
                     - group
                     - range
-                dimension:
-                  type: integer
-                  minimum: 1
                 global_config:
                   type: object
                   properties:

diff --git a/pkg/tools/benchmark/job/service/job.go b/pkg/tools/benchmark/job/service/job.go
@@ -82,7 +82,6 @@ func (jt jobType) String() string {
 
 type job struct {
 	eg                 errgroup.Group
-	dimension          int
 	dataset            *config.BenchmarkDataset
 	jobType            jobType
 	jobFunc            func(context.Context, chan error) error

diff --git a/pkg/tools/benchmark/job/service/option.go b/pkg/tools/benchmark/job/service/option.go
@@ -32,23 +32,11 @@ import (
 type Option func(j *job) error
 
 var defaultOpts = []Option{
-	// TODO: set default config for client
-	WithDimension(748),
 	WithBeforeJobDuration("30s"),
 	WithRPS(1000),
 	WithConcurencyLimit(200),
 }
 
-// WithDimension sets the vector's dimension for running benchmark job with dataset.
-func WithDimension(dim int) Option {
-	return func(j *job) error {
-		if dim > 0 {
-			j.dimension = dim
-		}
-		return nil
-	}
-}
-
 // WithInsertConfig sets the insert API config for running insert request job.
 func WithInsertConfig(c *config.InsertConfig) Option {
 	return func(j *job) error {

diff --git a/pkg/tools/benchmark/job/service/search.go b/pkg/tools/benchmark/job/service/search.go
@@ -66,10 +66,6 @@ func (j *job) search(ctx context.Context, ech chan error) error {
 			}
 			// idx is the modulo, which takes between <0, len(vecs)-1>.
 			idx := (iter - 1) % len(vecs)
-			if len(vecs[idx]) != j.dimension {
-				log.Warn("len(vecs) ", len(vecs[iter]), "is not matched with ", j.dimension)
-				return nil
-			}
 			res, err := j.client.Search(egctx, &payload.Search_Request{
 				Vector: vecs[idx],
 				Config: cfg,
@@ -121,10 +117,6 @@ func (j *job) search(ctx context.Context, ech chan error) error {
 				log.Debugf("[benchmark job] Start linear search: iter = %d", iter)
 				// idx is the modulo, which takes between <0, len(vecs)-1>.
 				idx := (iter - 1) % len(vecs)
-				if len(vecs[idx]) != j.dimension {
-					log.Warn("len(vecs) ", len(vecs[idx]), "is not matched with ", j.dimension)
-					return nil
-				}
 				res, err := j.client.LinearSearch(egctx, &payload.Search_Request{
 					Vector: vecs[idx],
 					Config: cfg,

diff --git a/pkg/tools/benchmark/job/usecase/benchmarkd.go b/pkg/tools/benchmark/job/usecase/benchmarkd.go
@@ -91,7 +91,6 @@ func New(cfg *config.Config) (r runner.Runner, err error) {
 		service.WithValdClient(vcli),
 		service.WithDataset(cfg.Job.Dataset),
 		service.WithJobTypeByString(cfg.Job.JobType),
-		service.WithDimension(cfg.Job.Dimension),
 		service.WithInsertConfig(cfg.Job.InsertConfig),
 		service.WithUpdateConfig(cfg.Job.UpdateConfig),
 		service.WithUpsertConfig(cfg.Job.UpsertConfig),