[WIP] feat: Implenet vald index export

cmd/index/job/exportation/main.go

@@ -13,7 +13,17 @@
 // limitations under the License.
 package main
 
-import "fmt"
+import (
+	"context"
+	"log"
+
+	"github.com/vdaas/vald/internal/errors"
+	"github.com/vdaas/vald/internal/info"
+	"github.com/vdaas/vald/internal/runner"
+	"github.com/vdaas/vald/internal/safety"
+	"github.com/vdaas/vald/pkg/index/job/exportation/config"
+	"github.com/vdaas/vald/pkg/index/job/exportation/usecase"
+)
 
 const (
 	maxVersion = "v0.0.10"
@@ -22,5 +32,28 @@ const (
 )
 
 func main() {
-	fmt.Println("hello world")
+	if err := safety.RecoverFunc(func() error {
+		return runner.Do(
+			context.Background(),
+			runner.WithName(name),
+			runner.WithVersion(info.Version, maxVersion, minVersion),
+			runner.WithConfigLoader(func(path string) (any, *config.GlobalConfig, error) {
+				cfg, err := config.NewConfig(path)
+				if err != nil {
+					return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration")
+				}
+				return cfg, &cfg.GlobalConfig, nil
+			}),
+			runner.WithDaemonInitializer(func(cfg any) (runner.Runner, error) {
+				c, ok := cfg.(*config.Data)
+				if !ok {
+					return nil, errors.ErrInvalidConfig
+				}
+				return usecase.New(c)
+			}),
+		)
+	})(); err != nil {
+		log.Fatal(err, info.Get())
+		return
+	}
 }

cmd/index/job/exportation/sample.yaml

@@ -0,0 +1,105 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+version: v0.0.0
+time_zone: JST
+logging:
+  format: raw
+  level: info
+  logger: glg
+server_config:
+  servers:
+    - name: grpc
+      host: 0.0.0.0
+      port: 8081
+      grpc:
+        bidirectional_stream_concurrency: 20
+        connection_timeout: ""
+        header_table_size: 0
+        initial_conn_window_size: 0
+        initial_window_size: 0
+        interceptors: []
+        keepalive:
+          max_conn_age: ""
+          max_conn_age_grace: ""
+          max_conn_idle: ""
+          time: ""
+          timeout: ""
+        max_header_list_size: 0
+        max_receive_message_size: 0
+        max_send_message_size: 0
+        read_buffer_size: 0
+        write_buffer_size: 0
+      mode: GRPC
+      probe_wait_time: 3s
+      restart: true
+  health_check_servers:
+    - name: readiness
+      host: 0.0.0.0
+      port: 3001
+      http:
+        handler_timeout: ""
+        idle_timeout: ""
+        read_header_timeout: ""
+        read_timeout: ""
+        shutdown_duration: 0s
+        write_timeout: ""
+      mode: ""
+      probe_wait_time: 3s
+  metrics_servers:
+  startup_strategy:
+    - grpc
+    - readiness
+  full_shutdown_duration: 600s
+  tls:
+    ca: /path/to/ca
+    cert: /path/to/cert
+    enabled: false
+    key: /path/to/key
+exporter:
+  concurrency: 1
+  kvs_background_sync_interval: 5s
+  kvs_background_compaction_interval: 5s
+observability:
+  enabled: false
+  otlp:
+    collector_endpoint: "otel-collector.monitoring.svc.cluster.local:4317"
+    trace_batch_timeout: "1s"
+    trace_export_timeout: "1m"
+    trace_max_export_batch_size: 1024
+    trace_max_queue_size: 256
+    metrics_export_interval: "1s"
+    metrics_export_timeout: "1m"
+    attribute:
+      namespace: "_MY_POD_NAMESPACE_"
+      pod_name: "_MY_POD_NAME_"
+      node_name: "_MY_NODE_NAME_"
+      service_name: "vald-index-deletion"
+  metrics:
+    enable_cgo: true
+    enable_goroutine: true
+    enable_memory: true
+    enable_version_info: true
+    version_info_labels:
+      - vald_version
+      - server_name
+      - git_commit
+      - build_time
+      - go_version
+      - go_os
+      - go_arch
+      - algorithm_info
+  trace:
+    enabled: true

internal/config/index_exporter.go

@@ -0,0 +1,30 @@
+// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package config
+
+// IndexExporter represents the configurations for index exportation.
+type IndexExporter struct {
+	// Concurrency represents indexing concurrency.
+	Concurrency int `json:"concurrency" yaml:"concurrency"`
+
+	// KVSBackgroundSyncInterval represents interval for checked id list kvs sync duration
+	KVSBackgroundSyncInterval string `json:"kvs_background_sync_interval" yaml:"kvs_background_sync_interval"`
+
+	// KVSBackgroundCompactionInterval represents interval for checked id list kvs compaction duration
+	KVSBackgroundCompactionInterval string `json:"kvs_background_compaction_interval" yaml:"kvs_background_compaction_interval"`
+}
+
+func (ic *IndexExporter) Bind() *IndexExporter {
+	return ic
+}

pkg/index/job/exportation/config/config.go

@@ -0,0 +1,66 @@
+// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package config
+
+import (
+	"github.com/vdaas/vald/internal/config"
+	"github.com/vdaas/vald/internal/errors"
+)
+
+// GlobalConfig is a type alias of config.GlobalConfig representing application base configurations.
+type GlobalConfig = config.GlobalConfig
+
+// Data represents the application configurations.
+type Data struct {
+	// GlobalConfig represents application base configurations.
+	config.GlobalConfig `json:",inline" yaml:",inline"`
+
+	// Server represent all server configurations
+	Server *config.Servers `json:"server_config" yaml:"server_config"`
+
+	// Observability represents observability configurations.
+	Observability *config.Observability `json:"observability" yaml:"observability"`
+
+	// Deletion represents auto indexing service configurations.
+	Exporter *config.IndexExporter `json:"exporter" yaml:"exporter"`
+}
+
+// NewConfig load configurations from file path.
+func NewConfig(path string) (cfg *Data, err error) {
+	cfg = new(Data)
+
+	if err = config.Read(path, &cfg); err != nil {
+		return nil, err
+	}
+
+	if cfg != nil {
+		_ = cfg.GlobalConfig.Bind()
+	} else {
+		return nil, errors.ErrInvalidConfig
+	}
+
+	if cfg.Server != nil {
+		_ = cfg.Server.Bind()
+	} else {
+		return nil, errors.ErrInvalidConfig
+	}
+
+	if cfg.Observability != nil {
+		_ = cfg.Observability.Bind()
+	} else {
+		cfg.Observability = new(config.Observability).Bind()
+	}
+
+	return cfg, nil
+}