diff --git a/ydb/library/yql/providers/generic/connector/app/client/client.go b/ydb/library/yql/providers/generic/connector/app/client/client.go deleted file mode 100644 index 1c4256d43774..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/client/client.go +++ /dev/null @@ -1,307 +0,0 @@ -package client - -import ( - "bytes" - "context" - "crypto/tls" - "crypto/x509" - "fmt" - "io" - "io/ioutil" - "os" - - "github.com/apache/arrow/go/v13/arrow/ipc" - "github.com/spf13/cobra" - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/protobuf/encoding/prototext" -) - -const ( - tableName = "primitives" - outputFormat = api_service_protos.TReadSplitsRequest_ARROW_IPC_STREAMING -) - -func newConfigFromPath(configPath string) (*config.ClientConfig, error) { - data, err := ioutil.ReadFile(configPath) - if err != nil { - return nil, fmt.Errorf("read file %v: %w", configPath, err) - } - - var cfg config.ClientConfig - - if err := prototext.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("prototext unmarshal `%v`: %w", string(data), err) - } - - return &cfg, nil -} - -func runClient(_ *cobra.Command, args []string) error { - configPath := args[0] - - cfg, err := newConfigFromPath(configPath) - if err != nil { - return fmt.Errorf("unknown instance: %w", err) - } - - logger, err := utils.NewDefaultLogger() - if err != nil { - return fmt.Errorf("new default logger: %w", err) - } - - if err := callServer(logger, cfg); err != nil { - return fmt.Errorf("call server: %w", err) - } - - return nil -} - -func makeConnection(logger log.Logger, cfg *config.ClientConfig) (*grpc.ClientConn, error) { - var opts []grpc.DialOption - - if cfg.Tls != nil { - logger.Info("client will use TLS connections") - - caCrt, err := os.ReadFile(cfg.Tls.Ca) - if err != nil { - return nil, err - } - - certPool := x509.NewCertPool() - if !certPool.AppendCertsFromPEM(caCrt) { - return nil, fmt.Errorf("failed to add server CA's certificate") - } - - tlsCfg := &tls.Config{ - RootCAs: certPool, - } - - opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))) - } else { - logger.Info("client will use insecure connections") - - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } - - conn, err := grpc.Dial(utils.EndpointToString(cfg.Endpoint), opts...) - if err != nil { - return nil, fmt.Errorf("grpc dial: %w", err) - } - - return conn, nil -} - -func callServer(logger log.Logger, cfg *config.ClientConfig) error { - conn, err := makeConnection(logger, cfg) - if err != nil { - return fmt.Errorf("grpc dial: %w", err) - } - - defer utils.LogCloserError(logger, conn, "connection close") - - connectorClient := api_service.NewConnectorClient(conn) - - var splits []*api_service_protos.TSplit - - switch cfg.DataSourceInstance.Kind { - case api_common.EDataSourceKind_CLICKHOUSE, api_common.EDataSourceKind_POSTGRESQL: - // DescribeTable - schema, err := describeTable(logger, connectorClient, cfg.DataSourceInstance) - if err != nil { - return fmt.Errorf("describe table: %w", err) - } - - // ListSplits - we want to SELECT * - splits, err = listSplits(logger, schema, connectorClient, cfg.DataSourceInstance) - if err != nil { - return fmt.Errorf("list splits: %w", err) - } - - case api_common.EDataSourceKind_S3: - return fmt.Errorf("unexpected data source kind %v", cfg.DataSourceInstance.Kind) - default: - return fmt.Errorf("unexpected data source kind %v", cfg.DataSourceInstance.Kind) - } - - // ReadSplits - if err := readSplits(logger, splits, outputFormat, connectorClient, cfg.DataSourceInstance); err != nil { - return fmt.Errorf("read splits: %w", err) - } - - return nil -} - -func describeTable( - logger log.Logger, - connectorClient api_service.ConnectorClient, - dsi *api_common.TDataSourceInstance, -) (*api_service_protos.TSchema, error) { - req := &api_service_protos.TDescribeTableRequest{Table: tableName, DataSourceInstance: dsi} - logger.Debug("DescribeTable", log.String("request", req.String())) - - resp, err := connectorClient.DescribeTable(context.TODO(), req) - if err != nil { - return nil, fmt.Errorf("describe table: %w", err) - } - - if utils.IsSuccess(resp.Error) { - logger.Debug("DescribeTable", log.String("response", resp.String())) - - return resp.Schema, nil - } - - logger.Error("DescribeTable", log.String("response", resp.String())) - - return nil, utils.NewSTDErrorFromAPIError(resp.Error) -} - -func listSplits( - logger log.Logger, - schema *api_service_protos.TSchema, - connectorClient api_service.ConnectorClient, - dsi *api_common.TDataSourceInstance, -) ([]*api_service_protos.TSplit, error) { - items := []*api_service_protos.TSelect_TWhat_TItem{} - - for _, column := range schema.Columns { - items = append(items, &api_service_protos.TSelect_TWhat_TItem{ - Payload: &api_service_protos.TSelect_TWhat_TItem_Column{Column: column}, - }) - } - - req := &api_service_protos.TListSplitsRequest{ - Selects: []*api_service_protos.TSelect{ - { - DataSourceInstance: dsi, - What: &api_service_protos.TSelect_TWhat{Items: items}, - From: &api_service_protos.TSelect_TFrom{Table: tableName}, - }, - }, - } - logger.Debug("ListSplits", log.String("request", req.String())) - - streamListSplits, err := connectorClient.ListSplits(context.TODO(), req) - if err != nil { - return nil, fmt.Errorf("list splits: %w", err) - } - - var splits []*api_service_protos.TSplit - - for { - resp, err := streamListSplits.Recv() - if err != nil { - if err == io.EOF { - break - } - - return nil, fmt.Errorf("stream list splits: %w", err) - } - - if !utils.IsSuccess(resp.Error) { - logger.Error("ListSplits", log.String("response", resp.String())) - - return splits, utils.NewSTDErrorFromAPIError(resp.Error) - } - - logger.Debug("ListSplits", log.String("response", resp.String())) - splits = append(splits, resp.Splits...) - } - - if len(splits) != 1 { - return nil, fmt.Errorf("too many splits") - } - - return splits, nil -} - -func readSplits( - logger log.Logger, - splits []*api_service_protos.TSplit, - format api_service_protos.TReadSplitsRequest_EFormat, - connectorClient api_service.ConnectorClient, - dsi *api_common.TDataSourceInstance, -) error { - req := &api_service_protos.TReadSplitsRequest{Splits: splits, Format: format, DataSourceInstance: dsi} - logger.Debug("ReadSplits", log.String("request", req.String())) - - streamReadSplits, err := connectorClient.ReadSplits(context.Background(), req) - if err != nil { - return fmt.Errorf("list splits: %w", err) - } - - var responses []*api_service_protos.TReadSplitsResponse - - for { - resp, err := streamReadSplits.Recv() - if err != nil { - if err == io.EOF { - break - } - - return fmt.Errorf("stream list splits: %w", err) - } - - if !utils.IsSuccess(resp.Error) { - return utils.NewSTDErrorFromAPIError(resp.Error) - } - - responses = append(responses, resp) - } - - if err := dumpReadResponses(logger, format, responses); err != nil { - return fmt.Errorf("dump read responses: %w", err) - } - - return nil -} - -func dumpReadResponses( - logger log.Logger, - format api_service_protos.TReadSplitsRequest_EFormat, - responses []*api_service_protos.TReadSplitsResponse, -) error { - if format == api_service_protos.TReadSplitsRequest_ARROW_IPC_STREAMING { - for _, resp := range responses { - buf := bytes.NewBuffer(resp.GetArrowIpcStreaming()) - - reader, err := ipc.NewReader(buf) - if err != nil { - return fmt.Errorf("new reader: %w", err) - } - - for reader.Next() { - record := reader.Record() - logger.Debug("schema", log.String("schema", record.Schema().String())) - - for i, column := range record.Columns() { - logger.Debug("column", log.Int("id", i), log.String("data", column.String())) - } - } - - reader.Release() - } - - return nil - } - - return fmt.Errorf("unknown format: %v", format) -} - -var Cmd = &cobra.Command{ - Use: "client", - Short: "client for Connector testing and debugging purposes", - Run: func(cmd *cobra.Command, args []string) { - if err := runClient(cmd, args); err != nil { - fmt.Println(err) - os.Exit(1) - } - }, -} diff --git a/ydb/library/yql/providers/generic/connector/app/client/doc.go b/ydb/library/yql/providers/generic/connector/app/client/doc.go deleted file mode 100644 index 60f37e4bae2d..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/client/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package client contains code of a simple CLI application -// that can be used for server debugging. -package client diff --git a/ydb/library/yql/providers/generic/connector/app/client/ya.make b/ydb/library/yql/providers/generic/connector/app/client/ya.make deleted file mode 100644 index 47bd0180b045..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/client/ya.make +++ /dev/null @@ -1,8 +0,0 @@ -GO_LIBRARY() - -SRCS( - client.go - doc.go -) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/config/client.proto b/ydb/library/yql/providers/generic/connector/app/config/client.proto deleted file mode 100644 index 166a518c5e3d..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/config/client.proto +++ /dev/null @@ -1,22 +0,0 @@ -syntax = "proto3"; -package NYql.Connector.App.Config; - -import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto"; -import "ydb/library/yql/providers/generic/connector/api/common/endpoint.proto"; - -option go_package = "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config"; - -// Connector client configuration -message ClientConfig { - // Connector service instance network address we connect to - NYql.NConnector.NApi.TEndpoint endpoint = 1; - // TLS credentials for Connector - ClientTLSConfig tls = 2; - // Data source instance we read data from - NYql.NConnector.NApi.TDataSourceInstance data_source_instance = 3; -} - -message ClientTLSConfig { - // CA certificate path - string ca = 1; -} diff --git a/ydb/library/yql/providers/generic/connector/app/config/server.proto b/ydb/library/yql/providers/generic/connector/app/config/server.proto deleted file mode 100644 index 0da9f25f2b63..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/config/server.proto +++ /dev/null @@ -1,106 +0,0 @@ -syntax = "proto3"; -package NYql.Connector.App.Config; - -import "ydb/library/yql/providers/generic/connector/api/common/endpoint.proto"; - -option go_package = "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config"; - -// Connector server configuration -message TServerConfig { - // TODO: remove it after YQ-2057 - NYql.NConnector.NApi.TEndpoint endpoint = 1 [deprecated=true]; - TServerTLSConfig tls = 2 [deprecated=true]; - - TConnectorServerConfig connector_server = 5; - // This is a rough restriction for YQ memory consumption until - // https://st.yandex-team.ru/YQ-2057 is implemented. - // Leave it empty if you want to avoid any memory limits. - TServerReadLimit read_limit = 3; - // Logger config - TLoggerConfig logger = 4; - // Go runtime profiler. - // Disabled if this part of config is empty. - TPprofServerConfig pprof_server = 6; - // Metrics server config - TMetricsServerConfig metrics_server = 7; - // Paging settings - TPagingConfig paging = 8; -} - -// TConnectorServerConfig - configuration of the main GRPC server -message TConnectorServerConfig { - // Network address server will be listening on - NYql.NConnector.NApi.TEndpoint endpoint = 1; - // TLS settings. - // Leave it empty for insecure connections. - TServerTLSConfig tls = 2; -} - -message TServerTLSConfig { - // TLS private key path - string key = 2; - // TLS public cert path - string cert = 3; - - reserved 1; -} - -// ServerReadLimit limitates the amount of data extracted from the data source on every read request. -message TServerReadLimit { - // The number of rows extracted from the data source - uint64 rows = 1; -} - -// TLogger represents logger configuration -message TLoggerConfig { - // Level of logging - ELogLevel log_level = 1; - // Is logging of queries enabled - bool enable_sql_query_logging = 2; -} - -// ELogLevel enumerates standard levels of logging -enum ELogLevel { - TRACE = 0; - DEBUG = 1; - INFO = 2; - WARN = 3; - ERROR = 4; - FATAL = 5; -} - -// TPprofServerConfig configures HTTP server delivering Go runtime profiler data -message TPprofServerConfig { - // Network address server will be listening on - NYql.NConnector.NApi.TEndpoint endpoint = 1; - // TLS settings. - // Leave it empty for insecure connections. - TServerTLSConfig tls = 2; -} - -// TMetricsConfig - configuration of the metrics service -message TMetricsServerConfig { - // Network address server will be listening on - NYql.NConnector.NApi.TEndpoint endpoint = 1; - // TLS settings. - // Leave it empty for insecure connections. - TServerTLSConfig tls = 2; -} - -// TPagingConfig configures the way of splitting of the data stream into the fragments (or pages) -// in order to return them as separate GRPC messages to the client. -message TPagingConfig { - // Configures the limit of a page size in rows. - // It may override other limits. Ignored if set to zero. - uint64 rows_per_page = 1; - - // Configures the limit of a page size in bytes. - // It may override other limits. Ignored if set to zero. - uint64 bytes_per_page = 2; - - // Connector service can read ahead some data from the data source before returning it to the client. - // This setting configures the number of pages that may reside in the service's address space - // waiting for the client readiness for the data consumption. - // Tune this carefully cause this may cause service OOMs. - uint32 prefetch_queue_capacity = 3; -} diff --git a/ydb/library/yql/providers/generic/connector/app/config/ya.make b/ydb/library/yql/providers/generic/connector/app/config/ya.make deleted file mode 100644 index df2e39997be0..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/config/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -PROTO_LIBRARY() - -ONLY_TAGS(GO_PROTO) - -SRCS( - client.proto - server.proto -) - -PEERDIR( - ydb/library/yql/providers/generic/connector/api/common -) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/main.go b/ydb/library/yql/providers/generic/connector/app/main.go deleted file mode 100644 index 0361f446881d..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/main.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/client" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server" -) - -var rootCmd = &cobra.Command{ - Use: "connector", - Short: "Connector for external data sources", - // Run: func(cmd *cobra.Command, args []string) {}, -} - -func init() { - rootCmd.AddCommand(server.Cmd) - rootCmd.AddCommand(client.Cmd) -} - -func main() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/cmd.go b/ydb/library/yql/providers/generic/connector/app/server/cmd.go deleted file mode 100644 index a5631ee06103..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/cmd.go +++ /dev/null @@ -1,30 +0,0 @@ -package server - -import ( - "fmt" - "log" - "os" - - "github.com/spf13/cobra" -) - -var Cmd = &cobra.Command{ - Use: "server", - Short: "Connector server", - Run: func(cmd *cobra.Command, args []string) { - if err := run(cmd, args); err != nil { - fmt.Println(err) - os.Exit(1) - } - }, -} - -const configFlag = "config" - -func init() { - Cmd.Flags().StringP(configFlag, "c", "", "path to server config file") - - if err := Cmd.MarkFlagRequired(configFlag); err != nil { - log.Fatal(err) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/config.go b/ydb/library/yql/providers/generic/connector/app/server/config.go deleted file mode 100644 index 098dba874731..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/config.go +++ /dev/null @@ -1,178 +0,0 @@ -package server - -import ( - "fmt" - "math" - "os" - - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "google.golang.org/protobuf/encoding/prototext" -) - -func fillServerConfigDefaults(c *config.TServerConfig) { - if c.Paging == nil { - c.Paging = &config.TPagingConfig{ - BytesPerPage: 4 * 1024 * 1024, - PrefetchQueueCapacity: 2, - } - } -} - -func validateServerConfig(c *config.TServerConfig) error { - if err := validateConnectorServerConfig(c.ConnectorServer); err != nil { - return fmt.Errorf("validate `connector_server`: %w", err) - } - - if err := validateServerReadLimit(c.ReadLimit); err != nil { - return fmt.Errorf("validate `read_limit`: %w", err) - } - - if err := validatePprofServerConfig(c.PprofServer); err != nil { - return fmt.Errorf("validate `pprof_server`: %w", err) - } - - if err := validatePagingConfig(c.Paging); err != nil { - return fmt.Errorf("validate `paging`: %w", err) - } - - return nil -} - -func validateConnectorServerConfig(c *config.TConnectorServerConfig) error { - if c == nil { - return fmt.Errorf("required section is missing") - } - - if err := validateEndpoint(c.Endpoint); err != nil { - return fmt.Errorf("validate `endpoint`: %w", err) - } - - if err := validateServerTLSConfig(c.Tls); err != nil { - return fmt.Errorf("validate `tls`: %w", err) - } - - return nil -} - -func validateEndpoint(c *api_common.TEndpoint) error { - if c == nil { - return fmt.Errorf("required section is missing") - } - - if c.Host == "" { - return fmt.Errorf("invalid value of field `host`: %v", c.Host) - } - - if c.Port == 0 || c.Port > math.MaxUint16 { - return fmt.Errorf("invalid value of field `port`: %v", c.Port) - } - - return nil -} - -func validateServerTLSConfig(c *config.TServerTLSConfig) error { - if c == nil { - // It's OK not to have TLS config section - return nil - } - - if err := fileMustExist(c.Key); err != nil { - return fmt.Errorf("invalid value of field `key`: %w", err) - } - - if err := fileMustExist(c.Cert); err != nil { - return fmt.Errorf("invalid value of field `cert`: %w", err) - } - - return nil -} - -func validateServerReadLimit(c *config.TServerReadLimit) error { - if c == nil { - // It's OK not to have read request memory limitation - return nil - } - - // but if it's not nil, one must set limits explicitly - if c.GetRows() == 0 { - return fmt.Errorf("invalid value of field `rows`") - } - - return nil -} - -func validatePprofServerConfig(c *config.TPprofServerConfig) error { - if c == nil { - // It's OK to disable profiler - return nil - } - - if err := validateEndpoint(c.Endpoint); err != nil { - return fmt.Errorf("validate `endpoint`: %w", err) - } - - if err := validateServerTLSConfig(c.Tls); err != nil { - return fmt.Errorf("validate `tls`: %w", err) - } - - return nil -} - -const maxInterconnectMessageSize = 50 * 1024 * 1024 - -func validatePagingConfig(c *config.TPagingConfig) error { - if c == nil { - return fmt.Errorf("required section is missing") - } - - limitIsSet := c.BytesPerPage != 0 || c.RowsPerPage != 0 - if !limitIsSet { - return fmt.Errorf("you must set either `bytes_per_page` or `rows_per_page` or both of them") - } - - if c.BytesPerPage > maxInterconnectMessageSize { - return fmt.Errorf("`bytes_per_page` limit exceeds the limits of interconnect system used by YDB engine") - } - - return nil -} - -func fileMustExist(path string) error { - info, err := os.Stat(path) - if os.IsNotExist(err) { - return fmt.Errorf("path '%s' does not exist", path) - } - - if info.IsDir() { - return fmt.Errorf("path '%s' is a directory", path) - } - - return nil -} - -func newConfigFromPath(configPath string) (*config.TServerConfig, error) { - data, err := os.ReadFile(configPath) - if err != nil { - return nil, fmt.Errorf("read file %v: %w", configPath, err) - } - - var cfg config.TServerConfig - - unmarshaller := prototext.UnmarshalOptions{ - // Do not emit an error if config contains outdated or too fresh fields - DiscardUnknown: true, - } - - if err := unmarshaller.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("prototext unmarshal `%v`: %w", string(data), err) - } - - fillServerConfigDefaults(&cfg) - - if err := validateServerConfig(&cfg); err != nil { - return nil, fmt.Errorf("validate config: %w", err) - } - - return &cfg, nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/data_source_collection.go b/ydb/library/yql/providers/generic/connector/app/server/data_source_collection.go deleted file mode 100644 index 07fbecdc060c..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/data_source_collection.go +++ /dev/null @@ -1,144 +0,0 @@ -package server - -import ( - "context" - "fmt" - - "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/s3" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/streaming" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type DataSourceCollection struct { - rdbms datasource.DataSourceFactory[any] - memoryAllocator memory.Allocator - readLimiterFactory *paging.ReadLimiterFactory - cfg *config.TServerConfig -} - -func (dsc *DataSourceCollection) DescribeTable( - ctx context.Context, logger log.Logger, request *api_service_protos.TDescribeTableRequest, -) (*api_service_protos.TDescribeTableResponse, error) { - kind := request.GetDataSourceInstance().GetKind() - - switch kind { - case api_common.EDataSourceKind_CLICKHOUSE, api_common.EDataSourceKind_POSTGRESQL: - ds, err := dsc.rdbms.Make(logger, kind) - if err != nil { - return nil, err - } - - return ds.DescribeTable(ctx, logger, request) - case api_common.EDataSourceKind_S3: - ds := s3.NewDataSource() - - return ds.DescribeTable(ctx, logger, request) - default: - return nil, fmt.Errorf("unsupported data source type '%v': %w", kind, utils.ErrDataSourceNotSupported) - } -} - -func (dsc *DataSourceCollection) DoReadSplit( - logger log.Logger, - stream api_service.Connector_ReadSplitsServer, - request *api_service_protos.TReadSplitsRequest, - split *api_service_protos.TSplit, -) error { - switch kind := request.GetDataSourceInstance().GetKind(); kind { - case api_common.EDataSourceKind_CLICKHOUSE, api_common.EDataSourceKind_POSTGRESQL: - ds, err := dsc.rdbms.Make(logger, kind) - if err != nil { - return err - } - - return readSplit[any](logger, stream, request, split, ds, dsc.memoryAllocator, dsc.readLimiterFactory, dsc.cfg) - case api_common.EDataSourceKind_S3: - ds := s3.NewDataSource() - - return readSplit[string](logger, stream, request, split, ds, dsc.memoryAllocator, dsc.readLimiterFactory, dsc.cfg) - default: - return fmt.Errorf("unsupported data source type '%v': %w", kind, utils.ErrDataSourceNotSupported) - } -} - -func readSplit[T utils.Acceptor]( - logger log.Logger, - stream api_service.Connector_ReadSplitsServer, - request *api_service_protos.TReadSplitsRequest, - split *api_service_protos.TSplit, - dataSource datasource.DataSource[T], - memoryAllocator memory.Allocator, - readLimiterFactory *paging.ReadLimiterFactory, - cfg *config.TServerConfig, -) error { - logger.Debug("split reading started", utils.SelectToFields(split.Select)...) - - columnarBufferFactory, err := paging.NewColumnarBufferFactory[T]( - logger, - memoryAllocator, - request.Format, - split.Select.What) - if err != nil { - return fmt.Errorf("new columnar buffer factory: %w", err) - } - - trafficTracker := paging.NewTrafficTracker[T](cfg.Paging) - - sink, err := paging.NewSink( - stream.Context(), - logger, - trafficTracker, - columnarBufferFactory, - readLimiterFactory.MakeReadLimiter(logger), - int(cfg.Paging.PrefetchQueueCapacity), - ) - if err != nil { - return fmt.Errorf("new sink: %w", err) - } - - streamer := streaming.NewStreamer( - logger, - stream, - request, - split, - sink, - dataSource, - ) - - if err := streamer.Run(); err != nil { - return fmt.Errorf("run paging streamer: %w", err) - } - - readStats := trafficTracker.DumpStats(true) - - logger.Debug( - "split reading finished", - log.UInt64("total_bytes", readStats.GetBytes()), - log.UInt64("total_rows", readStats.GetRows()), - ) - - return nil -} - -func NewDataSourceCollection( - queryLoggerFactory utils.QueryLoggerFactory, - memoryAllocator memory.Allocator, - readLimiterFactory *paging.ReadLimiterFactory, - cfg *config.TServerConfig, -) *DataSourceCollection { - return &DataSourceCollection{ - rdbms: rdbms.NewDataSourceFactory(queryLoggerFactory), - memoryAllocator: memoryAllocator, - readLimiterFactory: readLimiterFactory, - cfg: cfg, - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/doc.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/doc.go deleted file mode 100644 index dd6c95ceef91..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package datasource describes the interface that must be satisfied by any external data source -// accessible through the connector. -package datasource diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/interface.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/interface.go deleted file mode 100644 index 8f6e4eb57883..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/interface.go +++ /dev/null @@ -1,39 +0,0 @@ -package datasource - -import ( - "context" - - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type DataSourceFactory[T utils.Acceptor] interface { - Make( - logger log.Logger, - dataSourceType api_common.EDataSourceKind, - ) (DataSource[T], error) -} - -// DataSource is an abstraction over external data storage that is available for data and metadata extraction. -// All new data sources must implement this interface. -// The types of data extracted from the data source are parametrized via [T utils.Acceptor] interface. -type DataSource[T utils.Acceptor] interface { - // DescribeTable returns metadata about a table (or similair entity in non-relational data sources) - // located within a particular database in a data source cluster. - DescribeTable( - ctx context.Context, - logger log.Logger, - request *api_service_protos.TDescribeTableRequest, - ) (*api_service_protos.TDescribeTableResponse, error) - - // ReadSplit is a main method for reading data from the table. - ReadSplit( - ctx context.Context, - logger log.Logger, - split *api_service_protos.TSplit, - sink paging.Sink[T], - ) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/mock.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/mock.go deleted file mode 100644 index 14ddb3c9c77c..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/mock.go +++ /dev/null @@ -1,38 +0,0 @@ -package datasource - -import ( - "context" - - "github.com/stretchr/testify/mock" - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var _ DataSource[any] = (*DataSourceMock[any])(nil) - -type DataSourceMock[T utils.Acceptor] struct { - mock.Mock -} - -func (m *DataSourceMock[T]) DescribeTable( - ctx context.Context, - logger log.Logger, - request *api_service_protos.TDescribeTableRequest, -) (*api_service_protos.TDescribeTableResponse, error) { - panic("not implemented") // TODO: Implement -} - -func (m *DataSourceMock[T]) ReadSplit( - ctx context.Context, - logger log.Logger, - split *api_service_protos.TSplit, - pagingWriter paging.Sink[T], -) { - m.Called(split, pagingWriter) -} - -func (m *DataSourceMock[T]) TypeMapper() utils.TypeMapper { - panic("not implemented") // TODO: Implement -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/connection_manager.go deleted file mode 100644 index 4b6000879d4f..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/connection_manager.go +++ /dev/null @@ -1,147 +0,0 @@ -package clickhouse - -import ( - "context" - "crypto/tls" - "database/sql" - "fmt" - "time" - - "github.com/ClickHouse/clickhouse-go/v2" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -var _ rdbms_utils.Connection = (*Connection)(nil) - -type Connection struct { - *sql.DB - logger utils.QueryLogger -} - -type rows struct { - *sql.Rows -} - -func (r rows) MakeTransformer(ydbTypes []*Ydb.Type) (utils.RowTransformer[any], error) { - columns, err := r.ColumnTypes() - if err != nil { - return nil, fmt.Errorf("column types: %w", err) - } - - typeNames := make([]string, 0, len(columns)) - for _, column := range columns { - typeNames = append(typeNames, column.DatabaseTypeName()) - } - - transformer, err := transformerFromSQLTypes(typeNames, ydbTypes) - if err != nil { - return nil, fmt.Errorf("transformer from sql types: %w", err) - } - - return transformer, nil -} - -func (c Connection) Query(ctx context.Context, query string, args ...any) (rdbms_utils.Rows, error) { - c.logger.Dump(query, args...) - - out, err := c.DB.QueryContext(ctx, query, args...) - if err != nil { - return nil, fmt.Errorf("query context: %w", err) - } - - if err := out.Err(); err != nil { - defer func() { - if err := out.Close(); err != nil { - c.logger.Error("close rows", log.Error(err)) - } - }() - - return nil, fmt.Errorf("rows err: %w", err) - } - - return rows{Rows: out}, nil -} - -var _ rdbms_utils.ConnectionManager = (*connectionManager)(nil) - -type connectionManager struct { - rdbms_utils.ConnectionManagerBase - // TODO: cache of connections, remove unused connections with TTL -} - -func (c *connectionManager) Make( - ctx context.Context, - logger log.Logger, - dsi *api_common.TDataSourceInstance, -) (rdbms_utils.Connection, error) { - if dsi.GetCredentials().GetBasic() == nil { - return nil, fmt.Errorf("currently only basic auth is supported") - } - - var protocol clickhouse.Protocol - - switch dsi.Protocol { - case api_common.EProtocol_NATIVE: - protocol = clickhouse.Native - case api_common.EProtocol_HTTP: - protocol = clickhouse.HTTP - default: - return nil, fmt.Errorf("can not run ClickHouse connection with protocol '%v'", dsi.Protocol) - } - - opts := &clickhouse.Options{ - Addr: []string{utils.EndpointToString(dsi.GetEndpoint())}, - Auth: clickhouse.Auth{ - Database: dsi.Database, - Username: dsi.Credentials.GetBasic().Username, - Password: dsi.Credentials.GetBasic().Password, - }, - // Set this field to true if you want to see ClickHouse driver's debug output - Debug: false, - Debugf: func(format string, v ...any) { - logger.Debugf(format, v...) - }, - // TODO: make it configurable via Connector API - Compression: &clickhouse.Compression{ - Method: clickhouse.CompressionLZ4, - }, - Protocol: protocol, - } - - if dsi.UseTls { - opts.TLS = &tls.Config{ - InsecureSkipVerify: false, - } - } - - conn := clickhouse.OpenDB(opts) - if err := conn.Ping(); err != nil { - return nil, fmt.Errorf("conn ping: %w", err) - } - - const ( - maxIdleConns = 5 - maxOpenConns = 10 - connMaxLifetime = time.Hour - ) - - conn.SetMaxIdleConns(maxIdleConns) - conn.SetMaxOpenConns(maxOpenConns) - conn.SetConnMaxLifetime(connMaxLifetime) - - queryLogger := c.QueryLoggerFactory.Make(logger) - - return &Connection{DB: conn, logger: queryLogger}, nil -} - -func (c *connectionManager) Release(logger log.Logger, conn rdbms_utils.Connection) { - utils.LogCloserError(logger, conn, "close clickhouse connection") -} - -func NewConnectionManager(cfg rdbms_utils.ConnectionManagerBase) rdbms_utils.ConnectionManager { - return &connectionManager{ConnectionManagerBase: cfg} -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/doc.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/doc.go deleted file mode 100644 index aeeb567bdb74..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package clickhouse contains code specific for ClickHouse database. -package clickhouse diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/sql_formatter.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/sql_formatter.go deleted file mode 100644 index af793a40ab87..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/sql_formatter.go +++ /dev/null @@ -1,92 +0,0 @@ -package clickhouse - -import ( - "strings" - - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var _ rdbms_utils.SQLFormatter = (*sqlFormatter)(nil) - -type sqlFormatter struct { -} - -func (f *sqlFormatter) supportsType(typeID Ydb.Type_PrimitiveTypeId) bool { - switch typeID { - case Ydb.Type_BOOL: - return true - case Ydb.Type_INT8: - return true - case Ydb.Type_UINT8: - return true - case Ydb.Type_INT16: - return true - case Ydb.Type_UINT16: - return true - case Ydb.Type_INT32: - return true - case Ydb.Type_UINT32: - return true - case Ydb.Type_INT64: - return true - case Ydb.Type_UINT64: - return true - case Ydb.Type_FLOAT: - return true - case Ydb.Type_DOUBLE: - return true - default: - return false - } -} - -func (f *sqlFormatter) supportsConstantValueExpression(t *Ydb.Type) bool { - switch v := t.Type.(type) { - case *Ydb.Type_TypeId: - return f.supportsType(v.TypeId) - case *Ydb.Type_OptionalType: - return f.supportsConstantValueExpression(v.OptionalType.Item) - default: - return false - } -} - -func (f sqlFormatter) SupportsPushdownExpression(expression *api_service_protos.TExpression) bool { - switch e := expression.Payload.(type) { - case *api_service_protos.TExpression_Column: - return true - case *api_service_protos.TExpression_TypedValue: - return f.supportsConstantValueExpression(e.TypedValue.Type) - case *api_service_protos.TExpression_ArithmeticalExpression: - return false - case *api_service_protos.TExpression_Null: - return true - default: - return false - } -} - -func (f sqlFormatter) GetDescribeTableQuery(request *api_service_protos.TDescribeTableRequest) (string, []any) { - query := "SELECT name, type FROM system.columns WHERE table = ? and database = ?" - args := []any{request.Table, request.DataSourceInstance.Database} - - return query, args -} - -func (f sqlFormatter) GetPlaceholder(_ int) string { - return "?" -} - -func (f sqlFormatter) SanitiseIdentifier(ident string) string { - // https: //clickhouse.com/docs/en/sql-reference/syntax#identifiers - sanitizedIdent := strings.ReplaceAll(ident, string([]byte{0}), "") - sanitizedIdent = `"` + strings.ReplaceAll(sanitizedIdent, `"`, `""`) + `"` - - return sanitizedIdent -} - -func NewSQLFormatter() rdbms_utils.SQLFormatter { - return sqlFormatter{} -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/sql_formatter_test.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/sql_formatter_test.go deleted file mode 100644 index f15102e42754..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/sql_formatter_test.go +++ /dev/null @@ -1,426 +0,0 @@ -package clickhouse - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/require" - ydb "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func TestMakeDescribeTableQuery(t *testing.T) { - logger := utils.NewTestLogger(t) - formatter := NewSQLFormatter() - request := &api.TDescribeTableRequest{Table: "table", DataSourceInstance: &common.TDataSourceInstance{Database: "db"}} - - output, args := rdbms_utils.MakeDescribeTableQuery(logger, formatter, request) - require.Equal(t, "SELECT name, type FROM system.columns WHERE table = ? and database = ?", output) - require.Equal(t, args, []any{"table", "db"}) -} - -func TestMakeSQLFormatterQuery(t *testing.T) { - type testCase struct { - testName string - selectReq *api.TSelect - outputQuery string - outputArgs []any - err error - } - - logger := utils.NewTestLogger(t) - formatter := NewSQLFormatter() - - tcs := []testCase{ - { - testName: "empty_table_name", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "", - }, - What: &api.TSelect_TWhat{}, - }, - outputQuery: "", - outputArgs: nil, - err: utils.ErrEmptyTableName, - }, - { - testName: "empty_no columns", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: &api.TSelect_TWhat{}, - }, - outputQuery: `SELECT 0 FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "select_col", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: &api.TSelect_TWhat{ - Items: []*api.TSelect_TWhat_TItem{ - &api.TSelect_TWhat_TItem{ - Payload: &api.TSelect_TWhat_TItem_Column{ - Column: &ydb.Column{ - Name: "col", - Type: utils.NewPrimitiveType(ydb.Type_INT32), - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "col" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "is_null", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_IsNull{ - IsNull: &api.TPredicate_TIsNull{ - Value: utils.NewColumnExpression("col1"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE ("col1" IS NULL)`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "is_not_null", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_IsNotNull{ - IsNotNull: &api.TPredicate_TIsNotNull{ - Value: utils.NewColumnExpression("col2"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE ("col2" IS NOT NULL)`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "bool_column", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_BoolExpression{ - BoolExpression: &api.TPredicate_TBoolExpression{ - Value: utils.NewColumnExpression("col2"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE "col2"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "complex_filter", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Disjunction{ - Disjunction: &api.TPredicate_TDisjunction{ - Operands: []*api.TPredicate{ - &api.TPredicate{ - Payload: &api.TPredicate_Negation{ - Negation: &api.TPredicate_TNegation{ - Operand: &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_LE, - LeftValue: utils.NewColumnExpression("col2"), - RightValue: utils.NewInt32ValueExpression(42), - }, - }, - }, - }, - }, - }, - &api.TPredicate{ - Payload: &api.TPredicate_Conjunction{ - Conjunction: &api.TPredicate_TConjunction{ - Operands: []*api.TPredicate{ - &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_NE, - LeftValue: utils.NewColumnExpression("col1"), - RightValue: utils.NewUint64ValueExpression(0), - }, - }, - }, - &api.TPredicate{ - Payload: &api.TPredicate_IsNull{ - IsNull: &api.TPredicate_TIsNull{ - Value: utils.NewColumnExpression("col3"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE ((NOT ("col2" <= ?)) OR (("col1" <> ?) AND ("col3" IS NULL)))`, - outputArgs: []any{int32(42), uint64(0)}, - err: nil, - }, - { - testName: "unsupported_predicate", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Between{ - Between: &api.TPredicate_TBetween{ - Value: utils.NewColumnExpression("col2"), - Least: utils.NewColumnExpression("col1"), - Greatest: utils.NewColumnExpression("col3"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "unsupported_type", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col2"), - RightValue: utils.NewTextValueExpression("text"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "partial_filter_removes_and", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Conjunction{ - Conjunction: &api.TPredicate_TConjunction{ - Operands: []*api.TPredicate{ - &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col1"), - RightValue: utils.NewInt32ValueExpression(32), - }, - }, - }, - &api.TPredicate{ - // Not supported - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col2"), - RightValue: utils.NewTextValueExpression("text"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE ("col1" = ?)`, - outputArgs: []any{int32(32)}, - err: nil, - }, - { - testName: "partial_filter", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Conjunction{ - Conjunction: &api.TPredicate_TConjunction{ - Operands: []*api.TPredicate{ - &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col1"), - RightValue: utils.NewInt32ValueExpression(32), - }, - }, - }, - &api.TPredicate{ - // Not supported - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col2"), - RightValue: utils.NewTextValueExpression("text"), - }, - }, - }, - &api.TPredicate{ - Payload: &api.TPredicate_IsNull{ - IsNull: &api.TPredicate_TIsNull{ - Value: utils.NewColumnExpression("col3"), - }, - }, - }, - &api.TPredicate{ - Payload: &api.TPredicate_IsNotNull{ - IsNotNull: &api.TPredicate_TIsNotNull{ - Value: utils.NewColumnExpression("col4"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE (("col1" = ?) AND ("col3" IS NULL) AND ("col4" IS NOT NULL))`, - outputArgs: []any{int32(32)}, - err: nil, - }, - { - testName: "negative_sql_injection_by_table", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: `information_schema.columns; DROP TABLE information_schema.columns`, - }, - What: &api.TSelect_TWhat{}, - }, - outputQuery: `SELECT 0 FROM "information_schema.columns; DROP TABLE information_schema.columns"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "negative_sql_injection_by_col", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: &api.TSelect_TWhat{ - Items: []*api.TSelect_TWhat_TItem{ - &api.TSelect_TWhat_TItem{ - Payload: &api.TSelect_TWhat_TItem_Column{ - Column: &ydb.Column{ - Name: `0; DROP TABLE information_schema.columns`, - Type: utils.NewPrimitiveType(ydb.Type_INT32), - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "0; DROP TABLE information_schema.columns" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "negative_sql_injection_fake_quotes", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: &api.TSelect_TWhat{ - Items: []*api.TSelect_TWhat_TItem{ - &api.TSelect_TWhat_TItem{ - Payload: &api.TSelect_TWhat_TItem_Column{ - Column: &ydb.Column{ - Name: `0"; DROP TABLE information_schema.columns;`, - Type: utils.NewPrimitiveType(ydb.Type_INT32), - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "0""; DROP TABLE information_schema.columns;" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - } - - for _, tc := range tcs { - tc := tc - - t.Run(tc.testName, func(t *testing.T) { - outputQuery, outputArgs, err := rdbms_utils.MakeReadSplitQuery(logger, formatter, tc.selectReq) - require.Equal(t, tc.outputQuery, outputQuery) - require.Equal(t, tc.outputArgs, outputArgs) - - if tc.err != nil { - require.True(t, errors.Is(err, tc.err)) - } else { - require.NoError(t, err) - } - }) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/type_mapper.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/type_mapper.go deleted file mode 100644 index e021b49e46f7..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/type_mapper.go +++ /dev/null @@ -1,337 +0,0 @@ -package clickhouse - -import ( - "errors" - "fmt" - "regexp" - "time" - - "github.com/apache/arrow/go/v13/arrow/array" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var _ utils.TypeMapper = typeMapper{} - -type typeMapper struct { - isFixedString *regexp.Regexp - isDateTime *regexp.Regexp - isDateTime64 *regexp.Regexp - isNullable *regexp.Regexp -} - -func (tm typeMapper) SQLTypeToYDBColumn(columnName, typeName string, rules *api_service_protos.TTypeMappingSettings) (*Ydb.Column, error) { - var ( - ydbType *Ydb.Type - err error - ) - - // By default all columns in CH are non-nullable, so - // we wrap YDB types into Optional type only in such cases: - // - // 1. The column is explicitly defined as nullable; - // 2. The column type is a date/time. CH value ranges for date/time are much wider than YQL value ranges, - // so every time we encounter a value that is out of YQL ranges, we have to return NULL. - nullable := false - - if matches := tm.isNullable.FindStringSubmatch(typeName); len(matches) > 0 { - nullable = true - typeName = matches[1] - } - - // Reference table: https://wiki.yandex-team.ru/rtmapreduce/yql-streams-corner/connectors/lld-02-tipy-dannyx/ - switch { - case typeName == "Bool": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_BOOL}} - case typeName == "Int8": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT8}} - case typeName == "UInt8": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UINT8}} - case typeName == "Int16": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT16}} - case typeName == "UInt16": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UINT16}} - case typeName == "Int32": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT32}} - case typeName == "UInt32": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UINT32}} - case typeName == "Int64": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT64}} - case typeName == "UInt64": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UINT64}} - case typeName == "Float32": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_FLOAT}} - case typeName == "Float64": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_DOUBLE}} - // String/FixedString are binary in ClickHouse, so we map it to YDB's String instead of UTF8: - // https://ydb.tech/en/docs/yql/reference/types/primitive#string - // https://clickhouse.com/docs/en/sql-reference/data-types/string#encodings - case typeName == "String": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_STRING}} - case tm.isFixedString.MatchString(typeName): - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_STRING}} - case typeName == "Date": - var overflow bool - ydbType, overflow, err = makeYdbDateTimeType(Ydb.Type_DATE, rules.GetDateTimeFormat()) - nullable = overflow || nullable - case typeName == "Date32": - // NOTE: ClickHouse's Date32 value range is much more wide than YDB's Date value range - var overflow bool - ydbType, overflow, err = makeYdbDateTimeType(Ydb.Type_DATE, rules.GetDateTimeFormat()) - nullable = overflow || nullable - case tm.isDateTime64.MatchString(typeName): - // NOTE: ClickHouse's DateTime64 value range is much more wide than YDB's Timestamp value range - var overflow bool - ydbType, overflow, err = makeYdbDateTimeType(Ydb.Type_TIMESTAMP, rules.GetDateTimeFormat()) - nullable = overflow || nullable - case tm.isDateTime.MatchString(typeName): - var overflow bool - ydbType, overflow, err = makeYdbDateTimeType(Ydb.Type_DATETIME, rules.GetDateTimeFormat()) - nullable = overflow || nullable - default: - err = fmt.Errorf("convert type '%s': %w", typeName, utils.ErrDataTypeNotSupported) - } - - if err != nil { - return nil, err - } - - // If the column is nullable, wrap it into YQL's optional - if nullable { - ydbType = &Ydb.Type{Type: &Ydb.Type_OptionalType{OptionalType: &Ydb.OptionalType{Item: ydbType}}} - } - - return &Ydb.Column{ - Name: columnName, - Type: ydbType, - }, nil -} - -func makeYdbDateTimeType(ydbTypeID Ydb.Type_PrimitiveTypeId, format api_service_protos.EDateTimeFormat) (*Ydb.Type, bool, error) { - switch format { - case api_service_protos.EDateTimeFormat_YQL_FORMAT: - // type marked as nullable because ClickHouse's type value range is much more wide than YDB's type value range - return &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: ydbTypeID}}, true, nil - case api_service_protos.EDateTimeFormat_STRING_FORMAT: - return &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UTF8}}, false, nil - default: - return nil, false, fmt.Errorf("unexpected datetime format '%s': %w", format, utils.ErrDataTypeNotSupported) - } -} - -func transformerFromSQLTypes(typeNames []string, ydbTypes []*Ydb.Type) (utils.RowTransformer[any], error) { - acceptors := make([]any, 0, len(typeNames)) - appenders := make([]func(acceptor any, builder array.Builder) error, 0, len(typeNames)) - isNullable := regexp.MustCompile(`Nullable\((?P[\w\(\)]+)\)`) - isFixedString := regexp.MustCompile(`FixedString\([0-9]+\)`) - isDateTime := regexp.MustCompile(`DateTime(\('[\w,/]+'\))?`) - isDateTime64 := regexp.MustCompile(`DateTime64\(\d{1}(, '[\w,/]+')?\)`) - - for i, typeName := range typeNames { - if matches := isNullable.FindStringSubmatch(typeName); len(matches) > 0 { - typeName = matches[1] - } - - switch { - case typeName == "Bool": - acceptors = append(acceptors, new(*bool)) - appenders = append(appenders, appendValueToArrowBuilder[bool, uint8, *array.Uint8Builder, utils.BoolConverter]) - case typeName == "Int8": - acceptors = append(acceptors, new(*int8)) - appenders = append(appenders, appendValueToArrowBuilder[int8, int8, *array.Int8Builder, utils.Int8Converter]) - case typeName == "Int16": - acceptors = append(acceptors, new(*int16)) - appenders = append(appenders, appendValueToArrowBuilder[int16, int16, *array.Int16Builder, utils.Int16Converter]) - case typeName == "Int32": - acceptors = append(acceptors, new(*int32)) - appenders = append(appenders, appendValueToArrowBuilder[int32, int32, *array.Int32Builder, utils.Int32Converter]) - case typeName == "Int64": - acceptors = append(acceptors, new(*int64)) - appenders = append(appenders, appendValueToArrowBuilder[int64, int64, *array.Int64Builder, utils.Int64Converter]) - case typeName == "UInt8": - acceptors = append(acceptors, new(*uint8)) - appenders = append(appenders, appendValueToArrowBuilder[uint8, uint8, *array.Uint8Builder, utils.Uint8Converter]) - case typeName == "UInt16": - acceptors = append(acceptors, new(*uint16)) - appenders = append(appenders, appendValueToArrowBuilder[uint16, uint16, *array.Uint16Builder, utils.Uint16Converter]) - case typeName == "UInt32": - acceptors = append(acceptors, new(*uint32)) - appenders = append(appenders, appendValueToArrowBuilder[uint32, uint32, *array.Uint32Builder, utils.Uint32Converter]) - case typeName == "UInt64": - acceptors = append(acceptors, new(*uint64)) - appenders = append(appenders, appendValueToArrowBuilder[uint64, uint64, *array.Uint64Builder, utils.Uint64Converter]) - case typeName == "Float32": - acceptors = append(acceptors, new(*float32)) - appenders = append(appenders, appendValueToArrowBuilder[float32, float32, *array.Float32Builder, utils.Float32Converter]) - case typeName == "Float64": - acceptors = append(acceptors, new(*float64)) - appenders = append(appenders, appendValueToArrowBuilder[float64, float64, *array.Float64Builder, utils.Float64Converter]) - case typeName == "String", isFixedString.MatchString(typeName): - // Looks like []byte would be a better choice here, but clickhouse driver prefers string - acceptors = append(acceptors, new(*string)) - appenders = append(appenders, appendValueToArrowBuilder[string, []byte, *array.BinaryBuilder, utils.StringToBytesConverter]) - case typeName == "Date": - acceptors = append(acceptors, new(*time.Time)) - - ydbTypeID, err := utils.YdbTypeToYdbPrimitiveTypeID(ydbTypes[i]) - if err != nil { - return nil, fmt.Errorf("ydb type to ydb primitive type id: %w", err) - } - - switch ydbTypeID { - case Ydb.Type_UTF8: - appenders = append(appenders, appendValueToArrowBuilder[time.Time, string, *array.StringBuilder, dateToStringConverter]) - case Ydb.Type_DATE: - appenders = append(appenders, appendValueToArrowBuilder[time.Time, uint16, *array.Uint16Builder, utils.DateConverter]) - default: - return nil, fmt.Errorf("unexpected ydb type %v with sql type %s: %w", ydbTypes[i], typeName, utils.ErrDataTypeNotSupported) - } - case typeName == "Date32": - acceptors = append(acceptors, new(*time.Time)) - - ydbTypeID, err := utils.YdbTypeToYdbPrimitiveTypeID(ydbTypes[i]) - if err != nil { - return nil, fmt.Errorf("ydb type to ydb primitive type id: %w", err) - } - - switch ydbTypeID { - case Ydb.Type_UTF8: - appenders = append(appenders, appendValueToArrowBuilder[time.Time, string, *array.StringBuilder, date32ToStringConverter]) - case Ydb.Type_DATE: - appenders = append(appenders, appendValueToArrowBuilder[time.Time, uint16, *array.Uint16Builder, utils.DateConverter]) - default: - return nil, fmt.Errorf("unexpected ydb type %v with sql type %s: %w", ydbTypes[i], typeName, utils.ErrDataTypeNotSupported) - } - case isDateTime64.MatchString(typeName): - acceptors = append(acceptors, new(*time.Time)) - - ydbTypeID, err := utils.YdbTypeToYdbPrimitiveTypeID(ydbTypes[i]) - if err != nil { - return nil, fmt.Errorf("ydb type to ydb primitive type id: %w", err) - } - - switch ydbTypeID { - case Ydb.Type_UTF8: - appenders = append(appenders, appendValueToArrowBuilder[time.Time, string, *array.StringBuilder, dateTime64ToStringConverter]) - case Ydb.Type_TIMESTAMP: - appenders = append(appenders, appendValueToArrowBuilder[time.Time, uint64, *array.Uint64Builder, utils.TimestampConverter]) - default: - return nil, fmt.Errorf("unexpected ydb type %v with sql type %s: %w", ydbTypes[i], typeName, utils.ErrDataTypeNotSupported) - } - case isDateTime.MatchString(typeName): - acceptors = append(acceptors, new(*time.Time)) - - ydbTypeID, err := utils.YdbTypeToYdbPrimitiveTypeID(ydbTypes[i]) - if err != nil { - return nil, fmt.Errorf("ydb type to ydb primitive type id: %w", err) - } - - switch ydbTypeID { - case Ydb.Type_UTF8: - appenders = append(appenders, appendValueToArrowBuilder[time.Time, string, *array.StringBuilder, dateTimeToStringConverter]) - case Ydb.Type_DATETIME: - appenders = append(appenders, appendValueToArrowBuilder[time.Time, uint32, *array.Uint32Builder, utils.DatetimeConverter]) - default: - return nil, fmt.Errorf("unexpected ydb type %v with sql type %s: %w", ydbTypes[i], typeName, utils.ErrDataTypeNotSupported) - } - default: - return nil, fmt.Errorf("unknown type '%v'", typeName) - } - } - - return utils.NewRowTransformer[any](acceptors, appenders, nil), nil -} - -func appendValueToArrowBuilder[IN utils.ValueType, OUT utils.ValueType, AB utils.ArrowBuilder[OUT], CONV utils.ValueConverter[IN, OUT]]( - acceptor any, - builder array.Builder, -) error { - cast := acceptor.(**IN) - - if *cast == nil { - builder.AppendNull() - - return nil - } - - value := **cast - - var converter CONV - - out, err := converter.Convert(value) - if err != nil { - if errors.Is(err, utils.ErrValueOutOfTypeBounds) { - // TODO: write warning to logger - builder.AppendNull() - - return nil - } - - return fmt.Errorf("convert value %v: %w", value, err) - } - - //nolint:forcetypeassert - builder.(AB).Append(out) - - return nil -} - -// If time value is under of type bounds ClickHouse behaviour is undefined -// See note: https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions#tostartofmonth - -var ( - minClickHouseDate = time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) - maxClickHouseDate = time.Date(2149, time.June, 6, 0, 0, 0, 0, time.UTC) - minClickHouseDate32 = time.Date(1900, time.January, 1, 0, 0, 0, 0, time.UTC) - maxClickHouseDate32 = time.Date(2299, time.December, 31, 0, 0, 0, 0, time.UTC) - minClickHouseDatetime = time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) - maxClickHouseDatetime = time.Date(2106, time.February, 7, 6, 28, 15, 0, time.UTC) - minClickHouseDatetime64 = time.Date(1900, time.January, 1, 0, 0, 0, 0, time.UTC) - maxClickHouseDatetime64 = time.Date(2299, time.December, 31, 23, 59, 59, 99999999, time.UTC) -) - -func saturateDateTime(in, min, max time.Time) time.Time { - if in.Before(min) { - in = min - } - - if in.After(max) { - in = max - } - - return in -} - -type dateToStringConverter struct{} - -func (dateToStringConverter) Convert(in time.Time) (string, error) { - return utils.DateToStringConverter{}.Convert(saturateDateTime(in, minClickHouseDate, maxClickHouseDate)) -} - -type date32ToStringConverter struct{} - -func (date32ToStringConverter) Convert(in time.Time) (string, error) { - return utils.DateToStringConverter{}.Convert(saturateDateTime(in, minClickHouseDate32, maxClickHouseDate32)) -} - -type dateTimeToStringConverter struct{} - -func (dateTimeToStringConverter) Convert(in time.Time) (string, error) { - return utils.DatetimeToStringConverter{}.Convert(saturateDateTime(in, minClickHouseDatetime, maxClickHouseDatetime)) -} - -type dateTime64ToStringConverter struct{} - -func (dateTime64ToStringConverter) Convert(in time.Time) (string, error) { - return utils.TimestampToStringConverter{}.Convert(saturateDateTime(in, minClickHouseDatetime64, maxClickHouseDatetime64)) -} - -func NewTypeMapper() utils.TypeMapper { - return typeMapper{ - isFixedString: regexp.MustCompile(`FixedString\([0-9]+\)`), - isDateTime: regexp.MustCompile(`DateTime(\('[\w,/]+'\))?`), - isDateTime64: regexp.MustCompile(`DateTime64\(\d{1}(, '[\w,/]+')?\)`), - isNullable: regexp.MustCompile(`Nullable\((?P[\w\(\)]+)\)`), - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/ut/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/ut/ya.make deleted file mode 100644 index 4f4c779bb01c..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/ut/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -GO_TEST_FOR(ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse) - -SIZE(SMALL) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/ya.make deleted file mode 100644 index 46287aade567..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse/ya.make +++ /dev/null @@ -1,18 +0,0 @@ -GO_LIBRARY() - -SRCS( - connection_manager.go - doc.go - sql_formatter.go - type_mapper.go -) - -GO_TEST_SRCS( - sql_formatter_test.go -) - -END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source.go deleted file mode 100644 index 79405c5b512a..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source.go +++ /dev/null @@ -1,158 +0,0 @@ -package rdbms - -import ( - "context" - "fmt" - - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type Preset struct { - SQLFormatter rdbms_utils.SQLFormatter - ConnectionManager rdbms_utils.ConnectionManager - TypeMapper utils.TypeMapper -} - -var _ datasource.DataSource[any] = (*dataSourceImpl)(nil) - -type dataSourceImpl struct { - typeMapper utils.TypeMapper - sqlFormatter rdbms_utils.SQLFormatter - connectionManager rdbms_utils.ConnectionManager - logger log.Logger -} - -func (ds *dataSourceImpl) DescribeTable( - ctx context.Context, - logger log.Logger, - request *api_service_protos.TDescribeTableRequest, -) (*api_service_protos.TDescribeTableResponse, error) { - query, args := rdbms_utils.MakeDescribeTableQuery(logger, ds.sqlFormatter, request) - - conn, err := ds.connectionManager.Make(ctx, logger, request.DataSourceInstance) - if err != nil { - return nil, fmt.Errorf("make connection: %w", err) - } - - defer ds.connectionManager.Release(logger, conn) - - rows, err := conn.Query(ctx, query, args...) - if err != nil { - return nil, fmt.Errorf("query builder error: %w", err) - } - - defer func() { utils.LogCloserError(logger, rows, "close rows") }() - - var ( - columnName string - typeName string - ) - - sb := &schemaBuilder{typeMapper: ds.typeMapper, typeMappingSettings: request.TypeMappingSettings} - - for rows.Next() { - if err := rows.Scan(&columnName, &typeName); err != nil { - return nil, fmt.Errorf("rows scan: %w", err) - } - - if err := sb.addColumn(columnName, typeName); err != nil { - return nil, fmt.Errorf("add column to schema builder: %w", err) - } - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("rows iteration: %w", err) - } - - schema, err := sb.build(logger) - if err != nil { - return nil, fmt.Errorf("build schema: %w", err) - } - - return &api_service_protos.TDescribeTableResponse{Schema: schema}, nil -} - -func (ds *dataSourceImpl) doReadSplit( - ctx context.Context, - logger log.Logger, - split *api_service_protos.TSplit, - sink paging.Sink[any], -) error { - query, args, err := rdbms_utils.MakeReadSplitQuery(logger, ds.sqlFormatter, split.Select) - if err != nil { - return fmt.Errorf("make read split query: %w", err) - } - - conn, err := ds.connectionManager.Make(ctx, logger, split.Select.DataSourceInstance) - if err != nil { - return fmt.Errorf("make connection: %w", err) - } - - defer ds.connectionManager.Release(logger, conn) - - rows, err := conn.Query(ctx, query, args...) - if err != nil { - return fmt.Errorf("query '%s' error: %w", query, err) - } - - defer func() { utils.LogCloserError(logger, rows, "close rows") }() - - ydbTypes, err := utils.SelectWhatToYDBTypes(split.Select.What) - if err != nil { - return fmt.Errorf("convert Select.What to Ydb types: %w", err) - } - - transformer, err := rows.MakeTransformer(ydbTypes) - if err != nil { - return fmt.Errorf("make transformer: %w", err) - } - - // FIXME: use https://pkg.go.dev/database/sql#Rows.NextResultSet - // Very important! Possible data loss. - for rows.Next() { - if err := rows.Scan(transformer.GetAcceptors()...); err != nil { - return fmt.Errorf("rows scan error: %w", err) - } - - if err := sink.AddRow(transformer); err != nil { - return fmt.Errorf("add row to paging writer: %w", err) - } - } - - if err := rows.Err(); err != nil { - return fmt.Errorf("rows error: %w", err) - } - - return nil -} - -func (ds *dataSourceImpl) ReadSplit( - ctx context.Context, - logger log.Logger, - split *api_service_protos.TSplit, - sink paging.Sink[any], -) { - err := ds.doReadSplit(ctx, logger, split, sink) - if err != nil { - sink.AddError(err) - } - - sink.Finish() -} - -func NewDataSource( - logger log.Logger, - preset *Preset, -) datasource.DataSource[any] { - return &dataSourceImpl{ - logger: logger, - sqlFormatter: preset.SQLFormatter, - connectionManager: preset.ConnectionManager, - typeMapper: preset.TypeMapper, - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source_factory.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source_factory.go deleted file mode 100644 index 18bf26ae3dac..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source_factory.go +++ /dev/null @@ -1,53 +0,0 @@ -package rdbms - -import ( - "fmt" - - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -var _ datasource.DataSourceFactory[any] = (*dataSourceFactory)(nil) - -type dataSourceFactory struct { - clickhouse Preset - postgresql Preset -} - -func (dsf *dataSourceFactory) Make( - logger log.Logger, - dataSourceType api_common.EDataSourceKind, -) (datasource.DataSource[any], error) { - switch dataSourceType { - case api_common.EDataSourceKind_CLICKHOUSE: - return NewDataSource(logger, &dsf.clickhouse), nil - case api_common.EDataSourceKind_POSTGRESQL: - return NewDataSource(logger, &dsf.postgresql), nil - default: - return nil, fmt.Errorf("pick handler for data source type '%v': %w", dataSourceType, utils.ErrDataSourceNotSupported) - } -} - -func NewDataSourceFactory(qlf utils.QueryLoggerFactory) datasource.DataSourceFactory[any] { - connManagerCfg := rdbms_utils.ConnectionManagerBase{ - QueryLoggerFactory: qlf, - } - - return &dataSourceFactory{ - clickhouse: Preset{ - SQLFormatter: clickhouse.NewSQLFormatter(), - ConnectionManager: clickhouse.NewConnectionManager(connManagerCfg), - TypeMapper: clickhouse.NewTypeMapper(), - }, - postgresql: Preset{ - SQLFormatter: postgresql.NewSQLFormatter(), - ConnectionManager: postgresql.NewConnectionManager(connManagerCfg), - TypeMapper: postgresql.NewTypeMapper(), - }, - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source_test.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source_test.go deleted file mode 100644 index 310b2530afc7..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/data_source_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package rdbms - -import ( - "context" - "errors" - "fmt" - "testing" - - "github.com/stretchr/testify/mock" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func TestReadSplit(t *testing.T) { - ctx := context.Background() - split := &api_service_protos.TSplit{ - Select: &api_service_protos.TSelect{ - DataSourceInstance: &api_common.TDataSourceInstance{}, - What: &api_service_protos.TSelect_TWhat{ - Items: []*api_service_protos.TSelect_TWhat_TItem{ - { - Payload: &api_service_protos.TSelect_TWhat_TItem_Column{ - Column: &Ydb.Column{ - Name: "col1", - Type: &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT32}}, - }, - }, - }, - { - Payload: &api_service_protos.TSelect_TWhat_TItem_Column{ - Column: &Ydb.Column{ - Name: "col2", - Type: &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UTF8}}, - }, - }, - }, - }, - }, - From: &api_service_protos.TSelect_TFrom{ - Table: "example_1", - }, - }, - } - - t.Run("positive", func(t *testing.T) { - logger := utils.NewTestLogger(t) - - connectionManager := &rdbms_utils.ConnectionManagerMock{} - - preset := &Preset{ - ConnectionManager: connectionManager, - SQLFormatter: postgresql.NewSQLFormatter(), // TODO: parametrize - } - - connection := &rdbms_utils.ConnectionMock{} - connectionManager.On("Make", split.Select.DataSourceInstance).Return(connection, nil).Once() - connectionManager.On("Release", connection).Return().Once() - - rows := &rdbms_utils.RowsMock{ - PredefinedData: [][]any{ - {int32(1), "a"}, - {int32(2), "b"}, - }, - } - connection.On("Query", `SELECT "col1", "col2" FROM "example_1"`).Return(rows, nil).Once() - - transformer := &rdbms_utils.RowTransformerMock{ - Acceptors: []any{ - new(int32), - new(string), - }, - } - - rows.On("MakeTransformer", []*Ydb.Type{utils.NewPrimitiveType(Ydb.Type_INT32), utils.NewPrimitiveType(Ydb.Type_UTF8)}).Return(transformer, nil).Once() - rows.On("Next").Return(true).Times(2) - rows.On("Next").Return(false).Once() - rows.On("Scan", transformer.GetAcceptors()...).Return(nil).Times(2) - rows.On("Err").Return(nil).Once() - rows.On("Close").Return(nil).Once() - - sink := &paging.SinkMock{} - sink.On("AddRow", transformer).Return(nil).Times(2) - sink.On("Finish").Return().Once() - - dataSource := NewDataSource(logger, preset) - dataSource.ReadSplit(ctx, logger, split, sink) - - mock.AssertExpectationsForObjects(t, connectionManager, connection, rows, sink) - }) - - t.Run("scan error", func(t *testing.T) { - logger := utils.NewTestLogger(t) - - connectionManager := &rdbms_utils.ConnectionManagerMock{} - - preset := &Preset{ - ConnectionManager: connectionManager, - SQLFormatter: postgresql.NewSQLFormatter(), // TODO: parametrize - } - - connection := &rdbms_utils.ConnectionMock{} - connectionManager.On("Make", split.Select.DataSourceInstance).Return(connection, nil).Once() - connectionManager.On("Release", connection).Return().Once() - - rows := &rdbms_utils.RowsMock{ - PredefinedData: [][]any{ - {int32(1), "a"}, - {int32(2), "b"}, - }, - } - connection.On("Query", `SELECT "col1", "col2" FROM "example_1"`).Return(rows, nil).Once() - - transformer := &rdbms_utils.RowTransformerMock{ - Acceptors: []any{ - new(int32), - new(string), - }, - } - - scanErr := fmt.Errorf("scan failed") - - rows.On("MakeTransformer", []*Ydb.Type{utils.NewPrimitiveType(Ydb.Type_INT32), utils.NewPrimitiveType(Ydb.Type_UTF8)}).Return(transformer, nil).Once() - rows.On("Next").Return(true).Times(2) - rows.On("Scan", transformer.GetAcceptors()...).Return(nil).Once() - rows.On("Scan", transformer.GetAcceptors()...).Return(scanErr).Once() - rows.On("Close").Return(nil).Once() - - sink := &paging.SinkMock{} - sink.On("AddRow", transformer).Return(nil).Once() - sink.On("AddError", mock.MatchedBy(func(err error) bool { - return errors.Is(err, scanErr) - })).Return().Once() - sink.On("Finish").Return().Once() - - datasource := NewDataSource(logger, preset) - datasource.ReadSplit(ctx, logger, split, sink) - - mock.AssertExpectationsForObjects(t, connectionManager, connection, rows, sink) - }) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/doc.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/doc.go deleted file mode 100644 index 4261e7ef3335..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package rdbms contains generalized abstractions suitable for use with -// any relational database. -package rdbms diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/connection_manager.go deleted file mode 100644 index 7b1b34eeaa03..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/connection_manager.go +++ /dev/null @@ -1,124 +0,0 @@ -package postgresql - -import ( - "context" - "fmt" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" - _ "github.com/jackc/pgx/v5/stdlib" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -var _ rdbms_utils.Connection = (*Connection)(nil) - -type rows struct { - pgx.Rows -} - -func (r rows) Close() error { - r.Rows.Close() - - return nil -} - -func (r rows) MakeTransformer(ydbTypes []*Ydb.Type) (utils.RowTransformer[any], error) { - fields := r.FieldDescriptions() - - oids := make([]uint32, 0, len(fields)) - for _, field := range fields { - oids = append(oids, field.DataTypeOID) - } - - return transformerFromOIDs(oids, ydbTypes) -} - -type Connection struct { - *pgx.Conn - logger utils.QueryLogger -} - -func (c Connection) Close() error { - return c.Conn.Close(context.TODO()) -} - -func (c Connection) Query(ctx context.Context, query string, args ...any) (rdbms_utils.Rows, error) { - c.logger.Dump(query, args...) - - out, err := c.Conn.Query(ctx, query, args...) - - return rows{Rows: out}, err -} - -var _ rdbms_utils.ConnectionManager = (*connectionManager)(nil) - -type connectionManager struct { - rdbms_utils.ConnectionManagerBase - // TODO: cache of connections, remove unused connections with TTL -} - -func (c *connectionManager) Make( - ctx context.Context, - logger log.Logger, - dsi *api_common.TDataSourceInstance, -) (rdbms_utils.Connection, error) { - if dsi.GetCredentials().GetBasic() == nil { - return nil, fmt.Errorf("currently only basic auth is supported") - } - - if dsi.Protocol != api_common.EProtocol_NATIVE { - return nil, fmt.Errorf("can not create PostgreSQL connection with protocol '%v'", dsi.Protocol) - } - - if socketType, _ := pgconn.NetworkAddress(dsi.GetEndpoint().GetHost(), uint16(dsi.GetEndpoint().GetPort())); socketType != "tcp" { - return nil, fmt.Errorf("can not create PostgreSQL connection with socket type '%s'", socketType) - } - - connStr := "dbname=DBNAME user=USER password=PASSWORD host=HOST port=5432" - if dsi.UseTls { - connStr += " sslmode=verify-full" - } else { - connStr += " sslmode=disable" - } - - connCfg, err := pgx.ParseConfig(connStr) - if err != nil { - return nil, fmt.Errorf("parse connection config template: %w", err) - } - - connCfg.Database = dsi.Database - connCfg.Host = dsi.GetEndpoint().GetHost() - connCfg.Port = uint16(dsi.GetEndpoint().GetPort()) - connCfg.User = dsi.Credentials.GetBasic().GetUsername() - connCfg.Password = dsi.Credentials.GetBasic().GetPassword() - - if dsi.UseTls { - connCfg.TLSConfig.ServerName = dsi.GetEndpoint().GetHost() - } - - conn, err := pgx.ConnectConfig(ctx, connCfg) - if err != nil { - return nil, fmt.Errorf("open connection: %w", err) - } - - // set schema (public by default) - if _, err = conn.Exec(ctx, fmt.Sprintf("set search_path=%s", NewSQLFormatter().SanitiseIdentifier(dsi.GetPgOptions().GetSchema()))); err != nil { - return nil, fmt.Errorf("exec: %w", err) - } - - queryLogger := c.QueryLoggerFactory.Make(logger) - - return &Connection{conn, queryLogger}, nil -} - -func (c *connectionManager) Release(logger log.Logger, conn rdbms_utils.Connection) { - utils.LogCloserError(logger, conn, "close posgresql connection") -} - -func NewConnectionManager(cfg rdbms_utils.ConnectionManagerBase) rdbms_utils.ConnectionManager { - return &connectionManager{ConnectionManagerBase: cfg} -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/doc.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/doc.go deleted file mode 100644 index 9df86d415c13..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package postgresql contains code specific for PostgreSQL database. -package postgresql diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/sql_formatter.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/sql_formatter.go deleted file mode 100644 index 9d51967d6fb0..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/sql_formatter.go +++ /dev/null @@ -1,95 +0,0 @@ -package postgresql - -import ( - "fmt" - "strings" - - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var _ rdbms_utils.SQLFormatter = (*sqlFormatter)(nil) - -type sqlFormatter struct { -} - -func (f *sqlFormatter) supportsType(typeID Ydb.Type_PrimitiveTypeId) bool { - switch typeID { - case Ydb.Type_BOOL: - return true - case Ydb.Type_INT8: - return true - case Ydb.Type_UINT8: - return true - case Ydb.Type_INT16: - return true - case Ydb.Type_UINT16: - return true - case Ydb.Type_INT32: - return true - case Ydb.Type_UINT32: - return true - case Ydb.Type_INT64: - return true - case Ydb.Type_UINT64: - return true - case Ydb.Type_FLOAT: - return true - case Ydb.Type_DOUBLE: - return true - default: - return false - } -} - -func (f *sqlFormatter) supportsConstantValueExpression(t *Ydb.Type) bool { - switch v := t.Type.(type) { - case *Ydb.Type_TypeId: - return f.supportsType(v.TypeId) - case *Ydb.Type_OptionalType: - return f.supportsConstantValueExpression(v.OptionalType.Item) - default: - return false - } -} - -func (f sqlFormatter) SupportsPushdownExpression(expression *api_service_protos.TExpression) bool { - switch e := expression.Payload.(type) { - case *api_service_protos.TExpression_Column: - return true - case *api_service_protos.TExpression_TypedValue: - return f.supportsConstantValueExpression(e.TypedValue.Type) - case *api_service_protos.TExpression_ArithmeticalExpression: - return false - case *api_service_protos.TExpression_Null: - return true - default: - return false - } -} - -func (f sqlFormatter) GetDescribeTableQuery(request *api_service_protos.TDescribeTableRequest) (string, []any) { - schema := request.GetDataSourceInstance().GetPgOptions().GetSchema() - query := "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = $1 AND table_schema = $2" - args := []any{request.Table, schema} - - return query, args -} - -func (f sqlFormatter) GetPlaceholder(n int) string { - return fmt.Sprintf("$%d", n+1) -} - -func (f sqlFormatter) SanitiseIdentifier(ident string) string { - // https://github.com/jackc/pgx/blob/v5.4.3/conn.go#L93 - // https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS - sanitizedIdent := strings.ReplaceAll(ident, string([]byte{0}), "") - sanitizedIdent = `"` + strings.ReplaceAll(sanitizedIdent, `"`, `""`) + `"` - - return sanitizedIdent -} - -func NewSQLFormatter() rdbms_utils.SQLFormatter { - return sqlFormatter{} -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/sql_formatter_test.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/sql_formatter_test.go deleted file mode 100644 index f182c129b531..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/sql_formatter_test.go +++ /dev/null @@ -1,427 +0,0 @@ -package postgresql - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/require" - ydb "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func TestMakeDescribeTableQuery(t *testing.T) { - logger := utils.NewTestLogger(t) - formatter := NewSQLFormatter() - dsi := &common.TDataSourceInstance{Options: &common.TDataSourceInstance_PgOptions{PgOptions: &common.TPostgreSQLDataSourceOptions{Schema: "schema"}}} - request := &api.TDescribeTableRequest{Table: "table", DataSourceInstance: dsi} - - output, args := rdbms_utils.MakeDescribeTableQuery(logger, formatter, request) - require.Equal(t, "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = $1 AND table_schema = $2", output) - require.Equal(t, args, []any{"table", "schema"}) -} - -func TestMakeReadSplitQuery(t *testing.T) { - type testCase struct { - testName string - selectReq *api.TSelect - outputQuery string - outputArgs []any - err error - } - - logger := utils.NewTestLogger(t) - formatter := NewSQLFormatter() - - tcs := []testCase{ - { - testName: "empty_table_name", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "", - }, - What: &api.TSelect_TWhat{}, - }, - outputQuery: "", - outputArgs: nil, - err: utils.ErrEmptyTableName, - }, - { - testName: "empty_no columns", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: &api.TSelect_TWhat{}, - }, - outputQuery: `SELECT 0 FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "select_col", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: &api.TSelect_TWhat{ - Items: []*api.TSelect_TWhat_TItem{ - &api.TSelect_TWhat_TItem{ - Payload: &api.TSelect_TWhat_TItem_Column{ - Column: &ydb.Column{ - Name: "col", - Type: utils.NewPrimitiveType(ydb.Type_INT32), - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "col" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "is_null", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_IsNull{ - IsNull: &api.TPredicate_TIsNull{ - Value: utils.NewColumnExpression("col1"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE ("col1" IS NULL)`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "is_not_null", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_IsNotNull{ - IsNotNull: &api.TPredicate_TIsNotNull{ - Value: utils.NewColumnExpression("col2"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE ("col2" IS NOT NULL)`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "bool_column", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_BoolExpression{ - BoolExpression: &api.TPredicate_TBoolExpression{ - Value: utils.NewColumnExpression("col2"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE "col2"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "complex_filter", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Disjunction{ - Disjunction: &api.TPredicate_TDisjunction{ - Operands: []*api.TPredicate{ - &api.TPredicate{ - Payload: &api.TPredicate_Negation{ - Negation: &api.TPredicate_TNegation{ - Operand: &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_LE, - LeftValue: utils.NewColumnExpression("col2"), - RightValue: utils.NewInt32ValueExpression(42), - }, - }, - }, - }, - }, - }, - &api.TPredicate{ - Payload: &api.TPredicate_Conjunction{ - Conjunction: &api.TPredicate_TConjunction{ - Operands: []*api.TPredicate{ - &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_NE, - LeftValue: utils.NewColumnExpression("col1"), - RightValue: utils.NewUint64ValueExpression(0), - }, - }, - }, - &api.TPredicate{ - Payload: &api.TPredicate_IsNull{ - IsNull: &api.TPredicate_TIsNull{ - Value: utils.NewColumnExpression("col3"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE ((NOT ("col2" <= $1)) OR (("col1" <> $2) AND ("col3" IS NULL)))`, - outputArgs: []any{int32(42), uint64(0)}, - err: nil, - }, - { - testName: "unsupported_predicate", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Between{ - Between: &api.TPredicate_TBetween{ - Value: utils.NewColumnExpression("col2"), - Least: utils.NewColumnExpression("col1"), - Greatest: utils.NewColumnExpression("col3"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "unsupported_type", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col2"), - RightValue: utils.NewTextValueExpression("text"), - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "partial_filter_removes_and", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Conjunction{ - Conjunction: &api.TPredicate_TConjunction{ - Operands: []*api.TPredicate{ - &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col1"), - RightValue: utils.NewInt32ValueExpression(32), - }, - }, - }, - &api.TPredicate{ - // Not supported - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col2"), - RightValue: utils.NewTextValueExpression("text"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE ("col1" = $1)`, - outputArgs: []any{int32(32)}, - err: nil, - }, - { - testName: "partial_filter", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: utils.NewDefaultWhat(), - Where: &api.TSelect_TWhere{ - FilterTyped: &api.TPredicate{ - Payload: &api.TPredicate_Conjunction{ - Conjunction: &api.TPredicate_TConjunction{ - Operands: []*api.TPredicate{ - &api.TPredicate{ - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col1"), - RightValue: utils.NewInt32ValueExpression(32), - }, - }, - }, - &api.TPredicate{ - // Not supported - Payload: &api.TPredicate_Comparison{ - Comparison: &api.TPredicate_TComparison{ - Operation: api.TPredicate_TComparison_EQ, - LeftValue: utils.NewColumnExpression("col2"), - RightValue: utils.NewTextValueExpression("text"), - }, - }, - }, - &api.TPredicate{ - Payload: &api.TPredicate_IsNull{ - IsNull: &api.TPredicate_TIsNull{ - Value: utils.NewColumnExpression("col3"), - }, - }, - }, - &api.TPredicate{ - Payload: &api.TPredicate_IsNotNull{ - IsNotNull: &api.TPredicate_TIsNotNull{ - Value: utils.NewColumnExpression("col4"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "col0", "col1" FROM "tab" WHERE (("col1" = $1) AND ("col3" IS NULL) AND ("col4" IS NOT NULL))`, - outputArgs: []any{int32(32)}, - err: nil, - }, - { - testName: "negative_sql_injection_by_table", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: `information_schema.columns; DROP TABLE information_schema.columns`, - }, - What: &api.TSelect_TWhat{}, - }, - outputQuery: `SELECT 0 FROM "information_schema.columns; DROP TABLE information_schema.columns"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "negative_sql_injection_by_col", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: &api.TSelect_TWhat{ - Items: []*api.TSelect_TWhat_TItem{ - &api.TSelect_TWhat_TItem{ - Payload: &api.TSelect_TWhat_TItem_Column{ - Column: &ydb.Column{ - Name: `0; DROP TABLE information_schema.columns`, - Type: utils.NewPrimitiveType(ydb.Type_INT32), - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "0; DROP TABLE information_schema.columns" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - { - testName: "negative_sql_injection_fake_quotes", - selectReq: &api.TSelect{ - From: &api.TSelect_TFrom{ - Table: "tab", - }, - What: &api.TSelect_TWhat{ - Items: []*api.TSelect_TWhat_TItem{ - &api.TSelect_TWhat_TItem{ - Payload: &api.TSelect_TWhat_TItem_Column{ - Column: &ydb.Column{ - Name: `0"; DROP TABLE information_schema.columns;`, - Type: utils.NewPrimitiveType(ydb.Type_INT32), - }, - }, - }, - }, - }, - }, - outputQuery: `SELECT "0""; DROP TABLE information_schema.columns;" FROM "tab"`, - outputArgs: []any{}, - err: nil, - }, - } - - for _, tc := range tcs { - tc := tc - - t.Run(tc.testName, func(t *testing.T) { - output, outputArgs, err := rdbms_utils.MakeReadSplitQuery(logger, formatter, tc.selectReq) - require.Equal(t, tc.outputQuery, output) - require.Equal(t, tc.outputArgs, outputArgs) - - if tc.err != nil { - require.True(t, errors.Is(err, tc.err)) - } else { - require.NoError(t, err) - } - }) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/type_mapper.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/type_mapper.go deleted file mode 100644 index 2df05805a8c4..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/type_mapper.go +++ /dev/null @@ -1,235 +0,0 @@ -package postgresql - -import ( - "errors" - "fmt" - "time" - - "github.com/apache/arrow/go/v13/arrow/array" - "github.com/jackc/pgx/v5/pgtype" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var _ utils.TypeMapper = typeMapper{} - -type typeMapper struct{} - -func (tm typeMapper) SQLTypeToYDBColumn(columnName, typeName string, rules *api_service_protos.TTypeMappingSettings) (*Ydb.Column, error) { - var ydbType *Ydb.Type - - // Reference table: https://wiki.yandex-team.ru/rtmapreduce/yql-streams-corner/connectors/lld-02-tipy-dannyx/ - switch typeName { - case "boolean", "bool": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_BOOL}} - case "smallint", "int2", "smallserial", "serial2": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT16}} - case "integer", "int", "int4", "serial", "serial4": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT32}} - case "bigint", "int8", "bigserial", "serial8": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT64}} - case "real", "float4": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_FLOAT}} - case "double precision", "float8": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_DOUBLE}} - case "bytea": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_STRING}} - case "character", "character varying", "text": - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UTF8}} - case "date": - switch rules.GetDateTimeFormat() { - case api_service_protos.EDateTimeFormat_STRING_FORMAT: - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UTF8}} - case api_service_protos.EDateTimeFormat_YQL_FORMAT: - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_DATE}} - default: - return nil, fmt.Errorf("unexpected date format '%s': %w", rules.GetDateTimeFormat(), utils.ErrDataTypeNotSupported) - } - // TODO: PostgreSQL `time` data type has no direct counterparts in the YDB's type system; - // but it can be supported when the PG-compatible types is added to YDB: - // https://st.yandex-team.ru/YQ-2285 - // case "time": - case "timestamp without time zone": - switch rules.GetDateTimeFormat() { - case api_service_protos.EDateTimeFormat_STRING_FORMAT: - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UTF8}} - case api_service_protos.EDateTimeFormat_YQL_FORMAT: - ydbType = &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_TIMESTAMP}} - default: - return nil, fmt.Errorf("unexpected timestamp format '%s': %w", rules.GetDateTimeFormat(), utils.ErrDataTypeNotSupported) - } - default: - return nil, fmt.Errorf("convert type '%s': %w", typeName, utils.ErrDataTypeNotSupported) - } - - // In PostgreSQL all columns are actually nullable, hence we wrap every T in Optional. - // See this issue for details: https://st.yandex-team.ru/YQ-2256 - ydbType = &Ydb.Type{Type: &Ydb.Type_OptionalType{OptionalType: &Ydb.OptionalType{Item: ydbType}}} - - return &Ydb.Column{ - Name: columnName, - Type: ydbType, - }, nil -} - -func transformerFromOIDs(oids []uint32, ydbTypes []*Ydb.Type) (utils.RowTransformer[any], error) { - acceptors := make([]any, 0, len(oids)) - appenders := make([]func(acceptor any, builder array.Builder) error, 0, len(oids)) - - for i, oid := range oids { - switch oid { - case pgtype.BoolOID: - acceptors = append(acceptors, new(pgtype.Bool)) - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Bool) - - return appendValueToArrowBuilder[bool, uint8, *array.Uint8Builder, utils.BoolConverter](cast.Bool, builder, cast.Valid) - }) - case pgtype.Int2OID: - acceptors = append(acceptors, new(pgtype.Int2)) - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Int2) - - return appendValueToArrowBuilder[int16, int16, *array.Int16Builder, utils.Int16Converter](cast.Int16, builder, cast.Valid) - }) - case pgtype.Int4OID: - acceptors = append(acceptors, new(pgtype.Int4)) - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Int4) - - return appendValueToArrowBuilder[int32, int32, *array.Int32Builder, utils.Int32Converter](cast.Int32, builder, cast.Valid) - }) - case pgtype.Int8OID: - acceptors = append(acceptors, new(pgtype.Int8)) - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Int8) - - return appendValueToArrowBuilder[int64, int64, *array.Int64Builder, utils.Int64Converter](cast.Int64, builder, cast.Valid) - }) - case pgtype.Float4OID: - acceptors = append(acceptors, new(pgtype.Float4)) - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Float4) - - return appendValueToArrowBuilder[float32, float32, *array.Float32Builder, utils.Float32Converter](cast.Float32, builder, cast.Valid) - }) - case pgtype.Float8OID: - acceptors = append(acceptors, new(pgtype.Float8)) - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Float8) - - return appendValueToArrowBuilder[float64, float64, *array.Float64Builder, utils.Float64Converter](cast.Float64, builder, cast.Valid) - }) - case pgtype.TextOID, pgtype.BPCharOID, pgtype.VarcharOID: - acceptors = append(acceptors, new(pgtype.Text)) - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Text) - - return appendValueToArrowBuilder[string, string, *array.StringBuilder, utils.StringConverter](cast.String, builder, cast.Valid) - }) - case pgtype.ByteaOID: - acceptors = append(acceptors, new(*[]byte)) - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - // TODO: Bytea exists in the upstream library, but missing in jackx/pgx: - // https://github.com/jackc/pgtype/blob/v1.14.0/bytea.go - // https://github.com/jackc/pgx/blob/v5.3.1/pgtype/bytea.go - // https://github.com/jackc/pgx/issues/1714 - cast := acceptor.(**[]byte) - if *cast != nil { - builder.(*array.BinaryBuilder).Append(**cast) - } else { - builder.(*array.BinaryBuilder).AppendNull() - } - - return nil - }) - case pgtype.DateOID: - acceptors = append(acceptors, new(pgtype.Date)) - - ydbTypeID, err := utils.YdbTypeToYdbPrimitiveTypeID(ydbTypes[i]) - if err != nil { - return nil, fmt.Errorf("ydb type to ydb primitive type id: %w", err) - } - - switch ydbTypeID { - case Ydb.Type_UTF8: - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Date) - - return appendValueToArrowBuilder[time.Time, string, *array.StringBuilder, utils.DateToStringConverter](cast.Time, builder, cast.Valid) - }) - case Ydb.Type_DATE: - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Date) - - return appendValueToArrowBuilder[time.Time, uint16, *array.Uint16Builder, utils.DateConverter](cast.Time, builder, cast.Valid) - }) - default: - return nil, fmt.Errorf("unexpected ydb type %v with type oid %d: %w", ydbTypes[i], oid, utils.ErrDataTypeNotSupported) - } - case pgtype.TimestampOID: - acceptors = append(acceptors, new(pgtype.Timestamp)) - - ydbTypeID, err := utils.YdbTypeToYdbPrimitiveTypeID(ydbTypes[i]) - if err != nil { - return nil, fmt.Errorf("ydb type to ydb primitive type id: %w", err) - } - - switch ydbTypeID { - case Ydb.Type_UTF8: - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Timestamp) - - return appendValueToArrowBuilder[time.Time, string, *array.StringBuilder, utils.TimestampToStringConverter](cast.Time, builder, cast.Valid) - }) - case Ydb.Type_TIMESTAMP: - appenders = append(appenders, func(acceptor any, builder array.Builder) error { - cast := acceptor.(*pgtype.Timestamp) - - return appendValueToArrowBuilder[time.Time, uint64, *array.Uint64Builder, utils.TimestampConverter](cast.Time, builder, cast.Valid) - }) - default: - return nil, fmt.Errorf("unexpected ydb type %v with type oid %d: %w", ydbTypes[i], oid, utils.ErrDataTypeNotSupported) - } - default: - return nil, fmt.Errorf("convert type OID %d: %w", oid, utils.ErrDataTypeNotSupported) - } - } - - return utils.NewRowTransformer[any](acceptors, appenders, nil), nil -} - -func appendValueToArrowBuilder[IN utils.ValueType, OUT utils.ValueType, AB utils.ArrowBuilder[OUT], CONV utils.ValueConverter[IN, OUT]]( - value any, - builder array.Builder, - valid bool, -) error { - if !valid { - builder.AppendNull() - - return nil - } - - cast := value.(IN) - - var converter CONV - - out, err := converter.Convert(cast) - if err != nil { - if errors.Is(err, utils.ErrValueOutOfTypeBounds) { - // TODO: logger ? - builder.AppendNull() - - return nil - } - - return fmt.Errorf("convert value: %w", err) - } - - builder.(AB).Append(out) - - return nil -} - -func NewTypeMapper() utils.TypeMapper { return typeMapper{} } diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/ut/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/ut/ya.make deleted file mode 100644 index ec9d305cee13..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/ut/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -GO_TEST_FOR(ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql) - -SIZE(SMALL) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/ya.make deleted file mode 100644 index 46287aade567..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql/ya.make +++ /dev/null @@ -1,18 +0,0 @@ -GO_LIBRARY() - -SRCS( - connection_manager.go - doc.go - sql_formatter.go - type_mapper.go -) - -GO_TEST_SRCS( - sql_formatter_test.go -) - -END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/schema_builder.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/schema_builder.go deleted file mode 100644 index eccca8bb5b20..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/schema_builder.go +++ /dev/null @@ -1,69 +0,0 @@ -package rdbms - -import ( - "errors" - "fmt" - - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type schemaItem struct { - columnName string - columnType string - ydbColumn *Ydb.Column -} - -type schemaBuilder struct { - typeMapper utils.TypeMapper - typeMappingSettings *api_service_protos.TTypeMappingSettings - items []*schemaItem -} - -func (sb *schemaBuilder) addColumn(columnName, columnType string) error { - item := &schemaItem{ - columnName: columnName, - columnType: columnType, - } - - var err error - item.ydbColumn, err = sb.typeMapper.SQLTypeToYDBColumn(columnName, columnType, sb.typeMappingSettings) - - if err != nil && !errors.Is(err, utils.ErrDataTypeNotSupported) { - return fmt.Errorf("sql type to ydb column (%s, %s): %w", columnName, columnType, err) - } - - sb.items = append(sb.items, item) - - return nil -} - -func (sb *schemaBuilder) build(logger log.Logger) (*api_service_protos.TSchema, error) { - if len(sb.items) == 0 { - return nil, utils.ErrTableDoesNotExist - } - - var ( - schema api_service_protos.TSchema - unsupported []string - ) - - for _, item := range sb.items { - if item.ydbColumn == nil { - unsupported = append(unsupported, fmt.Sprintf("%s %s", item.columnName, item.columnType)) - } else { - schema.Columns = append(schema.Columns, item.ydbColumn) - } - } - - if len(unsupported) > 0 { - logger.Warn( - "the table schema was reduced because some column types are unsupported", - log.Strings("unsupported columns", unsupported), - ) - } - - return &schema, nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/schema_builder_test.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/schema_builder_test.go deleted file mode 100644 index e924f21e9d40..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/schema_builder_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package rdbms - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/postgresql" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - "google.golang.org/protobuf/proto" -) - -func TestSchemaBuilder(t *testing.T) { - t.Run("ClickHouse", func(t *testing.T) { - sb := &schemaBuilder{ - typeMapper: clickhouse.NewTypeMapper(), - } - - require.NoError(t, sb.addColumn("col1", "Int32")) // supported - require.NoError(t, sb.addColumn("col2", "String")) // supported - require.NoError(t, sb.addColumn("col3", "UUID")) // yet unsupported - - logger := utils.NewTestLogger(t) - schema, err := sb.build(logger) - require.NoError(t, err) - require.NotNil(t, schema) - - require.Len(t, schema.Columns, 2) - - require.Equal(t, schema.Columns[0].Name, "col1") - require.True( - t, - proto.Equal(schema.Columns[0].Type, &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT32}}), - schema.Columns[0].Type) - - require.Equal(t, schema.Columns[1].Name, "col2") - require.True( - t, - proto.Equal(schema.Columns[1].Type, &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_STRING}}), - schema.Columns[1].Type) - }) - - t.Run("PostgreSQL", func(t *testing.T) { - sb := &schemaBuilder{ - typeMapper: postgresql.NewTypeMapper(), - } - - require.NoError(t, sb.addColumn("col1", "bigint")) // supported - require.NoError(t, sb.addColumn("col2", "text")) // supported - require.NoError(t, sb.addColumn("col3", "time")) // yet unsupported - - logger := utils.NewTestLogger(t) - schema, err := sb.build(logger) - require.NoError(t, err) - require.NotNil(t, schema) - - require.Len(t, schema.Columns, 2) - - require.Equal(t, schema.Columns[0].Name, "col1") - require.True( - t, - proto.Equal( - schema.Columns[0].Type, - &Ydb.Type{Type: &Ydb.Type_OptionalType{OptionalType: &Ydb.OptionalType{Item: &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_INT64}}}}}, - ), - schema.Columns[0].Type) - - require.Equal(t, schema.Columns[1].Name, "col2") - require.True( - t, - proto.Equal( - schema.Columns[1].Type, - &Ydb.Type{Type: &Ydb.Type_OptionalType{OptionalType: &Ydb.OptionalType{Item: &Ydb.Type{Type: &Ydb.Type_TypeId{TypeId: Ydb.Type_UTF8}}}}}, - ), - schema.Columns[1].Type) - }) - - t.Run("NonExistingTable", func(t *testing.T) { - sb := &schemaBuilder{} - schema, err := sb.build(utils.NewTestLogger(t)) - require.ErrorIs(t, err, utils.ErrTableDoesNotExist) - require.Nil(t, schema) - }) - - t.Run("EmptyTable", func(t *testing.T) { - sb := &schemaBuilder{ - typeMapper: clickhouse.NewTypeMapper(), - } - - require.NoError(t, sb.addColumn("col1", "UUID")) // yet unsupported - - schema, err := sb.build(utils.NewTestLogger(t)) - require.NoError(t, err) - require.NotNil(t, schema) - require.Len(t, schema.Columns, 0) - }) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/ut/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/ut/ya.make deleted file mode 100644 index 02d67573beed..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/ut/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -GO_TEST_FOR(ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms) - -SIZE(SMALL) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/doc.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/doc.go deleted file mode 100644 index adebe5d27789..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package utils contains helper types and functions that can be used by any -// relational data source. -package utils diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/predicate_builder.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/predicate_builder.go deleted file mode 100644 index a7ff56b4e1c9..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/predicate_builder.go +++ /dev/null @@ -1,281 +0,0 @@ -package utils - -import ( - "fmt" - "strings" - - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func formatValue(formatter SQLFormatter, args []any, value *Ydb.TypedValue) (string, []any, error) { - switch v := value.Value.Value.(type) { - case *Ydb.Value_BoolValue: - return formatter.GetPlaceholder(len(args)), append(args, v.BoolValue), nil - case *Ydb.Value_Int32Value: - return formatter.GetPlaceholder(len(args)), append(args, v.Int32Value), nil - case *Ydb.Value_Uint32Value: - return formatter.GetPlaceholder(len(args)), append(args, v.Uint32Value), nil - case *Ydb.Value_Int64Value: - return formatter.GetPlaceholder(len(args)), append(args, v.Int64Value), nil - case *Ydb.Value_Uint64Value: - return formatter.GetPlaceholder(len(args)), append(args, v.Uint64Value), nil - case *Ydb.Value_FloatValue: - return formatter.GetPlaceholder(len(args)), append(args, v.FloatValue), nil - case *Ydb.Value_DoubleValue: - return formatter.GetPlaceholder(len(args)), append(args, v.DoubleValue), nil - default: - return "", args, fmt.Errorf("%w, type: %T", utils.ErrUnimplementedTypedValue, v) - } -} - -func formatColumn(formatter SQLFormatter, args []any, col string) (string, []any, error) { - return formatter.SanitiseIdentifier(col), args, nil -} - -func formatNull(formatter SQLFormatter, args []any, n *api_service_protos.TExpression_TNull) (string, []any, error) { - return "NULL", args, nil -} - -func formatArithmeticalExpression(formatter SQLFormatter, args []any, expression *api_service_protos.TExpression_TArithmeticalExpression) (string, []any, error) { - var operation string - - switch op := expression.Operation; op { - case api_service_protos.TExpression_TArithmeticalExpression_MUL: - operation = " * " - case api_service_protos.TExpression_TArithmeticalExpression_ADD: - operation = " + " - case api_service_protos.TExpression_TArithmeticalExpression_SUB: - operation = " - " - case api_service_protos.TExpression_TArithmeticalExpression_BIT_AND: - operation = " & " - case api_service_protos.TExpression_TArithmeticalExpression_BIT_OR: - operation = " | " - case api_service_protos.TExpression_TArithmeticalExpression_BIT_XOR: - operation = " ^ " - default: - return "", args, fmt.Errorf("%w, op: %d", utils.ErrUnimplementedArithmeticalExpression, op) - } - - left, args, err := formatExpression(formatter, args, expression.LeftValue) - if err != nil { - return "", args, fmt.Errorf("failed to format left argument: %w", err) - } - - right, args, err := formatExpression(formatter, args, expression.RightValue) - if err != nil { - return "", args, fmt.Errorf("failed to format right argument: %w", err) - } - - return fmt.Sprintf("(%s%s%s)", left, operation, right), args, nil -} - -func formatExpression(formatter SQLFormatter, args []any, expression *api_service_protos.TExpression) (string, []any, error) { - if !formatter.SupportsPushdownExpression(expression) { - return "", args, utils.ErrUnsupportedExpression - } - - switch e := expression.Payload.(type) { - case *api_service_protos.TExpression_Column: - return formatColumn(formatter, args, e.Column) - case *api_service_protos.TExpression_TypedValue: - return formatValue(formatter, args, e.TypedValue) - case *api_service_protos.TExpression_ArithmeticalExpression: - return formatArithmeticalExpression(formatter, args, e.ArithmeticalExpression) - case *api_service_protos.TExpression_Null: - return formatNull(formatter, args, e.Null) - default: - return "", args, fmt.Errorf("%w, type: %T", utils.ErrUnimplementedExpression, e) - } -} - -func formatComparison(formatter SQLFormatter, args []any, comparison *api_service_protos.TPredicate_TComparison) (string, []any, error) { - var operation string - - switch op := comparison.Operation; op { - case api_service_protos.TPredicate_TComparison_L: - operation = " < " - case api_service_protos.TPredicate_TComparison_LE: - operation = " <= " - case api_service_protos.TPredicate_TComparison_EQ: - operation = " = " - case api_service_protos.TPredicate_TComparison_NE: - operation = " <> " - case api_service_protos.TPredicate_TComparison_GE: - operation = " >= " - case api_service_protos.TPredicate_TComparison_G: - operation = " > " - default: - return "", args, fmt.Errorf("%w, op: %d", utils.ErrUnimplementedOperation, op) - } - - left, args, err := formatExpression(formatter, args, comparison.LeftValue) - if err != nil { - return "", args, fmt.Errorf("failed to format left argument: %w", err) - } - - right, args, err := formatExpression(formatter, args, comparison.RightValue) - if err != nil { - return "", args, fmt.Errorf("failed to format right argument: %w", err) - } - - return fmt.Sprintf("(%s%s%s)", left, operation, right), args, nil -} - -func formatNegation(formatter SQLFormatter, args []any, negation *api_service_protos.TPredicate_TNegation) (string, []any, error) { - pred, args, err := formatPredicate(formatter, args, negation.Operand, false) - if err != nil { - return "", args, fmt.Errorf("failed to format NOT statement: %w", err) - } - - return fmt.Sprintf("(NOT %s)", pred), args, nil -} - -func formatConjunction(formatter SQLFormatter, args []any, conjunction *api_service_protos.TPredicate_TConjunction, topLevel bool) (string, []any, error) { - var ( - sb strings.Builder - succeeded int32 = 0 - statement string - err error - first string - ) - - for _, predicate := range conjunction.Operands { - argsCut := args - statement, args, err = formatPredicate(formatter, args, predicate, false) - - if err != nil { - if !topLevel { - return "", args, fmt.Errorf("failed to format AND statement: %w", err) - } else { - args = argsCut - } - } else { - if succeeded > 0 { - if succeeded == 1 { - sb.WriteString("(") - sb.WriteString(first) - } - - sb.WriteString(" AND ") - sb.WriteString(statement) - } else { - first = statement - } - - succeeded++ - } - } - - if succeeded == 0 { - return "", args, fmt.Errorf("failed to format AND statement: %w", err) - } - - if succeeded == 1 { - sb.WriteString(first) - } else { - sb.WriteString(")") - } - - return sb.String(), args, nil -} - -func formatDisjunction(formatter SQLFormatter, args []any, disjunction *api_service_protos.TPredicate_TDisjunction) (string, []any, error) { - var ( - sb strings.Builder - cnt int32 = 0 - statement string - err error - first string - ) - - for _, predicate := range disjunction.Operands { - statement, args, err = formatPredicate(formatter, args, predicate, false) - if err != nil { - return "", args, fmt.Errorf("failed to format OR statement: %w", err) - } else { - if cnt > 0 { - if cnt == 1 { - sb.WriteString("(") - sb.WriteString(first) - } - - sb.WriteString(" OR ") - sb.WriteString(statement) - } else { - first = statement - } - - cnt++ - } - } - - if cnt == 0 { - return "", args, fmt.Errorf("failed to format OR statement: no operands") - } - - if cnt == 1 { - sb.WriteString(first) - } else { - sb.WriteString(")") - } - - return sb.String(), args, nil -} - -func formatIsNull(formatter SQLFormatter, args []any, isNull *api_service_protos.TPredicate_TIsNull) (string, []any, error) { - statement, args, err := formatExpression(formatter, args, isNull.Value) - if err != nil { - return "", args, fmt.Errorf("failed to format IS NULL statement: %w", err) - } - - return fmt.Sprintf("(%s IS NULL)", statement), args, nil -} - -func formatIsNotNull(formatter SQLFormatter, args []any, isNotNull *api_service_protos.TPredicate_TIsNotNull) (string, []any, error) { - statement, args, err := formatExpression(formatter, args, isNotNull.Value) - if err != nil { - return "", args, fmt.Errorf("failed to format IS NOT NULL statement: %w", err) - } - - return fmt.Sprintf("(%s IS NOT NULL)", statement), args, nil -} - -func formatPredicate(formatter SQLFormatter, args []any, predicate *api_service_protos.TPredicate, topLevel bool) (string, []any, error) { - switch p := predicate.Payload.(type) { - case *api_service_protos.TPredicate_Negation: - return formatNegation(formatter, args, p.Negation) - case *api_service_protos.TPredicate_Conjunction: - return formatConjunction(formatter, args, p.Conjunction, topLevel) - case *api_service_protos.TPredicate_Disjunction: - return formatDisjunction(formatter, args, p.Disjunction) - case *api_service_protos.TPredicate_IsNull: - return formatIsNull(formatter, args, p.IsNull) - case *api_service_protos.TPredicate_IsNotNull: - return formatIsNotNull(formatter, args, p.IsNotNull) - case *api_service_protos.TPredicate_Comparison: - return formatComparison(formatter, args, p.Comparison) - case *api_service_protos.TPredicate_BoolExpression: - return formatExpression(formatter, args, p.BoolExpression.Value) - default: - return "", args, fmt.Errorf("%w, type: %T", utils.ErrUnimplementedPredicateType, p) - } -} - -func formatWhereClause(formatter SQLFormatter, where *api_service_protos.TSelect_TWhere) (string, []any, error) { - if where.FilterTyped == nil { - return "", nil, utils.ErrUnimplemented - } - - args := make([]any, 0) - formatted, args, err := formatPredicate(formatter, args, where.FilterTyped, true) - - if err != nil { - return "", nil, err - } - - result := "WHERE " + formatted - - return result, args, nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/query_builder.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/query_builder.go deleted file mode 100644 index 871d4d546040..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/query_builder.go +++ /dev/null @@ -1,49 +0,0 @@ -package utils - -import ( - "fmt" - "strings" - - "github.com/ydb-platform/ydb/library/go/core/log" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func MakeDescribeTableQuery(logger log.Logger, formatter SQLFormatter, request *api_service_protos.TDescribeTableRequest) (string, []any) { - query, args := formatter.GetDescribeTableQuery(request) - - return query, args -} - -func MakeReadSplitQuery(logger log.Logger, formatter SQLFormatter, request *api_service_protos.TSelect) (string, []any, error) { - var ( - sb strings.Builder - args []any - ) - - selectPart, err := formatSelectColumns(formatter, request.What, request.GetFrom().GetTable(), true) - if err != nil { - return "", nil, fmt.Errorf("failed to format select statement: %w", err) - } - - sb.WriteString(selectPart) - - if request.Where != nil { - var clause string - - clause, args, err = formatWhereClause(formatter, request.Where) - if err != nil { - logger.Error("Failed to format WHERE clause", log.Error(err), log.String("where", request.Where.String())) - } else { - sb.WriteString(" ") - sb.WriteString(clause) - } - } - - query := sb.String() - - if args == nil { - args = []any{} - } - - return query, args, nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/select_helpers.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/select_helpers.go deleted file mode 100644 index fae3322a3d79..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/select_helpers.go +++ /dev/null @@ -1,63 +0,0 @@ -package utils - -import ( - "fmt" - "strings" - - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func selectWhatToYDBColumns(selectWhat *api_service_protos.TSelect_TWhat) ([]*Ydb.Column, error) { - var columns []*Ydb.Column - - for i, item := range selectWhat.Items { - column := item.GetColumn() - if column == nil { - return nil, fmt.Errorf("item #%d (%v) is not a column", i, item) - } - - columns = append(columns, column) - } - - return columns, nil -} - -func formatSelectColumns(formatter SQLFormatter, selectWhat *api_service_protos.TSelect_TWhat, tableName string, fakeZeroOnEmptyColumnsSet bool) (string, error) { - // SELECT $columns FROM $from - if tableName == "" { - return "", utils.ErrEmptyTableName - } - - var sb strings.Builder - - sb.WriteString("SELECT ") - - columns, err := selectWhatToYDBColumns(selectWhat) - if err != nil { - return "", fmt.Errorf("convert Select.What.Items to Ydb.Columns: %w", err) - } - - // for the case of empty column set select some constant for constructing a valid sql statement - if len(columns) == 0 { - if fakeZeroOnEmptyColumnsSet { - sb.WriteString("0") - } else { - return "", fmt.Errorf("empty columns set") - } - } else { - for i, column := range columns { - sb.WriteString(formatter.SanitiseIdentifier(column.GetName())) - - if i != len(columns)-1 { - sb.WriteString(", ") - } - } - } - - sb.WriteString(" FROM ") - sb.WriteString(formatter.SanitiseIdentifier(tableName)) - - return sb.String(), nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/sql.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/sql.go deleted file mode 100644 index 63c2efc194ce..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/sql.go +++ /dev/null @@ -1,46 +0,0 @@ -package utils - -import ( - "context" - - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type Connection interface { - Query(ctx context.Context, query string, args ...any) (Rows, error) - Close() error -} - -type Rows interface { - Close() error - Err() error - Next() bool - Scan(dest ...any) error - MakeTransformer(ydbTypes []*Ydb.Type) (utils.RowTransformer[any], error) -} - -type ConnectionManager interface { - Make(ctx context.Context, logger log.Logger, dataSourceInstance *api_common.TDataSourceInstance) (Connection, error) - Release(logger log.Logger, connection Connection) -} - -type ConnectionManagerBase struct { - QueryLoggerFactory utils.QueryLoggerFactory -} - -type SQLFormatter interface { - GetDescribeTableQuery(request *api_service_protos.TDescribeTableRequest) (string, []any) - - // Get placeholder for n'th argument (starting from 0) for prepared statement - GetPlaceholder(n int) string - - // Sanitize names of databases, tables, columns, views, schemas - SanitiseIdentifier(ident string) string - - // Support for high level expression (without subexpressions, they are checked separately) - SupportsPushdownExpression(expression *api_service_protos.TExpression) bool -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/sql_mock.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/sql_mock.go deleted file mode 100644 index b1799b5422a9..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/sql_mock.go +++ /dev/null @@ -1,117 +0,0 @@ -package utils - -import ( - "context" - - "github.com/apache/arrow/go/v13/arrow/array" - "github.com/stretchr/testify/mock" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -var _ Connection = (*ConnectionMock)(nil) - -type ConnectionMock struct { - mock.Mock -} - -func (m *ConnectionMock) Query(ctx context.Context, query string, params ...any) (Rows, error) { - called := []any{query} - called = append(called, params...) - args := m.Called(called...) - - return args.Get(0).(Rows), args.Error(1) -} - -func (m *ConnectionMock) Close() error { - return m.Called().Error(0) -} - -type ConnectionManagerMock struct { - mock.Mock -} - -func (m *ConnectionManagerMock) Make( - ctx context.Context, - logger log.Logger, - dataSourceInstance *api_common.TDataSourceInstance) (Connection, error) { - args := m.Called(dataSourceInstance) - - return args.Get(0).(Connection), args.Error(1) -} - -func (m *ConnectionManagerMock) Release(logger log.Logger, conn Connection) { - m.Called(conn) -} - -var _ Rows = (*RowsMock)(nil) - -type RowsMock struct { - mock.Mock - PredefinedData [][]any - scanCalls int -} - -func (m *RowsMock) Close() error { - return m.Called().Error(0) -} - -func (m *RowsMock) Err() error { - return m.Called().Error(0) -} - -func (m *RowsMock) Next() bool { - return m.Called().Bool(0) -} - -func (m *RowsMock) Scan(dest ...any) error { - args := m.Called(dest...) - - // mutate acceptors by reference - if m.scanCalls < len(m.PredefinedData) { - row := m.PredefinedData[m.scanCalls] - - for i, d := range dest { - switch t := d.(type) { - case **int32: - **t = row[i].(int32) - case **string: - **t = row[i].(string) - } - } - - m.scanCalls++ - } - - return args.Error(0) -} - -func (m *RowsMock) MakeTransformer(ydbType []*Ydb.Type) (utils.RowTransformer[any], error) { - args := m.Called(ydbType) - - return args.Get(0).(*RowTransformerMock), args.Error(1) -} - -var _ utils.RowTransformer[any] = (*RowTransformerMock)(nil) - -type RowTransformerMock struct { - mock.Mock - Acceptors []any -} - -func (t *RowTransformerMock) GetAcceptors() []any { return t.Acceptors } - -func (t *RowTransformerMock) SetAcceptors([]any) { - panic("not implemented") -} - -func (t *RowTransformerMock) AppendToArrowBuilders(builder []array.Builder) error { - builder[0].(*array.Int32Builder).Append(**t.Acceptors[0].(**int32)) - - cast := **t.Acceptors[1].(**string) - builder[1].(*array.BinaryBuilder).Append([]byte(cast)) - - return nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/ya.make deleted file mode 100644 index 5dbbd457aabc..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils/ya.make +++ /dev/null @@ -1,12 +0,0 @@ -GO_LIBRARY() - -SRCS( - doc.go - predicate_builder.go - query_builder.go - select_helpers.go - sql.go - sql_mock.go -) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/ya.make deleted file mode 100644 index 4acbab7adcc8..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/ya.make +++ /dev/null @@ -1,23 +0,0 @@ -GO_LIBRARY() - -SRCS( - data_source.go - data_source_factory.go - doc.go - schema_builder.go -) - -GO_TEST_SRCS( - data_source_test.go - schema_builder_test.go -) - -END() - -RECURSE( - clickhouse - postgresql - utils -) - -RECURSE_FOR_TESTS(ut) diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/data_source.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/data_source.go deleted file mode 100644 index 81cb969b86a6..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/data_source.go +++ /dev/null @@ -1,198 +0,0 @@ -package s3 - -import ( - "context" - "encoding/csv" - "fmt" - "io" - "strconv" - - "github.com/apache/arrow/go/v13/arrow/array" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var _ datasource.DataSource[string] = (*dataSource)(nil) - -type dataSource struct { -} - -func (ds *dataSource) DescribeTable(ctx context.Context, _ log.Logger, request *api_service_protos.TDescribeTableRequest) (*api_service_protos.TDescribeTableResponse, error) { - return nil, fmt.Errorf("table description is not implemented for schemaless data sources: %w", utils.ErrMethodNotSupported) -} - -func (ds *dataSource) ReadSplit(ctx context.Context, logger log.Logger, split *api_service_protos.TSplit, sink paging.Sink[string]) { - if err := ds.doReadSplit(ctx, logger, split, sink); err != nil { - sink.AddError(err) - } - - sink.Finish() -} - -func (ds *dataSource) doReadSplit( - ctx context.Context, - logger log.Logger, - split *api_service_protos.TSplit, - sink paging.Sink[string]) error { - conn := makeConnection() - - var ( - bucket string - key string - ) - - if bucket = split.Select.DataSourceInstance.GetS3Options().GetBucket(); bucket == "" { - return fmt.Errorf("empty field `bucket`: %w", utils.ErrInvalidRequest) - } - - if key = split.Select.From.GetObjectKey(); key == "" { - return fmt.Errorf("empty field `key`: %w", utils.ErrInvalidRequest) - } - - params := &s3.GetObjectInput{ - Bucket: &bucket, - Key: &key, - } - - response, err := conn.GetObject(ctx, params) - if err != nil { - return fmt.Errorf("get object: %w", err) - } - - defer response.Body.Close() - - csvReader := csv.NewReader(response.Body) - - if err := transformCSV(split.Select.What, split.Select.PredefinedSchema, csvReader, sink); err != nil { - return fmt.Errorf("transform csv: %w", err) - } - - return nil -} - -func makeAppender(ydbType *Ydb.Type) (func(acceptor string, builder array.Builder) error, error) { - var appender func(acceptor string, builder array.Builder) error - - typeID := ydbType.GetTypeId() - switch typeID { - case Ydb.Type_INT32: - appender = func(acceptor string, builder array.Builder) error { - value, err := strconv.Atoi(acceptor) - if err != nil { - return fmt.Errorf("strconv atoi '%v': %w", acceptor, err) - } - - builder.(*array.Int32Builder).Append(int32(value)) - - return nil - } - case Ydb.Type_STRING: - appender = func(acceptor string, builder array.Builder) error { - builder.(*array.StringBuilder).Append(acceptor) - - return nil - } - default: - return nil, fmt.Errorf("unexpected type %v: %w", typeID, utils.ErrDataTypeNotSupported) - } - - return appender, nil -} - -func prepareReading( - selectWhat *api_service_protos.TSelect_TWhat, - schema *api_service_protos.TSchema, -) ([]int, []func(acceptor string, builder array.Builder) error, error) { - result := make([]int, 0, len(selectWhat.Items)) - appenders := make([]func(acceptor string, builder array.Builder) error, 0, len(selectWhat.Items)) - - for _, item := range selectWhat.Items { - for i, column := range schema.Columns { - if item.GetColumn().Name == column.Name { - result = append(result, i) - - appender, err := makeAppender(column.Type) - if err != nil { - return nil, nil, fmt.Errorf("make appender for column #%d: %w", i, err) - } - - appenders = append(appenders, appender) - } - } - } - - if len(result) != len(selectWhat.Items) { - return nil, nil, fmt.Errorf( - "requested column with schema mismatch (wanted %d columns, found only %d): %w", - len(selectWhat.Items), len(result), utils.ErrInvalidRequest, - ) - } - - return result, appenders, nil -} - -func transformCSV( - selectWhat *api_service_protos.TSelect_TWhat, - schema *api_service_protos.TSchema, - csvReader *csv.Reader, - sink paging.Sink[string], -) error { - wantedColumnIds, appenders, err := prepareReading(selectWhat, schema) - if err != nil { - return fmt.Errorf("get wanted columns ids: %w", err) - } - - transformer := utils.NewRowTransformer[string](nil, appenders, wantedColumnIds) - - for { - row, err := csvReader.Read() - if err != nil { - if err == io.EOF { - break - } - - return fmt.Errorf("csv reader failure: %w", err) - } - - if len(schema.Columns) != len(row) { - return fmt.Errorf("schema and data mismatch: expected %d columns, got %d", len(schema.Columns), len(row)) - } - - // Save the row that was just read to make data accessible for other pipeline stages - transformer.SetAcceptors(row) - } - - return nil -} - -func makeConnection() *s3.Client { - resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { - return aws.Endpoint{ - PartitionID: "aws", - URL: "http://127.0.0.1:9000", - SigningRegion: "us-east-2", - HostnameImmutable: true, - }, nil - }) - - conn := s3.NewFromConfig(aws.Config{ - Region: "us-east-2", - Credentials: credentials.NewStaticCredentialsProvider("admin", "password", ""), - EndpointResolverWithOptions: resolver, - }, func(o *s3.Options) { - o.UsePathStyle = true - }) - - return conn -} - -func NewDataSource() datasource.DataSource[string] { - return &dataSource{} -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/doc.go b/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/doc.go deleted file mode 100644 index 51d37c88514c..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package s3 contains the implementation of S3 (Simple Storage Service) based data source -package s3 diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/ya.make deleted file mode 100644 index 9ca3b368decf..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/s3/ya.make +++ /dev/null @@ -1,8 +0,0 @@ -GO_LIBRARY() - -SRCS( - data_source.go - doc.go -) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/datasource/ya.make b/ydb/library/yql/providers/generic/connector/app/server/datasource/ya.make deleted file mode 100644 index 74443680b829..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/datasource/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -GO_LIBRARY() - -SRCS( - doc.go - interface.go - mock.go -) - -END() - -RECURSE( - rdbms - s3 -) diff --git a/ydb/library/yql/providers/generic/connector/app/server/doc.go b/ydb/library/yql/providers/generic/connector/app/server/doc.go deleted file mode 100644 index dbdca1745baa..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package server containes the code base of Connector GRPC service - -// the component of YDB's Federated Query system providing the unified interface -// to the external data sources. -package server diff --git a/ydb/library/yql/providers/generic/connector/app/server/grpc_metrics.go b/ydb/library/yql/providers/generic/connector/app/server/grpc_metrics.go deleted file mode 100644 index 9fbcd56c604d..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/grpc_metrics.go +++ /dev/null @@ -1,209 +0,0 @@ -package server - -import ( - "context" - "time" - - "github.com/ydb-platform/ydb/library/go/core/metrics" - "github.com/ydb-platform/ydb/library/go/core/metrics/solomon" - "google.golang.org/grpc" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" -) - -func UnaryServerMetrics(registry metrics.Registry) grpc.UnaryServerInterceptor { - requestCount := registry.CounterVec("requests_total", []string{"protocol", "endpoint"}) - requestDuration := registry.DurationHistogramVec("request_duration_seconds", metrics.MakeExponentialDurationBuckets(250*time.Microsecond, 1.5, 35), []string{"protocol", "endpoint"}) - panicsCount := registry.CounterVec("panics_total", []string{"protocol", "endpoint"}) - inflightRequests := registry.GaugeVec("inflight_requests", []string{"protocol", "endpoint"}) - statusCount := registry.CounterVec("status_total", []string{"protocol", "endpoint", "status"}) - requestBytes := registry.CounterVec("request_bytes", []string{"protocol", "endpoint"}) - responseBytes := registry.CounterVec("response_bytes", []string{"protocol", "endpoint"}) - - solomon.Rated(requestCount) - solomon.Rated(requestDuration) - solomon.Rated(panicsCount) - solomon.Rated(statusCount) - solomon.Rated(requestBytes) - solomon.Rated(responseBytes) - - return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ any, err error) { - deferFunc := func(startTime time.Time, opName string) { - requestDuration.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).RecordDuration(time.Since(startTime)) - - inflightRequests.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Add(-1) - - if p := recover(); p != nil { - panicsCount.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Inc() - panic(p) - } - } - - opName := info.FullMethod - - requestBytes.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Add(int64(proto.Size(req.(proto.Message)))) - - requestCount.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Inc() - - inflightRequests.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Add(1) - - startTime := time.Now() - defer deferFunc(startTime, opName) - - resp, err := handler(ctx, req) - - code := status.Code(err) - statusCount.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - "status": code.String(), - }).Inc() - - responseBytes.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Add(int64(proto.Size(resp.(proto.Message)))) - - return resp, err - } -} - -func StreamServerMetrics(registry metrics.Registry) grpc.StreamServerInterceptor { - streamCount := registry.CounterVec("streams_total", []string{"protocol", "endpoint"}) - streamDuration := registry.DurationHistogramVec("stream_duration_seconds", metrics.MakeExponentialDurationBuckets(250*time.Microsecond, 1.5, 35), []string{"protocol", "endpoint"}) - inflightStreams := registry.GaugeVec("inflight_streams", []string{"protocol", "endpoint"}) - panicsCount := registry.CounterVec("stream_panics_total", []string{"protocol", "endpoint"}) - sentStreamMessages := registry.CounterVec("sent_stream_messages_total", []string{"protocol", "endpoint"}) - receivedBytes := registry.CounterVec("received_bytes", []string{"protocol", "endpoint"}) - sentBytes := registry.CounterVec("sent_bytes", []string{"protocol", "endpoint"}) - statusCount := registry.CounterVec("stream_status_total", []string{"protocol", "endpoint", "status"}) - receivedStreamMessages := registry.CounterVec("received_stream_messages_total", []string{"protocol", "endpoint"}) - - solomon.Rated(streamCount) - solomon.Rated(streamDuration) - solomon.Rated(panicsCount) - solomon.Rated(sentStreamMessages) - solomon.Rated(receivedStreamMessages) - solomon.Rated(receivedBytes) - solomon.Rated(sentBytes) - solomon.Rated(statusCount) - - return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - deferFunc := func(startTime time.Time, opName string) { - streamDuration.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).RecordDuration(time.Since(startTime)) - - inflightStreams.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Add(-1) - - if p := recover(); p != nil { - panicsCount.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Inc() - panic(p) - } - } - - opName := info.FullMethod - - streamCount.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Inc() - - inflightStreams.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }).Add(1) - - startTime := time.Now() - defer deferFunc(startTime, opName) - - return handler(srv, serverStreamWithMessagesCount{ - ServerStream: ss, - sentStreamMessages: sentStreamMessages.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }), - receivedStreamMessages: receivedStreamMessages.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }), - sentBytes: sentBytes.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }), - receivedBytes: receivedBytes.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - }), - getStatusCounter: func(code string) metrics.Counter { - return statusCount.With(map[string]string{ - "protocol": "grpc", - "endpoint": opName, - "status": code, - }) - }, - }) - } -} - -type serverStreamWithMessagesCount struct { - grpc.ServerStream - sentStreamMessages metrics.Counter - receivedStreamMessages metrics.Counter - sentBytes metrics.Counter - receivedBytes metrics.Counter - getStatusCounter func(string) metrics.Counter -} - -func (s serverStreamWithMessagesCount) SendMsg(m any) error { - err := s.ServerStream.SendMsg(m) - - if err == nil { - s.sentStreamMessages.Inc() - s.sentBytes.Add(int64(proto.Size(m.(proto.Message)))) - } - - code := status.Code(err) - s.getStatusCounter(code.String()).Inc() - - return err -} - -func (s serverStreamWithMessagesCount) RecvMsg(m any) error { - err := s.ServerStream.RecvMsg(m) - - if err == nil { - s.receivedStreamMessages.Inc() - s.receivedBytes.Add(int64(proto.Size(m.(proto.Message)))) - } - - code := status.Code(err) - s.getStatusCounter(code.String()).Inc() - - return err -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/httppuller.go b/ydb/library/yql/providers/generic/connector/app/server/httppuller.go deleted file mode 100644 index b099fe4bca3b..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/httppuller.go +++ /dev/null @@ -1,126 +0,0 @@ -package server - -import ( - "context" - "fmt" - "io" - "net/http" - - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/library/go/core/log/nop" - "github.com/ydb-platform/ydb/library/go/core/metrics/solomon" - "github.com/ydb-platform/ydb/library/go/httputil/headers" -) - -type MetricsStreamer interface { - StreamJSON(context.Context, io.Writer) (int, error) - StreamSpack(context.Context, io.Writer, solomon.CompressionType) (int, error) -} - -type handler struct { - registry MetricsStreamer - streamFormat headers.ContentType - logger log.Logger -} - -type spackOption struct { -} - -func (*spackOption) isOption() {} - -func WithSpack() Option { - return &spackOption{} -} - -type Option interface { - isOption() -} - -// NewHTTPPullerHandler returns new HTTP handler to expose gathered metrics using metrics dumper -func NewHTTPPullerHandler(r MetricsStreamer, opts ...Option) http.Handler { - h := handler{ - registry: r, - streamFormat: headers.TypeApplicationJSON, - logger: &nop.Logger{}, - } - - for _, opt := range opts { - switch opt.(type) { - case *spackOption: - h.streamFormat = headers.TypeApplicationXSolomonSpack - default: - panic(fmt.Sprintf("unsupported option %T", opt)) - } - } - - return h -} - -func (h handler) okSpack(header http.Header) bool { - if h.streamFormat != headers.TypeApplicationXSolomonSpack { - return false - } - - for _, header := range header[headers.AcceptKey] { - types, err := headers.ParseAccept(header) - if err != nil { - h.logger.Warn("Can't parse accept header", log.Error(err), log.String("header", header)) - - continue - } - - for _, acceptableType := range types { - if acceptableType.Type == headers.TypeApplicationXSolomonSpack { - return true - } - } - } - - return false -} - -func (h handler) okLZ4Compression(header http.Header) bool { - for _, header := range header[headers.AcceptEncodingKey] { - encodings, err := headers.ParseAcceptEncoding(header) - - if err != nil { - h.logger.Warn("Can't parse accept-encoding header", log.Error(err), log.String("header", header)) - - continue - } - - for _, acceptableEncoding := range encodings { - if acceptableEncoding.Encoding == headers.EncodingLZ4 { - return true - } - } - } - - return false -} - -func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if h.okSpack(r.Header) { - compression := solomon.CompressionNone - - if h.okLZ4Compression(r.Header) { - compression = solomon.CompressionLz4 - } - - w.Header().Set(headers.ContentTypeKey, headers.TypeApplicationXSolomonSpack.String()) - _, err := h.registry.StreamSpack(r.Context(), w, compression) - - if err != nil { - h.logger.Error("Failed to write compressed spack", log.Error(err)) - } - - return - } - - w.Header().Set(headers.ContentTypeKey, headers.TypeApplicationJSON.String()) - _, err := h.registry.StreamJSON(r.Context(), w) - - if err != nil { - h.logger.Error("Failed to write json", log.Error(err)) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/launcher.go b/ydb/library/yql/providers/generic/connector/app/server/launcher.go deleted file mode 100644 index 4664eae7eb17..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/launcher.go +++ /dev/null @@ -1,128 +0,0 @@ -package server - -import ( - "fmt" - "os" - "os/signal" - "syscall" - - "github.com/spf13/cobra" - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/library/go/core/metrics/solomon" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -type service interface { - start() error - stop() -} - -type launcher struct { - services map[string]service - logger log.Logger -} - -func (l *launcher) start() <-chan error { - errChan := make(chan error, len(l.services)) - - for key := range l.services { - key := key - go func(key string) { - l.logger.Info("starting service", log.String("service", key)) - - // blocking call - errChan <- l.services[key].start() - }(key) - } - - return errChan -} - -func (l *launcher) stop() { - // TODO: make it concurrent - for key, s := range l.services { - l.logger.Info("stopping service", log.String("service", key)) - s.stop() - } -} - -const ( - connectorServiceKey = "connector" - pprofServiceKey = "pprof" - metricsKey = "metrics" -) - -func newLauncher(logger log.Logger, cfg *config.TServerConfig) (*launcher, error) { - l := &launcher{ - services: make(map[string]service, 2), - logger: logger, - } - - var err error - - registry := solomon.NewRegistry(&solomon.RegistryOpts{ - Separator: '.', - UseNameTag: true, - }) - - if cfg.MetricsServer != nil { - l.services[metricsKey] = newServiceMetrics( - log.With(logger, log.String("service", metricsKey)), - cfg.MetricsServer, registry) - } - - // init GRPC server - l.services[connectorServiceKey], err = newServiceConnector( - log.With(logger, log.String("service", connectorServiceKey)), - cfg, registry) - if err != nil { - return nil, fmt.Errorf("new connector server: %w", err) - } - - // init Pprof server - if cfg.PprofServer != nil { - l.services[pprofServiceKey] = newServicePprof( - log.With(logger, log.String("service", pprofServiceKey)), - cfg.PprofServer) - } - - return l, nil -} - -func run(cmd *cobra.Command, _ []string) error { - configPath, err := cmd.Flags().GetString(configFlag) - if err != nil { - return fmt.Errorf("get config flag: %v", err) - } - - cfg, err := newConfigFromPath(configPath) - if err != nil { - return fmt.Errorf("new config: %w", err) - } - - logger, err := utils.NewLoggerFromConfig(cfg.Logger) - if err != nil { - return fmt.Errorf("new logger from config: %w", err) - } - - l, err := newLauncher(logger, cfg) - if err != nil { - return fmt.Errorf("new launcher: %w", err) - } - - errChan := l.start() - - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM) - - select { - case err := <-errChan: - logger.Error("service fatal error", log.Error(err)) - case sig := <-signalChan: - logger.Info("interrupting signal", log.Any("value", sig)) - l.stop() - } - - return nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_arrow_ipc_streaming_default.go b/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_arrow_ipc_streaming_default.go deleted file mode 100644 index 7e5242981aee..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_arrow_ipc_streaming_default.go +++ /dev/null @@ -1,83 +0,0 @@ -package paging - -import ( - "bytes" - "fmt" - - "github.com/apache/arrow/go/v13/arrow" - "github.com/apache/arrow/go/v13/arrow/array" - "github.com/apache/arrow/go/v13/arrow/ipc" - "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type columnarBufferArrowIPCStreamingDefault[T utils.Acceptor] struct { - arrowAllocator memory.Allocator - builders []array.Builder - schema *arrow.Schema - logger log.Logger -} - -// AddRow saves a row obtained from the datasource into the buffer -func (cb *columnarBufferArrowIPCStreamingDefault[T]) addRow(transformer utils.RowTransformer[T]) error { - if len(cb.builders) != len(transformer.GetAcceptors()) { - return fmt.Errorf( - "expected %v values in a row, got %v instead", - len(cb.builders), len(transformer.GetAcceptors())) - } - - if err := transformer.AppendToArrowBuilders(cb.builders); err != nil { - return fmt.Errorf("append values to arrow builders: %w", err) - } - - return nil -} - -// ToResponse returns all the accumulated data and clears buffer -func (cb *columnarBufferArrowIPCStreamingDefault[T]) ToResponse() (*api_service_protos.TReadSplitsResponse, error) { - chunk := make([]arrow.Array, 0, len(cb.builders)) - - // prepare arrow record - for _, builder := range cb.builders { - chunk = append(chunk, builder.NewArray()) - } - - record := array.NewRecord(cb.schema, chunk, -1) - - for _, col := range chunk { - col.Release() - } - - // prepare arrow writer - var buf bytes.Buffer - - writer := ipc.NewWriter(&buf, ipc.WithSchema(cb.schema), ipc.WithAllocator(cb.arrowAllocator)) - - if err := writer.Write(record); err != nil { - return nil, fmt.Errorf("write record: %w", err) - } - - if err := writer.Close(); err != nil { - return nil, fmt.Errorf("close arrow writer: %w", err) - } - - out := &api_service_protos.TReadSplitsResponse{ - Payload: &api_service_protos.TReadSplitsResponse_ArrowIpcStreaming{ - ArrowIpcStreaming: buf.Bytes(), - }, - } - - return out, nil -} - -func (cb *columnarBufferArrowIPCStreamingDefault[T]) TotalRows() int { return cb.builders[0].Len() } - -// Frees resources if buffer is no longer used -func (cb *columnarBufferArrowIPCStreamingDefault[T]) Release() { - // cleanup builders - for _, b := range cb.builders { - b.Release() - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_arrow_ipc_streaming_empty_columns.go b/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_arrow_ipc_streaming_empty_columns.go deleted file mode 100644 index b0a89caae2f7..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_arrow_ipc_streaming_empty_columns.go +++ /dev/null @@ -1,65 +0,0 @@ -package paging - -import ( - "bytes" - "fmt" - - "github.com/apache/arrow/go/v13/arrow" - "github.com/apache/arrow/go/v13/arrow/array" - "github.com/apache/arrow/go/v13/arrow/ipc" - "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -// special implementation for buffer that writes schema with empty columns set -type columnarBufferArrowIPCStreamingEmptyColumns[T utils.Acceptor] struct { - arrowAllocator memory.Allocator - schema *arrow.Schema - rowsAdded int -} - -// AddRow saves a row obtained from the datasource into the buffer -func (cb *columnarBufferArrowIPCStreamingEmptyColumns[T]) addRow(transformer utils.RowTransformer[T]) error { - if len(transformer.GetAcceptors()) != 1 { - return fmt.Errorf("expected 1 value, got %v", len(transformer.GetAcceptors())) - } - - cb.rowsAdded++ - - return nil -} - -// ToResponse returns all the accumulated data and clears buffer -func (cb *columnarBufferArrowIPCStreamingEmptyColumns[T]) ToResponse() (*api_service_protos.TReadSplitsResponse, error) { - columns := make([]arrow.Array, 0) - - record := array.NewRecord(cb.schema, columns, int64(cb.rowsAdded)) - - // prepare arrow writer - var buf bytes.Buffer - - writer := ipc.NewWriter(&buf, ipc.WithSchema(cb.schema), ipc.WithAllocator(cb.arrowAllocator)) - - if err := writer.Write(record); err != nil { - return nil, fmt.Errorf("write record: %w", err) - } - - if err := writer.Close(); err != nil { - return nil, fmt.Errorf("close arrow writer: %w", err) - } - - out := &api_service_protos.TReadSplitsResponse{ - Payload: &api_service_protos.TReadSplitsResponse_ArrowIpcStreaming{ - ArrowIpcStreaming: buf.Bytes(), - }, - } - - return out, nil -} - -func (cb *columnarBufferArrowIPCStreamingEmptyColumns[T]) TotalRows() int { return int(cb.rowsAdded) } - -// Frees resources if buffer is no longer used -func (cb *columnarBufferArrowIPCStreamingEmptyColumns[T]) Release() { -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_factory.go b/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_factory.go deleted file mode 100644 index ec587539836c..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/columnar_buffer_factory.go +++ /dev/null @@ -1,74 +0,0 @@ -package paging - -import ( - "fmt" - - "github.com/apache/arrow/go/v13/arrow" - "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type columnarBufferFactoryImpl[T utils.Acceptor] struct { - arrowAllocator memory.Allocator - logger log.Logger - format api_service_protos.TReadSplitsRequest_EFormat - schema *arrow.Schema - ydbTypes []*Ydb.Type -} - -func (cbf *columnarBufferFactoryImpl[T]) MakeBuffer() (ColumnarBuffer[T], error) { - switch cbf.format { - case api_service_protos.TReadSplitsRequest_ARROW_IPC_STREAMING: - builders, err := utils.YdbTypesToArrowBuilders(cbf.ydbTypes, cbf.arrowAllocator) - if err != nil { - return nil, fmt.Errorf("convert Select.What to arrow.Schema: %w", err) - } - - if len(cbf.ydbTypes) == 0 { - return &columnarBufferArrowIPCStreamingEmptyColumns[T]{ - arrowAllocator: cbf.arrowAllocator, - schema: cbf.schema, - rowsAdded: 0, - }, nil - } - - return &columnarBufferArrowIPCStreamingDefault[T]{ - arrowAllocator: cbf.arrowAllocator, - builders: builders, - schema: cbf.schema, - logger: cbf.logger, - }, nil - default: - return nil, fmt.Errorf("unknown format: %v", cbf.format) - } -} - -func NewColumnarBufferFactory[T utils.Acceptor]( - logger log.Logger, - arrowAllocator memory.Allocator, - format api_service_protos.TReadSplitsRequest_EFormat, - selectWhat *api_service_protos.TSelect_TWhat, -) (ColumnarBufferFactory[T], error) { - ydbTypes, err := utils.SelectWhatToYDBTypes(selectWhat) - if err != nil { - return nil, fmt.Errorf("convert Select.What to Ydb types: %w", err) - } - - schema, err := utils.SelectWhatToArrowSchema(selectWhat) - if err != nil { - return nil, fmt.Errorf("convert Select.What to Arrow schema: %w", err) - } - - cbf := &columnarBufferFactoryImpl[T]{ - logger: logger, - arrowAllocator: arrowAllocator, - format: format, - schema: schema, - ydbTypes: ydbTypes, - } - - return cbf, nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/doc.go b/ydb/library/yql/providers/generic/connector/app/server/paging/doc.go deleted file mode 100644 index 702d6255f023..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package paging contains logic of splitting incoming data stream -// into the separate pages or blocks. -package paging diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/interface.go b/ydb/library/yql/providers/generic/connector/app/server/paging/interface.go deleted file mode 100644 index f0a200645633..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/interface.go +++ /dev/null @@ -1,42 +0,0 @@ -package paging - -import ( - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type ColumnarBuffer[T utils.Acceptor] interface { - // addRow saves a row obtained from the datasource into the columnar buffer - addRow(rowTransformer utils.RowTransformer[T]) error - // ToResponse returns all the accumulated data and clears buffer - ToResponse() (*api_service_protos.TReadSplitsResponse, error) - // Release frees resources if buffer is no longer used - Release() - // TotalRows return the number of rows accumulated - TotalRows() int -} - -type ColumnarBufferFactory[T utils.Acceptor] interface { - MakeBuffer() (ColumnarBuffer[T], error) -} - -// ReadResult is an algebraic data type containing: -// 1. a buffer (e. g. page) packed with data -// 2. result of read operation (potentially with error) -type ReadResult[T utils.Acceptor] struct { - ColumnarBuffer ColumnarBuffer[T] - Stats *api_service_protos.TReadSplitsResponse_TStats - Error error -} - -// Sink is a destination for a data stream that is read out of an external data source. -type Sink[T utils.Acceptor] interface { - // AddRow saves the row obtained from a stream incoming from an external data source. - AddRow(rowTransformer utils.RowTransformer[T]) error - // AddError propagates an error occured during the reading from the external data source. - AddError(err error) - // Finish reports the successful completion of reading the data stream. - Finish() - // ResultQueue returns a channel with results - ResultQueue() <-chan *ReadResult[T] -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/mock.go b/ydb/library/yql/providers/generic/connector/app/server/paging/mock.go deleted file mode 100644 index 7d016473e9d4..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/mock.go +++ /dev/null @@ -1,55 +0,0 @@ -package paging - -import ( - "github.com/stretchr/testify/mock" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var _ Sink[any] = (*SinkMock)(nil) - -type SinkMock struct { - mock.Mock -} - -func (m *SinkMock) AddRow(transformer utils.RowTransformer[any]) error { - args := m.Called(transformer) - - return args.Error(0) -} - -func (m *SinkMock) AddError(err error) { - m.Called(err) -} - -func (m *SinkMock) Finish() { - m.Called() -} - -func (m *SinkMock) ResultQueue() <-chan *ReadResult[any] { - return m.Called().Get(0).(chan *ReadResult[any]) -} - -var _ ColumnarBuffer[any] = (*ColumnarBufferMock)(nil) - -type ColumnarBufferMock struct { - mock.Mock -} - -func (m *ColumnarBufferMock) addRow(transformer utils.RowTransformer[any]) error { - panic("not implemented") // TODO: Implement -} - -func (m *ColumnarBufferMock) ToResponse() (*api_service_protos.TReadSplitsResponse, error) { - args := m.Called() - - return args.Get(0).(*api_service_protos.TReadSplitsResponse), args.Error(1) -} - -func (m *ColumnarBufferMock) Release() { - m.Called() -} - -func (m *ColumnarBufferMock) TotalRows() int { - panic("not implemented") // TODO: Implement -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/read_limiter.go b/ydb/library/yql/providers/generic/connector/app/server/paging/read_limiter.go deleted file mode 100644 index 3e7578c954a9..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/read_limiter.go +++ /dev/null @@ -1,55 +0,0 @@ -package paging - -import ( - "fmt" - - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -// ReadLimiter helps to limitate amount of data returned by Connector server in every read request. -// This is generally should be avoided after https://st.yandex-team.ru/YQ-2057 -type ReadLimiter interface { - addRow() error -} - -type readLimiterNoop struct { -} - -func (rl readLimiterNoop) addRow() error { return nil } - -type readLimiterRows struct { - rowsRead uint64 - rowsLimit uint64 -} - -func (rl *readLimiterRows) addRow() error { - if rl.rowsRead >= rl.rowsLimit { - return fmt.Errorf("can read only %d line(s) from data source per request: %w", - rl.rowsLimit, - utils.ErrReadLimitExceeded) - } - - rl.rowsRead++ - - return nil -} - -type ReadLimiterFactory struct { - cfg *config.TServerReadLimit -} - -func (rlf *ReadLimiterFactory) MakeReadLimiter(logger log.Logger) ReadLimiter { - if rlf.cfg == nil { - return readLimiterNoop{} - } - - logger.Warn(fmt.Sprintf("Server will return only first %d lines from the data source", rlf.cfg.GetRows())) - - return &readLimiterRows{rowsRead: 0, rowsLimit: rlf.cfg.GetRows()} -} - -func NewReadLimiterFactory(cfg *config.TServerReadLimit) *ReadLimiterFactory { - return &ReadLimiterFactory{cfg: cfg} -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/sink.go b/ydb/library/yql/providers/generic/connector/app/server/paging/sink.go deleted file mode 100644 index a2eae14ae6be..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/sink.go +++ /dev/null @@ -1,170 +0,0 @@ -//go:generate stringer -type=sinkState -output=sink_string.go -package paging - -import ( - "context" - "fmt" - - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type sinkState int8 - -const ( - operational sinkState = iota + 1 - failed - finished -) - -var _ Sink[any] = (*sinkImpl[any])(nil) -var _ Sink[string] = (*sinkImpl[string])(nil) - -type sinkImpl[T utils.Acceptor] struct { - currBuffer ColumnarBuffer[T] // accumulates incoming rows - resultQueue chan *ReadResult[T] // outgoing buffer queue - bufferFactory ColumnarBufferFactory[T] // creates new buffer - trafficTracker *TrafficTracker[T] // tracks the amount of data passed through the sink - readLimiter ReadLimiter // helps to restrict the number of rows read in every request - logger log.Logger // annotated logger - state sinkState // flag showing if it's ready to return data - ctx context.Context // client context -} - -func (s *sinkImpl[T]) AddRow(rowTransformer utils.RowTransformer[T]) error { - if s.state != operational { - panic(s.unexpectedState(operational)) - } - - if err := s.readLimiter.addRow(); err != nil { - return fmt.Errorf("add row to read limiter: %w", err) - } - - // Check if we can add one more data row - // without exceeding page size limit. - ok, err := s.trafficTracker.tryAddRow(rowTransformer.GetAcceptors()) - if err != nil { - return fmt.Errorf("add row to traffic tracker: %w", err) - } - - // If page is already too large, flush buffer to the channel and create a new one - if !ok { - if err := s.flush(true); err != nil { - return fmt.Errorf("flush: %w", err) - } - - _, err := s.trafficTracker.tryAddRow(rowTransformer.GetAcceptors()) - if err != nil { - return fmt.Errorf("add row to traffic tracker: %w", err) - } - } - - // Append row data to the columnar buffer - if err := s.currBuffer.addRow(rowTransformer); err != nil { - return fmt.Errorf("add row to buffer: %w", err) - } - - return nil -} - -func (s *sinkImpl[T]) AddError(err error) { - if s.state != operational { - panic(s.unexpectedState(operational)) - } - - s.respondWith(nil, nil, err) - - s.state = failed -} - -func (s *sinkImpl[T]) flush(makeNewBuffer bool) error { - if s.currBuffer.TotalRows() == 0 { - return nil - } - - stats := s.trafficTracker.DumpStats(false) - - // enqueue message to GRPC stream - s.respondWith(s.currBuffer, stats, nil) - - // create empty buffer and reset counters - s.currBuffer = nil - s.trafficTracker.refreshCounters() - - if makeNewBuffer { - var err error - - s.currBuffer, err = s.bufferFactory.MakeBuffer() - if err != nil { - return fmt.Errorf("make buffer: %w", err) - } - } - - return nil -} - -func (s *sinkImpl[T]) Finish() { - if s.state != operational && s.state != failed { - panic(s.unexpectedState(operational, failed)) - } - - // if there is some data left, send it to the channel - if s.state == operational { - err := s.flush(false) - if err != nil { - s.respondWith(nil, nil, fmt.Errorf("flush: %w", err)) - s.state = failed - } else { - s.state = finished - } - } - - // notify reader about the end of data - close(s.resultQueue) -} - -func (s *sinkImpl[T]) ResultQueue() <-chan *ReadResult[T] { - return s.resultQueue -} - -func (s *sinkImpl[T]) respondWith( - buf ColumnarBuffer[T], - stats *api_service_protos.TReadSplitsResponse_TStats, - err error) { - select { - case s.resultQueue <- &ReadResult[T]{ColumnarBuffer: buf, Stats: stats, Error: err}: - case <-s.ctx.Done(): - } -} - -func (s *sinkImpl[T]) unexpectedState(expected ...sinkState) error { - return fmt.Errorf( - "unexpected state '%v' (expected are '%v'): %w", - s.state, expected, utils.ErrInvariantViolation) -} - -func NewSink[T utils.Acceptor]( - ctx context.Context, - logger log.Logger, - trafficTracker *TrafficTracker[T], - columnarBufferFactory ColumnarBufferFactory[T], - readLimiter ReadLimiter, - resultQueueCapacity int, -) (Sink[T], error) { - buffer, err := columnarBufferFactory.MakeBuffer() - if err != nil { - return nil, fmt.Errorf("wrap buffer: %w", err) - } - - return &sinkImpl[T]{ - bufferFactory: columnarBufferFactory, - readLimiter: readLimiter, - resultQueue: make(chan *ReadResult[T], resultQueueCapacity), - trafficTracker: trafficTracker, - currBuffer: buffer, - logger: logger, - state: operational, - ctx: ctx, - }, nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/sink_string.go b/ydb/library/yql/providers/generic/connector/app/server/paging/sink_string.go deleted file mode 100644 index bea6becd6709..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/sink_string.go +++ /dev/null @@ -1,26 +0,0 @@ -// Code generated by "stringer -type=sinkState -output=sink_string.go"; DO NOT EDIT. - -package paging - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[operational-1] - _ = x[failed-2] - _ = x[finished-3] -} - -const _sinkState_name = "operationalfailedfinished" - -var _sinkState_index = [...]uint8{0, 11, 17, 25} - -func (i sinkState) String() string { - i -= 1 - if i < 0 || i >= sinkState(len(_sinkState_index)-1) { - return "sinkState(" + strconv.FormatInt(int64(i+1), 10) + ")" - } - return _sinkState_name[_sinkState_index[i]:_sinkState_index[i+1]] -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/size.go b/ydb/library/yql/providers/generic/connector/app/server/paging/size.go deleted file mode 100644 index 955c97a556f7..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/size.go +++ /dev/null @@ -1,128 +0,0 @@ -package paging - -import ( - "fmt" - "reflect" - "time" - - "github.com/jackc/pgx/v5/pgtype" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -// Some acceptors belong to fixed-size types (integers, floats, booleans), -// while the others contain variable length types (arrays, slices, maps, strings). -// We will save CPU if we calculate the size of acceptors of fixed types only once. -type acceptorKind int8 - -const ( - unknownSize acceptorKind = iota - fixedSize - variableSize -) - -type sizePattern[T utils.Acceptor] struct { - // Ordered numbers of acceptors of variable legnth types. - // Their size must be estimated every time. - varyingSizeIx []int - // We summarize the size of fixed size acceptors here. - // They never change between the rows. - fixedSizeTotal uint64 -} - -func (sp *sizePattern[T]) estimate(acceptors []T) (uint64, error) { - sizeTotal := sp.fixedSizeTotal - - for _, ix := range sp.varyingSizeIx { - sizeVariable, _, err := sizeOfValue(acceptors[ix]) - if err != nil { - return 0, fmt.Errorf("size of value #%d: %w", ix, err) - } - - sizeTotal += sizeVariable - } - - return sizeTotal, nil -} - -func newSizePattern[T utils.Acceptor](acceptors []T) (*sizePattern[T], error) { - sp := &sizePattern[T]{} - - for i, acceptor := range acceptors { - size, kind, err := sizeOfValue(acceptor) - if err != nil { - return nil, fmt.Errorf("estimate size of value #%d: %w", i, err) - } - - switch kind { - case fixedSize: - sp.fixedSizeTotal += size - case variableSize: - sp.varyingSizeIx = append(sp.varyingSizeIx, i) - default: - return nil, fmt.Errorf("unknown type kind: %w", err) - } - } - - return sp, nil -} - -func sizeOfValue(v any) (uint64, acceptorKind, error) { - reflected := reflect.ValueOf(v) - - // for nil values - value := reflect.Indirect(reflected) - - if value.Kind() == reflect.Ptr { - // unwrap double pointer - value = reflect.Indirect(value) - } - - if !value.IsValid() { - return 0, variableSize, nil - } - - // TODO: in order to support complicated and composite data types - // one should write reflection code in spite of - // https://github.com/DmitriyVTitov/size/blob/master/size.go - switch t := value.Interface().(type) { - case bool: - return 1, fixedSize, nil - case int8, uint8: - return 1, fixedSize, nil - case int16, uint16: - return 2, fixedSize, nil - case int32, uint32, float32: - return 4, fixedSize, nil - case int64, uint64, float64: - return 8, fixedSize, nil - case time.Time: - // time.Time and all its derivatives consist of two 8-byte ints: - // https://cs.opensource.google/go/go/+/refs/tags/go1.21.4:src/time/time.go;l=141-142 - // Location is ignored. - return 16, fixedSize, nil - case []byte: - return uint64(len(t)), variableSize, nil - case string: - return uint64(len(t)), variableSize, nil - case pgtype.Bool: - return 1, fixedSize, nil - case pgtype.Int2: - return 2, fixedSize, nil - case pgtype.Int4: - return 4, fixedSize, nil - case pgtype.Int8: - return 8, fixedSize, nil - case pgtype.Float4: - return 4, fixedSize, nil - case pgtype.Float8: - return 8, fixedSize, nil - case pgtype.Text: - return uint64(len(t.String)), variableSize, nil - case pgtype.Date: - return 16, fixedSize, nil - case pgtype.Timestamp: - return 16, fixedSize, nil - default: - return 0, 0, fmt.Errorf("value %v of unexpected data type %T: %w", t, t, utils.ErrDataTypeNotSupported) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/size_test.go b/ydb/library/yql/providers/generic/connector/app/server/paging/size_test.go deleted file mode 100644 index 91c79b3b31eb..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/size_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package paging - -import ( - "reflect" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -type testCaseSize[Type any] struct { - value Type - expectedSize uint64 - expectedKind acceptorKind -} - -func (tc testCaseSize[Type]) execute(t *testing.T) { - name := reflect.TypeOf(tc.value).Name() - - t.Run(name, func(t *testing.T) { - x0 := tc.value - x1 := new(Type) - *x1 = x0 - x2 := new(*Type) - *x2 = x1 - - size0, kind0, err := sizeOfValue(x0) - require.NoError(t, err) - require.Equal(t, size0, tc.expectedSize) - require.Equal(t, kind0, tc.expectedKind) - - size1, kind1, err := sizeOfValue(x1) - require.NoError(t, err) - require.Equal(t, size1, tc.expectedSize) - require.Equal(t, kind1, tc.expectedKind) - - size2, kind2, err := sizeOfValue(x2) - require.NoError(t, err) - require.Equal(t, size2, tc.expectedSize) - require.Equal(t, kind2, tc.expectedKind) - }) -} - -func TestSize(t *testing.T) { - type testCase interface { - execute(t *testing.T) - } - - testCases := []testCase{ - testCaseSize[int8]{value: 1, expectedSize: 1, expectedKind: fixedSize}, - testCaseSize[int16]{value: 1, expectedSize: 2, expectedKind: fixedSize}, - testCaseSize[int32]{value: 1, expectedSize: 4, expectedKind: fixedSize}, - testCaseSize[int64]{value: 1, expectedSize: 8, expectedKind: fixedSize}, - testCaseSize[uint8]{value: 1, expectedSize: 1, expectedKind: fixedSize}, - testCaseSize[uint16]{value: 1, expectedSize: 2, expectedKind: fixedSize}, - testCaseSize[uint32]{value: 1, expectedSize: 4, expectedKind: fixedSize}, - testCaseSize[uint64]{value: 1, expectedSize: 8, expectedKind: fixedSize}, - testCaseSize[float32]{value: 1.0, expectedSize: 4, expectedKind: fixedSize}, - testCaseSize[float64]{value: 1.0, expectedSize: 8, expectedKind: fixedSize}, - testCaseSize[string]{value: "abcde", expectedSize: 5, expectedKind: variableSize}, - testCaseSize[string]{value: "абвгд", expectedSize: 10, expectedKind: variableSize}, - testCaseSize[[]byte]{value: []byte("abcde"), expectedSize: 5, expectedKind: variableSize}, - testCaseSize[[]byte]{value: []byte("абвгд"), expectedSize: 10, expectedKind: variableSize}, - testCaseSize[time.Time]{value: time.Now().UTC(), expectedSize: 16, expectedKind: fixedSize}, - } - - for _, tc := range testCases { - tc.execute(t) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/traffic_tracker.go b/ydb/library/yql/providers/generic/connector/app/server/paging/traffic_tracker.go deleted file mode 100644 index 730fbcb9b6c0..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/traffic_tracker.go +++ /dev/null @@ -1,126 +0,0 @@ -package paging - -import ( - "fmt" - - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type TrafficTracker[T utils.Acceptor] struct { - pagination *config.TPagingConfig - sizePattern *sizePattern[T] - - // cumulative sums of bytes passed and rows handled since the start of the request - bytesTotal *utils.Counter[uint64] - rowsTotal *utils.Counter[uint64] - - // sums of bytes and rows accumulated since last flush - bytesCurr *utils.Counter[uint64] - rowsCurr *utils.Counter[uint64] -} - -// tryAddRow checks if the addition of the next row -// would exceed the limits on the page size. -// If there's enough space in buffer, it returns true and increases the internal counters. -// Otherwise it return false, but doesn't change internal state. -func (tt *TrafficTracker[T]) tryAddRow(acceptors []T) (bool, error) { - if err := tt.maybeInit(acceptors); err != nil { - return false, fmt.Errorf("maybe init: %w", err) - } - - totalBytes, err := tt.sizePattern.estimate(acceptors) - if err != nil { - return false, fmt.Errorf("size pattern estimate: %w", err) - } - - wouldBeEnough, err := tt.checkPageSizeLimit(totalBytes, 1) - if err != nil { - return false, fmt.Errorf("check page size limit: %w", err) - } - - if wouldBeEnough { - return false, nil - } - - tt.bytesCurr.Add(totalBytes) - tt.rowsCurr.Add(1) - - return true, nil -} - -func (tt *TrafficTracker[T]) maybeInit(acceptors []T) error { - if tt.sizePattern == nil { - // lazy initialization when the first row is ready - var err error - tt.sizePattern, err = newSizePattern(acceptors) - - if err != nil { - return fmt.Errorf("new size pattern: %w", err) - } - } - - return nil -} - -func (tt *TrafficTracker[T]) checkPageSizeLimit(bytesDelta, rowsDelta uint64) (bool, error) { - if tt.pagination.BytesPerPage != 0 { - // almost impossible case, but have to check - if bytesDelta > tt.pagination.BytesPerPage { - err := fmt.Errorf( - "single row size exceeds page size limit (%d > %d bytes): %w", - bytesDelta, - tt.pagination.BytesPerPage, - utils.ErrPageSizeExceeded) - - return true, err - } - - if tt.bytesCurr.Value()+bytesDelta > tt.pagination.BytesPerPage { - return true, nil - } - } - - if tt.pagination.RowsPerPage != 0 { - if tt.rowsCurr.Value()+rowsDelta > tt.pagination.RowsPerPage { - return true, nil - } - } - - return false, nil -} - -func (tt *TrafficTracker[T]) refreshCounters() { - tt.bytesCurr = tt.bytesTotal.MakeChild() - tt.rowsCurr = tt.rowsTotal.MakeChild() -} - -func (tt *TrafficTracker[T]) DumpStats(total bool) *api_service_protos.TReadSplitsResponse_TStats { - rowsCounter := tt.rowsCurr - bytesCounter := tt.bytesCurr - - if total { - rowsCounter = tt.rowsTotal - bytesCounter = tt.bytesTotal - } - - result := &api_service_protos.TReadSplitsResponse_TStats{ - Rows: rowsCounter.Value(), - Bytes: bytesCounter.Value(), - } - - return result -} - -func NewTrafficTracker[T utils.Acceptor](pagination *config.TPagingConfig) *TrafficTracker[T] { - tt := &TrafficTracker[T]{ - pagination: pagination, - bytesTotal: utils.NewCounter[uint64](), - rowsTotal: utils.NewCounter[uint64](), - } - - tt.refreshCounters() - - return tt -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/traffic_tracker_test.go b/ydb/library/yql/providers/generic/connector/app/server/paging/traffic_tracker_test.go deleted file mode 100644 index ececdc3cc8fa..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/traffic_tracker_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package paging - -import ( - "errors" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func TestTrafficTracker(t *testing.T) { - t.Run("separate pages by rows", func(t *testing.T) { - cfg := &config.TPagingConfig{ - RowsPerPage: 2, - } - - tt := NewTrafficTracker[any](cfg) - - col1Acceptor := new(int32) - col2Acceptor := new(string) - col3Acceptor := new(uint64) - - acceptors := []any{col1Acceptor, col2Acceptor, col3Acceptor} - - *col1Acceptor = 1 // 4 bytes - *col2Acceptor = "abcde" // 5 bytes - *col3Acceptor = 1 // 8 bytes - - ok, err := tt.tryAddRow(acceptors) - require.NoError(t, err) - require.True(t, ok) - - *col1Acceptor = 1 // 4 bytes - *col2Acceptor = "абвгд" // 10 bytes - *col3Acceptor = 1 // 8 bytes - - ok, err = tt.tryAddRow(acceptors) - require.NoError(t, err) - require.True(t, ok) - - actualStats := tt.DumpStats(false) - expectedStats := &api_service_protos.TReadSplitsResponse_TStats{ - Rows: 2, - Bytes: (4 + 5 + 8) + (4 + 10 + 8), - } - require.Equal(t, expectedStats, actualStats) - - // emulate buffer flushing - - tt.refreshCounters() - require.Zero(t, tt.bytesCurr.Value()) - require.Zero(t, tt.rowsCurr.Value()) - require.Equal(t, expectedStats.Bytes, tt.bytesTotal.Value()) - require.Equal(t, expectedStats.Rows, tt.rowsTotal.Value()) - - // add some more rows after flushing - - *col1Acceptor = 1 // 4 bytes - *col2Acceptor = "abcde" // 5 bytes - *col3Acceptor = 1 // 8 bytes - - ok, err = tt.tryAddRow(acceptors) - require.NoError(t, err) - require.True(t, ok) - - actualStats = tt.DumpStats(false) - expectedStats = &api_service_protos.TReadSplitsResponse_TStats{ - Rows: 1, - Bytes: (4 + 5 + 8), - } - require.Equal(t, expectedStats, actualStats) - - // global stats reflects previous events - actualStats = tt.DumpStats(true) - expectedStats = &api_service_protos.TReadSplitsResponse_TStats{ - Rows: 3, - Bytes: (4 + 5 + 8) + (4 + 10 + 8) + (4 + 5 + 8), - } - require.Equal(t, expectedStats, actualStats) - }) - - t.Run("separate pages by bytes", func(t *testing.T) { - cfg := &config.TPagingConfig{ - BytesPerPage: 40, - } - - tt := NewTrafficTracker[any](cfg) - - col1Acceptor := new(uint64) - col2Acceptor := new([]byte) - col3Acceptor := new(time.Time) - - acceptors := []any{col1Acceptor, col2Acceptor, col3Acceptor} - - *col1Acceptor = 1 // 8 bytes - *col2Acceptor = []byte{1, 2, 3, 4, 5} // 5 bytes - *col3Acceptor = time.Now().UTC() // approximately 16 bytes - - ok, err := tt.tryAddRow(acceptors) - require.NoError(t, err) - require.True(t, ok) // (8 + 5 + 16) < 40 - - ok, err = tt.tryAddRow(acceptors) - require.NoError(t, err) - require.False(t, ok) // 2 * (8 + 5 + 16) > 40 - - // only first addition resides in stats - actualStats := tt.DumpStats(false) - expectedStats := &api_service_protos.TReadSplitsResponse_TStats{ - Rows: 1, - Bytes: (8 + 5 + 16), - } - require.Equal(t, expectedStats, actualStats) - - tt.refreshCounters() - require.Zero(t, tt.bytesCurr.Value()) - require.Zero(t, tt.rowsCurr.Value()) - require.Equal(t, expectedStats.Bytes, tt.bytesTotal.Value()) - require.Equal(t, expectedStats.Rows, tt.rowsTotal.Value()) - }) - - t.Run("too small page", func(t *testing.T) { - cfg := &config.TPagingConfig{ - BytesPerPage: 1, - } - - tt := NewTrafficTracker[any](cfg) - col1Acceptor := new(int32) - acceptors := []any{col1Acceptor} - - *col1Acceptor = 1 // 4 bytes > 1 byte - - ok, err := tt.tryAddRow(acceptors) - require.True(t, errors.Is(err, utils.ErrPageSizeExceeded)) - require.False(t, ok) - }) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/ut/ya.make b/ydb/library/yql/providers/generic/connector/app/server/paging/ut/ya.make deleted file mode 100644 index 14e4aba38723..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/ut/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -GO_TEST_FOR(ydb/library/yql/providers/generic/connector/app/server/paging) - -SIZE(SMALL) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/paging/ya.make b/ydb/library/yql/providers/generic/connector/app/server/paging/ya.make deleted file mode 100644 index de3a25e86af0..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/paging/ya.make +++ /dev/null @@ -1,24 +0,0 @@ -GO_LIBRARY() - -SRCS( - columnar_buffer_arrow_ipc_streaming_default.go - columnar_buffer_arrow_ipc_streaming_empty_columns.go - columnar_buffer_factory.go - doc.go - interface.go - mock.go - read_limiter.go - sink.go - sink_string.go - size.go - traffic_tracker.go -) - -GO_TEST_SRCS( - size_test.go - traffic_tracker_test.go -) - -END() - -RECURSE_FOR_TESTS(ut) diff --git a/ydb/library/yql/providers/generic/connector/app/server/service_connector.go b/ydb/library/yql/providers/generic/connector/app/server/service_connector.go deleted file mode 100644 index 1ba92e54d58e..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/service_connector.go +++ /dev/null @@ -1,298 +0,0 @@ -package server - -import ( - "context" - "crypto/tls" - "fmt" - "net" - - "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/library/go/core/metrics/solomon" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/reflection" -) - -type serviceConnector struct { - api_service.UnimplementedConnectorServer - dataSourceCollection *DataSourceCollection - cfg *config.TServerConfig - grpcServer *grpc.Server - listener net.Listener - logger log.Logger -} - -func (s *serviceConnector) ListTables(_ *api_service_protos.TListTablesRequest, _ api_service.Connector_ListTablesServer) error { - return nil -} - -func (s *serviceConnector) DescribeTable( - ctx context.Context, - request *api_service_protos.TDescribeTableRequest, -) (*api_service_protos.TDescribeTableResponse, error) { - logger := utils.AnnotateLogger(s.logger, "DescribeTable", request.DataSourceInstance) - logger.Info("request handling started", log.String("table", request.GetTable())) - - if err := ValidateDescribeTableRequest(logger, request); err != nil { - logger.Error("request handling failed", log.Error(err)) - - return &api_service_protos.TDescribeTableResponse{ - Error: utils.NewAPIErrorFromStdError(err), - }, nil - } - - out, err := s.dataSourceCollection.DescribeTable(ctx, logger, request) - if err != nil { - logger.Error("request handling failed", log.Error(err)) - - out = &api_service_protos.TDescribeTableResponse{Error: utils.NewAPIErrorFromStdError(err)} - - return out, nil - } - - out.Error = utils.NewSuccess() - logger.Info("request handling finished", log.String("response", out.String())) - - return out, nil -} - -func (s *serviceConnector) ListSplits(request *api_service_protos.TListSplitsRequest, stream api_service.Connector_ListSplitsServer) error { - logger := utils.AnnotateLogger(s.logger, "ListSplits", nil) - logger.Info("request handling started", log.Int("total selects", len(request.Selects))) - - if err := ValidateListSplitsRequest(logger, request); err != nil { - return s.doListSplitsResponse(logger, stream, - &api_service_protos.TListSplitsResponse{Error: utils.NewAPIErrorFromStdError(err)}) - } - - // Make a trivial copy of requested selects - totalSplits := 0 - - for _, slct := range request.Selects { - if err := s.doListSplitsHandleSelect(stream, slct, &totalSplits); err != nil { - logger.Error("request handling failed", log.Error(err)) - - return err - } - } - - logger.Info("request handling finished", log.Int("total_splits", totalSplits)) - - return nil -} - -func (s *serviceConnector) doListSplitsHandleSelect( - stream api_service.Connector_ListSplitsServer, - slct *api_service_protos.TSelect, - totalSplits *int, -) error { - logger := utils.AnnotateLogger(s.logger, "ListSplits", slct.DataSourceInstance) - - args := []log.Field{ - log.Int("split_id", *totalSplits), - } - args = append(args, utils.SelectToFields(slct)...) - - logger.Debug("responding selects", args...) - - resp := &api_service_protos.TListSplitsResponse{ - Error: utils.NewSuccess(), - Splits: []*api_service_protos.TSplit{{Select: slct}}, - } - - for _, split := range resp.Splits { - args := []log.Field{ - log.Int("split_id", *totalSplits), - } - args = append(args, utils.SelectToFields(split.Select)...) - - logger.Debug("responding split", args...) - - *totalSplits++ - } - - if err := s.doListSplitsResponse(logger, stream, resp); err != nil { - return err - } - - return nil -} - -func (s *serviceConnector) doListSplitsResponse( - logger log.Logger, - stream api_service.Connector_ListSplitsServer, - response *api_service_protos.TListSplitsResponse, -) error { - if !utils.IsSuccess(response.Error) { - logger.Error("request handling failed", utils.APIErrorToLogFields(response.Error)...) - } - - if err := stream.Send(response); err != nil { - logger.Error("send channel failed", log.Error(err)) - - return err - } - - return nil -} - -func (s *serviceConnector) ReadSplits( - request *api_service_protos.TReadSplitsRequest, - stream api_service.Connector_ReadSplitsServer) error { - logger := utils.AnnotateLogger(s.logger, "ReadSplits", request.DataSourceInstance) - logger.Info("request handling started", log.Int("total_splits", len(request.Splits))) - - err := s.doReadSplits(logger, request, stream) - if err != nil { - logger.Error("request handling failed", log.Error(err)) - - response := &api_service_protos.TReadSplitsResponse{Error: utils.NewAPIErrorFromStdError(err)} - - if err := stream.Send(response); err != nil { - return fmt.Errorf("stream send: %w", err) - } - } - - logger.Info("request handling finished") - - return nil -} - -func (s *serviceConnector) doReadSplits( - logger log.Logger, - request *api_service_protos.TReadSplitsRequest, - stream api_service.Connector_ReadSplitsServer, -) error { - if err := ValidateReadSplitsRequest(logger, request); err != nil { - return fmt.Errorf("validate read splits request: %w", err) - } - - for i, split := range request.Splits { - splitLogger := log.With(logger, log.Int("split_id", i)) - - err := s.dataSourceCollection.DoReadSplit( - splitLogger, - stream, - request, - split, - ) - - if err != nil { - return fmt.Errorf("read split %d: %w", i, err) - } - } - - return nil -} - -func (s *serviceConnector) start() error { - s.logger.Debug("starting GRPC server", log.String("address", s.listener.Addr().String())) - - if err := s.grpcServer.Serve(s.listener); err != nil { - return fmt.Errorf("listener serve: %w", err) - } - - return nil -} - -func makeGRPCOptions(logger log.Logger, cfg *config.TServerConfig, registry *solomon.Registry) ([]grpc.ServerOption, error) { - var ( - opts []grpc.ServerOption - tlsConfig *config.TServerTLSConfig - ) - - unaryInterceptors := []grpc.UnaryServerInterceptor{UnaryServerMetrics(registry)} - - streamInterceptors := []grpc.StreamServerInterceptor{StreamServerMetrics(registry)} - - opts = append(opts, grpc.ChainUnaryInterceptor(unaryInterceptors...), grpc.ChainStreamInterceptor(streamInterceptors...)) - - // TODO: drop deprecated fields after YQ-2057 - switch { - case cfg.GetConnectorServer().GetTls() != nil: - tlsConfig = cfg.GetConnectorServer().GetTls() - case cfg.GetTls() != nil: - tlsConfig = cfg.GetTls() - default: - logger.Warn("server will use insecure connections") - - return opts, nil - } - - logger.Info("server will use TLS connections") - - logger.Debug("reading key pair", log.String("cert", tlsConfig.Cert), log.String("key", tlsConfig.Key)) - - cert, err := tls.LoadX509KeyPair(tlsConfig.Cert, tlsConfig.Key) - if err != nil { - return nil, fmt.Errorf("LoadX509KeyPair: %w", err) - } - - // for security reasons we do not allow TLS < 1.2, see YQ-1877 - creds := credentials.NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}) - opts = append(opts, grpc.Creds(creds)) - - return opts, nil -} - -func (s *serviceConnector) stop() { - s.grpcServer.GracefulStop() -} - -func newServiceConnector( - logger log.Logger, - cfg *config.TServerConfig, - registry *solomon.Registry, -) (service, error) { - queryLoggerFactory := utils.NewQueryLoggerFactory(cfg.Logger) - - // TODO: drop deprecated fields after YQ-2057 - var endpoint *api_common.TEndpoint - - switch { - case cfg.GetConnectorServer().GetEndpoint() != nil: - endpoint = cfg.ConnectorServer.GetEndpoint() - case cfg.GetEndpoint() != nil: - logger.Warn("Using deprecated field `endpoint` from config. Please update your config.") - - endpoint = cfg.GetEndpoint() - default: - return nil, fmt.Errorf("invalid config: no endpoint") - } - - listener, err := net.Listen("tcp", utils.EndpointToString(endpoint)) - if err != nil { - return nil, fmt.Errorf("net listen: %w", err) - } - - options, err := makeGRPCOptions(logger, cfg, registry) - if err != nil { - return nil, fmt.Errorf("make GRPC options: %w", err) - } - - grpcServer := grpc.NewServer(options...) - reflection.Register(grpcServer) - - s := &serviceConnector{ - dataSourceCollection: NewDataSourceCollection( - queryLoggerFactory, - memory.DefaultAllocator, - paging.NewReadLimiterFactory(cfg.ReadLimit), - cfg), - logger: logger, - grpcServer: grpcServer, - listener: listener, - cfg: cfg, - } - - api_service.RegisterConnectorServer(grpcServer, s) - - return s, nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/service_metrics.go b/ydb/library/yql/providers/generic/connector/app/server/service_metrics.go deleted file mode 100644 index 1533d76e5a23..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/service_metrics.go +++ /dev/null @@ -1,57 +0,0 @@ -package server - -import ( - "context" - "fmt" - "net/http" - - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/library/go/core/metrics/solomon" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -type serviceMetrics struct { - httpServer *http.Server - logger log.Logger - registry *solomon.Registry -} - -func (s *serviceMetrics) start() error { - s.logger.Debug("starting HTTP metrics server", log.String("address", s.httpServer.Addr)) - - if err := s.httpServer.ListenAndServe(); err != nil { - return fmt.Errorf("http metrics server listen and serve: %w", err) - } - - return nil -} - -func (s *serviceMetrics) stop() { - ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) - defer cancel() - - err := s.httpServer.Shutdown(ctx) - if err != nil { - s.logger.Error("shutdown http metrics server", log.Error(err)) - } -} - -func newServiceMetrics(logger log.Logger, cfg *config.TMetricsServerConfig, registry *solomon.Registry) service { - mux := http.NewServeMux() - mux.Handle("/metrics", NewHTTPPullerHandler(registry, WithSpack())) - - httpServer := &http.Server{ - Addr: utils.EndpointToString(cfg.Endpoint), - Handler: mux, - } - - // TODO: TLS - logger.Warn("metrics server will use insecure connections") - - return &serviceMetrics{ - httpServer: httpServer, - logger: logger, - registry: registry, - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/service_pprof.go b/ydb/library/yql/providers/generic/connector/app/server/service_pprof.go deleted file mode 100644 index 96c50fcfab67..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/service_pprof.go +++ /dev/null @@ -1,62 +0,0 @@ -package server - -import ( - "context" - "fmt" - "net/http" - "net/http/pprof" - "time" - - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" -) - -type servicePprof struct { - httpServer *http.Server - logger log.Logger -} - -func (s *servicePprof) start() error { - s.logger.Debug("starting HTTP server", log.String("address", s.httpServer.Addr)) - - if err := s.httpServer.ListenAndServe(); err != nil { - return fmt.Errorf("http server listen and server: %w", err) - } - - return nil -} - -const shutdownTimeout = 5 * time.Second - -func (s *servicePprof) stop() { - ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) - defer cancel() - - err := s.httpServer.Shutdown(ctx) - if err != nil { - s.logger.Error("shutdown http server", log.Error(err)) - } -} - -func newServicePprof(logger log.Logger, cfg *config.TPprofServerConfig) service { - mux := http.NewServeMux() - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - - httpServer := &http.Server{ - Addr: utils.EndpointToString(cfg.Endpoint), - Handler: mux, - } - - // TODO: TLS - logger.Warn("server will use insecure connections") - - return &servicePprof{ - httpServer: httpServer, - logger: logger, - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/streaming/doc.go b/ydb/library/yql/providers/generic/connector/app/server/streaming/doc.go deleted file mode 100644 index e9974f9e04e2..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/streaming/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package streaming contains the logic of converting the data stream -// coming from the data source into the data stream sent over the network to the client. -package streaming diff --git a/ydb/library/yql/providers/generic/connector/app/server/streaming/streamer.go b/ydb/library/yql/providers/generic/connector/app/server/streaming/streamer.go deleted file mode 100644 index 416e3df76611..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/streaming/streamer.go +++ /dev/null @@ -1,117 +0,0 @@ -package streaming - -import ( - "context" - "fmt" - "sync" - - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type Streamer[T utils.Acceptor] struct { - stream api_service.Connector_ReadSplitsServer - request *api_service_protos.TReadSplitsRequest - dataSource datasource.DataSource[T] - split *api_service_protos.TSplit - sink paging.Sink[T] - logger log.Logger - ctx context.Context // clone of a stream context - cancel context.CancelFunc -} - -func (s *Streamer[T]) writeDataToStream() error { - // exit from this function will cause publisher's goroutine termination as well - defer s.cancel() - - for { - select { - case result, ok := <-s.sink.ResultQueue(): - if !ok { - // correct termination - return nil - } - - if result.Error != nil { - return fmt.Errorf("read result: %w", result.Error) - } - - // handle next data block - if err := s.sendResultToStream(result); err != nil { - return fmt.Errorf("send buffer to stream: %w", err) - } - case <-s.stream.Context().Done(): - // handle request termination - return s.stream.Context().Err() - } - } -} - -func (s *Streamer[T]) sendResultToStream(result *paging.ReadResult[T]) error { - // buffer must be explicitly marked as unused, - // otherwise memory will leak - defer result.ColumnarBuffer.Release() - - resp, err := result.ColumnarBuffer.ToResponse() - if err != nil { - return fmt.Errorf("buffer to response: %w", err) - } - - resp.Stats = result.Stats - - utils.DumpReadSplitsResponse(s.logger, resp) - - if err := s.stream.Send(resp); err != nil { - return fmt.Errorf("stream send: %w", err) - } - - return nil -} - -func (s *Streamer[T]) Run() error { - wg := &sync.WaitGroup{} - wg.Add(1) - - defer wg.Wait() - - // Launch reading from the data source. - // Subsriber goroutine controls publisher goroutine lifetime. - go func() { - defer wg.Done() - - s.dataSource.ReadSplit(s.ctx, s.logger, s.split, s.sink) - }() - - // pass received blocks into the GRPC channel - if err := s.writeDataToStream(); err != nil { - return fmt.Errorf("write data to stream: %w", err) - } - - return nil -} - -func NewStreamer[T utils.Acceptor]( - logger log.Logger, - stream api_service.Connector_ReadSplitsServer, - request *api_service_protos.TReadSplitsRequest, - split *api_service_protos.TSplit, - sink paging.Sink[T], - dataSource datasource.DataSource[T], -) *Streamer[T] { - ctx, cancel := context.WithCancel(stream.Context()) - - return &Streamer[T]{ - logger: logger, - stream: stream, - split: split, - request: request, - dataSource: dataSource, - sink: sink, - ctx: ctx, - cancel: cancel, - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/streaming/streamer_test.go b/ydb/library/yql/providers/generic/connector/app/server/streaming/streamer_test.go deleted file mode 100644 index 2b3e1f76214a..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/streaming/streamer_test.go +++ /dev/null @@ -1,339 +0,0 @@ -package streaming - -import ( - "bytes" - "context" - "errors" - "fmt" - "testing" - - "github.com/apache/arrow/go/v13/arrow" - "github.com/apache/arrow/go/v13/arrow/array" - "github.com/apache/arrow/go/v13/arrow/ipc" - "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/clickhouse" - rdbms_utils "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/datasource/rdbms/utils" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/paging" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var _ api_service.Connector_ReadSplitsServer = (*streamMock)(nil) - -type streamMock struct { - mock.Mock - api_service.Connector_ReadSplitsServer - logger log.Logger -} - -func (m *streamMock) Context() context.Context { - args := m.Called() - - return args.Get(0).(context.Context) -} - -func (m *streamMock) Send(response *api_service_protos.TReadSplitsResponse) error { - args := m.Called(response) - - return args.Error(0) -} - -func (m *streamMock) makeSendMatcher( - t *testing.T, - split *api_service_protos.TSplit, - expectedColumnarBlock [][]any, - expectedRowCount int, -) func(response *api_service_protos.TReadSplitsResponse) bool { - return func(response *api_service_protos.TReadSplitsResponse) bool { - // Check values in data blocks - buf := bytes.NewBuffer(response.GetArrowIpcStreaming()) - - reader, err := ipc.NewReader(buf) - require.NoError(t, err) - - for reader.Next() { - record := reader.Record() - - require.Equal(t, len(split.Select.What.Items), len(record.Columns())) - - if record.NumRows() != int64(expectedRowCount) { - return false - } - - col0 := record.Column(0).(*array.Int32) - require.Equal(t, &arrow.Int32Type{}, col0.DataType()) - - for i := 0; i < len(expectedColumnarBlock[0]); i++ { - if expectedColumnarBlock[0][i] != col0.Value(i) { - return false - } - } - - col1 := record.Column(1).(*array.Binary) - require.Equal(t, &arrow.BinaryType{}, col1.DataType()) - - for i := 0; i < len(expectedColumnarBlock[1]); i++ { - if !bytes.Equal([]byte(expectedColumnarBlock[1][i].(string)), col1.Value(i)) { - return false - } - } - } - - reader.Release() - - // Check stats - require.NotNil(t, response.Stats) - require.Equal(t, uint64(len(expectedColumnarBlock[0])), response.Stats.Rows) - - // TODO: come up with more elegant way of expected data size computing - var expectedBytes int - - expectedBytes += len(expectedColumnarBlock[0]) * 4 // int32 -> 4 bytes - for _, val := range expectedColumnarBlock[1] { - expectedBytes += len(val.(string)) - } - - require.Equal(t, uint64(expectedBytes), response.Stats.Bytes) - - return true - } -} - -type testCaseStreaming struct { - src [][]any - rowsPerPage int - bufferQueueCapacity int - scanErr error - sendErr error -} - -func (tc testCaseStreaming) name() string { - return fmt.Sprintf( - "totalRows_%d_rowsPerBlock_%d_bufferQueueCapacity_%d_scanErr_%v_sendErr_%v", - len(tc.src), tc.rowsPerPage, tc.bufferQueueCapacity, tc.scanErr != nil, tc.sendErr != nil) -} - -func (tc testCaseStreaming) messageParams() (sentMessages, rowsInLastMessage int) { - modulo := len(tc.src) % tc.rowsPerPage - - if modulo == 0 { - sentMessages = len(tc.src) / tc.rowsPerPage - rowsInLastMessage = tc.rowsPerPage - } else { - sentMessages = len(tc.src)/tc.rowsPerPage + 1 - rowsInLastMessage = modulo - } - - if tc.scanErr != nil { - sentMessages-- - rowsInLastMessage = tc.rowsPerPage - } - - return -} - -func (tc testCaseStreaming) execute(t *testing.T) { - logger := utils.NewTestLogger(t) - request := &api_service_protos.TReadSplitsRequest{} - split := utils.MakeTestSplit() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stream := &streamMock{logger: logger} - - stream.On("Context").Return(ctx) - - connection := &rdbms_utils.ConnectionMock{} - - connectionManager := &rdbms_utils.ConnectionManagerMock{} - connectionManager.On("Make", split.Select.DataSourceInstance).Return(connection, nil).Once() - connectionManager.On("Release", connection).Return().Once() - - rows := &rdbms_utils.RowsMock{ - PredefinedData: tc.src, - } - connection.On("Query", `SELECT "col0", "col1" FROM "example_1"`).Return(rows, nil).Once() - - col0Acceptor := new(*int32) - *col0Acceptor = new(int32) - col1Acceptor := new(*string) - *col1Acceptor = new(string) - - transformer := &rdbms_utils.RowTransformerMock{ - Acceptors: []any{ - col0Acceptor, - col1Acceptor, - }, - } - - if tc.scanErr == nil { - rows.On("MakeTransformer", []*Ydb.Type{utils.NewPrimitiveType(Ydb.Type_INT32), utils.NewPrimitiveType(Ydb.Type_STRING)}).Return(transformer, nil).Once() - rows.On("Next").Return(true).Times(len(rows.PredefinedData)) - rows.On("Next").Return(false).Once() - rows.On("Scan", transformer.GetAcceptors()...).Return(nil).Times(len(rows.PredefinedData)) - rows.On("Err").Return(nil).Once() - rows.On("Close").Return(nil).Once() - } else { - rows.On("MakeTransformer", []*Ydb.Type{utils.NewPrimitiveType(Ydb.Type_INT32), utils.NewPrimitiveType(Ydb.Type_STRING)}).Return(transformer, nil).Once() - rows.On("Next").Return(true).Times(len(rows.PredefinedData) + 1) - rows.On("Scan", transformer.GetAcceptors()...).Return(nil).Times(len(rows.PredefinedData)) - // instead of the last message, an error occurs - rows.On("Scan", transformer.GetAcceptors()...).Return(tc.scanErr).Once() - rows.On("Err").Return(nil).Once() - rows.On("Close").Return(nil).Once() - } - - totalMessages, rowsInLastMessage := tc.messageParams() - - expectedColumnarBlocks := utils.DataConverter{}.RowsToColumnBlocks(rows.PredefinedData, tc.rowsPerPage) - - if tc.sendErr == nil { - for sendCallID := 0; sendCallID < totalMessages; sendCallID++ { - expectedColumnarBlock := expectedColumnarBlocks[sendCallID] - - rowsInMessage := tc.rowsPerPage - if sendCallID == totalMessages-1 { - rowsInMessage = rowsInLastMessage - } - - matcher := stream.makeSendMatcher(t, split, expectedColumnarBlock, rowsInMessage) - - stream.On("Send", mock.MatchedBy(matcher)).Return(nil).Once() - } - } else { - // the first attempt to send response is failed - stream.On("Send", mock.MatchedBy(func(response *api_service_protos.TReadSplitsResponse) bool { - cancel() // emulate real behavior of GRPC - - return true - })).Return(tc.sendErr).Once() - } - - typeMapper := clickhouse.NewTypeMapper() - - dataSourcePreset := &rdbms.Preset{ - SQLFormatter: clickhouse.NewSQLFormatter(), - ConnectionManager: connectionManager, - TypeMapper: typeMapper, - } - - dataSource := rdbms.NewDataSource(logger, dataSourcePreset) - - columnarBufferFactory, err := paging.NewColumnarBufferFactory[any]( - logger, - memory.NewGoAllocator(), - api_service_protos.TReadSplitsRequest_ARROW_IPC_STREAMING, - split.Select.What) - require.NoError(t, err) - - pagingCfg := &config.TPagingConfig{RowsPerPage: uint64(tc.rowsPerPage)} - trafficTracker := paging.NewTrafficTracker[any](pagingCfg) - readLimiterFactory := paging.NewReadLimiterFactory(nil) - sink, err := paging.NewSink( - ctx, - logger, - trafficTracker, - columnarBufferFactory, - readLimiterFactory.MakeReadLimiter(logger), - tc.bufferQueueCapacity, - ) - require.NoError(t, err) - - streamer := NewStreamer(logger, stream, request, split, sink, dataSource) - - err = streamer.Run() - - switch { - case tc.scanErr != nil: - require.True(t, errors.Is(err, tc.scanErr)) - case tc.sendErr != nil: - require.True(t, errors.Is(err, tc.sendErr)) - default: - require.NoError(t, err) - } - - mocks := []interface{}{stream, connectionManager, connection, transformer} - - mock.AssertExpectationsForObjects(t, mocks...) -} - -func TestStreaming(t *testing.T) { - srcValues := [][][]any{ - { - {int32(1), "a"}, - {int32(2), "b"}, - {int32(3), "c"}, - {int32(4), "d"}, - }, - { - {int32(1), "a"}, - {int32(2), "b"}, - {int32(3), "c"}, - {int32(4), "d"}, - {int32(5), "e"}, - }, - } - rowsPerBlockValues := []int{2} - bufferQueueCapacityValues := []int{ - 0, - 1, - 10, - } - - var testCases []testCaseStreaming - - for _, src := range srcValues { - for _, rowsPerBlock := range rowsPerBlockValues { - for _, bufferQueueCapacity := range bufferQueueCapacityValues { - tc := testCaseStreaming{ - src: src, - rowsPerPage: rowsPerBlock, - bufferQueueCapacity: bufferQueueCapacity, - } - - testCases = append(testCases, tc) - } - } - } - - t.Run("positive", func(t *testing.T) { - for _, tc := range testCases { - tc := tc - t.Run(tc.name(), func(t *testing.T) { - tc.execute(t) - }) - } - }) - - t.Run("scan error", func(t *testing.T) { - scanErr := fmt.Errorf("scan error") - - for _, tc := range testCases { - tc := tc - tc.scanErr = scanErr - t.Run(tc.name(), func(t *testing.T) { - tc.execute(t) - }) - } - }) - - t.Run("send error", func(t *testing.T) { - sendErr := fmt.Errorf("stream send error") - - for _, tc := range testCases { - tc := tc - tc.sendErr = sendErr - t.Run(tc.name(), func(t *testing.T) { - tc.execute(t) - }) - } - }) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/streaming/ut/ya.make b/ydb/library/yql/providers/generic/connector/app/server/streaming/ut/ya.make deleted file mode 100644 index 7b6078c7b95e..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/streaming/ut/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -GO_TEST_FOR(ydb/library/yql/providers/generic/connector/app/server/streaming) - -SIZE(SMALL) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/streaming/ya.make b/ydb/library/yql/providers/generic/connector/app/server/streaming/ya.make deleted file mode 100644 index 283c8ce7a211..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/streaming/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -GO_LIBRARY() - -SRCS( - doc.go - streamer.go -) - -GO_TEST_SRCS( - streamer_test.go -) - -END() - -RECURSE_FOR_TESTS(ut) diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go b/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go deleted file mode 100644 index 380dda9a93bf..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go +++ /dev/null @@ -1,181 +0,0 @@ -package utils - -import ( - "fmt" - - "github.com/apache/arrow/go/v13/arrow" - "github.com/apache/arrow/go/v13/arrow/array" - "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type ArrowBuilder[VT ValueType] interface { - AppendNull() - Append(value VT) -} - -func SelectWhatToArrowSchema(selectWhat *api_service_protos.TSelect_TWhat) (*arrow.Schema, error) { - fields := make([]arrow.Field, 0, len(selectWhat.Items)) - - for i, item := range selectWhat.Items { - column := item.GetColumn() - if column == nil { - return nil, fmt.Errorf("item #%d (%v) is not a column", i, item) - } - - ydbType := column.GetType() - - var ( - field arrow.Field - err error - ) - - // Reference table: https://wiki.yandex-team.ru/rtmapreduce/yql-streams-corner/connectors/lld-02-tipy-dannyx - switch t := ydbType.Type.(type) { - // Primitive types - case *Ydb.Type_TypeId: - field, err = ydbTypeToArrowField(t.TypeId, column) - if err != nil { - return nil, fmt.Errorf("primitive YDB type to arrow field: %w", err) - } - case *Ydb.Type_OptionalType: - field, err = ydbTypeToArrowField(t.OptionalType.Item.GetTypeId(), column) - if err != nil { - return nil, fmt.Errorf("optional YDB type to arrow field: %w", err) - } - default: - return nil, fmt.Errorf("only primitive and optional types are supported, got '%T' instead: %w", t, ErrDataTypeNotSupported) - } - - fields = append(fields, field) - } - - schema := arrow.NewSchema(fields, nil) - - return schema, nil -} - -func YdbTypesToArrowBuilders(ydbTypes []*Ydb.Type, arrowAllocator memory.Allocator) ([]array.Builder, error) { - var ( - builders []array.Builder - builder array.Builder - err error - ) - - for _, ydbType := range ydbTypes { - switch t := ydbType.Type.(type) { - // Primitive types - case *Ydb.Type_TypeId: - builder, err = ydbTypeToArrowBuilder(t.TypeId, arrowAllocator) - if err != nil { - return nil, fmt.Errorf("primitive YDB type to Arrow builder: %w", err) - } - case *Ydb.Type_OptionalType: - builder, err = ydbTypeToArrowBuilder(t.OptionalType.Item.GetTypeId(), arrowAllocator) - if err != nil { - return nil, fmt.Errorf("optional YDB type to Arrow builder: %w", err) - } - default: - return nil, fmt.Errorf("only primitive and optional types are supported, got '%T' instead: %w", t, ErrDataTypeNotSupported) - } - - builders = append(builders, builder) - } - - return builders, nil -} - -func ydbTypeToArrowBuilder(typeID Ydb.Type_PrimitiveTypeId, arrowAllocator memory.Allocator) (array.Builder, error) { - var builder array.Builder - - switch typeID { - case Ydb.Type_BOOL: - // NOTE: for some reason YDB bool type is mapped to Arrow uint8 - // https://st.yandex-team.ru/YQL-15332 - builder = array.NewUint8Builder(arrowAllocator) - case Ydb.Type_INT8: - builder = array.NewInt8Builder(arrowAllocator) - case Ydb.Type_UINT8: - builder = array.NewUint8Builder(arrowAllocator) - case Ydb.Type_INT16: - builder = array.NewInt16Builder(arrowAllocator) - case Ydb.Type_UINT16: - builder = array.NewUint16Builder(arrowAllocator) - case Ydb.Type_INT32: - builder = array.NewInt32Builder(arrowAllocator) - case Ydb.Type_UINT32: - builder = array.NewUint32Builder(arrowAllocator) - case Ydb.Type_INT64: - builder = array.NewInt64Builder(arrowAllocator) - case Ydb.Type_UINT64: - builder = array.NewUint64Builder(arrowAllocator) - case Ydb.Type_FLOAT: - builder = array.NewFloat32Builder(arrowAllocator) - case Ydb.Type_DOUBLE: - builder = array.NewFloat64Builder(arrowAllocator) - case Ydb.Type_STRING: - builder = array.NewBinaryBuilder(arrowAllocator, arrow.BinaryTypes.Binary) - case Ydb.Type_UTF8: - // TODO: what about LargeString? - // https://arrow.apache.org/docs/cpp/api/datatype.html#_CPPv4N5arrow4Type4type12LARGE_STRINGE - builder = array.NewStringBuilder(arrowAllocator) - case Ydb.Type_DATE: - builder = array.NewUint16Builder(arrowAllocator) - case Ydb.Type_DATETIME: - builder = array.NewUint32Builder(arrowAllocator) - case Ydb.Type_TIMESTAMP: - builder = array.NewUint64Builder(arrowAllocator) - default: - return nil, fmt.Errorf("register type '%v': %w", typeID, ErrDataTypeNotSupported) - } - - return builder, nil -} - -func ydbTypeToArrowField(typeID Ydb.Type_PrimitiveTypeId, column *Ydb.Column) (arrow.Field, error) { - var field arrow.Field - - switch typeID { - case Ydb.Type_BOOL: - // NOTE: for some reason YDB bool type is mapped to Arrow uint8 - // https://st.yandex-team.ru/YQL-15332 - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Uint8} - case Ydb.Type_INT8: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Int8} - case Ydb.Type_UINT8: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Uint8} - case Ydb.Type_INT16: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Int16} - case Ydb.Type_UINT16: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Uint16} - case Ydb.Type_INT32: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Int32} - case Ydb.Type_UINT32: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Uint32} - case Ydb.Type_INT64: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Int64} - case Ydb.Type_UINT64: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Uint64} - case Ydb.Type_FLOAT: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Float32} - case Ydb.Type_DOUBLE: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Float64} - case Ydb.Type_STRING: - field = arrow.Field{Name: column.Name, Type: arrow.BinaryTypes.Binary} - case Ydb.Type_UTF8: - // TODO: what about LargeString? - // https://arrow.apache.org/docs/cpp/api/datatype.html#_CPPv4N5arrow4Type4type12LARGE_STRINGE - field = arrow.Field{Name: column.Name, Type: arrow.BinaryTypes.String} - case Ydb.Type_DATE: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Uint16} - case Ydb.Type_DATETIME: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Uint32} - case Ydb.Type_TIMESTAMP: - field = arrow.Field{Name: column.Name, Type: arrow.PrimitiveTypes.Uint64} - default: - return arrow.Field{}, fmt.Errorf("register type '%v': %w", typeID, ErrDataTypeNotSupported) - } - - return field, nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/converters.go b/ydb/library/yql/providers/generic/connector/app/server/utils/converters.go deleted file mode 100644 index a62c8f8cfb0f..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/converters.go +++ /dev/null @@ -1,135 +0,0 @@ -package utils - -import ( - "fmt" - "time" -) - -type ValueType interface { - bool | int8 | int16 | int32 | int64 | uint8 | uint16 | uint32 | uint64 | float32 | float64 | string | []byte | time.Time -} - -type ValueConverter[IN ValueType, OUT ValueType] interface { - Convert(in IN) (OUT, error) -} - -type BoolConverter struct{} - -func (BoolConverter) Convert(in bool) (uint8, error) { - // For a some reason, Bool values are converted to Arrow Uint8 rather than to Arrow native Bool. - // See https://st.yandex-team.ru/YQL-15332 for more details. - if in { - return 1, nil - } - - return 0, nil -} - -type Int8Converter struct{} - -func (Int8Converter) Convert(in int8) (int8, error) { return in, nil } - -type Int16Converter struct{} - -func (Int16Converter) Convert(in int16) (int16, error) { return in, nil } - -type Int32Converter struct{} - -func (Int32Converter) Convert(in int32) (int32, error) { return in, nil } - -type Int64Converter struct{} - -func (Int64Converter) Convert(in int64) (int64, error) { return in, nil } - -type Uint8Converter struct{} - -func (Uint8Converter) Convert(in uint8) (uint8, error) { return in, nil } - -type Uint16Converter struct{} - -func (Uint16Converter) Convert(in uint16) (uint16, error) { return in, nil } - -type Uint32Converter struct{} - -func (Uint32Converter) Convert(in uint32) (uint32, error) { return in, nil } - -type Uint64Converter struct{} - -func (Uint64Converter) Convert(in uint64) (uint64, error) { return in, nil } - -type Float32Converter struct{} - -func (Float32Converter) Convert(in float32) (float32, error) { return in, nil } - -type Float64Converter struct{} - -func (Float64Converter) Convert(in float64) (float64, error) { return in, nil } - -type StringConverter struct{} - -func (StringConverter) Convert(in string) (string, error) { return in, nil } - -type StringToBytesConverter struct{} - -func (StringToBytesConverter) Convert(in string) ([]byte, error) { return []byte(in), nil } - -type BytesConverter struct{} - -func (BytesConverter) Convert(in []byte) ([]byte, error) { return in, nil } - -type DateConverter struct{} - -func (DateConverter) Convert(in time.Time) (uint16, error) { - inTime := time.Time(in) - out, err := TimeToYDBDate(&inTime) - - if err != nil { - return 0, fmt.Errorf("convert time to YDB Date: %w", err) - } - - return out, nil -} - -type DateToStringConverter struct{} - -func (DateToStringConverter) Convert(in time.Time) (string, error) { - return time.Time(in).Format("2006-01-02"), nil -} - -type DatetimeConverter struct{} - -func (DatetimeConverter) Convert(in time.Time) (uint32, error) { - inTime := time.Time(in) - out, err := TimeToYDBDatetime(&inTime) - - if err != nil { - return 0, fmt.Errorf("convert time to YDB Datetime: %w", err) - } - - return out, nil -} - -type DatetimeToStringConverter struct{} - -func (DatetimeToStringConverter) Convert(in time.Time) (string, error) { - return time.Time(in).UTC().Format("2006-01-02T15:04:05Z"), nil -} - -type TimestampConverter struct{} - -func (TimestampConverter) Convert(in time.Time) (uint64, error) { - inTime := time.Time(in) - out, err := TimeToYDBTimestamp(&inTime) - - if err != nil { - return 0, fmt.Errorf("convert time to YDB Timestamp: %w", err) - } - - return out, nil -} - -type TimestampToStringConverter struct{} - -func (TimestampToStringConverter) Convert(in time.Time) (string, error) { - return time.Time(in).UTC().Format("2006-01-02T15:04:05.000Z"), nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/counter.go b/ydb/library/yql/providers/generic/connector/app/server/utils/counter.go deleted file mode 100644 index 67690b968904..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/counter.go +++ /dev/null @@ -1,32 +0,0 @@ -package utils - -import "golang.org/x/exp/constraints" - -type number interface { - constraints.Integer | constraints.Float -} - -type Counter[T number] struct { - parent *Counter[T] - value T -} - -func (c *Counter[T]) Add(delta T) { - if c.parent != nil { - c.parent.value += delta - } - - c.value += delta -} - -func (c *Counter[T]) Value() T { - return c.value -} - -func (c *Counter[T]) MakeChild() *Counter[T] { - return &Counter[T]{parent: c} -} - -func NewCounter[T number]() *Counter[T] { - return &Counter[T]{} -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/counter_test.go b/ydb/library/yql/providers/generic/connector/app/server/utils/counter_test.go deleted file mode 100644 index e8836a1cd33f..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/counter_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package utils - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCounter(t *testing.T) { - t.Run("simple", func(t *testing.T) { - counter := NewCounter[int64]() - counter.Add(1) - counter.Add(-1) - require.Equal(t, int64(0), counter.Value()) - }) - - t.Run("hierarchy", func(t *testing.T) { - parent := NewCounter[uint64]() - require.Equal(t, uint64(0), parent.Value()) - - child1 := parent.MakeChild() - child2 := parent.MakeChild() - - child1.Add(1) - require.Equal(t, uint64(1), child1.Value()) - - child2.Add(1) - require.Equal(t, uint64(1), child1.Value()) - - require.Equal(t, uint64(2), parent.Value()) - }) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go b/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go deleted file mode 100644 index 9925b62fe708..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package utils contains various helpers and utility functions -package utils diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/endpoint.go b/ydb/library/yql/providers/generic/connector/app/server/utils/endpoint.go deleted file mode 100644 index 56fbde80dae5..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/endpoint.go +++ /dev/null @@ -1,11 +0,0 @@ -package utils - -import ( - "fmt" - - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" -) - -func EndpointToString(ep *api_common.TEndpoint) string { - return fmt.Sprintf("%s:%d", ep.GetHost(), ep.GetPort()) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/errors.go b/ydb/library/yql/providers/generic/connector/app/server/utils/errors.go deleted file mode 100644 index 6f89adb787a4..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/errors.go +++ /dev/null @@ -1,103 +0,0 @@ -package utils - -import ( - "errors" - "fmt" - - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - "github.com/ydb-platform/ydb/library/go/core/log" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -var ( - ErrTableDoesNotExist = fmt.Errorf("table does not exist") - ErrDataSourceNotSupported = fmt.Errorf("data source not supported") - ErrDataTypeNotSupported = fmt.Errorf("data type not supported") - ErrMethodNotSupported = fmt.Errorf("method not supported") - ErrReadLimitExceeded = fmt.Errorf("read limit exceeded") - ErrInvalidRequest = fmt.Errorf("invalid request") - ErrValueOutOfTypeBounds = fmt.Errorf("value is out of possible range of values for the type") - ErrUnimplemented = fmt.Errorf("unimplemented") - ErrUnimplementedTypedValue = fmt.Errorf("unimplemented typed value") - ErrUnimplementedExpression = fmt.Errorf("unimplemented expression") - ErrUnsupportedExpression = fmt.Errorf("expression is not supported") - ErrUnimplementedOperation = fmt.Errorf("unimplemented operation") - ErrUnimplementedPredicateType = fmt.Errorf("unimplemented predicate type") - ErrInvariantViolation = fmt.Errorf("implementation error (invariant violation)") - ErrUnimplementedArithmeticalExpression = fmt.Errorf("unimplemented arithmetical expression") - ErrEmptyTableName = fmt.Errorf("empty table name") - ErrPageSizeExceeded = fmt.Errorf("page size exceeded, check service configuration") -) - -func NewSuccess() *api_service_protos.TError { - return &api_service_protos.TError{ - Status: Ydb.StatusIds_SUCCESS, - Message: "succeeded", - } -} - -func IsSuccess(apiErr *api_service_protos.TError) bool { - if apiErr.Status == Ydb.StatusIds_STATUS_CODE_UNSPECIFIED { - panic("status uninitialized") - } - - return apiErr.Status == Ydb.StatusIds_SUCCESS -} - -func NewAPIErrorFromStdError(err error) *api_service_protos.TError { - var status Ydb.StatusIds_StatusCode - - switch { - case errors.Is(err, ErrTableDoesNotExist): - status = Ydb.StatusIds_NOT_FOUND - case errors.Is(err, ErrReadLimitExceeded): - // Return BAD_REQUEST to avoid retrying - status = Ydb.StatusIds_BAD_REQUEST - case errors.Is(err, ErrInvalidRequest): - status = Ydb.StatusIds_BAD_REQUEST - case errors.Is(err, ErrDataSourceNotSupported): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrDataTypeNotSupported): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrValueOutOfTypeBounds): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrUnimplementedTypedValue): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrUnimplementedExpression): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrUnimplementedOperation): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrUnimplementedPredicateType): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrUnimplemented): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrUnimplementedArithmeticalExpression): - status = Ydb.StatusIds_UNSUPPORTED - case errors.Is(err, ErrEmptyTableName): - status = Ydb.StatusIds_BAD_REQUEST - case errors.Is(err, ErrPageSizeExceeded): - status = Ydb.StatusIds_INTERNAL_ERROR - default: - status = Ydb.StatusIds_INTERNAL_ERROR - } - - return &api_service_protos.TError{ - Status: status, - Message: err.Error(), - } -} - -func APIErrorToLogFields(apiErr *api_service_protos.TError) []log.Field { - return []log.Field{ - log.String("message", apiErr.Message), - log.String("status", apiErr.Status.String()), - } -} - -func NewSTDErrorFromAPIError(apiErr *api_service_protos.TError) error { - if IsSuccess(apiErr) { - return nil - } - - return errors.New(apiErr.Message) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go deleted file mode 100644 index ab3d16a0b5ec..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go +++ /dev/null @@ -1,142 +0,0 @@ -package utils - -import ( - "fmt" - "io" - "testing" - - "github.com/ydb-platform/ydb/library/go/core/log" - "github.com/ydb-platform/ydb/library/go/core/log/zap" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/config" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" - "go.uber.org/zap/zapcore" - "go.uber.org/zap/zaptest" -) - -// TODO: it's better to do this in GRPC middleware - -func AnnotateLogger(logger log.Logger, method string, dsi *api_common.TDataSourceInstance) log.Logger { - logger = log.With(logger, log.String("method", method)) - - if dsi != nil { - logger = log.With(logger, - log.String("data_source_kind", api_common.EDataSourceKind_name[int32(dsi.Kind)]), - log.String("host", dsi.Endpoint.Host), - log.UInt32("port", dsi.Endpoint.Port), - log.String("database", dsi.Database), - log.Bool("use_tls", dsi.UseTls), - log.String("protocol", dsi.Protocol.String()), - // TODO: can we print just a login without a password? - ) - } - - return logger -} - -func LogCloserError(logger log.Logger, closer io.Closer, msg string) { - if err := closer.Close(); err != nil { - logger.Error(msg, log.Error(err)) - } -} - -func NewLoggerFromConfig(cfg *config.TLoggerConfig) (log.Logger, error) { - if cfg == nil { - return NewDefaultLogger() - } - - loggerCfg := zap.NewDeployConfig() - loggerCfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder - loggerCfg.Encoding = "console" - loggerCfg.Level.SetLevel(convertToZapLogLevel(cfg.GetLogLevel())) - - zapLogger, err := loggerCfg.Build() - if err != nil { - return nil, fmt.Errorf("new logger: %w", err) - } - - return &zap.Logger{L: zapLogger}, nil -} - -func NewDefaultLogger() (log.Logger, error) { - return NewLoggerFromConfig(&config.TLoggerConfig{LogLevel: config.ELogLevel_TRACE}) -} - -func NewTestLogger(t *testing.T) log.Logger { return &zap.Logger{L: zaptest.NewLogger(t)} } - -func DumpReadSplitsResponse(logger log.Logger, resp *api_service_protos.TReadSplitsResponse) { - switch t := resp.GetPayload().(type) { - case *api_service_protos.TReadSplitsResponse_ArrowIpcStreaming: - if dump := resp.GetArrowIpcStreaming(); dump != nil { - logger.Debug("response", log.Int("arrow_blob_length", len(dump))) - } - case *api_service_protos.TReadSplitsResponse_ColumnSet: - for i := range t.ColumnSet.Data { - data := t.ColumnSet.Data[i] - meta := t.ColumnSet.Meta[i] - - logger.Debug("response", log.Int("column_id", i), log.String("meta", meta.String()), log.String("data", data.String())) - } - default: - panic(fmt.Sprintf("unexpected message type %v", t)) - } -} - -func SelectToFields(slct *api_service_protos.TSelect) []log.Field { - result := []log.Field{ - log.Any("from", slct.From), - log.Any("what", slct.What), - log.Any("where", slct.Where), - } - - return result -} - -type QueryLoggerFactory struct { - enableQueryLogging bool -} - -func NewQueryLoggerFactory(cfg *config.TLoggerConfig) QueryLoggerFactory { - enabled := cfg.GetEnableSqlQueryLogging() - - return QueryLoggerFactory{enableQueryLogging: enabled} -} - -func (f *QueryLoggerFactory) Make(logger log.Logger) QueryLogger { - return QueryLogger{Logger: logger, enabled: f.enableQueryLogging} -} - -type QueryLogger struct { - log.Logger - enabled bool -} - -func (ql *QueryLogger) Dump(query string, args ...any) { - if !ql.enabled { - return - } - - logFields := []log.Field{log.String("query", query)} - if len(args) > 0 { - logFields = append(logFields, log.Any("args", args)) - } - - ql.Debug("execute SQL query", logFields...) -} - -func convertToZapLogLevel(lvl config.ELogLevel) zapcore.Level { - switch lvl { - case config.ELogLevel_TRACE, config.ELogLevel_DEBUG: - return zapcore.DebugLevel - case config.ELogLevel_INFO: - return zapcore.InfoLevel - case config.ELogLevel_WARN: - return zapcore.WarnLevel - case config.ELogLevel_ERROR: - return zapcore.ErrorLevel - case config.ELogLevel_FATAL: - return zapcore.FatalLevel - } - - return zapcore.InvalidLevel -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/protobuf.go b/ydb/library/yql/providers/generic/connector/app/server/utils/protobuf.go deleted file mode 100644 index b6528a96a07e..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/protobuf.go +++ /dev/null @@ -1,26 +0,0 @@ -package utils - -import ( - "fmt" - "io" - - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" -) - -func DumpProtoMessageToJSON(msg proto.Message, stream io.Writer) error { - opts := protojson.MarshalOptions{ - Indent: " ", - } - - data, err := opts.Marshal(msg) - if err != nil { - return fmt.Errorf("protojson marshal: %w", err) - } - - if _, err := stream.Write(data); err != nil { - return fmt.Errorf("stream write: %w", err) - } - - return nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/row_transformer.go b/ydb/library/yql/providers/generic/connector/app/server/utils/row_transformer.go deleted file mode 100644 index b8951f086590..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/row_transformer.go +++ /dev/null @@ -1,68 +0,0 @@ -package utils - -import ( - "fmt" - - "github.com/apache/arrow/go/v13/arrow/array" -) - -// Acceptor is a fundamental type class that is used during data extraction from the data source -type Acceptor interface { - any | string -} - -// RowTransformer is a container for values taken extracted from a single table row. -// RowTransformer also knows how to convert them into columnar reprsentation with Arrow builders. -type RowTransformer[T Acceptor] interface { - AppendToArrowBuilders(builders []array.Builder) error - SetAcceptors(acceptors []T) - GetAcceptors() []T -} - -type RowTransformerDefault[T Acceptor] struct { - // The row data itself. - acceptors []T - // Collection of functions responsible for appending certain row items to the corresponding columns. - appenders []func(acceptor T, builder array.Builder) error - // Sometimes row containes more data than necessary. - // This array specifies what particular row items to convert into the columnar format. - wantedColumnIDs []int -} - -func (rt *RowTransformerDefault[T]) AppendToArrowBuilders(builders []array.Builder) error { - if len(rt.wantedColumnIDs) != 0 { - for i, columnID := range rt.wantedColumnIDs { - if err := rt.appenders[i](rt.acceptors[columnID], builders[i]); err != nil { - return fmt.Errorf( - "append acceptor %#v of %d column to arrow builder %#v: %w", - rt.acceptors[columnID], i, builders[i], err) - } - } - } else { - for i, acceptor := range rt.acceptors { - if err := rt.appenders[i](acceptor, builders[i]); err != nil { - return fmt.Errorf( - "append acceptor %#v of %d column to arrow builder %#v: %w", - acceptor, i, builders[i], err) - } - } - } - - return nil -} - -func (rt *RowTransformerDefault[T]) SetAcceptors(acceptors []T) { rt.acceptors = acceptors } - -func (rt *RowTransformerDefault[T]) GetAcceptors() []T { return rt.acceptors } - -func NewRowTransformer[T Acceptor]( - acceptors []T, - appenders []func(acceptor T, builder array.Builder) error, - wantedColumnIDs []int, -) RowTransformer[T] { - return &RowTransformerDefault[T]{ - acceptors: acceptors, - appenders: appenders, - wantedColumnIDs: wantedColumnIDs, - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/select_helpers.go b/ydb/library/yql/providers/generic/connector/app/server/utils/select_helpers.go deleted file mode 100644 index db1c4bbd3c1d..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/select_helpers.go +++ /dev/null @@ -1,39 +0,0 @@ -package utils - -import ( - "fmt" - - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func SelectWhatToYDBTypes(selectWhat *api_service_protos.TSelect_TWhat) ([]*Ydb.Type, error) { - var ydbTypes []*Ydb.Type - - for i, item := range selectWhat.Items { - ydbType := item.GetColumn().GetType() - if ydbType == nil { - return nil, fmt.Errorf("item #%d (%v) is not a column", i, item) - } - - ydbTypes = append(ydbTypes, ydbType) - } - - return ydbTypes, nil -} - -func YdbTypeToYdbPrimitiveTypeID(ydbType *Ydb.Type) (Ydb.Type_PrimitiveTypeId, error) { - switch t := ydbType.Type.(type) { - case *Ydb.Type_TypeId: - return t.TypeId, nil - case *Ydb.Type_OptionalType: - switch t.OptionalType.Item.Type.(type) { - case *Ydb.Type_TypeId: - return t.OptionalType.Item.GetTypeId(), nil - default: - return Ydb.Type_PRIMITIVE_TYPE_ID_UNSPECIFIED, fmt.Errorf("unexpected type %v: %w", t.OptionalType.Item, ErrDataTypeNotSupported) - } - default: - return Ydb.Type_PRIMITIVE_TYPE_ID_UNSPECIFIED, fmt.Errorf("unexpected type %v: %w", t, ErrDataTypeNotSupported) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/time.go b/ydb/library/yql/providers/generic/connector/app/server/utils/time.go deleted file mode 100644 index d21310262fa5..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/time.go +++ /dev/null @@ -1,42 +0,0 @@ -package utils - -import ( - "fmt" - "time" -) - -var ( - // According to https://ydb.tech/en/docs/yql/reference/types/primitive#datetime - minYDBTime = time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) - maxYDBTime = time.Date(2106, time.January, 1, 0, 0, 0, 0, time.UTC) -) - -func TimeToYDBDate(t *time.Time) (uint16, error) { - if t.Before(minYDBTime) || t.After(maxYDBTime) { - return 0, fmt.Errorf("convert '%v' to YDB Date: %w", t, ErrValueOutOfTypeBounds) - } - - days := t.Sub(minYDBTime).Hours() / 24 - - return uint16(days), nil -} - -func TimeToYDBDatetime(t *time.Time) (uint32, error) { - if t.Before(minYDBTime) || t.After(maxYDBTime) { - return 0, fmt.Errorf("convert '%v' to YDB Date: %w", t, ErrValueOutOfTypeBounds) - } - - seconds := t.Unix() - - return uint32(seconds), nil -} - -func TimeToYDBTimestamp(t *time.Time) (uint64, error) { - if t.Before(minYDBTime) || t.After(maxYDBTime) { - return 0, fmt.Errorf("convert '%v' to YDB Date: %w", t, ErrValueOutOfTypeBounds) - } - - seconds := t.UnixMicro() - - return uint64(seconds), nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/time_test.go b/ydb/library/yql/providers/generic/connector/app/server/utils/time_test.go deleted file mode 100644 index aac6cb8c9f16..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/time_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package utils - -import ( - "errors" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestTimeToYDBDate(t *testing.T) { - type testCase struct { - input time.Time - output uint16 - err error - } - - tcs := []testCase{ - { - input: time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC), - output: 0, - err: nil, - }, - { - input: time.Date(1970, 01, 02, 00, 00, 00, 00, time.UTC), - output: 1, - err: nil, - }, - { - input: time.Date(1969, 12, 31, 23, 59, 00, 00, time.UTC), - output: 0, - err: ErrValueOutOfTypeBounds, - }, - { - input: time.Date(9999, 01, 01, 00, 00, 00, 00, time.UTC), - output: 0, - err: ErrValueOutOfTypeBounds, - }, - } - - for _, tc := range tcs { - tc := tc - - t.Run(tc.input.String(), func(t *testing.T) { - output, err := TimeToYDBDate(&tc.input) - require.Equal(t, tc.output, output) - - if tc.err != nil { - require.True(t, errors.Is(tc.err, ErrValueOutOfTypeBounds)) - } else { - require.NoError(t, err) - } - }) - } -} - -func TestTimeToYDBDatetime(t *testing.T) { - type testCase struct { - input time.Time - output uint32 - err error - } - - tcs := []testCase{ - { - input: time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC), - output: 0, - err: nil, - }, - { - input: time.Date(1970, 01, 02, 00, 00, 00, 00, time.UTC), - output: 86400, - err: nil, - }, - { - input: time.Date(1969, 12, 31, 23, 59, 00, 00, time.UTC), - output: 0, - err: ErrValueOutOfTypeBounds, - }, - { - input: time.Date(9999, 01, 01, 00, 00, 00, 00, time.UTC), - output: 0, - err: ErrValueOutOfTypeBounds, - }, - } - - for _, tc := range tcs { - tc := tc - - t.Run(tc.input.String(), func(t *testing.T) { - output, err := TimeToYDBDatetime(&tc.input) - require.Equal(t, tc.output, output) - - if tc.err != nil { - require.True(t, errors.Is(tc.err, ErrValueOutOfTypeBounds)) - } else { - require.NoError(t, err) - } - }) - } -} - -func TestTimeToYDBTimestamp(t *testing.T) { - type testCase struct { - input time.Time - output uint64 - err error - } - - tcs := []testCase{ - { - input: time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC), - output: 0, - err: nil, - }, - { - input: time.Date(1970, 01, 02, 00, 00, 00, 00, time.UTC), - output: 86400000000, - err: nil, - }, - { - input: time.Date(1969, 12, 31, 23, 59, 00, 00, time.UTC), - output: 0, - err: ErrValueOutOfTypeBounds, - }, - { - input: time.Date(29427, 01, 01, 00, 00, 00, 00, time.UTC), - output: 0, - err: ErrValueOutOfTypeBounds, - }, - } - - for _, tc := range tcs { - tc := tc - - t.Run(tc.input.String(), func(t *testing.T) { - output, err := TimeToYDBTimestamp(&tc.input) - require.Equal(t, tc.output, output) - - if tc.err != nil { - require.True(t, errors.Is(tc.err, ErrValueOutOfTypeBounds)) - } else { - require.NoError(t, err) - } - }) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/type_mapper.go b/ydb/library/yql/providers/generic/connector/app/server/utils/type_mapper.go deleted file mode 100644 index ff4ed84c4e8f..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/type_mapper.go +++ /dev/null @@ -1,10 +0,0 @@ -package utils - -import ( - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -type TypeMapper interface { - SQLTypeToYDBColumn(columnName, typeName string, rules *api_service_protos.TTypeMappingSettings) (*Ydb.Column, error) -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/unit_test_helpers.go b/ydb/library/yql/providers/generic/connector/app/server/utils/unit_test_helpers.go deleted file mode 100644 index f8b3fe408e73..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/unit_test_helpers.go +++ /dev/null @@ -1,141 +0,0 @@ -package utils - -import ( - "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func NewPrimitiveType(t Ydb.Type_PrimitiveTypeId) *Ydb.Type { - return &Ydb.Type{ - Type: &Ydb.Type_TypeId{ - TypeId: t, - }, - } -} - -// NewDefaultWhat generates default What field with a pair of columns -func NewDefaultWhat() *api_service_protos.TSelect_TWhat { - return &api_service_protos.TSelect_TWhat{ - Items: []*api_service_protos.TSelect_TWhat_TItem{ - &api_service_protos.TSelect_TWhat_TItem{ - Payload: &api_service_protos.TSelect_TWhat_TItem_Column{ - Column: &Ydb.Column{ - Name: "col0", - Type: NewPrimitiveType(Ydb.Type_INT32), - }, - }, - }, - &api_service_protos.TSelect_TWhat_TItem{ - Payload: &api_service_protos.TSelect_TWhat_TItem_Column{ - Column: &Ydb.Column{ - Name: "col1", - Type: NewPrimitiveType(Ydb.Type_STRING), - }, - }, - }, - }, - } -} - -func NewColumnExpression(name string) *api_service_protos.TExpression { - return &api_service_protos.TExpression{ - Payload: &api_service_protos.TExpression_Column{ - Column: name, - }, - } -} - -func NewInt32ValueExpression(val int32) *api_service_protos.TExpression { - return &api_service_protos.TExpression{ - Payload: &api_service_protos.TExpression_TypedValue{ - TypedValue: &Ydb.TypedValue{ - Type: NewPrimitiveType(Ydb.Type_INT32), - Value: &Ydb.Value{ - Value: &Ydb.Value_Int32Value{ - Int32Value: val, - }, - }, - }, - }, - } -} - -func NewUint64ValueExpression(val uint64) *api_service_protos.TExpression { - return &api_service_protos.TExpression{ - Payload: &api_service_protos.TExpression_TypedValue{ - TypedValue: &Ydb.TypedValue{ - Type: NewPrimitiveType(Ydb.Type_UINT64), - Value: &Ydb.Value{ - Value: &Ydb.Value_Uint64Value{ - Uint64Value: val, - }, - }, - }, - }, - } -} - -func NewTextValueExpression(val string) *api_service_protos.TExpression { - return &api_service_protos.TExpression{ - Payload: &api_service_protos.TExpression_TypedValue{ - TypedValue: &Ydb.TypedValue{ - Type: NewPrimitiveType(Ydb.Type_UTF8), - Value: &Ydb.Value{ - Value: &Ydb.Value_TextValue{ - TextValue: val, - }, - }, - }, - }, - } -} - -func MakeTestSplit() *api_service_protos.TSplit { - return &api_service_protos.TSplit{ - Select: &api_service_protos.TSelect{ - DataSourceInstance: &api_common.TDataSourceInstance{}, - What: NewDefaultWhat(), - From: &api_service_protos.TSelect_TFrom{ - Table: "example_1", - }, - }, - } -} - -// DataConverter should be used only from unit tests -type DataConverter struct{} - -func (dc DataConverter) RowsToColumnBlocks(input [][]any, rowsPerBlock int) [][][]any { - var ( - totalColumns = len(input[0]) - results [][][]any - ) - - for i := 0; i < len(input); i += rowsPerBlock { - start := i - - end := start + rowsPerBlock - if end > len(input) { - end = len(input) - } - - result := dc.rowGroupToColumnBlock(input, totalColumns, start, end) - - results = append(results, result) - } - - return results -} - -func (dc DataConverter) rowGroupToColumnBlock(input [][]any, totalColumns, start, end int) [][]any { - columnarData := make([][]any, totalColumns) - - for columnID := range columnarData { - for rowID := range input[start:end] { - columnarData[columnID] = append(columnarData[columnID], input[rowID+start][columnID]) - } - } - - return columnarData -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/unit_test_helpers_test.go b/ydb/library/yql/providers/generic/connector/app/server/utils/unit_test_helpers_test.go deleted file mode 100644 index f08aaff5fd3f..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/unit_test_helpers_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package utils - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestDataConverter(t *testing.T) { - dc := DataConverter{} - - type testCase struct { - src [][]any - dst [][][]any - rowsPerBlock int - } - - testCases := []testCase{ - { - src: [][]any{ - {int32(1), "a"}, - {int32(2), "b"}, - {int32(3), "c"}, - {int32(4), "d"}, - }, - dst: [][][]any{ - { - {int32(1), int32(2)}, - {"a", "b"}, - }, - { - {int32(3), int32(4)}, - {"c", "d"}, - }, - }, - rowsPerBlock: 2, - }, - { - src: [][]any{ - {int32(1), "a"}, - {int32(2), "b"}, - {int32(3), "c"}, - {int32(4), "d"}, - {int32(5), "e"}, - }, - dst: [][][]any{ - { - {int32(1), int32(2)}, - {"a", "b"}, - }, - { - {int32(3), int32(4)}, - {"c", "d"}, - }, - { - {int32(5)}, - {"e"}, - }, - }, - rowsPerBlock: 2, - }, - } - - for _, tc := range testCases { - tc := tc - t.Run(fmt.Sprintf("rowsPerRecord_%d", tc.rowsPerBlock), func(t *testing.T) { - actual := dc.RowsToColumnBlocks(tc.src, tc.rowsPerBlock) - require.Equal(t, tc.dst, actual) - }) - } -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/ut/ya.make b/ydb/library/yql/providers/generic/connector/app/server/utils/ut/ya.make deleted file mode 100644 index 227f12138eb6..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/ut/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -GO_TEST_FOR(ydb/library/yql/providers/generic/connector/app/server/utils) - -SIZE(SMALL) - -END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make b/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make deleted file mode 100644 index 44b534d94ab0..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make +++ /dev/null @@ -1,29 +0,0 @@ -GO_LIBRARY() - -SRCS( - arrow_helpers.go - converters.go - counter.go - doc.go - endpoint.go - errors.go - logger.go - protobuf.go - row_transformer.go - select_helpers.go - time.go - type_mapper.go - unit_test_helpers.go -) - -GO_TEST_SRCS( - counter_test.go - time_test.go - unit_test_helpers_test.go -) - -END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/library/yql/providers/generic/connector/app/server/validate.go b/ydb/library/yql/providers/generic/connector/app/server/validate.go deleted file mode 100644 index 0bd008e4ffac..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/validate.go +++ /dev/null @@ -1,106 +0,0 @@ -package server - -import ( - "fmt" - - "github.com/ydb-platform/ydb/library/go/core/log" - api_common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils" - api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" -) - -func ValidateDescribeTableRequest(logger log.Logger, request *api_service_protos.TDescribeTableRequest) error { - if err := validateDataSourceInstance(logger, request.GetDataSourceInstance()); err != nil { - return fmt.Errorf("validate data source instance: %w", err) - } - - if request.GetTable() == "" { - return fmt.Errorf("empty table: %w", utils.ErrInvalidRequest) - } - - return nil -} - -func ValidateListSplitsRequest(logger log.Logger, request *api_service_protos.TListSplitsRequest) error { - if len(request.Selects) == 0 { - return fmt.Errorf("empty select list: %w", utils.ErrInvalidRequest) - } - - for i, slct := range request.Selects { - if err := validateSelect(logger, slct); err != nil { - return fmt.Errorf("validate select %d: %w", i, err) - } - } - - return nil -} - -func ValidateReadSplitsRequest(logger log.Logger, request *api_service_protos.TReadSplitsRequest) error { - if err := validateDataSourceInstance(logger, request.GetDataSourceInstance()); err != nil { - return fmt.Errorf("validate data source instance: %w", err) - } - - if len(request.Splits) == 0 { - return fmt.Errorf("splits are empty: %w", utils.ErrInvalidRequest) - } - - return nil -} - -func validateSelect(logger log.Logger, slct *api_service_protos.TSelect) error { - if slct == nil { - return fmt.Errorf("select is empty: %w", utils.ErrInvalidRequest) - } - - if err := validateDataSourceInstance(logger, slct.GetDataSourceInstance()); err != nil { - return fmt.Errorf("validate data source instance: %w", err) - } - - return nil -} - -func validateDataSourceInstance(logger log.Logger, dsi *api_common.TDataSourceInstance) error { - if dsi.GetKind() == api_common.EDataSourceKind_DATA_SOURCE_KIND_UNSPECIFIED { - return fmt.Errorf("empty kind: %w", utils.ErrInvalidRequest) - } - - if dsi.Endpoint == nil { - return fmt.Errorf("endpoint is empty: %w", utils.ErrInvalidRequest) - } - - if dsi.Endpoint.Host == "" { - return fmt.Errorf("endpoint.host is empty: %w", utils.ErrInvalidRequest) - } - - if dsi.Endpoint.Port == 0 { - return fmt.Errorf("endpoint.port is empty: %w", utils.ErrInvalidRequest) - } - - if dsi.Database == "" { - return fmt.Errorf("database field is empty: %w", utils.ErrInvalidRequest) - } - - if dsi.UseTls { - logger.Info("connector will use secure connection to access data source") - } else { - logger.Warn("connector will use insecure connection to access data source") - } - - if dsi.Protocol == api_common.EProtocol_PROTOCOL_UNSPECIFIED { - return fmt.Errorf("protocol field is empty: %w", utils.ErrInvalidRequest) - } - - switch dsi.GetKind() { - case api_common.EDataSourceKind_POSTGRESQL: - if dsi.GetPgOptions().Schema == "" { - return fmt.Errorf("schema field is empty: %w", utils.ErrInvalidRequest) - } - - case api_common.EDataSourceKind_CLICKHOUSE, api_common.EDataSourceKind_S3: - break - default: - return fmt.Errorf("unsupported data source: %w", utils.ErrInvalidRequest) - } - - return nil -} diff --git a/ydb/library/yql/providers/generic/connector/app/server/ya.make b/ydb/library/yql/providers/generic/connector/app/server/ya.make deleted file mode 100644 index aa08da13a269..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/server/ya.make +++ /dev/null @@ -1,24 +0,0 @@ -GO_LIBRARY() - -SRCS( - cmd.go - config.go - data_source_collection.go - doc.go - grpc_metrics.go - httppuller.go - launcher.go - service_connector.go - service_metrics.go - service_pprof.go - validate.go -) - -END() - -RECURSE( - datasource - paging - streaming - utils -) diff --git a/ydb/library/yql/providers/generic/connector/app/ya.make b/ydb/library/yql/providers/generic/connector/app/ya.make deleted file mode 100644 index 44d6e2b96208..000000000000 --- a/ydb/library/yql/providers/generic/connector/app/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -GO_PROGRAM(yq-connector) - -SRCS(main.go) - -END() - -RECURSE( - client - config - server -) diff --git a/ydb/library/yql/providers/generic/connector/debug/clickhouse/config/z_log_disable.xml b/ydb/library/yql/providers/generic/connector/debug/clickhouse/config/z_log_disable.xml deleted file mode 100644 index eb11758f05be..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/clickhouse/config/z_log_disable.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/ydb/library/yql/providers/generic/connector/debug/clickhouse/init/init_db.sh b/ydb/library/yql/providers/generic/connector/debug/clickhouse/init/init_db.sh deleted file mode 100644 index 2f6f3b957cdb..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/clickhouse/init/init_db.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -set -e - -clickhouse client -n <<-EOSQL - DROP TABLE IF EXISTS dqrun.example_1; - CREATE TABLE dqrun.example_1 (id Int32, col1 String, col2 Int32) ENGINE = MergeTree ORDER BY id; - INSERT INTO dqrun.example_1 (*) VALUES (1, 'ch_example_1_a', 10); - INSERT INTO dqrun.example_1 (*) VALUES (2, 'ch_example_1_b', 20); - INSERT INTO dqrun.example_1 (*) VALUES (3, 'ch_example_1_c', 30); - INSERT INTO dqrun.example_1 (*) VALUES (4, 'ch_example_1_d', 40); - INSERT INTO dqrun.example_1 (*) VALUES (5, 'ch_example_1_e', 50); -EOSQL - -clickhouse client -n <<-EOSQL - DROP TABLE IF EXISTS dqrun.date_time; - CREATE TABLE dqrun.date_time ( - col_date Date, - col_datetime DateTime, - col_datetime_zone DateTime('Europe/Moscow'), - col_date64 DateTime64(3), - col_date64_zone DateTime64(3, 'Asia/Istanbul') - ) ENGINE = TinyLog; - INSERT INTO dqrun.date_time (*) VALUES (now(), now(), now(), now(), now()); - INSERT INTO dqrun.date_time (*) VALUES ('2004-01-10', '2004-01-10 00:05:00', '2004-01-10 00:05:00', '2004-01-10T00:05:00.321', '2004-01-10T00:05:00.321'); - INSERT INTO dqrun.date_time (*) VALUES ('1950-01-10', '1950-01-10 00:05:00', '1950-01-10 00:05:00', '2100-01-10T00:05:00.321', '2004-01-10T00:05:00.321'); -EOSQL - -clickhouse client -n <<-EOSQL - DROP TABLE IF EXISTS dqrun.primitives; - CREATE TABLE dqrun.primitives ( - col_01_boolean Boolean, - col_02_int8 Int8, - col_03_uint8 UInt8, - col_04_int16 Int16, - col_05_uint16 UInt16, - col_06_int32 Int32, - col_07_uint32 UInt32, - col_08_int64 Int64, - col_09_uint64 UInt64, - col_10_float32 Float32, - col_11_float64 Float64, - col_12_string String, - col_13_string FixedString(13), - col_14_date Date, - col_16_datetime DateTime, - col_17_date64 DateTime64(3) - ) ENGINE = TinyLog; - INSERT INTO dqrun.primitives (*) VALUES (True, 2, 3, 4, 5, 6, 7, 8, 9, 10.10, 11.11, 'az', 'az', now(), now(), '1988-11-20 12:55:08.123'); - INSERT INTO dqrun.primitives (*) VALUES (False, -2, 3, -4, 5, -6, 7, -8, 9, -10.10, -11.11, 'буки', 'буки', now(), now(), now()); -EOSQL - -clickhouse client -n <<-EOSQL - DROP TABLE IF EXISTS dqrun.optional; - CREATE TABLE dqrun.optional ( - col_01_boolean Nullable(Boolean), - col_02_int8 Nullable(Int8), - col_03_uint8 Nullable(UInt8), - col_04_int16 Nullable(Int16), - col_05_uint16 Nullable(UInt16), - col_06_int32 Nullable(Int32), - col_07_uint32 Nullable(UInt32), - col_08_int64 Nullable(Int64), - col_09_uint64 Nullable(UInt64), - col_10_float32 Nullable(Float32), - col_11_float64 Nullable(Float64), - col_12_string Nullable(String), - col_13_string Nullable(FixedString(13)), - col_14_date Nullable(Date), - col_15_datetime Nullable(DateTime), - col_16_date64 Nullable(DateTime64(3)) - ) ENGINE = TinyLog; - INSERT INTO dqrun.optional (*) VALUES (True, 2, 3, 4, 5, 6, 7, 8, 9, 10.10, 11.11, 'az', 'az', now(), now(), now()); - INSERT INTO dqrun.optional (*) VALUES (False, -2, 3, -4, 5, -6, 7, -8, 9, -10.10, -11.11, 'буки', 'буки', now(), now(), now()); - INSERT INTO dqrun.optional (*) VALUES (NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -EOSQL - -clickhouse client -n <<-EOSQL - DROP TABLE IF EXISTS dqrun.unsupported; - CREATE TABLE dqrun.unsupported (id Int32, col1 String, col2 UUID) ENGINE = TinyLog; - INSERT INTO dqrun.unsupported (*) VALUES (1, 'ch_az', generateUUIDv4()); - INSERT INTO dqrun.unsupported (*) VALUES (2, 'ch_buki', generateUUIDv4()); - INSERT INTO dqrun.unsupported (*) VALUES (3, 'ch_vedi', generateUUIDv4()); - INSERT INTO dqrun.unsupported (*) VALUES (4, 'ch_glagol', generateUUIDv4()); - INSERT INTO dqrun.unsupported (*) VALUES (5, 'ch_dobro', generateUUIDv4()); -EOSQL - -clickhouse client -n <<-EOSQL - DROP TABLE IF EXISTS dqrun.benchmark_1g; - CREATE TABLE dqrun.benchmark_1g (id UInt64, col Text) ENGINE = MergeTree ORDER BY id; - INSERT INTO dqrun.benchmark_1g SELECT *, randomPrintableASCII(randUniform(1024, 1024)) FROM numbers(1048576); -EOSQL diff --git a/ydb/library/yql/providers/generic/connector/debug/clickhouse/init_table_datetime.sh b/ydb/library/yql/providers/generic/connector/debug/clickhouse/init_table_datetime.sh deleted file mode 100644 index 48ceff8fdeab..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/clickhouse/init_table_datetime.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -ex - -URL='http://crab:qwerty12345@localhost:8123/?database=dqrun' - -SCRIPTS=( -"DROP TABLE IF EXISTS date_time;" -"CREATE TABLE date_time ( - col_date Date, - col_date32 Date32, - col_datetime DateTime, - col_datetime64 DateTime64(3) -) ENGINE = TinyLog;" -"INSERT INTO date_time (*) VALUES ('1950-01-10', '1850-01-10', '1950-01-10 12:23:45', '1850-01-10 12:23:45.678');" -"INSERT INTO date_time (*) VALUES ('1970-01-10', '1950-01-10', '1970-01-10 12:23:45', '1950-01-10 12:23:45.678');" -"INSERT INTO date_time (*) VALUES ('2004-01-10', '2004-01-10', '2004-01-10 12:23:45', '2004-01-10 12:23:45.678');" -"INSERT INTO date_time (*) VALUES ('2110-01-10', '2110-01-10', '2106-01-10 12:23:45', '2110-01-10 12:23:45.678');" -"INSERT INTO date_time (*) VALUES ('2150-01-10', '2300-01-10', '2107-01-10 12:23:45', '2300-01-10 12:23:45.678');" -) - -for ((i = 0; i < ${#SCRIPTS[@]}; i++)) -do - echo "${SCRIPTS[$i]}" | curl "${URL}" --data-binary @- -done - -echo "SELECT * FROM date_time" | curl "${URL}" --data-binary @- diff --git a/ydb/library/yql/providers/generic/connector/debug/clickhouse/maintain/show_table_size.sh b/ydb/library/yql/providers/generic/connector/debug/clickhouse/maintain/show_table_size.sh deleted file mode 100755 index a81926c5fb26..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/clickhouse/maintain/show_table_size.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -ex - -URL='http://admin:password@localhost:8123/?database=dqrun' - -#echo "SELECT table, formatReadableSize(sum(bytes)) as size, min(min_date) as min_date, max(max_date) as max_date -# FROM system.parts WHERE active GROUP BY table" | curl "${URL}" --data-binary @- - -echo "SELECT - database, - table, - formatReadableSize(sum(data_compressed_bytes) AS size) AS compressed, - formatReadableSize(sum(data_uncompressed_bytes) AS usize) AS uncompressed, - round(usize / size, 2) AS compr_rate, - sum(rows) AS rows, - count() AS part_count - FROM system.parts - WHERE (active = 1) AND (database LIKE '%') AND (table LIKE '%') - GROUP BY database, table ORDER BY size DESC;" | curl "${URL}" --data-binary @- diff --git a/ydb/library/yql/providers/generic/connector/debug/config/client/ch.local.txt b/ydb/library/yql/providers/generic/connector/debug/config/client/ch.local.txt deleted file mode 100644 index 5cf862e32a99..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/config/client/ch.local.txt +++ /dev/null @@ -1,19 +0,0 @@ -endpoint { - host: "localhost" - port: 50051 -} - -data_source_instance { - kind: CLICKHOUSE - endpoint { - host: "localhost" - port: 9000 - } - database: "dqrun" - credentials { - basic { - username: "crab" - password: "qwerty12345" - } - } -} diff --git a/ydb/library/yql/providers/generic/connector/debug/config/client/pg.cloud.txt b/ydb/library/yql/providers/generic/connector/debug/config/client/pg.cloud.txt deleted file mode 100644 index e516a510bc03..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/config/client/pg.cloud.txt +++ /dev/null @@ -1,23 +0,0 @@ -endpoint { - host: "connector.yql-streaming.cloud.yandex.net" - port: 50051 -} - -data_source_instance { - kind: CLICKHOUSE - endpoint { - host: "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net" - port: 9000 - } - database: "dqrun" - credentials { - basic { - username: "admin" - password: "password" - } - } -} - -tls { - ca: "/home/vitalyisaev/arcadia/kikimr/deployment/terraform/yc-crt/YandexInternalRootCA.crt" -} diff --git a/ydb/library/yql/providers/generic/connector/debug/config/client/pg.local.txt b/ydb/library/yql/providers/generic/connector/debug/config/client/pg.local.txt deleted file mode 100644 index 88ef9cd1e2c2..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/config/client/pg.local.txt +++ /dev/null @@ -1,19 +0,0 @@ -endpoint { - host: "localhost" - port: 50051 -} - -data_source_instance { - kind: POSTGRESQL - endpoint { - host: "localhost" - port: 5432 - } - database: "dqrun" - credentials { - basic { - username: "crab" - password: "qwerty12345" - } - } -} diff --git a/ydb/library/yql/providers/generic/connector/debug/config/client/s3.local.txt b/ydb/library/yql/providers/generic/connector/debug/config/client/s3.local.txt deleted file mode 100644 index 9e9822f4cba9..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/config/client/s3.local.txt +++ /dev/null @@ -1,20 +0,0 @@ -endpoint { - host: "localhost" - port: 50051 -} - -data_source_instance { - kind: S3 - endpoint { - host: "localhost" - port: 9000 - } - database: "dqrun" - credentials { - basic { - username: "crab" - password: "qwerty12345" - } - } - protocol: HTTP -} diff --git a/ydb/library/yql/providers/generic/connector/debug/config/server/server.no_tls.txt b/ydb/library/yql/providers/generic/connector/debug/config/server/server.no_tls.txt deleted file mode 100644 index f2732c366f62..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/config/server/server.no_tls.txt +++ /dev/null @@ -1,23 +0,0 @@ -connector_server { - endpoint { - host: "0.0.0.0" - port: 50051 - } -} - -logger { - log_level: TRACE - enable_sql_query_logging: true -} - -pprof_server { - endpoint { - host: "0.0.0.0" - port: 6060 - } -} - -paging { - bytes_per_page: 4194304 - prefetch_queue_capacity: 2 -} diff --git a/ydb/library/yql/providers/generic/connector/debug/config/server/server.tls.txt b/ydb/library/yql/providers/generic/connector/debug/config/server/server.tls.txt deleted file mode 100644 index d5f3498a4908..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/config/server/server.tls.txt +++ /dev/null @@ -1,15 +0,0 @@ -endpoint { - host: "0.0.0.0" - port: 50051 -} - -tls { - ca: "/home/vitalyisaev/arcadia/junk/vitalyisaev/connectors/tls/ca.crt" - key: "/home/vitalyisaev/arcadia/junk/vitalyisaev/connectors/tls/server.key" - cert: "/home/vitalyisaev/arcadia/junk/vitalyisaev/connectors/tls/server.crt" -} - -logger { - log_level: TRACE - enable_sql_query_logging: true -} diff --git a/ydb/library/yql/providers/generic/connector/debug/docker-compose.yaml b/ydb/library/yql/providers/generic/connector/debug/docker-compose.yaml deleted file mode 100644 index 5ce59dd5482d..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/docker-compose.yaml +++ /dev/null @@ -1,58 +0,0 @@ -version: '3' - -services: - - clickhouse: - image: clickhouse/clickhouse-server - container_name: yq-connector-clickhouse - ports: - - '8123:8123' - - '9000:9000' - - '9009:9009' - environment: - CLICKHOUSE_DB: dqrun - CLICKHOUSE_USER: admin - CLICKHOUSE_PASSWORD: password - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 - volumes: - - /tmp/yq-connector/clickhouse/data:/var/lib/clickhouse - - /tmp/yq-connector/clickhouse/logs:/var/log/clickhouse-server - - ./clickhouse/init:/docker-entrypoint-initdb.d - - ./clickhouse/config/z_log_disable.xml:/etc/clickhouse-server/config.d/z_log_disable.xml - ulimits: - nproc: 65535 - nofile: - soft: 262144 - hard: 262144 - - postgresql: - image: postgres - container_name: yq-connector-postgresql - ports: - - '5432:5432' - environment: - POSTGRES_DB: dqrun - POSTGRES_USER: admin - POSTGRES_PASSWORD: password - PGDATA: /var/lib/postgresql/data/pgdata - volumes: - - /tmp/yq-connector/postgresql/data/:/var/lib/postgresql/data/ - - ./postgresql/init:/docker-entrypoint-initdb.d - - minio: - image: minio/minio:latest - container_name: yq-connector-minio - command: server --console-address ':9001' /data/ - ports: - - '29000:9000' - - '29001:9001' - environment: - MINIO_ROOT_USER: admin - MINIO_ROOT_PASSWORD: password - volumes: - - /tmp/yq-connector/minio/data:/data - healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:29000/minio/health/live'] - interval: 30s - timeout: 20s - retries: 3 diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.cloud.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.cloud.yql deleted file mode 100644 index 8bdfcff780f1..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.cloud.yql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM clickhouse_streaming.example_1; diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.local.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.local.yql deleted file mode 100644 index 3709b65b7aec..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.local.yql +++ /dev/null @@ -1,4 +0,0 @@ --- SELECT col_15_date32 FROM rtmr_dev00_clickhouse_native.primitives; --- SELECT MIN(LEN(col)) FROM rtmr_dev00_clickhouse_native.benchmark_1g; --- SELECT MIN(LEN(col1)+ LEN(col2)) FROM rtmr_dev00_clickhouse_native.benchmark_2g; -SELECT * FROM rtmr_dev00_clickhouse_native.example_1; diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.single.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.single.yql deleted file mode 100644 index cb090ffc6dc5..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.single.yql +++ /dev/null @@ -1,3 +0,0 @@ -pragma UseBlocks; - -SELECT id, id * id AS id_square FROM rtmr_dev00_clickhouse.example_1; diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/debug.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/debug.yql deleted file mode 100644 index 4672cd32949a..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/debug.yql +++ /dev/null @@ -1 +0,0 @@ -SELECT col_14_date, col_16_datetime , col_17_date64 FROM rtmr_dev00_clickhouse.primitives; \ No newline at end of file diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.cloud.conf b/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.cloud.conf deleted file mode 100644 index 51d3cabcb38d..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.cloud.conf +++ /dev/null @@ -1,95 +0,0 @@ -Generic { - Connector { - Endpoint { - host: "connector.yql-streaming.cloud.yandex.net" - port: 50051 - } - UseSsl: true - } - - ClusterMapping { - Kind: CLICKHOUSE, - Name: "clickhouse_streaming" - DatabaseName: "dqrun" - DatabaseId: "c9ql09h4firghvrv49jt" - Credentials { - basic { - username: "admin" - password: "password" - } - } - Protocol: HTTP - UseSsl: true - } - - ClusterMapping { - Kind: POSTGRESQL, - Name: "postgresql_streaming" - DatabaseName: "dqrun" - DatabaseId: "c9qb2bjghs8onbncpamk" - Credentials { - basic { - username: "crab" - password: "password" - } - } - Protocol: NATIVE - } - MdbGateway: "https://mdb.api.cloud.yandex.net:443" -} - -DbResolver { - YdbMvpEndpoint: "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod" -} - -Dq { - DefaultSettings { - Name: "EnableComputeActor" - Value: "1" - } - - DefaultSettings { - Name: "ComputeActorType" - Value: "async" - } - - DefaultSettings { - Name: "AnalyzeQuery" - Value: "true" - } - - DefaultSettings { - Name: "MaxTasksPerStage" - Value: "200" - } - - DefaultSettings { - Name: "MaxTasksPerOperation" - Value: "200" - } - - DefaultSettings { - Name: "EnableInsert" - Value: "true" - } - - DefaultSettings { - Name: "_EnablePrecompute" - Value: "true" - } - - DefaultSettings { - Name: "UseAggPhases" - Value: "true" - } - - DefaultSettings { - Name: "HashJoinMode" - Value: "grace" - } - - DefaultSettings { - Name: "UseFastPickleTransport" - Value: "true" - } -} diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.local.conf b/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.local.conf deleted file mode 100644 index 046e019b3eb0..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.local.conf +++ /dev/null @@ -1,120 +0,0 @@ -Generic { - Connector { - Endpoint { - host: "localhost" - port: 50051 - } - UseSsl: false - } - - ClusterMapping { - Kind: CLICKHOUSE, - Name: "rtmr_dev00_clickhouse_http" - DatabaseName: "dqrun" - Endpoint { - host: "localhost" - port: 8123 - } - Credentials { - basic { - username: "admin" - password: "password" - } - } - UseSsl: false - Protocol: HTTP - } - - ClusterMapping { - Kind: CLICKHOUSE, - Name: "rtmr_dev00_clickhouse_native" - DatabaseName: "dqrun" - Endpoint { - host: "localhost" - port: 9000 - } - Credentials { - basic { - username: "admin" - password: "password" - } - } - UseSsl: false - Protocol: NATIVE - } - - ClusterMapping { - Kind: POSTGRESQL, - Name: "rtmr_dev00_postgresql" - DatabaseName: "dqrun" - Endpoint { - host: "localhost" - port: 5432 - } - Credentials { - basic { - username: "admin" - password: "password" - } - } - UseSsl: false - Protocol: NATIVE - } - - DefaultSettings { - Name: "DateTimeFormat" - Value: "string" - } -} - -Dq { - DefaultSettings { - Name: "EnableComputeActor" - Value: "1" - } - - DefaultSettings { - Name: "ComputeActorType" - Value: "async" - } - - DefaultSettings { - Name: "AnalyzeQuery" - Value: "true" - } - - DefaultSettings { - Name: "MaxTasksPerStage" - Value: "200" - } - - DefaultSettings { - Name: "MaxTasksPerOperation" - Value: "200" - } - - DefaultSettings { - Name: "EnableInsert" - Value: "true" - } - - DefaultSettings { - Name: "_EnablePrecompute" - Value: "true" - } - - DefaultSettings { - Name: "UseAggPhases" - Value: "true" - } - - DefaultSettings { - Name: "HashJoinMode" - Value: "grace" - } - - DefaultSettings { - Name: "UseFastPickleTransport" - Value: "true" - } -} diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.different.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.different.yql deleted file mode 100644 index 5246a8aee295..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.different.yql +++ /dev/null @@ -1,8 +0,0 @@ -pragma UseBlocks; - -SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2 - FROM clickhouse_streaming.example_1 as ch - JOIN postgresql_streaming.example_2 as pg -ON ch.id = pg.id -ORDER BY id; - diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_1.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_1.yql deleted file mode 100644 index 2ce5316eb27d..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_1.yql +++ /dev/null @@ -1,8 +0,0 @@ -pragma UseBlocks; - -SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2 - FROM rtmr_dev00_clickhouse.example_1 as ch - JOIN rtmr_dev00_postgresql.example_1 as pg -ON ch.id = pg.id -ORDER BY id; - diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_2.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_2.yql deleted file mode 100644 index 368427363062..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_2.yql +++ /dev/null @@ -1,8 +0,0 @@ -pragma UseBlocks; - -SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2 - FROM clickhouse_streaming.example_2 as ch - JOIN postgresql_streaming.example_2 as pg -ON ch.id = pg.id -ORDER BY id; - diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.benchmark_1g.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.benchmark_1g.yql deleted file mode 100644 index e7680ef875f0..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.benchmark_1g.yql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT MAX(LENGTH(ch.col) - LENGTH(pg.col)) - FROM rtmr_dev00_clickhouse.benchmark_1g as ch - JOIN rtmr_dev00_postgresql.benchmark_1g as pg -ON ch.id = pg.id; diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.different.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.different.yql deleted file mode 100644 index 09be54575c89..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.different.yql +++ /dev/null @@ -1,8 +0,0 @@ -pragma UseBlocks; - -SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2 - FROM rtmr_dev00_clickhouse.example_1 as ch - JOIN rtmr_dev00_postgresql.example_2 as pg -ON ch.id = pg.id -ORDER BY id; - diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.example_1.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.example_1.yql deleted file mode 100644 index bc53e7ccc7fa..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.example_1.yql +++ /dev/null @@ -1,8 +0,0 @@ -pragma UseBlocks; - -SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2 - FROM rtmr_dev00_clickhouse_native.example_1 as ch - JOIN rtmr_dev00_postgresql.example_1 as pg -ON ch.id = pg.id -ORDER BY id; - diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.cloud.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.cloud.yql deleted file mode 100644 index bf4e0166058c..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.cloud.yql +++ /dev/null @@ -1,2 +0,0 @@ -pragma UseBlocks; -SELECT * FROM postgresql_streaming.example_1; diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.local.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.local.yql deleted file mode 100644 index 6927470ee649..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.local.yql +++ /dev/null @@ -1,2 +0,0 @@ --- SELECT col_19_bytea FROM rtmr_dev00_postgresql.primitives; -SELECT MAX(id + LENGTH(col)) FROM rtmr_dev00_postgresql.benchmark_1g; diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.single.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.single.yql deleted file mode 100644 index 22996a40abaf..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.single.yql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT COL1 FROM rtmr_dev00_postgresql.column_projection; - diff --git a/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-rtmp-dev00.txt b/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-rtmp-dev00.txt deleted file mode 100644 index a17d84b1434d..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-rtmp-dev00.txt +++ /dev/null @@ -1,24 +0,0 @@ -CREATE OBJECT pg_local_password (TYPE SECRET) WITH (value = qwerty12345); - -CREATE EXTERNAL DATA SOURCE pg_local WITH ( - SOURCE_TYPE="PostgreSQL", - LOCATION="rtmr-dev00.search.yandex.net:5432", - AUTH_METHOD="BASIC", - LOGIN="crab", - PASSWORD_SECRET_NAME="pg_local_password", - USE_TLS="FALSE", - PROTOCOL="NATIVE" -); - -CREATE OBJECT ch_local_password (TYPE SECRET) WITH (value = qwerty12345); - -CREATE EXTERNAL DATA SOURCE ch_local WITH ( - SOURCE_TYPE="ClickHouse", - LOCATION="rtmr-dev00.search.yandex.net:9000", - AUTH_METHOD="BASIC", - LOGIN="crab", - PASSWORD_SECRET_NAME="ch_local_password", - DATABASE_NAME="dqrun", - USE_TLS="FALSE", - PROTOCOL="NATIVE" -); \ No newline at end of file diff --git a/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-streaming.txt b/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-streaming.txt deleted file mode 100644 index 8fbe9e68df77..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-streaming.txt +++ /dev/null @@ -1,25 +0,0 @@ --- streaming instances - -CREATE OBJECT clickhouse_streaming_password (TYPE SECRET) WITH (value=password); -CREATE EXTERNAL DATA SOURCE clickhouse_streaming WITH ( - SOURCE_TYPE="ClickHouse", - MDB_CLUSTER_ID="c9ql09h4firghvrv49jt", - DATABASE_NAME="dqrun", - PROTOCOL="HTTP", - AUTH_METHOD="BASIC", - LOGIN="admin", - PASSWORD_SECRET_NAME="clickhouse_streaming_password", - USE_TLS="TRUE" -); - -CREATE OBJECT postgresql_streaming_password (TYPE SECRET) WITH (value=password); -CREATE EXTERNAL DATA SOURCE postgresql_streaming WITH ( - SOURCE_TYPE="PostgreSQL", - MDB_CLUSTER_ID="c9qb2bjghs8onbncpamk", - DATABASE_NAME="dqrun", - PROTOCOL="NATIVE", - AUTH_METHOD="BASIC", - LOGIN="crab", - PASSWORD_SECRET_NAME="postgresql_streaming_password", - USE_TLS="TRUE" -); diff --git a/ydb/library/yql/providers/generic/connector/debug/kqprun/script.benchmark_1g.txt b/ydb/library/yql/providers/generic/connector/debug/kqprun/script.benchmark_1g.txt deleted file mode 100644 index 2f2ac911da8b..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/kqprun/script.benchmark_1g.txt +++ /dev/null @@ -1,5 +0,0 @@ -SELECT MAX(length(ch.col) + length(pg.col)) - FROM clickhouse_cloud.benchmark_1g as ch - JOIN postgresql_cloud.benchmark_1g as pg -ON ch.id = pg.id; - diff --git a/ydb/library/yql/providers/generic/connector/debug/kqprun/script.example.txt b/ydb/library/yql/providers/generic/connector/debug/kqprun/script.example.txt deleted file mode 100644 index 7ca381c451c9..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/kqprun/script.example.txt +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM ch_local.primitives; diff --git a/ydb/library/yql/providers/generic/connector/debug/postgresql/init/init_db.sh b/ydb/library/yql/providers/generic/connector/debug/postgresql/init/init_db.sh deleted file mode 100644 index ca0e4eaaf24a..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/postgresql/init/init_db.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -set -e - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - DROP TABLE IF EXISTS example_1; - CREATE TABLE example_1 (id integer, col1 text, col2 integer); - INSERT INTO example_1 VALUES (1, 'pg_example_1_a', 10); - INSERT INTO example_1 VALUES (2, 'pg_example_1_b', 20); - INSERT INTO example_1 VALUES (3, 'pg_example_1_c', 30); - INSERT INTO example_1 VALUES (4, 'pg_example_1_d', 40); - INSERT INTO example_1 VALUES (5, 'pg_example_1_e', 50); - INSERT INTO example_1 VALUES (6, NULL, 1); -EOSQL - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - DROP TABLE IF EXISTS example_2; - CREATE TABLE example_2 (col_01_bool bool, col_02_smallint smallint, col_03_int2 int2, col_04_smallserial smallserial, col_05_serial2 serial2, col_06_integer integer, col_07_int int, col_08_int4 int4, col_09_serial serial, col_10_serial4 serial4, col_11_bigint bigint, col_12_int8 int8, col_13_bigserial bigserial, col_14_serial8 serial8, col_15_real real, col_16_float4 float4, col_17_double_precision double precision, col_18_float8 float8, col_19_bytea bytea, col_20_character character (5), col_21_character_varying character varying (5), col_22_text text, col_23_date date, col_24_timestamp timestamp without time zone); - INSERT INTO example_2 (col_01_bool, col_02_smallint, col_03_int2, col_04_smallserial, col_05_serial2, col_06_integer, col_07_int, col_08_int4, col_09_serial, col_10_serial4, col_11_bigint, col_12_int8, col_13_bigserial, col_14_serial8, col_15_real, col_16_float4, col_17_double_precision, col_18_float8, col_19_bytea, col_20_character, col_21_character_varying, col_22_text, col_23_date, col_24_timestamp) VALUES(False, 2, 3, 1, 1, 6, 7, 8, 1, 1, 11, 12, 1, 1, 15.15, 16.16, 17.17, 18.18, 'az', 'az ', 'az ', 'az', '2023-08-09', '2023-08-09 13:19:11'); - INSERT INTO example_2 (col_01_bool, col_02_smallint, col_03_int2, col_04_smallserial, col_05_serial2, col_06_integer, col_07_int, col_08_int4, col_09_serial, col_10_serial4, col_11_bigint, col_12_int8, col_13_bigserial, col_14_serial8, col_15_real, col_16_float4, col_17_double_precision, col_18_float8, col_19_bytea, col_20_character, col_21_character_varying, col_22_text, col_23_date, col_24_timestamp) VALUES(NULL, NULL, NULL, 3, 3, NULL, NULL, NULL, 3, 3, NULL, NULL, 3, 3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -EOSQL - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - DROP TABLE IF EXISTS empty; - CREATE TABLE empty (id integer, col1 text, col2 integer); -EOSQL - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - DROP TABLE IF EXISTS primitives; - CREATE TABLE primitives ( - col_01_bool bool, - col_02_smallint smallint, - col_03_int2 int2, - col_04_smallserial smallserial, - col_05_serial2 serial2, - col_06_integer integer, - col_07_int int, - col_08_int4 int4, - col_09_serial serial, - col_10_serial4 serial4, - col_11_bigint bigint, - col_12_int8 int8, - col_13_bigserial bigserial, - col_14_serial8 serial8, - col_15_real real, - col_16_float4 float4, - col_17_double_precision double precision, - col_18_float8 float8, - col_19_bytea bytea, - col_20_character_n character(20), - col_21_character_varying_n character varying(21), - col_22_text text, - col_23_timestamp timestamp, - col_24_date date - ); - INSERT INTO primitives VALUES ( - true, 2, 3, DEFAULT, DEFAULT, 6, 7, 8, DEFAULT, DEFAULT, 11, 12, DEFAULT, DEFAULT, - 15.15, 16.16, 17.17, 18.18, 'az', 'az', 'az', 'az', - current_timestamp, current_timestamp); - INSERT INTO primitives VALUES ( - false, -2, -3, DEFAULT, DEFAULT, -6, -7, -8, DEFAULT, DEFAULT, -11, -12, DEFAULT, DEFAULT, - -15.15, -16.16, -17.17, -18.18, 'буки', 'буки', 'буки', 'буки', - current_timestamp, current_timestamp); - INSERT INTO primitives VALUES ( - NULL, NULL, NULL, DEFAULT, DEFAULT, NULL, - NULL, NULL, DEFAULT, DEFAULT, NULL, NULL, - DEFAULT, DEFAULT, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL - ); -EOSQL - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - DROP TABLE IF EXISTS benchmark_1g; - CREATE TABLE benchmark_1g (id bigserial, col varchar(1024)); - INSERT INTO benchmark_1g SELECT generate_series(1,1048576) AS id, REPEAT(md5(random()::text), 32) AS col; -EOSQL diff --git a/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/list_schemas.sh b/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/list_schemas.sh deleted file mode 100755 index 7797eb5d29f7..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/list_schemas.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -ex - -export PGPASSWORD=qwerty12345 - -SCRIPT=" -select schema_name -from information_schema.schemata; -" - -sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}" - - -SCRIPT=" -select nspname -from pg_catalog.pg_namespace; -" - -sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}" diff --git a/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/show_table_size.sh b/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/show_table_size.sh deleted file mode 100755 index c92a8b5b2b40..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/show_table_size.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -ex - -export PGPASSWORD=qwerty12345 - -SCRIPT=" -DROP TABLE IF EXISTS benchmark_1g; -CREATE TABLE benchmark_1g (id bigserial, col varchar(1024)); -INSERT INTO benchmark_1g SELECT generate_series(1,1048576) AS id, REPEAT(md5(random()::text), 32) AS col; -" - -sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}" - -SCRIPT=" -SELECT pg_size_pretty(pg_total_relation_size('public.benchmark_1g')); -" - -sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}" diff --git a/ydb/library/yql/providers/generic/connector/debug/tls/.arcignore b/ydb/library/yql/providers/generic/connector/debug/tls/.arcignore deleted file mode 100644 index 832b3e16ad7f..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/tls/.arcignore +++ /dev/null @@ -1,5 +0,0 @@ -*.crt -*.csr -*.ext -*.key -*.pem diff --git a/ydb/library/yql/providers/generic/connector/debug/tls/generate.sh b/ydb/library/yql/providers/generic/connector/debug/tls/generate.sh deleted file mode 100755 index 6f613be21df9..000000000000 --- a/ydb/library/yql/providers/generic/connector/debug/tls/generate.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -set -ex - -# Clean keys for previous run -rm ca.key ca.crt \ - server.key server.csr server.crt server.pem server.ext \ - client.key client.csr client.crt client.pem || true - - -HOSTNAME=$(hostname -f) -SUBJECT="/C=RU/L=Moscow/O=yq-connector/OU=yandex/CN=${HOSTNAME}/emailAddress=vitalyisaev@yandex-team.ru" -SUBJECT_ALT_NAME="subjectAltName = DNS:${HOSTNAME},DNS:localhost" - -# Generate self signed root CA cert -openssl req -nodes -x509 -newkey rsa:2048 -keyout ca.key -out ca.crt -subj "${SUBJECT}" - -# Server key pair -openssl req -newkey rsa:2048 -nodes -keyout server.key -subj "${SUBJECT}" -out server.csr -echo -n "${SUBJECT_ALT_NAME}" > server.ext -openssl x509 -req -extfile server.ext -days 365 -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -cat server.key server.crt > server.pem - -# Client key pair -openssl req -nodes -newkey rsa:2048 -keyout client.key -out client.csr -subj "${SUBJECT}" -addext "${SUBJECT_ALT_NAME}" -openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAserial ca.srl -out client.crt -cat client.key client.crt > client.pem diff --git a/ydb/library/yql/providers/generic/connector/generate/main.py b/ydb/library/yql/providers/generic/connector/generate/main.py deleted file mode 100755 index 7a7995e1963e..000000000000 --- a/ydb/library/yql/providers/generic/connector/generate/main.py +++ /dev/null @@ -1,136 +0,0 @@ -import subprocess -from pathlib import Path -from typing import List - - -class YDBProtoFile: - """ - YDBProtoFile is a proto file lying within YDB directory that - we have to patch in order to generate valid GRPC for connector. - """ - - src_initial: str - src_patched: str - filepath: Path - - def __init__(self, filepath: Path, go_package: str): - self.filepath = filepath - - # preserve original content - with open(filepath, 'r') as f: - self.src_initial = f.read() - - # prepare patched version - lines_initial = self.src_initial.splitlines() - - import_line = f'option go_package = "{go_package}";' - import_line_pos = 5 - - lines_patched = lines_initial[:import_line_pos] + [import_line] + lines_initial[import_line_pos:] - self.src_patched = '\n'.join(lines_patched) - - def patch(self): - with open(self.filepath, 'w') as f: - f.write(self.src_patched) - - def revert(self): - with open(self.filepath, 'w') as f: - f.write(self.src_initial) - - -def __call_subprocess(cmd: List[str]): - formatted = "\n".join(map(str, cmd)) - print(f'Running command:\n{formatted}') - - process = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout, stderr = process.communicate() - exit_code = process.wait() - - if exit_code != 0: - raise Exception( - f'Subprocess failure: exit_code={exit_code} stdout={stdout.decode("utf-8")}, stderr={stderr.decode("utf-8")}' - ) - - if stdout: - print(stdout.decode('utf-8')) - if stderr: - print(stderr.decode('utf-8')) - return stdout - - -def get_arc_root() -> Path: - out = __call_subprocess(['arc', 'root']) - return Path(out.decode('utf-8').strip()) - - -def build_protoc(arc_root: Path) -> Path: - __call_subprocess([arc_root.joinpath('ya'), 'make', arc_root.joinpath('contrib/tools/protoc')]) - return arc_root.joinpath('contrib/tools/protoc/protoc') - - -def build_protoc_gen_go(arc_root: Path) -> Path: - __call_subprocess( - [arc_root.joinpath('ya'), 'make', arc_root.joinpath('vendor/google.golang.org/protobuf/cmd/protoc-gen-go')] - ) - return arc_root.joinpath('vendor/google.golang.org/protobuf/cmd/protoc-gen-go/protoc-gen-go') - - -def build_protoc_gen_go_grpc(arc_root: Path) -> Path: - __call_subprocess( - [arc_root.joinpath('ya'), 'make', arc_root.joinpath('vendor/google.golang.org/grpc/cmd/protoc-gen-go-grpc')] - ) - return arc_root.joinpath('vendor/google.golang.org/grpc/cmd/protoc-gen-go-grpc/protoc-gen-go-grpc') - - -def run_protoc(arc_root: Path): - # compile protoc from Arcadia - protoc_binary = build_protoc(arc_root) - protoc_gen_go_binary = build_protoc_gen_go(arc_root) - protoc_gen_go_grpc_binary = build_protoc_gen_go_grpc(arc_root) - - # look for project protofiles - source_dir = arc_root.joinpath('ydb/library/yql/providers/generic/connector/api/service') - target_dir = arc_root.joinpath('ydb/library/yql/providers/generic/connector/libgo/service') - proto_files = source_dir.rglob('*.proto') - - # build protoc args - cmd = [ - protoc_binary, - f'--plugin=protoc-gen-go={protoc_gen_go_binary}', - f'--plugin=protoc-gen-go-grpc={protoc_gen_go_grpc_binary}', - f'--go_out={target_dir}', - '--go_opt=module=a.yandex-team.ru/ydb/library/yql/providers/generic/connector/libgo/service', - f'--go-grpc_out={target_dir}', - '--go-grpc_opt=module=a.yandex-team.ru/ydb/library/yql/providers/generic/connector/libgo/service', - f'-I{arc_root}', - f'-I{arc_root.joinpath("contrib/libs/protobuf/src/")}', - ] - cmd.extend(proto_files) - __call_subprocess(cmd) - - -def main(): - # derive Arcadia's root - arc_root = get_arc_root() - - # YDB's protofiles this project depends on - ydb_source_params = [ - ('ydb/public/api/protos/ydb_value.proto', 'github.com/ydb-platform/ydb-go-genproto/protos/Ydb'), - ('ydb/public/api/protos/ydb_status_codes.proto', 'github.com/ydb-platform/ydb-go-genproto/protos/Ydb'), - ('ydb/public/api/protos/ydb_issue_message.proto', 'github.com/ydb-platform/ydb-go-genproto/protos/Ydb_Issue'), - ] - - ydb_source_files = [YDBProtoFile(arc_root.joinpath(param[0]), param[1]) for param in ydb_source_params] - - # Patch YDB sources - for f in ydb_source_files: - f.patch() - - try: - # Generate Connector GRPC API - run_protoc(arc_root) - finally: - # pass - # Revert changes in YDB sources - for f in ydb_source_files: - f.revert() diff --git a/ydb/library/yql/providers/generic/connector/generate/ya.make b/ydb/library/yql/providers/generic/connector/generate/ya.make deleted file mode 100644 index 034c6ac580e4..000000000000 --- a/ydb/library/yql/providers/generic/connector/generate/ya.make +++ /dev/null @@ -1,8 +0,0 @@ -PY3_PROGRAM() - -PY_SRCS( - MAIN main.py -) - - -END() diff --git a/ydb/library/yql/providers/generic/connector/libgo/service/connector.pb.go b/ydb/library/yql/providers/generic/connector/libgo/service/connector.pb.go deleted file mode 100644 index eb648bb2925f..000000000000 --- a/ydb/library/yql/providers/generic/connector/libgo/service/connector.pb.go +++ /dev/null @@ -1,119 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.31.0 -// protoc v3.19.0 -// source: ydb/library/yql/providers/generic/connector/api/service/connector.proto - -package service - -import ( - protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -var File_ydb_library_yql_providers_generic_connector_api_service_connector_proto protoreflect.FileDescriptor - -var file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_rawDesc = []byte{ - 0x0a, 0x47, 0x79, 0x64, 0x62, 0x2f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x2f, 0x79, 0x71, - 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x69, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, - 0x69, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x14, 0x4e, 0x59, 0x71, 0x6c, 0x2e, - 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x1a, - 0x4e, 0x79, 0x64, 0x62, 0x2f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x2f, 0x79, 0x71, 0x6c, - 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x69, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x69, - 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, - 0xa6, 0x03, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x63, 0x0a, - 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x28, 0x2e, 0x4e, 0x59, - 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, - 0x70, 0x69, 0x2e, 0x54, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x4c, 0x69, - 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x0d, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x62, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2c, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x62, - 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, - 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x12, 0x28, 0x2e, 0x4e, - 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, - 0x41, 0x70, 0x69, 0x2e, 0x54, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x0a, 0x52, 0x65, 0x61, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, - 0x73, 0x12, 0x28, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x52, 0x65, 0x61, 0x64, 0x53, 0x70, - 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x4e, 0x59, - 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, - 0x70, 0x69, 0x2e, 0x54, 0x52, 0x65, 0x61, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x4c, 0x5a, 0x4a, 0x61, 0x2e, 0x79, 0x61, - 0x6e, 0x64, 0x65, 0x78, 0x2d, 0x74, 0x65, 0x61, 0x6d, 0x2e, 0x72, 0x75, 0x2f, 0x79, 0x64, 0x62, - 0x2f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x2f, 0x79, 0x71, 0x6c, 0x2f, 0x70, 0x72, 0x6f, - 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x2f, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x6c, 0x69, 0x62, 0x67, 0x6f, 0x2f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_goTypes = []interface{}{ - (*protos.TListTablesRequest)(nil), // 0: NYql.NConnector.NApi.TListTablesRequest - (*protos.TDescribeTableRequest)(nil), // 1: NYql.NConnector.NApi.TDescribeTableRequest - (*protos.TListSplitsRequest)(nil), // 2: NYql.NConnector.NApi.TListSplitsRequest - (*protos.TReadSplitsRequest)(nil), // 3: NYql.NConnector.NApi.TReadSplitsRequest - (*protos.TListTablesResponse)(nil), // 4: NYql.NConnector.NApi.TListTablesResponse - (*protos.TDescribeTableResponse)(nil), // 5: NYql.NConnector.NApi.TDescribeTableResponse - (*protos.TListSplitsResponse)(nil), // 6: NYql.NConnector.NApi.TListSplitsResponse - (*protos.TReadSplitsResponse)(nil), // 7: NYql.NConnector.NApi.TReadSplitsResponse -} -var file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_depIdxs = []int32{ - 0, // 0: NYql.NConnector.NApi.Connector.ListTables:input_type -> NYql.NConnector.NApi.TListTablesRequest - 1, // 1: NYql.NConnector.NApi.Connector.DescribeTable:input_type -> NYql.NConnector.NApi.TDescribeTableRequest - 2, // 2: NYql.NConnector.NApi.Connector.ListSplits:input_type -> NYql.NConnector.NApi.TListSplitsRequest - 3, // 3: NYql.NConnector.NApi.Connector.ReadSplits:input_type -> NYql.NConnector.NApi.TReadSplitsRequest - 4, // 4: NYql.NConnector.NApi.Connector.ListTables:output_type -> NYql.NConnector.NApi.TListTablesResponse - 5, // 5: NYql.NConnector.NApi.Connector.DescribeTable:output_type -> NYql.NConnector.NApi.TDescribeTableResponse - 6, // 6: NYql.NConnector.NApi.Connector.ListSplits:output_type -> NYql.NConnector.NApi.TListSplitsResponse - 7, // 7: NYql.NConnector.NApi.Connector.ReadSplits:output_type -> NYql.NConnector.NApi.TReadSplitsResponse - 4, // [4:8] is the sub-list for method output_type - 0, // [0:4] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_init() } -func file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_init() { - if File_ydb_library_yql_providers_generic_connector_api_service_connector_proto != nil { - return - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_rawDesc, - NumEnums: 0, - NumMessages: 0, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_goTypes, - DependencyIndexes: file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_depIdxs, - }.Build() - File_ydb_library_yql_providers_generic_connector_api_service_connector_proto = out.File - file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_rawDesc = nil - file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_goTypes = nil - file_ydb_library_yql_providers_generic_connector_api_service_connector_proto_depIdxs = nil -} diff --git a/ydb/library/yql/providers/generic/connector/libgo/service/connector_grpc.pb.go b/ydb/library/yql/providers/generic/connector/libgo/service/connector_grpc.pb.go deleted file mode 100644 index 18ed5e9d46ff..000000000000 --- a/ydb/library/yql/providers/generic/connector/libgo/service/connector_grpc.pb.go +++ /dev/null @@ -1,313 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. -// versions: -// - protoc-gen-go-grpc v1.3.0 -// - protoc v3.19.0 -// source: ydb/library/yql/providers/generic/connector/api/service/connector.proto - -package service - -import ( - protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos" - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -const ( - Connector_ListTables_FullMethodName = "/NYql.NConnector.NApi.Connector/ListTables" - Connector_DescribeTable_FullMethodName = "/NYql.NConnector.NApi.Connector/DescribeTable" - Connector_ListSplits_FullMethodName = "/NYql.NConnector.NApi.Connector/ListSplits" - Connector_ReadSplits_FullMethodName = "/NYql.NConnector.NApi.Connector/ReadSplits" -) - -// ConnectorClient is the client API for Connector service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type ConnectorClient interface { - // ListTables returns the list of tables existing in a particular database. - ListTables(ctx context.Context, in *protos.TListTablesRequest, opts ...grpc.CallOption) (Connector_ListTablesClient, error) - // DescribeTable returns table's schema. - DescribeTable(ctx context.Context, in *protos.TDescribeTableRequest, opts ...grpc.CallOption) (*protos.TDescribeTableResponse, error) - // ListSplits asks Connector to partition the data that are going to be read - // into elementary parts suitable for parallel reading. - ListSplits(ctx context.Context, in *protos.TListSplitsRequest, opts ...grpc.CallOption) (Connector_ListSplitsClient, error) - // ReadSplits reads data associated with splits. - ReadSplits(ctx context.Context, in *protos.TReadSplitsRequest, opts ...grpc.CallOption) (Connector_ReadSplitsClient, error) -} - -type connectorClient struct { - cc grpc.ClientConnInterface -} - -func NewConnectorClient(cc grpc.ClientConnInterface) ConnectorClient { - return &connectorClient{cc} -} - -func (c *connectorClient) ListTables(ctx context.Context, in *protos.TListTablesRequest, opts ...grpc.CallOption) (Connector_ListTablesClient, error) { - stream, err := c.cc.NewStream(ctx, &Connector_ServiceDesc.Streams[0], Connector_ListTables_FullMethodName, opts...) - if err != nil { - return nil, err - } - x := &connectorListTablesClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Connector_ListTablesClient interface { - Recv() (*protos.TListTablesResponse, error) - grpc.ClientStream -} - -type connectorListTablesClient struct { - grpc.ClientStream -} - -func (x *connectorListTablesClient) Recv() (*protos.TListTablesResponse, error) { - m := new(protos.TListTablesResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *connectorClient) DescribeTable(ctx context.Context, in *protos.TDescribeTableRequest, opts ...grpc.CallOption) (*protos.TDescribeTableResponse, error) { - out := new(protos.TDescribeTableResponse) - err := c.cc.Invoke(ctx, Connector_DescribeTable_FullMethodName, in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *connectorClient) ListSplits(ctx context.Context, in *protos.TListSplitsRequest, opts ...grpc.CallOption) (Connector_ListSplitsClient, error) { - stream, err := c.cc.NewStream(ctx, &Connector_ServiceDesc.Streams[1], Connector_ListSplits_FullMethodName, opts...) - if err != nil { - return nil, err - } - x := &connectorListSplitsClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Connector_ListSplitsClient interface { - Recv() (*protos.TListSplitsResponse, error) - grpc.ClientStream -} - -type connectorListSplitsClient struct { - grpc.ClientStream -} - -func (x *connectorListSplitsClient) Recv() (*protos.TListSplitsResponse, error) { - m := new(protos.TListSplitsResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *connectorClient) ReadSplits(ctx context.Context, in *protos.TReadSplitsRequest, opts ...grpc.CallOption) (Connector_ReadSplitsClient, error) { - stream, err := c.cc.NewStream(ctx, &Connector_ServiceDesc.Streams[2], Connector_ReadSplits_FullMethodName, opts...) - if err != nil { - return nil, err - } - x := &connectorReadSplitsClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Connector_ReadSplitsClient interface { - Recv() (*protos.TReadSplitsResponse, error) - grpc.ClientStream -} - -type connectorReadSplitsClient struct { - grpc.ClientStream -} - -func (x *connectorReadSplitsClient) Recv() (*protos.TReadSplitsResponse, error) { - m := new(protos.TReadSplitsResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// ConnectorServer is the server API for Connector service. -// All implementations must embed UnimplementedConnectorServer -// for forward compatibility -type ConnectorServer interface { - // ListTables returns the list of tables existing in a particular database. - ListTables(*protos.TListTablesRequest, Connector_ListTablesServer) error - // DescribeTable returns table's schema. - DescribeTable(context.Context, *protos.TDescribeTableRequest) (*protos.TDescribeTableResponse, error) - // ListSplits asks Connector to partition the data that are going to be read - // into elementary parts suitable for parallel reading. - ListSplits(*protos.TListSplitsRequest, Connector_ListSplitsServer) error - // ReadSplits reads data associated with splits. - ReadSplits(*protos.TReadSplitsRequest, Connector_ReadSplitsServer) error - mustEmbedUnimplementedConnectorServer() -} - -// UnimplementedConnectorServer must be embedded to have forward compatible implementations. -type UnimplementedConnectorServer struct { -} - -func (UnimplementedConnectorServer) ListTables(*protos.TListTablesRequest, Connector_ListTablesServer) error { - return status.Errorf(codes.Unimplemented, "method ListTables not implemented") -} -func (UnimplementedConnectorServer) DescribeTable(context.Context, *protos.TDescribeTableRequest) (*protos.TDescribeTableResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DescribeTable not implemented") -} -func (UnimplementedConnectorServer) ListSplits(*protos.TListSplitsRequest, Connector_ListSplitsServer) error { - return status.Errorf(codes.Unimplemented, "method ListSplits not implemented") -} -func (UnimplementedConnectorServer) ReadSplits(*protos.TReadSplitsRequest, Connector_ReadSplitsServer) error { - return status.Errorf(codes.Unimplemented, "method ReadSplits not implemented") -} -func (UnimplementedConnectorServer) mustEmbedUnimplementedConnectorServer() {} - -// UnsafeConnectorServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to ConnectorServer will -// result in compilation errors. -type UnsafeConnectorServer interface { - mustEmbedUnimplementedConnectorServer() -} - -func RegisterConnectorServer(s grpc.ServiceRegistrar, srv ConnectorServer) { - s.RegisterService(&Connector_ServiceDesc, srv) -} - -func _Connector_ListTables_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(protos.TListTablesRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(ConnectorServer).ListTables(m, &connectorListTablesServer{stream}) -} - -type Connector_ListTablesServer interface { - Send(*protos.TListTablesResponse) error - grpc.ServerStream -} - -type connectorListTablesServer struct { - grpc.ServerStream -} - -func (x *connectorListTablesServer) Send(m *protos.TListTablesResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _Connector_DescribeTable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(protos.TDescribeTableRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ConnectorServer).DescribeTable(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Connector_DescribeTable_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ConnectorServer).DescribeTable(ctx, req.(*protos.TDescribeTableRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Connector_ListSplits_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(protos.TListSplitsRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(ConnectorServer).ListSplits(m, &connectorListSplitsServer{stream}) -} - -type Connector_ListSplitsServer interface { - Send(*protos.TListSplitsResponse) error - grpc.ServerStream -} - -type connectorListSplitsServer struct { - grpc.ServerStream -} - -func (x *connectorListSplitsServer) Send(m *protos.TListSplitsResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _Connector_ReadSplits_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(protos.TReadSplitsRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(ConnectorServer).ReadSplits(m, &connectorReadSplitsServer{stream}) -} - -type Connector_ReadSplitsServer interface { - Send(*protos.TReadSplitsResponse) error - grpc.ServerStream -} - -type connectorReadSplitsServer struct { - grpc.ServerStream -} - -func (x *connectorReadSplitsServer) Send(m *protos.TReadSplitsResponse) error { - return x.ServerStream.SendMsg(m) -} - -// Connector_ServiceDesc is the grpc.ServiceDesc for Connector service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var Connector_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "NYql.NConnector.NApi.Connector", - HandlerType: (*ConnectorServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "DescribeTable", - Handler: _Connector_DescribeTable_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "ListTables", - Handler: _Connector_ListTables_Handler, - ServerStreams: true, - }, - { - StreamName: "ListSplits", - Handler: _Connector_ListSplits_Handler, - ServerStreams: true, - }, - { - StreamName: "ReadSplits", - Handler: _Connector_ReadSplits_Handler, - ServerStreams: true, - }, - }, - Metadata: "ydb/library/yql/providers/generic/connector/api/service/connector.proto", -} diff --git a/ydb/library/yql/providers/generic/connector/libgo/service/protos/connector.pb.go b/ydb/library/yql/providers/generic/connector/libgo/service/protos/connector.pb.go deleted file mode 100644 index 36dfc5f544db..000000000000 --- a/ydb/library/yql/providers/generic/connector/libgo/service/protos/connector.pb.go +++ /dev/null @@ -1,3861 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.31.0 -// protoc v3.19.0 -// source: ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto - -package protos - -import ( - common "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/api/common" - Ydb "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" - Ydb_Issue "github.com/ydb-platform/ydb-go-genproto/protos/Ydb_Issue" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type EDateTimeFormat int32 - -const ( - EDateTimeFormat_DATE_TIME_FORMAT_UNSPECIFIED EDateTimeFormat = 0 - EDateTimeFormat_STRING_FORMAT EDateTimeFormat = 1 - EDateTimeFormat_YQL_FORMAT EDateTimeFormat = 2 -) - -// Enum value maps for EDateTimeFormat. -var ( - EDateTimeFormat_name = map[int32]string{ - 0: "DATE_TIME_FORMAT_UNSPECIFIED", - 1: "STRING_FORMAT", - 2: "YQL_FORMAT", - } - EDateTimeFormat_value = map[string]int32{ - "DATE_TIME_FORMAT_UNSPECIFIED": 0, - "STRING_FORMAT": 1, - "YQL_FORMAT": 2, - } -) - -func (x EDateTimeFormat) Enum() *EDateTimeFormat { - p := new(EDateTimeFormat) - *p = x - return p -} - -func (x EDateTimeFormat) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (EDateTimeFormat) Descriptor() protoreflect.EnumDescriptor { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[0].Descriptor() -} - -func (EDateTimeFormat) Type() protoreflect.EnumType { - return &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[0] -} - -func (x EDateTimeFormat) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use EDateTimeFormat.Descriptor instead. -func (EDateTimeFormat) EnumDescriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{0} -} - -type TReadSplitsRequest_EMode int32 - -const ( - TReadSplitsRequest_MODE_UNSPECIFIED TReadSplitsRequest_EMode = 0 - // Connector will read splits in a single thread one by one. - // The data will be returned in the order corresponding to the order of requested splits. - TReadSplitsRequest_ORDERED TReadSplitsRequest_EMode = 1 - // Connector may read different splits concurrently and send the split fragments to the response stream - // as soon as the data is obtained from the data source. Thus the stream is multiplexed between splits. - TReadSplitsRequest_UNORDERED TReadSplitsRequest_EMode = 2 -) - -// Enum value maps for TReadSplitsRequest_EMode. -var ( - TReadSplitsRequest_EMode_name = map[int32]string{ - 0: "MODE_UNSPECIFIED", - 1: "ORDERED", - 2: "UNORDERED", - } - TReadSplitsRequest_EMode_value = map[string]int32{ - "MODE_UNSPECIFIED": 0, - "ORDERED": 1, - "UNORDERED": 2, - } -) - -func (x TReadSplitsRequest_EMode) Enum() *TReadSplitsRequest_EMode { - p := new(TReadSplitsRequest_EMode) - *p = x - return p -} - -func (x TReadSplitsRequest_EMode) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (TReadSplitsRequest_EMode) Descriptor() protoreflect.EnumDescriptor { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[1].Descriptor() -} - -func (TReadSplitsRequest_EMode) Type() protoreflect.EnumType { - return &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[1] -} - -func (x TReadSplitsRequest_EMode) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use TReadSplitsRequest_EMode.Descriptor instead. -func (TReadSplitsRequest_EMode) EnumDescriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{10, 0} -} - -type TReadSplitsRequest_EFormat int32 - -const ( - TReadSplitsRequest_FORMAT_UNSPECIFIED TReadSplitsRequest_EFormat = 0 - // Arrow IPC Streaming format: - // https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format - TReadSplitsRequest_ARROW_IPC_STREAMING TReadSplitsRequest_EFormat = 2 -) - -// Enum value maps for TReadSplitsRequest_EFormat. -var ( - TReadSplitsRequest_EFormat_name = map[int32]string{ - 0: "FORMAT_UNSPECIFIED", - 2: "ARROW_IPC_STREAMING", - } - TReadSplitsRequest_EFormat_value = map[string]int32{ - "FORMAT_UNSPECIFIED": 0, - "ARROW_IPC_STREAMING": 2, - } -) - -func (x TReadSplitsRequest_EFormat) Enum() *TReadSplitsRequest_EFormat { - p := new(TReadSplitsRequest_EFormat) - *p = x - return p -} - -func (x TReadSplitsRequest_EFormat) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (TReadSplitsRequest_EFormat) Descriptor() protoreflect.EnumDescriptor { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[2].Descriptor() -} - -func (TReadSplitsRequest_EFormat) Type() protoreflect.EnumType { - return &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[2] -} - -func (x TReadSplitsRequest_EFormat) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use TReadSplitsRequest_EFormat.Descriptor instead. -func (TReadSplitsRequest_EFormat) EnumDescriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{10, 1} -} - -// An operation code. -type TExpression_TArithmeticalExpression_EOperation int32 - -const ( - TExpression_TArithmeticalExpression_EXPRESSION_OPERATION_UNSPECIFIED TExpression_TArithmeticalExpression_EOperation = 0 - TExpression_TArithmeticalExpression_MUL TExpression_TArithmeticalExpression_EOperation = 1 // left_value * right_value - TExpression_TArithmeticalExpression_ADD TExpression_TArithmeticalExpression_EOperation = 2 // left_value + right_value - TExpression_TArithmeticalExpression_SUB TExpression_TArithmeticalExpression_EOperation = 3 // left_value - right_value - TExpression_TArithmeticalExpression_BIT_AND TExpression_TArithmeticalExpression_EOperation = 4 // left_value & right_value - TExpression_TArithmeticalExpression_BIT_OR TExpression_TArithmeticalExpression_EOperation = 5 // left_value | right_value - TExpression_TArithmeticalExpression_BIT_XOR TExpression_TArithmeticalExpression_EOperation = 6 // left_value ^ right_value -) - -// Enum value maps for TExpression_TArithmeticalExpression_EOperation. -var ( - TExpression_TArithmeticalExpression_EOperation_name = map[int32]string{ - 0: "EXPRESSION_OPERATION_UNSPECIFIED", - 1: "MUL", - 2: "ADD", - 3: "SUB", - 4: "BIT_AND", - 5: "BIT_OR", - 6: "BIT_XOR", - } - TExpression_TArithmeticalExpression_EOperation_value = map[string]int32{ - "EXPRESSION_OPERATION_UNSPECIFIED": 0, - "MUL": 1, - "ADD": 2, - "SUB": 3, - "BIT_AND": 4, - "BIT_OR": 5, - "BIT_XOR": 6, - } -) - -func (x TExpression_TArithmeticalExpression_EOperation) Enum() *TExpression_TArithmeticalExpression_EOperation { - p := new(TExpression_TArithmeticalExpression_EOperation) - *p = x - return p -} - -func (x TExpression_TArithmeticalExpression_EOperation) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (TExpression_TArithmeticalExpression_EOperation) Descriptor() protoreflect.EnumDescriptor { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[3].Descriptor() -} - -func (TExpression_TArithmeticalExpression_EOperation) Type() protoreflect.EnumType { - return &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[3] -} - -func (x TExpression_TArithmeticalExpression_EOperation) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use TExpression_TArithmeticalExpression_EOperation.Descriptor instead. -func (TExpression_TArithmeticalExpression_EOperation) EnumDescriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{13, 0, 0} -} - -// An operation code. -type TPredicate_TComparison_EOperation int32 - -const ( - TPredicate_TComparison_COMPARISON_OPERATION_UNSPECIFIED TPredicate_TComparison_EOperation = 0 - TPredicate_TComparison_L TPredicate_TComparison_EOperation = 1 // "$column < value" - TPredicate_TComparison_LE TPredicate_TComparison_EOperation = 2 // "$column <= value" - TPredicate_TComparison_EQ TPredicate_TComparison_EOperation = 3 // "$column = value" - TPredicate_TComparison_NE TPredicate_TComparison_EOperation = 4 // "$column != value" - TPredicate_TComparison_GE TPredicate_TComparison_EOperation = 5 // "$column >= value" - TPredicate_TComparison_G TPredicate_TComparison_EOperation = 6 // "$column > value" -) - -// Enum value maps for TPredicate_TComparison_EOperation. -var ( - TPredicate_TComparison_EOperation_name = map[int32]string{ - 0: "COMPARISON_OPERATION_UNSPECIFIED", - 1: "L", - 2: "LE", - 3: "EQ", - 4: "NE", - 5: "GE", - 6: "G", - } - TPredicate_TComparison_EOperation_value = map[string]int32{ - "COMPARISON_OPERATION_UNSPECIFIED": 0, - "L": 1, - "LE": 2, - "EQ": 3, - "NE": 4, - "GE": 5, - "G": 6, - } -) - -func (x TPredicate_TComparison_EOperation) Enum() *TPredicate_TComparison_EOperation { - p := new(TPredicate_TComparison_EOperation) - *p = x - return p -} - -func (x TPredicate_TComparison_EOperation) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (TPredicate_TComparison_EOperation) Descriptor() protoreflect.EnumDescriptor { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[4].Descriptor() -} - -func (TPredicate_TComparison_EOperation) Type() protoreflect.EnumType { - return &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes[4] -} - -func (x TPredicate_TComparison_EOperation) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use TPredicate_TComparison_EOperation.Descriptor instead. -func (TPredicate_TComparison_EOperation) EnumDescriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 8, 0} -} - -// TListTablesRequest requests the list of tables in a particular database of the data source -type TListTablesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Data source instance to connect - DataSourceInstance *common.TDataSourceInstance `protobuf:"bytes,1,opt,name=data_source_instance,json=dataSourceInstance,proto3" json:"data_source_instance,omitempty"` - // There may be a huge number of tables in the data source, - // and here are ways to extract only necessary ones: - // - // Types that are assignable to Filtering: - // - // *TListTablesRequest_Pattern - Filtering isTListTablesRequest_Filtering `protobuf_oneof:"filtering"` -} - -func (x *TListTablesRequest) Reset() { - *x = TListTablesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TListTablesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TListTablesRequest) ProtoMessage() {} - -func (x *TListTablesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TListTablesRequest.ProtoReflect.Descriptor instead. -func (*TListTablesRequest) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{0} -} - -func (x *TListTablesRequest) GetDataSourceInstance() *common.TDataSourceInstance { - if x != nil { - return x.DataSourceInstance - } - return nil -} - -func (m *TListTablesRequest) GetFiltering() isTListTablesRequest_Filtering { - if m != nil { - return m.Filtering - } - return nil -} - -func (x *TListTablesRequest) GetPattern() string { - if x, ok := x.GetFiltering().(*TListTablesRequest_Pattern); ok { - return x.Pattern - } - return "" -} - -type isTListTablesRequest_Filtering interface { - isTListTablesRequest_Filtering() -} - -type TListTablesRequest_Pattern struct { - // Regexp to filter table names - Pattern string `protobuf:"bytes,2,opt,name=pattern,proto3,oneof"` -} - -func (*TListTablesRequest_Pattern) isTListTablesRequest_Filtering() {} - -// TListTablesResponse returns the list of tables in a particular database of the data source -type TListTablesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Table names list - Tables []string `protobuf:"bytes,1,rep,name=tables,proto3" json:"tables,omitempty"` - // Call result - Error *TError `protobuf:"bytes,100,opt,name=error,proto3" json:"error,omitempty"` -} - -func (x *TListTablesResponse) Reset() { - *x = TListTablesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TListTablesResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TListTablesResponse) ProtoMessage() {} - -func (x *TListTablesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TListTablesResponse.ProtoReflect.Descriptor instead. -func (*TListTablesResponse) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{1} -} - -func (x *TListTablesResponse) GetTables() []string { - if x != nil { - return x.Tables - } - return nil -} - -func (x *TListTablesResponse) GetError() *TError { - if x != nil { - return x.Error - } - return nil -} - -// TDescribeTableRequest requests table metadata -type TDescribeTableRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Data source instance to connect - DataSourceInstance *common.TDataSourceInstance `protobuf:"bytes,1,opt,name=data_source_instance,json=dataSourceInstance,proto3" json:"data_source_instance,omitempty"` - // Table to describe - Table string `protobuf:"bytes,2,opt,name=table,proto3" json:"table,omitempty"` - // Rules for type mapping - TypeMappingSettings *TTypeMappingSettings `protobuf:"bytes,3,opt,name=type_mapping_settings,json=typeMappingSettings,proto3" json:"type_mapping_settings,omitempty"` -} - -func (x *TDescribeTableRequest) Reset() { - *x = TDescribeTableRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TDescribeTableRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TDescribeTableRequest) ProtoMessage() {} - -func (x *TDescribeTableRequest) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TDescribeTableRequest.ProtoReflect.Descriptor instead. -func (*TDescribeTableRequest) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{2} -} - -func (x *TDescribeTableRequest) GetDataSourceInstance() *common.TDataSourceInstance { - if x != nil { - return x.DataSourceInstance - } - return nil -} - -func (x *TDescribeTableRequest) GetTable() string { - if x != nil { - return x.Table - } - return "" -} - -func (x *TDescribeTableRequest) GetTypeMappingSettings() *TTypeMappingSettings { - if x != nil { - return x.TypeMappingSettings - } - return nil -} - -type TTypeMappingSettings struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Determines the format of date or time representation - DateTimeFormat EDateTimeFormat `protobuf:"varint,1,opt,name=date_time_format,json=dateTimeFormat,proto3,enum=NYql.NConnector.NApi.EDateTimeFormat" json:"date_time_format,omitempty"` -} - -func (x *TTypeMappingSettings) Reset() { - *x = TTypeMappingSettings{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TTypeMappingSettings) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TTypeMappingSettings) ProtoMessage() {} - -func (x *TTypeMappingSettings) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TTypeMappingSettings.ProtoReflect.Descriptor instead. -func (*TTypeMappingSettings) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{3} -} - -func (x *TTypeMappingSettings) GetDateTimeFormat() EDateTimeFormat { - if x != nil { - return x.DateTimeFormat - } - return EDateTimeFormat_DATE_TIME_FORMAT_UNSPECIFIED -} - -// TDescribeTableResponse returns table metadata -type TDescribeTableResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The whole schema of a table - Schema *TSchema `protobuf:"bytes,1,opt,name=schema,proto3" json:"schema,omitempty"` - // Call result - Error *TError `protobuf:"bytes,100,opt,name=error,proto3" json:"error,omitempty"` -} - -func (x *TDescribeTableResponse) Reset() { - *x = TDescribeTableResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TDescribeTableResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TDescribeTableResponse) ProtoMessage() {} - -func (x *TDescribeTableResponse) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TDescribeTableResponse.ProtoReflect.Descriptor instead. -func (*TDescribeTableResponse) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{4} -} - -func (x *TDescribeTableResponse) GetSchema() *TSchema { - if x != nil { - return x.Schema - } - return nil -} - -func (x *TDescribeTableResponse) GetError() *TError { - if x != nil { - return x.Error - } - return nil -} - -// TSchema represents the schema of the table -type TSchema struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Columns with YQL types - Columns []*Ydb.Column `protobuf:"bytes,1,rep,name=columns,proto3" json:"columns,omitempty"` // TODO: optional metadata? -} - -func (x *TSchema) Reset() { - *x = TSchema{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TSchema) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TSchema) ProtoMessage() {} - -func (x *TSchema) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TSchema.ProtoReflect.Descriptor instead. -func (*TSchema) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{5} -} - -func (x *TSchema) GetColumns() []*Ydb.Column { - if x != nil { - return x.Columns - } - return nil -} - -// TListSplitRequest asks Connector to split the requested data into elementary parts. -type TListSplitsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // YQ engine may want to read data from different tables simultaneously. - // Perhaps Connector will provide consistency guarantees across the tables some day. - Selects []*TSelect `protobuf:"bytes,2,rep,name=selects,proto3" json:"selects,omitempty"` - // Defines the number of splits (and, as a consequence, affects the size of the split). - // If you don't want splitting, set 1. - MaxSplitCount uint32 `protobuf:"varint,3,opt,name=max_split_count,json=maxSplitCount,proto3" json:"max_split_count,omitempty"` - // Connector will try to divide the data into the splits of this size, - // but the exact match is not guaranteed. - // Also this setting can be overridden by max_split_count. - SplitSize uint64 `protobuf:"varint,4,opt,name=split_size,json=splitSize,proto3" json:"split_size,omitempty"` - // Sometimes YQ doesn't know the exact size of the data set, - // so it asks Connector to split the data into the splits of $split_size, - // and the $max_split_count = MaxUINT32. - // But if the data is too large, and too many splits will be generated, - // this may exceed the memory available for YQ. - // In such case, it's better to fail fast. This limit helps to implement it: - SplitNumberLimit uint64 `protobuf:"varint,5,opt,name=split_number_limit,json=splitNumberLimit,proto3" json:"split_number_limit,omitempty"` -} - -func (x *TListSplitsRequest) Reset() { - *x = TListSplitsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TListSplitsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TListSplitsRequest) ProtoMessage() {} - -func (x *TListSplitsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TListSplitsRequest.ProtoReflect.Descriptor instead. -func (*TListSplitsRequest) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{6} -} - -func (x *TListSplitsRequest) GetSelects() []*TSelect { - if x != nil { - return x.Selects - } - return nil -} - -func (x *TListSplitsRequest) GetMaxSplitCount() uint32 { - if x != nil { - return x.MaxSplitCount - } - return 0 -} - -func (x *TListSplitsRequest) GetSplitSize() uint64 { - if x != nil { - return x.SplitSize - } - return 0 -} - -func (x *TListSplitsRequest) GetSplitNumberLimit() uint64 { - if x != nil { - return x.SplitNumberLimit - } - return 0 -} - -// TListSplitResponse returns the list of splits for a particular set of table partitions -type TListSplitsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // the list of splits for concurrent reading - Splits []*TSplit `protobuf:"bytes,1,rep,name=splits,proto3" json:"splits,omitempty"` - // Call result - Error *TError `protobuf:"bytes,100,opt,name=error,proto3" json:"error,omitempty"` -} - -func (x *TListSplitsResponse) Reset() { - *x = TListSplitsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TListSplitsResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TListSplitsResponse) ProtoMessage() {} - -func (x *TListSplitsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TListSplitsResponse.ProtoReflect.Descriptor instead. -func (*TListSplitsResponse) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{7} -} - -func (x *TListSplitsResponse) GetSplits() []*TSplit { - if x != nil { - return x.Splits - } - return nil -} - -func (x *TListSplitsResponse) GetError() *TError { - if x != nil { - return x.Error - } - return nil -} - -// Select describes what to read from the data source. -// -// In RDBMS systems this call internally transforms into SQL expression using this template: -// SELECT $what -// FROM $from -// WHERE $filter -// LIMIT $limit [OFFSET $offset] -// TODO: support JOIN, ORDER BY, GROUP BY -type TSelect struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Data source instance to connect - DataSourceInstance *common.TDataSourceInstance `protobuf:"bytes,1,opt,name=data_source_instance,json=dataSourceInstance,proto3" json:"data_source_instance,omitempty"` - // Transforms into SELECT $what. - What *TSelect_TWhat `protobuf:"bytes,2,opt,name=what,proto3" json:"what,omitempty"` - // Transforms into FROM $from - From *TSelect_TFrom `protobuf:"bytes,3,opt,name=from,proto3" json:"from,omitempty"` - // Transforms into WHERE $filter. - // Optional field. - Where *TSelect_TWhere `protobuf:"bytes,4,opt,name=where,proto3" json:"where,omitempty"` - // Transforms into LIMIT $limit [OFFSET $offset]. - // Optional field. - Limit *TSelect_TLimit `protobuf:"bytes,5,opt,name=limit,proto3" json:"limit,omitempty"` - // For schemaless data sources, when it's hard for us to infer schema for the query result, - // user can supply the scheme himself. - // Optional field that must be empty for schematized data sources. - PredefinedSchema *TSchema `protobuf:"bytes,6,opt,name=predefined_schema,json=predefinedSchema,proto3" json:"predefined_schema,omitempty"` -} - -func (x *TSelect) Reset() { - *x = TSelect{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TSelect) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TSelect) ProtoMessage() {} - -func (x *TSelect) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TSelect.ProtoReflect.Descriptor instead. -func (*TSelect) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{8} -} - -func (x *TSelect) GetDataSourceInstance() *common.TDataSourceInstance { - if x != nil { - return x.DataSourceInstance - } - return nil -} - -func (x *TSelect) GetWhat() *TSelect_TWhat { - if x != nil { - return x.What - } - return nil -} - -func (x *TSelect) GetFrom() *TSelect_TFrom { - if x != nil { - return x.From - } - return nil -} - -func (x *TSelect) GetWhere() *TSelect_TWhere { - if x != nil { - return x.Where - } - return nil -} - -func (x *TSelect) GetLimit() *TSelect_TLimit { - if x != nil { - return x.Limit - } - return nil -} - -func (x *TSelect) GetPredefinedSchema() *TSchema { - if x != nil { - return x.PredefinedSchema - } - return nil -} - -// Split is a certain part of a table. In general, it should be much smaller than a partition. -// It also represents a unit of a parallel work for YQ engine. -type TSplit struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Every split contains the description of SELECT it was generated for. - Select *TSelect `protobuf:"bytes,1,opt,name=select,proto3" json:"select,omitempty"` - // Types that are assignable to Payload: - // - // *TSplit_Description - Payload isTSplit_Payload `protobuf_oneof:"payload"` -} - -func (x *TSplit) Reset() { - *x = TSplit{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TSplit) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TSplit) ProtoMessage() {} - -func (x *TSplit) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TSplit.ProtoReflect.Descriptor instead. -func (*TSplit) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{9} -} - -func (x *TSplit) GetSelect() *TSelect { - if x != nil { - return x.Select - } - return nil -} - -func (m *TSplit) GetPayload() isTSplit_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *TSplit) GetDescription() []byte { - if x, ok := x.GetPayload().(*TSplit_Description); ok { - return x.Description - } - return nil -} - -type isTSplit_Payload interface { - isTSplit_Payload() -} - -type TSplit_Description struct { - // Different data sources may use different ways to describe a split, - // and we don't want YQ to dig into its internals (at least now), - // so we make the description opaque for YQ. - Description []byte `protobuf:"bytes,2,opt,name=description,proto3,oneof"` -} - -func (*TSplit_Description) isTSplit_Payload() {} - -// ReadDataRequest reads the data associated with a particular table split. -type TReadSplitsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Data source instance to connect - DataSourceInstance *common.TDataSourceInstance `protobuf:"bytes,1,opt,name=data_source_instance,json=dataSourceInstance,proto3" json:"data_source_instance,omitempty"` - // Splits that YQ engine would like to read. - Splits []*TSplit `protobuf:"bytes,2,rep,name=splits,proto3" json:"splits,omitempty"` - // Determines the mode of data extraction - Mode TReadSplitsRequest_EMode `protobuf:"varint,3,opt,name=mode,proto3,enum=NYql.NConnector.NApi.TReadSplitsRequest_EMode" json:"mode,omitempty"` - // Determines the format of data representation - Format TReadSplitsRequest_EFormat `protobuf:"varint,4,opt,name=format,proto3,enum=NYql.NConnector.NApi.TReadSplitsRequest_EFormat" json:"format,omitempty"` - // Specifies the location of split from where to start reading. - // If stream has been recently interrupted, YQ may retry reading the split from the interrupted block - // instead of reading the split from scratch. - // If empty, the connector will return the split data from the very beginning. - Continuation *TContinuation `protobuf:"bytes,6,opt,name=continuation,proto3" json:"continuation,omitempty"` -} - -func (x *TReadSplitsRequest) Reset() { - *x = TReadSplitsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TReadSplitsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TReadSplitsRequest) ProtoMessage() {} - -func (x *TReadSplitsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TReadSplitsRequest.ProtoReflect.Descriptor instead. -func (*TReadSplitsRequest) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{10} -} - -func (x *TReadSplitsRequest) GetDataSourceInstance() *common.TDataSourceInstance { - if x != nil { - return x.DataSourceInstance - } - return nil -} - -func (x *TReadSplitsRequest) GetSplits() []*TSplit { - if x != nil { - return x.Splits - } - return nil -} - -func (x *TReadSplitsRequest) GetMode() TReadSplitsRequest_EMode { - if x != nil { - return x.Mode - } - return TReadSplitsRequest_MODE_UNSPECIFIED -} - -func (x *TReadSplitsRequest) GetFormat() TReadSplitsRequest_EFormat { - if x != nil { - return x.Format - } - return TReadSplitsRequest_FORMAT_UNSPECIFIED -} - -func (x *TReadSplitsRequest) GetContinuation() *TContinuation { - if x != nil { - return x.Continuation - } - return nil -} - -// ReadDataResponse returns the data corresponding to a particular split -type TReadSplitsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // There may be various formats to represent data - // - // Types that are assignable to Payload: - // - // *TReadSplitsResponse_ColumnSet - // *TReadSplitsResponse_ArrowIpcStreaming - Payload isTReadSplitsResponse_Payload `protobuf_oneof:"payload"` - // Since multiple splits can be read within one request, it's important to - // match the received data with the requested split. - SplitIndexNumber uint32 `protobuf:"varint,3,opt,name=split_index_number,json=splitIndexNumber,proto3" json:"split_index_number,omitempty"` - // Specifies the location where the next block starts. - // If stream has been interrupted, YQ may retry reading using the Continuation message - // received for the last time. - Continuation *TContinuation `protobuf:"bytes,4,opt,name=continuation,proto3" json:"continuation,omitempty"` - Stats *TReadSplitsResponse_TStats `protobuf:"bytes,5,opt,name=stats,proto3" json:"stats,omitempty"` - // Call result - Error *TError `protobuf:"bytes,100,opt,name=error,proto3" json:"error,omitempty"` -} - -func (x *TReadSplitsResponse) Reset() { - *x = TReadSplitsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TReadSplitsResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TReadSplitsResponse) ProtoMessage() {} - -func (x *TReadSplitsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TReadSplitsResponse.ProtoReflect.Descriptor instead. -func (*TReadSplitsResponse) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{11} -} - -func (m *TReadSplitsResponse) GetPayload() isTReadSplitsResponse_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *TReadSplitsResponse) GetColumnSet() *TReadSplitsResponse_TColumnSet { - if x, ok := x.GetPayload().(*TReadSplitsResponse_ColumnSet); ok { - return x.ColumnSet - } - return nil -} - -func (x *TReadSplitsResponse) GetArrowIpcStreaming() []byte { - if x, ok := x.GetPayload().(*TReadSplitsResponse_ArrowIpcStreaming); ok { - return x.ArrowIpcStreaming - } - return nil -} - -func (x *TReadSplitsResponse) GetSplitIndexNumber() uint32 { - if x != nil { - return x.SplitIndexNumber - } - return 0 -} - -func (x *TReadSplitsResponse) GetContinuation() *TContinuation { - if x != nil { - return x.Continuation - } - return nil -} - -func (x *TReadSplitsResponse) GetStats() *TReadSplitsResponse_TStats { - if x != nil { - return x.Stats - } - return nil -} - -func (x *TReadSplitsResponse) GetError() *TError { - if x != nil { - return x.Error - } - return nil -} - -type isTReadSplitsResponse_Payload interface { - isTReadSplitsResponse_Payload() -} - -type TReadSplitsResponse_ColumnSet struct { - // Columnar data in protobuf format with YDB types. - // Use it only for debugging, don't use in production. - ColumnSet *TReadSplitsResponse_TColumnSet `protobuf:"bytes,1,opt,name=column_set,json=columnSet,proto3,oneof"` -} - -type TReadSplitsResponse_ArrowIpcStreaming struct { - // Data in Arrow IPC streaming format. - ArrowIpcStreaming []byte `protobuf:"bytes,2,opt,name=arrow_ipc_streaming,json=arrowIpcStreaming,proto3,oneof"` -} - -func (*TReadSplitsResponse_ColumnSet) isTReadSplitsResponse_Payload() {} - -func (*TReadSplitsResponse_ArrowIpcStreaming) isTReadSplitsResponse_Payload() {} - -// Continuation is a special type useful for the request retry. -// In case if split reading was interrupted, -// the engine does not have to read all the split data from the very beginning, -// it can specify the location from where it wants to reread the data instead. -type TContinuation struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Payload: - // - // *TContinuation_Description - Payload isTContinuation_Payload `protobuf_oneof:"payload"` -} - -func (x *TContinuation) Reset() { - *x = TContinuation{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TContinuation) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TContinuation) ProtoMessage() {} - -func (x *TContinuation) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TContinuation.ProtoReflect.Descriptor instead. -func (*TContinuation) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{12} -} - -func (m *TContinuation) GetPayload() isTContinuation_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *TContinuation) GetDescription() []byte { - if x, ok := x.GetPayload().(*TContinuation_Description); ok { - return x.Description - } - return nil -} - -type isTContinuation_Payload interface { - isTContinuation_Payload() -} - -type TContinuation_Description struct { - // In general description should be opaque to YQ. - Description []byte `protobuf:"bytes,1,opt,name=description,proto3,oneof"` -} - -func (*TContinuation_Description) isTContinuation_Payload() {} - -// Expression with value -// Can be a column, a constant or a result of, for example, -// some arithmetical operation -type TExpression struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Payload: - // - // *TExpression_TypedValue - // *TExpression_Column - // *TExpression_ArithmeticalExpression - // *TExpression_Null - Payload isTExpression_Payload `protobuf_oneof:"payload"` -} - -func (x *TExpression) Reset() { - *x = TExpression{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TExpression) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TExpression) ProtoMessage() {} - -func (x *TExpression) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TExpression.ProtoReflect.Descriptor instead. -func (*TExpression) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{13} -} - -func (m *TExpression) GetPayload() isTExpression_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *TExpression) GetTypedValue() *Ydb.TypedValue { - if x, ok := x.GetPayload().(*TExpression_TypedValue); ok { - return x.TypedValue - } - return nil -} - -func (x *TExpression) GetColumn() string { - if x, ok := x.GetPayload().(*TExpression_Column); ok { - return x.Column - } - return "" -} - -func (x *TExpression) GetArithmeticalExpression() *TExpression_TArithmeticalExpression { - if x, ok := x.GetPayload().(*TExpression_ArithmeticalExpression); ok { - return x.ArithmeticalExpression - } - return nil -} - -func (x *TExpression) GetNull() *TExpression_TNull { - if x, ok := x.GetPayload().(*TExpression_Null); ok { - return x.Null - } - return nil -} - -type isTExpression_Payload interface { - isTExpression_Payload() -} - -type TExpression_TypedValue struct { - // A scalar value - TypedValue *Ydb.TypedValue `protobuf:"bytes,1,opt,name=typed_value,json=typedValue,proto3,oneof"` -} - -type TExpression_Column struct { - // A name of another column to compare with - Column string `protobuf:"bytes,2,opt,name=column,proto3,oneof"` -} - -type TExpression_ArithmeticalExpression struct { - ArithmeticalExpression *TExpression_TArithmeticalExpression `protobuf:"bytes,3,opt,name=arithmetical_expression,json=arithmeticalExpression,proto3,oneof"` -} - -type TExpression_Null struct { - Null *TExpression_TNull `protobuf:"bytes,4,opt,name=null,proto3,oneof"` -} - -func (*TExpression_TypedValue) isTExpression_Payload() {} - -func (*TExpression_Column) isTExpression_Payload() {} - -func (*TExpression_ArithmeticalExpression) isTExpression_Payload() {} - -func (*TExpression_Null) isTExpression_Payload() {} - -// Predicate -type TPredicate struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Payload: - // - // *TPredicate_Negation - // *TPredicate_Conjunction - // *TPredicate_Disjunction - // *TPredicate_Between - // *TPredicate_In - // *TPredicate_IsNull - // *TPredicate_IsNotNull - // *TPredicate_Comparison - // *TPredicate_BoolExpression - Payload isTPredicate_Payload `protobuf_oneof:"payload"` -} - -func (x *TPredicate) Reset() { - *x = TPredicate{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate) ProtoMessage() {} - -func (x *TPredicate) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate.ProtoReflect.Descriptor instead. -func (*TPredicate) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14} -} - -func (m *TPredicate) GetPayload() isTPredicate_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *TPredicate) GetNegation() *TPredicate_TNegation { - if x, ok := x.GetPayload().(*TPredicate_Negation); ok { - return x.Negation - } - return nil -} - -func (x *TPredicate) GetConjunction() *TPredicate_TConjunction { - if x, ok := x.GetPayload().(*TPredicate_Conjunction); ok { - return x.Conjunction - } - return nil -} - -func (x *TPredicate) GetDisjunction() *TPredicate_TDisjunction { - if x, ok := x.GetPayload().(*TPredicate_Disjunction); ok { - return x.Disjunction - } - return nil -} - -func (x *TPredicate) GetBetween() *TPredicate_TBetween { - if x, ok := x.GetPayload().(*TPredicate_Between); ok { - return x.Between - } - return nil -} - -func (x *TPredicate) GetIn() *TPredicate_TIn { - if x, ok := x.GetPayload().(*TPredicate_In); ok { - return x.In - } - return nil -} - -func (x *TPredicate) GetIsNull() *TPredicate_TIsNull { - if x, ok := x.GetPayload().(*TPredicate_IsNull); ok { - return x.IsNull - } - return nil -} - -func (x *TPredicate) GetIsNotNull() *TPredicate_TIsNotNull { - if x, ok := x.GetPayload().(*TPredicate_IsNotNull); ok { - return x.IsNotNull - } - return nil -} - -func (x *TPredicate) GetComparison() *TPredicate_TComparison { - if x, ok := x.GetPayload().(*TPredicate_Comparison); ok { - return x.Comparison - } - return nil -} - -func (x *TPredicate) GetBoolExpression() *TPredicate_TBoolExpression { - if x, ok := x.GetPayload().(*TPredicate_BoolExpression); ok { - return x.BoolExpression - } - return nil -} - -type isTPredicate_Payload interface { - isTPredicate_Payload() -} - -type TPredicate_Negation struct { - Negation *TPredicate_TNegation `protobuf:"bytes,1,opt,name=negation,proto3,oneof"` -} - -type TPredicate_Conjunction struct { - Conjunction *TPredicate_TConjunction `protobuf:"bytes,2,opt,name=conjunction,proto3,oneof"` -} - -type TPredicate_Disjunction struct { - Disjunction *TPredicate_TDisjunction `protobuf:"bytes,3,opt,name=disjunction,proto3,oneof"` -} - -type TPredicate_Between struct { - Between *TPredicate_TBetween `protobuf:"bytes,4,opt,name=between,proto3,oneof"` -} - -type TPredicate_In struct { - In *TPredicate_TIn `protobuf:"bytes,5,opt,name=in,proto3,oneof"` -} - -type TPredicate_IsNull struct { - IsNull *TPredicate_TIsNull `protobuf:"bytes,6,opt,name=is_null,json=isNull,proto3,oneof"` -} - -type TPredicate_IsNotNull struct { - IsNotNull *TPredicate_TIsNotNull `protobuf:"bytes,7,opt,name=is_not_null,json=isNotNull,proto3,oneof"` -} - -type TPredicate_Comparison struct { - Comparison *TPredicate_TComparison `protobuf:"bytes,8,opt,name=comparison,proto3,oneof"` -} - -type TPredicate_BoolExpression struct { - BoolExpression *TPredicate_TBoolExpression `protobuf:"bytes,9,opt,name=bool_expression,json=boolExpression,proto3,oneof"` -} - -func (*TPredicate_Negation) isTPredicate_Payload() {} - -func (*TPredicate_Conjunction) isTPredicate_Payload() {} - -func (*TPredicate_Disjunction) isTPredicate_Payload() {} - -func (*TPredicate_Between) isTPredicate_Payload() {} - -func (*TPredicate_In) isTPredicate_Payload() {} - -func (*TPredicate_IsNull) isTPredicate_Payload() {} - -func (*TPredicate_IsNotNull) isTPredicate_Payload() {} - -func (*TPredicate_Comparison) isTPredicate_Payload() {} - -func (*TPredicate_BoolExpression) isTPredicate_Payload() {} - -// Special type to describe the result of any operation -type TError struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // High-level code - Status Ydb.StatusIds_StatusCode `protobuf:"varint,1,opt,name=status,proto3,enum=Ydb.StatusIds_StatusCode" json:"status,omitempty"` - // Error message - Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` - // Detailed explanation of a problem; - // must be empty if status == SUCCESS - Issues []*Ydb_Issue.IssueMessage `protobuf:"bytes,3,rep,name=issues,proto3" json:"issues,omitempty"` -} - -func (x *TError) Reset() { - *x = TError{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TError) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TError) ProtoMessage() {} - -func (x *TError) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TError.ProtoReflect.Descriptor instead. -func (*TError) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{15} -} - -func (x *TError) GetStatus() Ydb.StatusIds_StatusCode { - if x != nil { - return x.Status - } - return Ydb.StatusIds_StatusCode(0) -} - -func (x *TError) GetMessage() string { - if x != nil { - return x.Message - } - return "" -} - -func (x *TError) GetIssues() []*Ydb_Issue.IssueMessage { - if x != nil { - return x.Issues - } - return nil -} - -// TAst is an internal representation of an YQL request. -// Advanced connectors may use it for the full-fledged implementations of the push down. -type TAst struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Payload: - // - // *TAst_Atom - // *TAst_List - Payload isTAst_Payload `protobuf_oneof:"payload"` -} - -func (x *TAst) Reset() { - *x = TAst{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TAst) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TAst) ProtoMessage() {} - -func (x *TAst) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[16] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TAst.ProtoReflect.Descriptor instead. -func (*TAst) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{16} -} - -func (m *TAst) GetPayload() isTAst_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *TAst) GetAtom() []byte { - if x, ok := x.GetPayload().(*TAst_Atom); ok { - return x.Atom - } - return nil -} - -func (x *TAst) GetList() *TAst_TList { - if x, ok := x.GetPayload().(*TAst_List); ok { - return x.List - } - return nil -} - -type isTAst_Payload interface { - isTAst_Payload() -} - -type TAst_Atom struct { - Atom []byte `protobuf:"bytes,1,opt,name=atom,proto3,oneof"` -} - -type TAst_List struct { - List *TAst_TList `protobuf:"bytes,2,opt,name=list,proto3,oneof"` -} - -func (*TAst_Atom) isTAst_Payload() {} - -func (*TAst_List) isTAst_Payload() {} - -// Describes what particularly to get from the data source -type TSelect_TWhat struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // NOTE: this API intentionally makes it not possible to request 'SELECT *'. - // YQ must provide all the column names explicitly. - // - // Еmpty list means that YQ wants to get empty tuples in the response. - // On the connector's side this request will be transformed into something like - // SELECT 1 FROM $table (...) - Items []*TSelect_TWhat_TItem `protobuf:"bytes,1,rep,name=items,proto3" json:"items,omitempty"` -} - -func (x *TSelect_TWhat) Reset() { - *x = TSelect_TWhat{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[17] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TSelect_TWhat) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TSelect_TWhat) ProtoMessage() {} - -func (x *TSelect_TWhat) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[17] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TSelect_TWhat.ProtoReflect.Descriptor instead. -func (*TSelect_TWhat) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{8, 0} -} - -func (x *TSelect_TWhat) GetItems() []*TSelect_TWhat_TItem { - if x != nil { - return x.Items - } - return nil -} - -type TSelect_TFrom struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Table name for RDBMs - Table string `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"` - // Unique identifier of an object stored within S3 - ObjectKey string `protobuf:"bytes,2,opt,name=object_key,json=objectKey,proto3" json:"object_key,omitempty"` -} - -func (x *TSelect_TFrom) Reset() { - *x = TSelect_TFrom{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TSelect_TFrom) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TSelect_TFrom) ProtoMessage() {} - -func (x *TSelect_TFrom) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[18] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TSelect_TFrom.ProtoReflect.Descriptor instead. -func (*TSelect_TFrom) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{8, 1} -} - -func (x *TSelect_TFrom) GetTable() string { - if x != nil { - return x.Table - } - return "" -} - -func (x *TSelect_TFrom) GetObjectKey() string { - if x != nil { - return x.ObjectKey - } - return "" -} - -type TSelect_TWhere struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Strongly typed tree of predicates - FilterTyped *TPredicate `protobuf:"bytes,1,opt,name=filter_typed,json=filterTyped,proto3" json:"filter_typed,omitempty"` - // An internal representation of YQL request part describing filters. - // Advanced connectors may use it for the full-fledged implementations of the push down. - FilterRaw *TAst `protobuf:"bytes,2,opt,name=filter_raw,json=filterRaw,proto3" json:"filter_raw,omitempty"` -} - -func (x *TSelect_TWhere) Reset() { - *x = TSelect_TWhere{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[19] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TSelect_TWhere) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TSelect_TWhere) ProtoMessage() {} - -func (x *TSelect_TWhere) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[19] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TSelect_TWhere.ProtoReflect.Descriptor instead. -func (*TSelect_TWhere) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{8, 2} -} - -func (x *TSelect_TWhere) GetFilterTyped() *TPredicate { - if x != nil { - return x.FilterTyped - } - return nil -} - -func (x *TSelect_TWhere) GetFilterRaw() *TAst { - if x != nil { - return x.FilterRaw - } - return nil -} - -type TSelect_TLimit struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Limit uint64 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` - Offset uint64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` -} - -func (x *TSelect_TLimit) Reset() { - *x = TSelect_TLimit{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TSelect_TLimit) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TSelect_TLimit) ProtoMessage() {} - -func (x *TSelect_TLimit) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TSelect_TLimit.ProtoReflect.Descriptor instead. -func (*TSelect_TLimit) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{8, 3} -} - -func (x *TSelect_TLimit) GetLimit() uint64 { - if x != nil { - return x.Limit - } - return 0 -} - -func (x *TSelect_TLimit) GetOffset() uint64 { - if x != nil { - return x.Offset - } - return 0 -} - -type TSelect_TWhat_TItem struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // YQ can read particular table columns or call aggregate functions, for example. - // - // Types that are assignable to Payload: - // - // *TSelect_TWhat_TItem_Column - Payload isTSelect_TWhat_TItem_Payload `protobuf_oneof:"payload"` -} - -func (x *TSelect_TWhat_TItem) Reset() { - *x = TSelect_TWhat_TItem{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[21] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TSelect_TWhat_TItem) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TSelect_TWhat_TItem) ProtoMessage() {} - -func (x *TSelect_TWhat_TItem) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[21] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TSelect_TWhat_TItem.ProtoReflect.Descriptor instead. -func (*TSelect_TWhat_TItem) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{8, 0, 0} -} - -func (m *TSelect_TWhat_TItem) GetPayload() isTSelect_TWhat_TItem_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *TSelect_TWhat_TItem) GetColumn() *Ydb.Column { - if x, ok := x.GetPayload().(*TSelect_TWhat_TItem_Column); ok { - return x.Column - } - return nil -} - -type isTSelect_TWhat_TItem_Payload interface { - isTSelect_TWhat_TItem_Payload() -} - -type TSelect_TWhat_TItem_Column struct { - // a column to read - Column *Ydb.Column `protobuf:"bytes,1,opt,name=column,proto3,oneof"` -} - -func (*TSelect_TWhat_TItem_Column) isTSelect_TWhat_TItem_Payload() {} - -// Protobuf columnar representation of data. -// Use it only for debugging, don't use in production. -type TReadSplitsResponse_TColumnSet struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Meta []*Ydb.Column `protobuf:"bytes,1,rep,name=meta,proto3" json:"meta,omitempty"` - Data []*TReadSplitsResponse_TColumnSet_TColumn `protobuf:"bytes,2,rep,name=data,proto3" json:"data,omitempty"` -} - -func (x *TReadSplitsResponse_TColumnSet) Reset() { - *x = TReadSplitsResponse_TColumnSet{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[22] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TReadSplitsResponse_TColumnSet) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TReadSplitsResponse_TColumnSet) ProtoMessage() {} - -func (x *TReadSplitsResponse_TColumnSet) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[22] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TReadSplitsResponse_TColumnSet.ProtoReflect.Descriptor instead. -func (*TReadSplitsResponse_TColumnSet) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{11, 0} -} - -func (x *TReadSplitsResponse_TColumnSet) GetMeta() []*Ydb.Column { - if x != nil { - return x.Meta - } - return nil -} - -func (x *TReadSplitsResponse_TColumnSet) GetData() []*TReadSplitsResponse_TColumnSet_TColumn { - if x != nil { - return x.Data - } - return nil -} - -// Contains information about the page (a particular block of data -// returned by the Connector within a ReadSplits stream). -type TReadSplitsResponse_TStats struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Number of rows read from the data source in order to make this page. - Rows uint64 `protobuf:"varint,1,opt,name=rows,proto3" json:"rows,omitempty"` - // Number of bytes read from the data source in order to make this page. - // (measured in terms of Go type system). - Bytes uint64 `protobuf:"varint,2,opt,name=bytes,proto3" json:"bytes,omitempty"` -} - -func (x *TReadSplitsResponse_TStats) Reset() { - *x = TReadSplitsResponse_TStats{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[23] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TReadSplitsResponse_TStats) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TReadSplitsResponse_TStats) ProtoMessage() {} - -func (x *TReadSplitsResponse_TStats) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[23] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TReadSplitsResponse_TStats.ProtoReflect.Descriptor instead. -func (*TReadSplitsResponse_TStats) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{11, 1} -} - -func (x *TReadSplitsResponse_TStats) GetRows() uint64 { - if x != nil { - return x.Rows - } - return 0 -} - -func (x *TReadSplitsResponse_TStats) GetBytes() uint64 { - if x != nil { - return x.Bytes - } - return 0 -} - -type TReadSplitsResponse_TColumnSet_TColumn struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Data []*Ydb.Value `protobuf:"bytes,1,rep,name=data,proto3" json:"data,omitempty"` -} - -func (x *TReadSplitsResponse_TColumnSet_TColumn) Reset() { - *x = TReadSplitsResponse_TColumnSet_TColumn{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[24] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TReadSplitsResponse_TColumnSet_TColumn) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TReadSplitsResponse_TColumnSet_TColumn) ProtoMessage() {} - -func (x *TReadSplitsResponse_TColumnSet_TColumn) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[24] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TReadSplitsResponse_TColumnSet_TColumn.ProtoReflect.Descriptor instead. -func (*TReadSplitsResponse_TColumnSet_TColumn) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{11, 0, 0} -} - -func (x *TReadSplitsResponse_TColumnSet_TColumn) GetData() []*Ydb.Value { - if x != nil { - return x.Data - } - return nil -} - -type TExpression_TArithmeticalExpression struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Operation TExpression_TArithmeticalExpression_EOperation `protobuf:"varint,1,opt,name=operation,proto3,enum=NYql.NConnector.NApi.TExpression_TArithmeticalExpression_EOperation" json:"operation,omitempty"` - LeftValue *TExpression `protobuf:"bytes,2,opt,name=left_value,json=leftValue,proto3" json:"left_value,omitempty"` - RightValue *TExpression `protobuf:"bytes,3,opt,name=right_value,json=rightValue,proto3" json:"right_value,omitempty"` -} - -func (x *TExpression_TArithmeticalExpression) Reset() { - *x = TExpression_TArithmeticalExpression{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[25] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TExpression_TArithmeticalExpression) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TExpression_TArithmeticalExpression) ProtoMessage() {} - -func (x *TExpression_TArithmeticalExpression) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[25] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TExpression_TArithmeticalExpression.ProtoReflect.Descriptor instead. -func (*TExpression_TArithmeticalExpression) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{13, 0} -} - -func (x *TExpression_TArithmeticalExpression) GetOperation() TExpression_TArithmeticalExpression_EOperation { - if x != nil { - return x.Operation - } - return TExpression_TArithmeticalExpression_EXPRESSION_OPERATION_UNSPECIFIED -} - -func (x *TExpression_TArithmeticalExpression) GetLeftValue() *TExpression { - if x != nil { - return x.LeftValue - } - return nil -} - -func (x *TExpression_TArithmeticalExpression) GetRightValue() *TExpression { - if x != nil { - return x.RightValue - } - return nil -} - -type TExpression_TNull struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *TExpression_TNull) Reset() { - *x = TExpression_TNull{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[26] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TExpression_TNull) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TExpression_TNull) ProtoMessage() {} - -func (x *TExpression_TNull) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[26] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TExpression_TNull.ProtoReflect.Descriptor instead. -func (*TExpression_TNull) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{13, 1} -} - -// NOT -type TPredicate_TNegation struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Operand *TPredicate `protobuf:"bytes,1,opt,name=operand,proto3" json:"operand,omitempty"` -} - -func (x *TPredicate_TNegation) Reset() { - *x = TPredicate_TNegation{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[27] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TNegation) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TNegation) ProtoMessage() {} - -func (x *TPredicate_TNegation) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[27] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TNegation.ProtoReflect.Descriptor instead. -func (*TPredicate_TNegation) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 0} -} - -func (x *TPredicate_TNegation) GetOperand() *TPredicate { - if x != nil { - return x.Operand - } - return nil -} - -// AND -type TPredicate_TConjunction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Operands []*TPredicate `protobuf:"bytes,1,rep,name=operands,proto3" json:"operands,omitempty"` -} - -func (x *TPredicate_TConjunction) Reset() { - *x = TPredicate_TConjunction{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[28] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TConjunction) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TConjunction) ProtoMessage() {} - -func (x *TPredicate_TConjunction) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[28] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TConjunction.ProtoReflect.Descriptor instead. -func (*TPredicate_TConjunction) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 1} -} - -func (x *TPredicate_TConjunction) GetOperands() []*TPredicate { - if x != nil { - return x.Operands - } - return nil -} - -// OR -type TPredicate_TDisjunction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Operands []*TPredicate `protobuf:"bytes,1,rep,name=operands,proto3" json:"operands,omitempty"` -} - -func (x *TPredicate_TDisjunction) Reset() { - *x = TPredicate_TDisjunction{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[29] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TDisjunction) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TDisjunction) ProtoMessage() {} - -func (x *TPredicate_TDisjunction) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[29] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TDisjunction.ProtoReflect.Descriptor instead. -func (*TPredicate_TDisjunction) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 2} -} - -func (x *TPredicate_TDisjunction) GetOperands() []*TPredicate { - if x != nil { - return x.Operands - } - return nil -} - -// "$column BETWEEN $least AND $greatest" -type TPredicate_TBetween struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value *TExpression `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` - Least *TExpression `protobuf:"bytes,2,opt,name=least,proto3" json:"least,omitempty"` - Greatest *TExpression `protobuf:"bytes,3,opt,name=greatest,proto3" json:"greatest,omitempty"` -} - -func (x *TPredicate_TBetween) Reset() { - *x = TPredicate_TBetween{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[30] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TBetween) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TBetween) ProtoMessage() {} - -func (x *TPredicate_TBetween) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[30] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TBetween.ProtoReflect.Descriptor instead. -func (*TPredicate_TBetween) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 3} -} - -func (x *TPredicate_TBetween) GetValue() *TExpression { - if x != nil { - return x.Value - } - return nil -} - -func (x *TPredicate_TBetween) GetLeast() *TExpression { - if x != nil { - return x.Least - } - return nil -} - -func (x *TPredicate_TBetween) GetGreatest() *TExpression { - if x != nil { - return x.Greatest - } - return nil -} - -// "$column IN $(set)" -type TPredicate_TIn struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value *TExpression `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` - Set []*TExpression `protobuf:"bytes,2,rep,name=set,proto3" json:"set,omitempty"` -} - -func (x *TPredicate_TIn) Reset() { - *x = TPredicate_TIn{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[31] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TIn) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TIn) ProtoMessage() {} - -func (x *TPredicate_TIn) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[31] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TIn.ProtoReflect.Descriptor instead. -func (*TPredicate_TIn) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 4} -} - -func (x *TPredicate_TIn) GetValue() *TExpression { - if x != nil { - return x.Value - } - return nil -} - -func (x *TPredicate_TIn) GetSet() []*TExpression { - if x != nil { - return x.Set - } - return nil -} - -// "$column IS NULL" -type TPredicate_TIsNull struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value *TExpression `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` -} - -func (x *TPredicate_TIsNull) Reset() { - *x = TPredicate_TIsNull{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[32] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TIsNull) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TIsNull) ProtoMessage() {} - -func (x *TPredicate_TIsNull) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[32] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TIsNull.ProtoReflect.Descriptor instead. -func (*TPredicate_TIsNull) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 5} -} - -func (x *TPredicate_TIsNull) GetValue() *TExpression { - if x != nil { - return x.Value - } - return nil -} - -// "$column IS NOT NULL" -// TODO: maybe it is better to express with TNegation here -type TPredicate_TIsNotNull struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value *TExpression `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` -} - -func (x *TPredicate_TIsNotNull) Reset() { - *x = TPredicate_TIsNotNull{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[33] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TIsNotNull) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TIsNotNull) ProtoMessage() {} - -func (x *TPredicate_TIsNotNull) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[33] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TIsNotNull.ProtoReflect.Descriptor instead. -func (*TPredicate_TIsNotNull) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 6} -} - -func (x *TPredicate_TIsNotNull) GetValue() *TExpression { - if x != nil { - return x.Value - } - return nil -} - -// Expression wich has bool type -// For example, bool column -type TPredicate_TBoolExpression struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value *TExpression `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` -} - -func (x *TPredicate_TBoolExpression) Reset() { - *x = TPredicate_TBoolExpression{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[34] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TBoolExpression) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TBoolExpression) ProtoMessage() {} - -func (x *TPredicate_TBoolExpression) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[34] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TBoolExpression.ProtoReflect.Descriptor instead. -func (*TPredicate_TBoolExpression) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 7} -} - -func (x *TPredicate_TBoolExpression) GetValue() *TExpression { - if x != nil { - return x.Value - } - return nil -} - -// A subset of comparators corresponding to the binary logical operators -type TPredicate_TComparison struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Operation TPredicate_TComparison_EOperation `protobuf:"varint,1,opt,name=operation,proto3,enum=NYql.NConnector.NApi.TPredicate_TComparison_EOperation" json:"operation,omitempty"` - LeftValue *TExpression `protobuf:"bytes,2,opt,name=left_value,json=leftValue,proto3" json:"left_value,omitempty"` - RightValue *TExpression `protobuf:"bytes,3,opt,name=right_value,json=rightValue,proto3" json:"right_value,omitempty"` -} - -func (x *TPredicate_TComparison) Reset() { - *x = TPredicate_TComparison{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[35] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TPredicate_TComparison) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TPredicate_TComparison) ProtoMessage() {} - -func (x *TPredicate_TComparison) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[35] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TPredicate_TComparison.ProtoReflect.Descriptor instead. -func (*TPredicate_TComparison) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{14, 8} -} - -func (x *TPredicate_TComparison) GetOperation() TPredicate_TComparison_EOperation { - if x != nil { - return x.Operation - } - return TPredicate_TComparison_COMPARISON_OPERATION_UNSPECIFIED -} - -func (x *TPredicate_TComparison) GetLeftValue() *TExpression { - if x != nil { - return x.LeftValue - } - return nil -} - -func (x *TPredicate_TComparison) GetRightValue() *TExpression { - if x != nil { - return x.RightValue - } - return nil -} - -type TAst_TList struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Children []*TAst `protobuf:"bytes,1,rep,name=children,proto3" json:"children,omitempty"` -} - -func (x *TAst_TList) Reset() { - *x = TAst_TList{} - if protoimpl.UnsafeEnabled { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[36] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TAst_TList) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TAst_TList) ProtoMessage() {} - -func (x *TAst_TList) ProtoReflect() protoreflect.Message { - mi := &file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[36] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TAst_TList.ProtoReflect.Descriptor instead. -func (*TAst_TList) Descriptor() ([]byte, []int) { - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP(), []int{16, 0} -} - -func (x *TAst_TList) GetChildren() []*TAst { - if x != nil { - return x.Children - } - return nil -} - -var File_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto protoreflect.FileDescriptor - -var file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDesc = []byte{ - 0x0a, 0x4e, 0x79, 0x64, 0x62, 0x2f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x2f, 0x79, 0x71, - 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x69, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, - 0x69, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, - 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x12, 0x14, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x1a, 0x25, 0x79, 0x64, 0x62, 0x2f, 0x70, 0x75, 0x62, 0x6c, - 0x69, 0x63, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x79, 0x64, - 0x62, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2c, 0x79, - 0x64, 0x62, 0x2f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x79, 0x64, 0x62, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, - 0x63, 0x6f, 0x64, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2d, 0x79, 0x64, 0x62, - 0x2f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x73, 0x2f, 0x79, 0x64, 0x62, 0x5f, 0x69, 0x73, 0x73, 0x75, 0x65, 0x5f, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x48, 0x79, 0x64, 0x62, 0x2f, - 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x2f, 0x79, 0x71, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x2f, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x9a, 0x01, 0x0a, 0x12, 0x54, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x5b, 0x0a, 0x14, 0x64, - 0x61, 0x74, 0x61, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x4e, 0x59, 0x71, 0x6c, - 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, - 0x2e, 0x54, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x52, 0x12, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x74, - 0x65, 0x72, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x70, 0x61, 0x74, - 0x74, 0x65, 0x72, 0x6e, 0x42, 0x0b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x69, 0x6e, - 0x67, 0x22, 0x61, 0x0a, 0x13, 0x54, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, - 0x12, 0x32, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x64, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x22, 0xea, 0x01, 0x0a, 0x15, 0x54, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, - 0x62, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x5b, - 0x0a, 0x14, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x4e, - 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, - 0x41, 0x70, 0x69, 0x2e, 0x54, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x12, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x12, 0x5e, 0x0a, 0x15, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2a, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x54, 0x79, 0x70, 0x65, 0x4d, 0x61, 0x70, - 0x70, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x13, 0x74, 0x79, - 0x70, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, - 0x73, 0x22, 0x67, 0x0a, 0x14, 0x54, 0x54, 0x79, 0x70, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x4f, 0x0a, 0x10, 0x64, 0x61, 0x74, - 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x45, 0x44, 0x61, 0x74, 0x65, - 0x54, 0x69, 0x6d, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0e, 0x64, 0x61, 0x74, 0x65, - 0x54, 0x69, 0x6d, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0x83, 0x01, 0x0a, 0x16, 0x54, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x53, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x32, 0x0a, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x64, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x4e, 0x59, - 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, - 0x70, 0x69, 0x2e, 0x54, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x22, 0x30, 0x0a, 0x07, 0x54, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x25, 0x0a, 0x07, 0x63, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x59, - 0x64, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x73, 0x22, 0xc8, 0x01, 0x0a, 0x12, 0x54, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x70, 0x6c, 0x69, - 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x07, 0x73, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x4e, 0x59, 0x71, - 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, - 0x69, 0x2e, 0x54, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x07, 0x73, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x6d, 0x61, 0x78, - 0x53, 0x70, 0x6c, 0x69, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, - 0x6c, 0x69, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, - 0x73, 0x70, 0x6c, 0x69, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x70, 0x6c, - 0x69, 0x74, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x4e, 0x75, 0x6d, 0x62, - 0x65, 0x72, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x7f, 0x0a, - 0x13, 0x54, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x53, 0x70, 0x6c, - 0x69, 0x74, 0x52, 0x06, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x64, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x4e, 0x59, 0x71, 0x6c, - 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, - 0x2e, 0x54, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa3, - 0x06, 0x0a, 0x07, 0x54, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x12, 0x5b, 0x0a, 0x14, 0x64, 0x61, - 0x74, 0x61, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, - 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, - 0x54, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x52, 0x12, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x37, 0x0a, 0x04, 0x77, 0x68, 0x61, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x53, 0x65, - 0x6c, 0x65, 0x63, 0x74, 0x2e, 0x54, 0x57, 0x68, 0x61, 0x74, 0x52, 0x04, 0x77, 0x68, 0x61, 0x74, - 0x12, 0x37, 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, - 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x2e, 0x54, 0x46, - 0x72, 0x6f, 0x6d, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12, 0x3a, 0x0a, 0x05, 0x77, 0x68, 0x65, - 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, - 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, - 0x54, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x2e, 0x54, 0x57, 0x68, 0x65, 0x72, 0x65, 0x52, 0x05, - 0x77, 0x68, 0x65, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x53, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x2e, 0x54, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, - 0x74, 0x12, 0x4a, 0x0a, 0x11, 0x70, 0x72, 0x65, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x5f, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x4e, - 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, - 0x41, 0x70, 0x69, 0x2e, 0x54, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x10, 0x70, 0x72, 0x65, - 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x1a, 0x83, 0x01, - 0x0a, 0x05, 0x54, 0x57, 0x68, 0x61, 0x74, 0x12, 0x3f, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x53, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x2e, 0x54, 0x57, 0x68, 0x61, 0x74, 0x2e, 0x54, 0x49, 0x74, 0x65, - 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x1a, 0x39, 0x0a, 0x05, 0x54, 0x49, 0x74, 0x65, - 0x6d, 0x12, 0x25, 0x0a, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0b, 0x2e, 0x59, 0x64, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x48, 0x00, - 0x52, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x1a, 0x3c, 0x0a, 0x05, 0x54, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x14, 0x0a, 0x05, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6b, 0x65, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4b, 0x65, - 0x79, 0x1a, 0x88, 0x01, 0x0a, 0x06, 0x54, 0x57, 0x68, 0x65, 0x72, 0x65, 0x12, 0x43, 0x0a, 0x0c, - 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, - 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x5f, 0x72, 0x61, 0x77, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x41, 0x73, - 0x74, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x61, 0x77, 0x1a, 0x36, 0x0a, 0x06, - 0x54, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x22, 0x6e, 0x0a, 0x06, 0x54, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, 0x35, - 0x0a, 0x06, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, - 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x06, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x12, 0x22, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x81, 0x04, 0x0a, 0x12, 0x54, 0x52, 0x65, 0x61, 0x64, 0x53, 0x70, - 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x5b, 0x0a, 0x14, 0x64, - 0x61, 0x74, 0x61, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x4e, 0x59, 0x71, 0x6c, - 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, - 0x2e, 0x54, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x52, 0x12, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x73, 0x70, 0x6c, 0x69, - 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, - 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, - 0x54, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x06, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x12, 0x42, - 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x4e, - 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, - 0x41, 0x70, 0x69, 0x2e, 0x54, 0x52, 0x65, 0x61, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6d, 0x6f, - 0x64, 0x65, 0x12, 0x48, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x52, 0x65, 0x61, 0x64, 0x53, - 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x46, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x47, 0x0a, 0x0c, - 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x43, 0x6f, 0x6e, 0x74, 0x69, - 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x39, 0x0a, 0x05, 0x45, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x14, - 0x0a, 0x10, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x52, 0x44, 0x45, 0x52, 0x45, 0x44, 0x10, - 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x4f, 0x52, 0x44, 0x45, 0x52, 0x45, 0x44, 0x10, 0x02, - 0x22, 0x40, 0x0a, 0x07, 0x45, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x16, 0x0a, 0x12, 0x46, - 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x41, 0x52, 0x52, 0x4f, 0x57, 0x5f, 0x49, 0x50, 0x43, - 0x5f, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x22, 0x04, 0x08, 0x01, - 0x10, 0x01, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0xfd, 0x04, 0x0a, 0x13, 0x54, 0x52, 0x65, - 0x61, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x55, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x52, 0x65, 0x61, - 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x54, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x53, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x63, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x53, 0x65, 0x74, 0x12, 0x30, 0x0a, 0x13, 0x61, 0x72, 0x72, 0x6f, 0x77, - 0x5f, 0x69, 0x70, 0x63, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x11, 0x61, 0x72, 0x72, 0x6f, 0x77, 0x49, 0x70, 0x63, - 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x70, 0x6c, - 0x69, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x47, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x69, - 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, - 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x46, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x30, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x52, 0x65, 0x61, 0x64, 0x53, 0x70, 0x6c, 0x69, - 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x53, 0x74, 0x61, 0x74, - 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x18, 0x64, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x1a, 0xaa, 0x01, 0x0a, - 0x0a, 0x54, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x53, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x04, 0x6d, - 0x65, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x59, 0x64, 0x62, 0x2e, - 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x12, 0x50, 0x0a, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x4e, 0x59, 0x71, - 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, - 0x69, 0x2e, 0x54, 0x52, 0x65, 0x61, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x53, 0x65, 0x74, - 0x2e, 0x54, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x29, - 0x0a, 0x07, 0x54, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x1e, 0x0a, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x59, 0x64, 0x62, 0x2e, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x32, 0x0a, 0x06, 0x54, 0x53, 0x74, - 0x61, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x42, 0x09, 0x0a, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x3e, 0x0a, 0x0d, 0x54, 0x43, 0x6f, 0x6e, - 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0b, 0x64, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, - 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x09, 0x0a, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x9f, 0x05, 0x0a, 0x0b, 0x54, 0x45, 0x78, - 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x0b, 0x74, 0x79, 0x70, 0x65, - 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, - 0x59, 0x64, 0x62, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x00, - 0x52, 0x0a, 0x74, 0x79, 0x70, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x18, 0x0a, 0x06, - 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, - 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x74, 0x0a, 0x17, 0x61, 0x72, 0x69, 0x74, 0x68, 0x6d, - 0x65, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, - 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x54, 0x41, 0x72, 0x69, 0x74, - 0x68, 0x6d, 0x65, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x16, 0x61, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x65, 0x74, 0x69, 0x63, - 0x61, 0x6c, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3d, 0x0a, 0x04, - 0x6e, 0x75, 0x6c, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x4e, 0x59, 0x71, - 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, - 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x54, 0x4e, - 0x75, 0x6c, 0x6c, 0x48, 0x00, 0x52, 0x04, 0x6e, 0x75, 0x6c, 0x6c, 0x1a, 0xf8, 0x02, 0x0a, 0x17, - 0x54, 0x41, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x65, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x45, 0x78, 0x70, - 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x62, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x44, 0x2e, 0x4e, 0x59, 0x71, - 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, - 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x54, 0x41, - 0x72, 0x69, 0x74, 0x68, 0x6d, 0x65, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x45, 0x78, 0x70, 0x72, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x40, 0x0a, 0x0a, 0x6c, - 0x65, 0x66, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x65, 0x66, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x42, 0x0a, - 0x0b, 0x72, 0x69, 0x67, 0x68, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x72, 0x69, 0x67, 0x68, 0x74, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x22, 0x73, 0x0a, 0x0a, 0x45, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x24, 0x0a, 0x20, 0x45, 0x58, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4f, 0x50, - 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, - 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4d, 0x55, 0x4c, 0x10, 0x01, 0x12, 0x07, - 0x0a, 0x03, 0x41, 0x44, 0x44, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x55, 0x42, 0x10, 0x03, - 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x49, 0x54, 0x5f, 0x41, 0x4e, 0x44, 0x10, 0x04, 0x12, 0x0a, 0x0a, - 0x06, 0x42, 0x49, 0x54, 0x5f, 0x4f, 0x52, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x49, 0x54, - 0x5f, 0x58, 0x4f, 0x52, 0x10, 0x06, 0x1a, 0x07, 0x0a, 0x05, 0x54, 0x4e, 0x75, 0x6c, 0x6c, 0x42, - 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x85, 0x0e, 0x0a, 0x0a, 0x54, - 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x48, 0x0a, 0x08, 0x6e, 0x65, 0x67, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x4e, 0x59, - 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, - 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4e, - 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x08, 0x6e, 0x65, 0x67, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x51, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x6a, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, - 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, - 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x43, 0x6f, 0x6e, 0x6a, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x6a, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x51, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x6a, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x4e, 0x59, - 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, - 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x44, - 0x69, 0x73, 0x6a, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x69, - 0x73, 0x6a, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x45, 0x0a, 0x07, 0x62, 0x65, 0x74, - 0x77, 0x65, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x4e, 0x59, 0x71, - 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, - 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x42, 0x65, - 0x74, 0x77, 0x65, 0x65, 0x6e, 0x48, 0x00, 0x52, 0x07, 0x62, 0x65, 0x74, 0x77, 0x65, 0x65, 0x6e, - 0x12, 0x36, 0x0a, 0x02, 0x69, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x4e, - 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, - 0x41, 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, - 0x49, 0x6e, 0x48, 0x00, 0x52, 0x02, 0x69, 0x6e, 0x12, 0x43, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x6e, - 0x75, 0x6c, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x4e, 0x59, 0x71, 0x6c, - 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, - 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x49, 0x73, 0x4e, - 0x75, 0x6c, 0x6c, 0x48, 0x00, 0x52, 0x06, 0x69, 0x73, 0x4e, 0x75, 0x6c, 0x6c, 0x12, 0x4d, 0x0a, - 0x0b, 0x69, 0x73, 0x5f, 0x6e, 0x6f, 0x74, 0x5f, 0x6e, 0x75, 0x6c, 0x6c, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x49, 0x73, 0x4e, 0x6f, 0x74, 0x4e, 0x75, 0x6c, 0x6c, 0x48, - 0x00, 0x52, 0x09, 0x69, 0x73, 0x4e, 0x6f, 0x74, 0x4e, 0x75, 0x6c, 0x6c, 0x12, 0x4e, 0x0a, 0x0a, - 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2c, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x2e, 0x54, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x48, 0x00, - 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x12, 0x5b, 0x0a, 0x0f, - 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, - 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x42, 0x6f, 0x6f, 0x6c, 0x45, 0x78, 0x70, - 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0e, 0x62, 0x6f, 0x6f, 0x6c, 0x45, - 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x47, 0x0a, 0x09, 0x54, 0x4e, 0x65, - 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x07, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x6e, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, - 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x07, 0x6f, 0x70, 0x65, 0x72, 0x61, - 0x6e, 0x64, 0x1a, 0x4c, 0x0a, 0x0c, 0x54, 0x43, 0x6f, 0x6e, 0x6a, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x3c, 0x0a, 0x08, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x6e, 0x64, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, - 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x08, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x6e, 0x64, 0x73, - 0x1a, 0x4c, 0x0a, 0x0c, 0x54, 0x44, 0x69, 0x73, 0x6a, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x3c, 0x0a, 0x08, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x52, 0x08, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x6e, 0x64, 0x73, 0x1a, 0xbb, - 0x01, 0x0a, 0x08, 0x54, 0x42, 0x65, 0x74, 0x77, 0x65, 0x65, 0x6e, 0x12, 0x37, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, 0x71, - 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, - 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x37, 0x0a, 0x05, 0x6c, 0x65, 0x61, 0x73, 0x74, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x6c, 0x65, 0x61, 0x73, 0x74, 0x12, 0x3d, 0x0a, - 0x08, 0x67, 0x72, 0x65, 0x61, 0x74, 0x65, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x08, 0x67, 0x72, 0x65, 0x61, 0x74, 0x65, 0x73, 0x74, 0x1a, 0x73, 0x0a, 0x03, - 0x54, 0x49, 0x6e, 0x12, 0x37, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x33, 0x0a, 0x03, - 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, - 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, - 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x73, 0x65, - 0x74, 0x1a, 0x42, 0x0a, 0x07, 0x54, 0x49, 0x73, 0x4e, 0x75, 0x6c, 0x6c, 0x12, 0x37, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, - 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, - 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x45, 0x0a, 0x0a, 0x54, 0x49, 0x73, 0x4e, 0x6f, 0x74, 0x4e, - 0x75, 0x6c, 0x6c, 0x12, 0x37, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x4a, 0x0a, 0x0f, - 0x54, 0x42, 0x6f, 0x6f, 0x6c, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x37, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, - 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0xcc, 0x02, 0x0a, 0x0b, 0x54, 0x43, 0x6f, - 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, 0x4e, 0x59, - 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, - 0x70, 0x69, 0x2e, 0x54, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x43, - 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x2e, 0x45, 0x4f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x40, 0x0a, 0x0a, 0x6c, 0x65, 0x66, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, 0x78, 0x70, 0x72, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x65, 0x66, 0x74, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x42, 0x0a, 0x0b, 0x72, 0x69, 0x67, 0x68, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x45, - 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x72, 0x69, 0x67, 0x68, 0x74, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x60, 0x0a, 0x0a, 0x45, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x20, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, 0x49, 0x53, 0x4f, - 0x4e, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x05, 0x0a, 0x01, 0x4c, 0x10, 0x01, - 0x12, 0x06, 0x0a, 0x02, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x45, 0x51, 0x10, 0x03, - 0x12, 0x06, 0x0a, 0x02, 0x4e, 0x45, 0x10, 0x04, 0x12, 0x06, 0x0a, 0x02, 0x47, 0x45, 0x10, 0x05, - 0x12, 0x05, 0x0a, 0x01, 0x47, 0x10, 0x06, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x22, 0x86, 0x01, 0x0a, 0x06, 0x54, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x31, 0x0a, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, - 0x59, 0x64, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x64, 0x73, 0x2e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2f, 0x0a, 0x06, 0x69, 0x73, - 0x73, 0x75, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x59, 0x64, 0x62, - 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x52, 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x04, - 0x54, 0x41, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x04, 0x61, 0x74, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x48, 0x00, 0x52, 0x04, 0x61, 0x74, 0x6f, 0x6d, 0x12, 0x36, 0x0a, 0x04, 0x6c, 0x69, - 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, - 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x41, 0x70, 0x69, 0x2e, - 0x54, 0x41, 0x73, 0x74, 0x2e, 0x54, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x04, 0x6c, 0x69, - 0x73, 0x74, 0x1a, 0x3f, 0x0a, 0x05, 0x54, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x36, 0x0a, 0x08, 0x63, - 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x4e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, - 0x4e, 0x41, 0x70, 0x69, 0x2e, 0x54, 0x41, 0x73, 0x74, 0x52, 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, - 0x72, 0x65, 0x6e, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2a, 0x56, - 0x0a, 0x0f, 0x45, 0x44, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x12, 0x20, 0x0a, 0x1c, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x5f, 0x46, - 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x46, 0x4f, - 0x52, 0x4d, 0x41, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x59, 0x51, 0x4c, 0x5f, 0x46, 0x4f, - 0x52, 0x4d, 0x41, 0x54, 0x10, 0x02, 0x42, 0x53, 0x5a, 0x51, 0x61, 0x2e, 0x79, 0x61, 0x6e, 0x64, - 0x65, 0x78, 0x2d, 0x74, 0x65, 0x61, 0x6d, 0x2e, 0x72, 0x75, 0x2f, 0x79, 0x64, 0x62, 0x2f, 0x6c, - 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x2f, 0x79, 0x71, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, - 0x64, 0x65, 0x72, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x2f, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x6c, 0x69, 0x62, 0x67, 0x6f, 0x2f, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, -} - -var ( - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescOnce sync.Once - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescData = file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDesc -) - -func file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescGZIP() []byte { - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescOnce.Do(func() { - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescData = protoimpl.X.CompressGZIP(file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescData) - }) - return file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDescData -} - -var file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes = make([]protoimpl.MessageInfo, 37) -var file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_goTypes = []interface{}{ - (EDateTimeFormat)(0), // 0: NYql.NConnector.NApi.EDateTimeFormat - (TReadSplitsRequest_EMode)(0), // 1: NYql.NConnector.NApi.TReadSplitsRequest.EMode - (TReadSplitsRequest_EFormat)(0), // 2: NYql.NConnector.NApi.TReadSplitsRequest.EFormat - (TExpression_TArithmeticalExpression_EOperation)(0), // 3: NYql.NConnector.NApi.TExpression.TArithmeticalExpression.EOperation - (TPredicate_TComparison_EOperation)(0), // 4: NYql.NConnector.NApi.TPredicate.TComparison.EOperation - (*TListTablesRequest)(nil), // 5: NYql.NConnector.NApi.TListTablesRequest - (*TListTablesResponse)(nil), // 6: NYql.NConnector.NApi.TListTablesResponse - (*TDescribeTableRequest)(nil), // 7: NYql.NConnector.NApi.TDescribeTableRequest - (*TTypeMappingSettings)(nil), // 8: NYql.NConnector.NApi.TTypeMappingSettings - (*TDescribeTableResponse)(nil), // 9: NYql.NConnector.NApi.TDescribeTableResponse - (*TSchema)(nil), // 10: NYql.NConnector.NApi.TSchema - (*TListSplitsRequest)(nil), // 11: NYql.NConnector.NApi.TListSplitsRequest - (*TListSplitsResponse)(nil), // 12: NYql.NConnector.NApi.TListSplitsResponse - (*TSelect)(nil), // 13: NYql.NConnector.NApi.TSelect - (*TSplit)(nil), // 14: NYql.NConnector.NApi.TSplit - (*TReadSplitsRequest)(nil), // 15: NYql.NConnector.NApi.TReadSplitsRequest - (*TReadSplitsResponse)(nil), // 16: NYql.NConnector.NApi.TReadSplitsResponse - (*TContinuation)(nil), // 17: NYql.NConnector.NApi.TContinuation - (*TExpression)(nil), // 18: NYql.NConnector.NApi.TExpression - (*TPredicate)(nil), // 19: NYql.NConnector.NApi.TPredicate - (*TError)(nil), // 20: NYql.NConnector.NApi.TError - (*TAst)(nil), // 21: NYql.NConnector.NApi.TAst - (*TSelect_TWhat)(nil), // 22: NYql.NConnector.NApi.TSelect.TWhat - (*TSelect_TFrom)(nil), // 23: NYql.NConnector.NApi.TSelect.TFrom - (*TSelect_TWhere)(nil), // 24: NYql.NConnector.NApi.TSelect.TWhere - (*TSelect_TLimit)(nil), // 25: NYql.NConnector.NApi.TSelect.TLimit - (*TSelect_TWhat_TItem)(nil), // 26: NYql.NConnector.NApi.TSelect.TWhat.TItem - (*TReadSplitsResponse_TColumnSet)(nil), // 27: NYql.NConnector.NApi.TReadSplitsResponse.TColumnSet - (*TReadSplitsResponse_TStats)(nil), // 28: NYql.NConnector.NApi.TReadSplitsResponse.TStats - (*TReadSplitsResponse_TColumnSet_TColumn)(nil), // 29: NYql.NConnector.NApi.TReadSplitsResponse.TColumnSet.TColumn - (*TExpression_TArithmeticalExpression)(nil), // 30: NYql.NConnector.NApi.TExpression.TArithmeticalExpression - (*TExpression_TNull)(nil), // 31: NYql.NConnector.NApi.TExpression.TNull - (*TPredicate_TNegation)(nil), // 32: NYql.NConnector.NApi.TPredicate.TNegation - (*TPredicate_TConjunction)(nil), // 33: NYql.NConnector.NApi.TPredicate.TConjunction - (*TPredicate_TDisjunction)(nil), // 34: NYql.NConnector.NApi.TPredicate.TDisjunction - (*TPredicate_TBetween)(nil), // 35: NYql.NConnector.NApi.TPredicate.TBetween - (*TPredicate_TIn)(nil), // 36: NYql.NConnector.NApi.TPredicate.TIn - (*TPredicate_TIsNull)(nil), // 37: NYql.NConnector.NApi.TPredicate.TIsNull - (*TPredicate_TIsNotNull)(nil), // 38: NYql.NConnector.NApi.TPredicate.TIsNotNull - (*TPredicate_TBoolExpression)(nil), // 39: NYql.NConnector.NApi.TPredicate.TBoolExpression - (*TPredicate_TComparison)(nil), // 40: NYql.NConnector.NApi.TPredicate.TComparison - (*TAst_TList)(nil), // 41: NYql.NConnector.NApi.TAst.TList - (*common.TDataSourceInstance)(nil), // 42: NYql.NConnector.NApi.TDataSourceInstance - (*Ydb.Column)(nil), // 43: Ydb.Column - (*Ydb.TypedValue)(nil), // 44: Ydb.TypedValue - (Ydb.StatusIds_StatusCode)(0), // 45: Ydb.StatusIds.StatusCode - (*Ydb_Issue.IssueMessage)(nil), // 46: Ydb.Issue.IssueMessage - (*Ydb.Value)(nil), // 47: Ydb.Value -} -var file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_depIdxs = []int32{ - 42, // 0: NYql.NConnector.NApi.TListTablesRequest.data_source_instance:type_name -> NYql.NConnector.NApi.TDataSourceInstance - 20, // 1: NYql.NConnector.NApi.TListTablesResponse.error:type_name -> NYql.NConnector.NApi.TError - 42, // 2: NYql.NConnector.NApi.TDescribeTableRequest.data_source_instance:type_name -> NYql.NConnector.NApi.TDataSourceInstance - 8, // 3: NYql.NConnector.NApi.TDescribeTableRequest.type_mapping_settings:type_name -> NYql.NConnector.NApi.TTypeMappingSettings - 0, // 4: NYql.NConnector.NApi.TTypeMappingSettings.date_time_format:type_name -> NYql.NConnector.NApi.EDateTimeFormat - 10, // 5: NYql.NConnector.NApi.TDescribeTableResponse.schema:type_name -> NYql.NConnector.NApi.TSchema - 20, // 6: NYql.NConnector.NApi.TDescribeTableResponse.error:type_name -> NYql.NConnector.NApi.TError - 43, // 7: NYql.NConnector.NApi.TSchema.columns:type_name -> Ydb.Column - 13, // 8: NYql.NConnector.NApi.TListSplitsRequest.selects:type_name -> NYql.NConnector.NApi.TSelect - 14, // 9: NYql.NConnector.NApi.TListSplitsResponse.splits:type_name -> NYql.NConnector.NApi.TSplit - 20, // 10: NYql.NConnector.NApi.TListSplitsResponse.error:type_name -> NYql.NConnector.NApi.TError - 42, // 11: NYql.NConnector.NApi.TSelect.data_source_instance:type_name -> NYql.NConnector.NApi.TDataSourceInstance - 22, // 12: NYql.NConnector.NApi.TSelect.what:type_name -> NYql.NConnector.NApi.TSelect.TWhat - 23, // 13: NYql.NConnector.NApi.TSelect.from:type_name -> NYql.NConnector.NApi.TSelect.TFrom - 24, // 14: NYql.NConnector.NApi.TSelect.where:type_name -> NYql.NConnector.NApi.TSelect.TWhere - 25, // 15: NYql.NConnector.NApi.TSelect.limit:type_name -> NYql.NConnector.NApi.TSelect.TLimit - 10, // 16: NYql.NConnector.NApi.TSelect.predefined_schema:type_name -> NYql.NConnector.NApi.TSchema - 13, // 17: NYql.NConnector.NApi.TSplit.select:type_name -> NYql.NConnector.NApi.TSelect - 42, // 18: NYql.NConnector.NApi.TReadSplitsRequest.data_source_instance:type_name -> NYql.NConnector.NApi.TDataSourceInstance - 14, // 19: NYql.NConnector.NApi.TReadSplitsRequest.splits:type_name -> NYql.NConnector.NApi.TSplit - 1, // 20: NYql.NConnector.NApi.TReadSplitsRequest.mode:type_name -> NYql.NConnector.NApi.TReadSplitsRequest.EMode - 2, // 21: NYql.NConnector.NApi.TReadSplitsRequest.format:type_name -> NYql.NConnector.NApi.TReadSplitsRequest.EFormat - 17, // 22: NYql.NConnector.NApi.TReadSplitsRequest.continuation:type_name -> NYql.NConnector.NApi.TContinuation - 27, // 23: NYql.NConnector.NApi.TReadSplitsResponse.column_set:type_name -> NYql.NConnector.NApi.TReadSplitsResponse.TColumnSet - 17, // 24: NYql.NConnector.NApi.TReadSplitsResponse.continuation:type_name -> NYql.NConnector.NApi.TContinuation - 28, // 25: NYql.NConnector.NApi.TReadSplitsResponse.stats:type_name -> NYql.NConnector.NApi.TReadSplitsResponse.TStats - 20, // 26: NYql.NConnector.NApi.TReadSplitsResponse.error:type_name -> NYql.NConnector.NApi.TError - 44, // 27: NYql.NConnector.NApi.TExpression.typed_value:type_name -> Ydb.TypedValue - 30, // 28: NYql.NConnector.NApi.TExpression.arithmetical_expression:type_name -> NYql.NConnector.NApi.TExpression.TArithmeticalExpression - 31, // 29: NYql.NConnector.NApi.TExpression.null:type_name -> NYql.NConnector.NApi.TExpression.TNull - 32, // 30: NYql.NConnector.NApi.TPredicate.negation:type_name -> NYql.NConnector.NApi.TPredicate.TNegation - 33, // 31: NYql.NConnector.NApi.TPredicate.conjunction:type_name -> NYql.NConnector.NApi.TPredicate.TConjunction - 34, // 32: NYql.NConnector.NApi.TPredicate.disjunction:type_name -> NYql.NConnector.NApi.TPredicate.TDisjunction - 35, // 33: NYql.NConnector.NApi.TPredicate.between:type_name -> NYql.NConnector.NApi.TPredicate.TBetween - 36, // 34: NYql.NConnector.NApi.TPredicate.in:type_name -> NYql.NConnector.NApi.TPredicate.TIn - 37, // 35: NYql.NConnector.NApi.TPredicate.is_null:type_name -> NYql.NConnector.NApi.TPredicate.TIsNull - 38, // 36: NYql.NConnector.NApi.TPredicate.is_not_null:type_name -> NYql.NConnector.NApi.TPredicate.TIsNotNull - 40, // 37: NYql.NConnector.NApi.TPredicate.comparison:type_name -> NYql.NConnector.NApi.TPredicate.TComparison - 39, // 38: NYql.NConnector.NApi.TPredicate.bool_expression:type_name -> NYql.NConnector.NApi.TPredicate.TBoolExpression - 45, // 39: NYql.NConnector.NApi.TError.status:type_name -> Ydb.StatusIds.StatusCode - 46, // 40: NYql.NConnector.NApi.TError.issues:type_name -> Ydb.Issue.IssueMessage - 41, // 41: NYql.NConnector.NApi.TAst.list:type_name -> NYql.NConnector.NApi.TAst.TList - 26, // 42: NYql.NConnector.NApi.TSelect.TWhat.items:type_name -> NYql.NConnector.NApi.TSelect.TWhat.TItem - 19, // 43: NYql.NConnector.NApi.TSelect.TWhere.filter_typed:type_name -> NYql.NConnector.NApi.TPredicate - 21, // 44: NYql.NConnector.NApi.TSelect.TWhere.filter_raw:type_name -> NYql.NConnector.NApi.TAst - 43, // 45: NYql.NConnector.NApi.TSelect.TWhat.TItem.column:type_name -> Ydb.Column - 43, // 46: NYql.NConnector.NApi.TReadSplitsResponse.TColumnSet.meta:type_name -> Ydb.Column - 29, // 47: NYql.NConnector.NApi.TReadSplitsResponse.TColumnSet.data:type_name -> NYql.NConnector.NApi.TReadSplitsResponse.TColumnSet.TColumn - 47, // 48: NYql.NConnector.NApi.TReadSplitsResponse.TColumnSet.TColumn.data:type_name -> Ydb.Value - 3, // 49: NYql.NConnector.NApi.TExpression.TArithmeticalExpression.operation:type_name -> NYql.NConnector.NApi.TExpression.TArithmeticalExpression.EOperation - 18, // 50: NYql.NConnector.NApi.TExpression.TArithmeticalExpression.left_value:type_name -> NYql.NConnector.NApi.TExpression - 18, // 51: NYql.NConnector.NApi.TExpression.TArithmeticalExpression.right_value:type_name -> NYql.NConnector.NApi.TExpression - 19, // 52: NYql.NConnector.NApi.TPredicate.TNegation.operand:type_name -> NYql.NConnector.NApi.TPredicate - 19, // 53: NYql.NConnector.NApi.TPredicate.TConjunction.operands:type_name -> NYql.NConnector.NApi.TPredicate - 19, // 54: NYql.NConnector.NApi.TPredicate.TDisjunction.operands:type_name -> NYql.NConnector.NApi.TPredicate - 18, // 55: NYql.NConnector.NApi.TPredicate.TBetween.value:type_name -> NYql.NConnector.NApi.TExpression - 18, // 56: NYql.NConnector.NApi.TPredicate.TBetween.least:type_name -> NYql.NConnector.NApi.TExpression - 18, // 57: NYql.NConnector.NApi.TPredicate.TBetween.greatest:type_name -> NYql.NConnector.NApi.TExpression - 18, // 58: NYql.NConnector.NApi.TPredicate.TIn.value:type_name -> NYql.NConnector.NApi.TExpression - 18, // 59: NYql.NConnector.NApi.TPredicate.TIn.set:type_name -> NYql.NConnector.NApi.TExpression - 18, // 60: NYql.NConnector.NApi.TPredicate.TIsNull.value:type_name -> NYql.NConnector.NApi.TExpression - 18, // 61: NYql.NConnector.NApi.TPredicate.TIsNotNull.value:type_name -> NYql.NConnector.NApi.TExpression - 18, // 62: NYql.NConnector.NApi.TPredicate.TBoolExpression.value:type_name -> NYql.NConnector.NApi.TExpression - 4, // 63: NYql.NConnector.NApi.TPredicate.TComparison.operation:type_name -> NYql.NConnector.NApi.TPredicate.TComparison.EOperation - 18, // 64: NYql.NConnector.NApi.TPredicate.TComparison.left_value:type_name -> NYql.NConnector.NApi.TExpression - 18, // 65: NYql.NConnector.NApi.TPredicate.TComparison.right_value:type_name -> NYql.NConnector.NApi.TExpression - 21, // 66: NYql.NConnector.NApi.TAst.TList.children:type_name -> NYql.NConnector.NApi.TAst - 67, // [67:67] is the sub-list for method output_type - 67, // [67:67] is the sub-list for method input_type - 67, // [67:67] is the sub-list for extension type_name - 67, // [67:67] is the sub-list for extension extendee - 0, // [0:67] is the sub-list for field type_name -} - -func init() { - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_init() -} -func file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_init() { - if File_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TListTablesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TListTablesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TDescribeTableRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TTypeMappingSettings); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TDescribeTableResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TSchema); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TListSplitsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TListSplitsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TSelect); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TSplit); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TReadSplitsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TReadSplitsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TContinuation); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TExpression); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TError); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TAst); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TSelect_TWhat); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TSelect_TFrom); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TSelect_TWhere); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TSelect_TLimit); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TSelect_TWhat_TItem); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TReadSplitsResponse_TColumnSet); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TReadSplitsResponse_TStats); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TReadSplitsResponse_TColumnSet_TColumn); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TExpression_TArithmeticalExpression); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TExpression_TNull); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TNegation); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TConjunction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TDisjunction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TBetween); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TIn); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TIsNull); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TIsNotNull); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TBoolExpression); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPredicate_TComparison); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TAst_TList); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[0].OneofWrappers = []interface{}{ - (*TListTablesRequest_Pattern)(nil), - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[9].OneofWrappers = []interface{}{ - (*TSplit_Description)(nil), - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[11].OneofWrappers = []interface{}{ - (*TReadSplitsResponse_ColumnSet)(nil), - (*TReadSplitsResponse_ArrowIpcStreaming)(nil), - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[12].OneofWrappers = []interface{}{ - (*TContinuation_Description)(nil), - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[13].OneofWrappers = []interface{}{ - (*TExpression_TypedValue)(nil), - (*TExpression_Column)(nil), - (*TExpression_ArithmeticalExpression)(nil), - (*TExpression_Null)(nil), - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[14].OneofWrappers = []interface{}{ - (*TPredicate_Negation)(nil), - (*TPredicate_Conjunction)(nil), - (*TPredicate_Disjunction)(nil), - (*TPredicate_Between)(nil), - (*TPredicate_In)(nil), - (*TPredicate_IsNull)(nil), - (*TPredicate_IsNotNull)(nil), - (*TPredicate_Comparison)(nil), - (*TPredicate_BoolExpression)(nil), - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[16].OneofWrappers = []interface{}{ - (*TAst_Atom)(nil), - (*TAst_List)(nil), - } - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes[21].OneofWrappers = []interface{}{ - (*TSelect_TWhat_TItem_Column)(nil), - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDesc, - NumEnums: 5, - NumMessages: 37, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_goTypes, - DependencyIndexes: file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_depIdxs, - EnumInfos: file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_enumTypes, - MessageInfos: file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_msgTypes, - }.Build() - File_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto = out.File - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_rawDesc = nil - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_goTypes = nil - file_ydb_library_yql_providers_generic_connector_api_service_protos_connector_proto_depIdxs = nil -} diff --git a/ydb/library/yql/providers/generic/connector/libgo/service/protos/ya.make b/ydb/library/yql/providers/generic/connector/libgo/service/protos/ya.make deleted file mode 100644 index a34aeebc4a1a..000000000000 --- a/ydb/library/yql/providers/generic/connector/libgo/service/protos/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -GO_LIBRARY() - -SRCS(connector.pb.go) - -END() diff --git a/ydb/library/yql/providers/generic/connector/libgo/service/ya.make b/ydb/library/yql/providers/generic/connector/libgo/service/ya.make deleted file mode 100644 index 741dc878f307..000000000000 --- a/ydb/library/yql/providers/generic/connector/libgo/service/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -GO_LIBRARY() - -PEERDIR(ydb/library/yql/providers/generic/connector/libgo/service/protos) - -SRCS( - connector.pb.go - connector_grpc.pb.go -) - -END() - -RECURSE( - protos -) diff --git a/ydb/library/yql/providers/generic/connector/libgo/ya.make b/ydb/library/yql/providers/generic/connector/libgo/ya.make deleted file mode 100644 index 66f1c16bea83..000000000000 --- a/ydb/library/yql/providers/generic/connector/libgo/ya.make +++ /dev/null @@ -1 +0,0 @@ -RECURSE(service) diff --git a/ydb/library/yql/providers/generic/connector/recipe/__main__.py b/ydb/library/yql/providers/generic/connector/recipe/__main__.py deleted file mode 100644 index 3fe692664a21..000000000000 --- a/ydb/library/yql/providers/generic/connector/recipe/__main__.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -This recipe was inspired by: -https://a.yandex-team.ru/arcadia/library/recipes/clickhouse/recipe/__init__.py -""" - -import os -import logging -import socket -from pathlib import Path -from typing import Dict, Final -import tempfile - -import jinja2 - -import yatest.common as yat -from library.python.testing.recipe import declare_recipe, set_env -from library.recipes.common import find_free_ports, start_daemon - -logger = logging.getLogger('connector.recipe') - -CONNECTOR_PID_FILE: Final = 'recipe.connector.pid' - - -def start(argv): - logger.debug('Start arguments: %s', argv) - - connector = yat.build_path("ydb/library/yql/providers/generic/connector/app/yq-connector") - - options = { - "grpc_host": "0.0.0.0", - "grpc_port": find_free_ports(1)[0], - "paging_bytes_per_page": 1024, - "paging_prefetch_queue_capacity": 2, - } - - config_path = _render_config(options) - - logger.info('Starting connector server...') - - start_daemon( - command=[connector, 'server', f'--config={config_path}'], - pid_file_name=yat.work_path(CONNECTOR_PID_FILE), - is_alive_check=lambda: _is_alive_check(host=options["grpc_host"], port=options["grpc_port"]), - environment=_update_environment(options), - ) - - logger.info('Connector server started') - - -def _render_config( - options: Dict, -) -> Path: - template_ = ''' - connector_server { - endpoint { - host: "{{grpc_host}}", - port: {{grpc_port}} - } - } - - logger { - log_level: TRACE - enable_sql_query_logging: true - } - - paging { - bytes_per_page: {{paging_bytes_per_page}} - prefetch_queue_capacity: {{paging_prefetch_queue_capacity}} - } - ''' - template = jinja2.Environment(loader=jinja2.BaseLoader).from_string(template_) - - content = template.render(**options) - tmp = tempfile.NamedTemporaryFile(delete=False) - with open(tmp.name, 'w') as f: - f.write(content) - - return tmp.name - - -def _update_environment(options: Dict): - variables = { - 'YDB_CONNECTOR_RECIPE_GRPC_HOST': options['grpc_host'], - 'YDB_CONNECTOR_RECIPE_GRPC_PORT': str(options['grpc_port']), - 'YDB_CONNECTOR_RECIPE_GRPC_PAGING_BYTES_PER_PAGE': str(options['paging_bytes_per_page']), - 'YDB_CONNECTOR_RECIPE_GRPC_PAGING_PREFETCH_QUEUE_CAPACITY': str(options['paging_prefetch_queue_capacity']), - } - - for variable in variables.items(): - (k, v) = variable - # k = prefix + k - set_env(k, v) - - environment = os.environ.copy() - environment.update(variables) - - return environment - - -def _is_alive_check(host: str, port: str, timeout=2) -> bool: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # presumably - sock.settimeout(timeout) - try: - sock.connect((host, port)) - except Exception as e: - logger.error(e) - return False - else: - sock.close() - return True - - -def stop(argv): - logger.debug('Start arguments: %s', argv) - logger.info('Terminating Connector server...') - try: - with open(yat.work_path(CONNECTOR_PID_FILE)) as fin: - pid = fin.read() - except IOError: - logger.error('Can not find server PID') - else: - logger.info('Terminate Connector server PID: %s', pid) - os.kill(int(pid), 9) - logger.info('Server terminated.') - - -if __name__ == "__main__": - declare_recipe(start, stop) diff --git a/ydb/library/yql/providers/generic/connector/recipe/recipe.inc b/ydb/library/yql/providers/generic/connector/recipe/recipe.inc deleted file mode 100644 index 6bb42535dc53..000000000000 --- a/ydb/library/yql/providers/generic/connector/recipe/recipe.inc +++ /dev/null @@ -1,6 +0,0 @@ -DEPENDS(ydb/library/yql/providers/generic/connector/recipe) - -USE_RECIPE(ydb/library/yql/providers/generic/connector/recipe/recipe) - -DEPENDS(ydb/library/yql/providers/generic/connector/app) - diff --git a/ydb/library/yql/providers/generic/connector/recipe/ya.make b/ydb/library/yql/providers/generic/connector/recipe/ya.make deleted file mode 100644 index eb483a56d73d..000000000000 --- a/ydb/library/yql/providers/generic/connector/recipe/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -PY3_PROGRAM() - -STYLE_PYTHON() - -PY_SRCS(__main__.py) - -PEERDIR( - contrib/python/Jinja2 - library/python/testing/recipe - library/python/testing/yatest_common - library/recipes/common -) - -END() diff --git a/ydb/library/yql/providers/generic/connector/tests/ya.make b/ydb/library/yql/providers/generic/connector/tests/ya.make index 3946cce94c4f..1ed277324a1b 100644 --- a/ydb/library/yql/providers/generic/connector/tests/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/ya.make @@ -3,7 +3,7 @@ PY3TEST() STYLE_PYTHON() NO_CHECK_IMPORTS() -SIZE(LARGE) +SIZE(MEDIUM) IF (AUTOCHECK) # Split tests to chunks only when they're running on different machines with distbuild, @@ -51,7 +51,6 @@ PEERDIR( ) DEPENDS( - ydb/library/yql/providers/generic/connector/app ydb/library/yql/tools/dqrun ydb/tests/tools/kqprun ) diff --git a/ydb/library/yql/providers/generic/connector/ya.make b/ydb/library/yql/providers/generic/connector/ya.make index 4ed432cbd144..66fc2678cde8 100644 --- a/ydb/library/yql/providers/generic/connector/ya.make +++ b/ydb/library/yql/providers/generic/connector/ya.make @@ -1,10 +1,6 @@ RECURSE( api - app - generate libcpp - libgo - recipe ) RECURSE_FOR_TESTS(