From 87b41353ded5699c4e5b669587d6c3cd7c56e333 Mon Sep 17 00:00:00 2001 From: xhe Date: Tue, 27 Dec 2022 16:41:02 +0800 Subject: [PATCH] *: adapt to toml Signed-off-by: xhe --- cmd/tiproxy/main.go | 2 +- conf/namespace/example.toml | 7 ++++ conf/namespace/example.yaml | 8 ---- conf/namespace/test.toml | 8 ++++ conf/namespace/test.yaml | 9 ----- conf/proxy.toml | 74 ++++++++++++++++++++++++++++++++++++ conf/proxy.yaml | 59 ---------------------------- docker/Dockerfile | 2 +- lib/cli/main.go | 2 +- lib/cli/namespace.go | 32 +++++++++------- lib/config/namespace.go | 20 ++++++++++ lib/config/namespace_test.go | 11 +++--- lib/config/proxy.go | 9 +++-- lib/util/security/cert.go | 8 ++-- lib/util/security/tls.go | 14 +++---- 15 files changed, 153 insertions(+), 112 deletions(-) create mode 100644 conf/namespace/example.toml delete mode 100644 conf/namespace/example.yaml create mode 100644 conf/namespace/test.toml delete mode 100644 conf/namespace/test.yaml create mode 100644 conf/proxy.toml delete mode 100644 conf/proxy.yaml diff --git a/cmd/tiproxy/main.go b/cmd/tiproxy/main.go index c1f5dc18..8ad477f5 100644 --- a/cmd/tiproxy/main.go +++ b/cmd/tiproxy/main.go @@ -32,7 +32,7 @@ func main() { Short: "start the proxy server", } - configFile := rootCmd.PersistentFlags().String("config", "conf/proxy.yaml", "proxy config file path") + configFile := rootCmd.PersistentFlags().String("config", "conf/proxy.toml", "proxy config file path") logEncoder := rootCmd.PersistentFlags().String("log_encoder", "tidb", "log in format of tidb, console, or json") logLevel := rootCmd.PersistentFlags().String("log_level", "", "log level") _ = rootCmd.PersistentFlags().String("cluster_name", "tiproxy", "default cluster name, used to generate node name and differential clusters in dns discovery") diff --git a/conf/namespace/example.toml b/conf/namespace/example.toml new file mode 100644 index 00000000..1816b03c --- /dev/null +++ b/conf/namespace/example.toml @@ -0,0 +1,7 @@ +namespace = "default" + +[frontend] + +[backend] +instances = [ "127.0.0.1:4000" ] +selector-type = "random" diff --git a/conf/namespace/example.yaml b/conf/namespace/example.yaml deleted file mode 100644 index dbeaa97f..00000000 --- a/conf/namespace/example.yaml +++ /dev/null @@ -1,8 +0,0 @@ -namespace: "default" -frontend: - security: -backend: - instances: - - "127.0.0.1:4000" - selector-type: "random" - security: diff --git a/conf/namespace/test.toml b/conf/namespace/test.toml new file mode 100644 index 00000000..7df36c1b --- /dev/null +++ b/conf/namespace/test.toml @@ -0,0 +1,8 @@ +namespace = "ns2" + +[frontend] +user = "xhe" + +[backend] +instances = [ "127.0.0.1:7000" ] +selector-type = "random" diff --git a/conf/namespace/test.yaml b/conf/namespace/test.yaml deleted file mode 100644 index f0c8e987..00000000 --- a/conf/namespace/test.yaml +++ /dev/null @@ -1,9 +0,0 @@ -namespace: "ns2" -frontend: - user: "xhe" - security: -backend: - instances: - - "127.0.0.1:7000" - selector-type: "random" - security: diff --git a/conf/proxy.toml b/conf/proxy.toml new file mode 100644 index 00000000..5ed3c090 --- /dev/null +++ b/conf/proxy.toml @@ -0,0 +1,74 @@ +workdir = "./work" + +[proxy] +addr = "0.0.0.0:6000" +tcp-keep-alive = true +max-connections = 1000 +# pd-addrs = "127.0.0.1:2379" +# require-backend-tls = true +# proxy-protocol = "v2" +# graceful-wait-before-shutdown = 10 + +[metrics] + +[api] +addr = "0.0.0.0:3080" +enable-basic-auth = false +user = "" +password = "" + +[log] +level = "info" +encoder = "tidb" + +[log.log-file] +filename = "" +max-size = 300 +max-days = 1 +max-backups = 1 + +[security] +rsa-key-size = 4_096 +# tls object is either of type server, client, or peer +# [xxxx] +# ca = "ca.pem" +# cert = "c.pem" +# key = "k.pem" +# auto-certs = true +# skip-ca = trure +# client object: +# 1. requires: ca or skip-ca(skip verify server certs) +# 2. optionally: cert/key will be used if server asks +# 3. useless/forbid: auto-certs +# server object: +# 1. requires: cert/key or auto-certs(generate a temporary cert, mostly for testing) +# 2. optionally: ca will enable server-side client verification. +# 3. useless/forbid: skip-ca +# peer object: +# 1. requires: cert/key/ca or auto-certs +# 2. useless/forbid: skip-ca + + # client object + [security.cluster-tls] + # access to other components like TiDB or PD, will use this + # skip-ca = true + + # client object + [security.sql-tls] + # access to other components like TiDB or PD, will use this + skip-ca = true + + # server object + [security.server-tls] + # proxy SQL or HTTP port will use this + # auto-certs: true + + # peer object + [security.peer-tls] + # internal communication between proxies + # auto-certs: true + +[advance] +# ignore-wrong-namespace = true +# peer-port = "3081" +# watch-interval = "10m" diff --git a/conf/proxy.yaml b/conf/proxy.yaml deleted file mode 100644 index fc7f1652..00000000 --- a/conf/proxy.yaml +++ /dev/null @@ -1,59 +0,0 @@ -workdir: "./work" -proxy: - addr: "0.0.0.0:6000" - tcp-keep-alive: true - max-connections: 1000 - pd-addrs: "127.0.0.1:2379" - # require-backend-tls: true - # proxy-protocol: "v2" - # graceful-wait-before-shutdown: 10 -metrics: -api: - addr: "0.0.0.0:3080" - enable-basic-auth: false - user: "" - password: "" -log: - level: "info" - encoder: "tidb" - log-file: - filename: "" - max-size: 300 - max-days: 1 - max-backups: 1 -security: - rsa-key-size: 4096 - # tls object is either of type server, client, or peer - # xxxx: - # ca: ca.pem - # cert: c.pem - # key: k.pem - # auto-certs: true - # skip-ca: trure - # client object: - # 1. requires: ca or skip-ca(skip verify server certs) - # 2. optionally: cert/key will be used if server asks - # 3. useless/forbid: auto-certs - # server object: - # 1. requires: cert/key or auto-certs(generate a temporary cert, mostly for testing) - # 2. optionally: ca will enable server-side client verification. - # 3. useless/forbid: skip-ca - # peer object: - # 1. requires: cert/key/ca or auto-certs - # 2. useless/forbid: skip-ca - cluster-tls: # client object - # access to other components like TiDB or PD, will use this - # skip-ca: true - sql-tls: # client object - # access to TiDB sql port, it has a standalone TLS configuration - skip-ca: true - server-tls: # server object - # proxy SQL or HTTP port will use this - # auto-certs: true - peer-tls: # peer object - # internal communication between proxies - # auto-certs: true -advance: - # ignore-wrong-namespace: true - # peer-port: "3081" - # watch-interval: "10m" diff --git a/docker/Dockerfile b/docker/Dockerfile index 9b19a78d..bd32302a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,4 +17,4 @@ EXPOSE 3080 EXPOSE 3081 EXPOSE 6000 -ENTRYPOINT ["/bin/tiproxy", "-conf", "/etc/proxy/proxy.yaml"] +ENTRYPOINT ["/bin/tiproxy", "-conf", "/etc/proxy/proxy.toml"] diff --git a/lib/cli/main.go b/lib/cli/main.go index 94682b52..59499043 100644 --- a/lib/cli/main.go +++ b/lib/cli/main.go @@ -37,7 +37,7 @@ func GetRootCmd(tlsConfig *tls.Config) *cobra.Command { curls := rootCmd.PersistentFlags().StringArray("curls", []string{"localhost:3080"}, "API gateway addresses") logEncoder := rootCmd.PersistentFlags().String("log_encoder", "tidb", "log in format of tidb, console, or json") - logLevel := rootCmd.PersistentFlags().String("log_level", "info", "log level") + logLevel := rootCmd.PersistentFlags().String("log_level", "warn", "log level") insecure := rootCmd.PersistentFlags().BoolP("insecure", "k", false, "enable TLS without CA, useful for testing, or for expired certs") caPath := rootCmd.PersistentFlags().String("ca", "", "CA to verify server certificates, set to 'skip' if want to enable SSL without verification") certPath := rootCmd.PersistentFlags().String("cert", "", "cert for server-side client authentication") diff --git a/lib/cli/namespace.go b/lib/cli/namespace.go index 2c1c98b7..6007ceef 100644 --- a/lib/cli/namespace.go +++ b/lib/cli/namespace.go @@ -18,14 +18,15 @@ import ( "bytes" "encoding/json" "fmt" + "io" "net/http" "os" "path/filepath" "strings" + "github.com/BurntSushi/toml" "github.com/pingcap/TiProxy/lib/config" "github.com/spf13/cobra" - "gopkg.in/yaml.v3" ) const ( @@ -69,12 +70,11 @@ func GetNamespaceCmd(ctx *Context) *cobra.Command { if err := json.Unmarshal([]byte(resp), &nscs); err != nil { return err } - nscsbytes, err := yaml.Marshal(&nscs) - if err != nil { - return err + nscsmap := make(map[string]config.Namespace, len(nscs)) + for _, nsc := range nscs { + nscsmap[nsc.Namespace] = nsc } - cmd.Println(string(nscsbytes)) - return nil + return toml.NewEncoder(cmd.OutOrStdout()).Encode(nscsmap) }, }, ) @@ -115,7 +115,7 @@ func GetNamespaceCmd(ctx *Context) *cobra.Command { if err := json.Unmarshal([]byte(resp), &nsc); err != nil { return err } - nscbytes, err := yaml.Marshal(&nsc) + nscbytes, err := nsc.ToBytes() if err != nil { return err } @@ -141,11 +141,15 @@ func GetNamespaceCmd(ctx *Context) *cobra.Command { defer f.Close() in = f } - var nsc config.Namespace - if err := yaml.NewDecoder(in).Decode(&nsc); err != nil { + nscbytes, err := io.ReadAll(in) + if err != nil { return err } - nscbytes, err := json.Marshal(&nsc) + nsc, err := config.NewNamespace(nscbytes) + if err != nil { + return err + } + nscbytes, err = json.Marshal(nsc) if err != nil { return err } @@ -171,7 +175,7 @@ func GetNamespaceCmd(ctx *Context) *cobra.Command { return cmd.Help() } - nFiles, err := listAllFiles(args[0], ".yaml") + nFiles, err := listAllFiles(args[0], ".toml") if err != nil { return err } @@ -181,11 +185,11 @@ func GetNamespaceCmd(ctx *Context) *cobra.Command { if err != nil { return err } - var nsc config.Namespace - if err := yaml.Unmarshal(fileData, &nsc); err != nil { + nsc, err := config.NewNamespace(fileData) + if err != nil { return err } - nscbytes, err := json.Marshal(&nsc) + nscbytes, err := json.Marshal(nsc) if err != nil { return err } diff --git a/lib/config/namespace.go b/lib/config/namespace.go index 1934c77b..975528a0 100644 --- a/lib/config/namespace.go +++ b/lib/config/namespace.go @@ -15,6 +15,12 @@ package config +import ( + "bytes" + + "github.com/BurntSushi/toml" +) + type Namespace struct { Namespace string `yaml:"namespace" json:"namespace" toml:"namespace"` Frontend FrontendNamespace `yaml:"frontend" json:"frontend" toml:"frontend"` @@ -31,3 +37,17 @@ type BackendNamespace struct { SelectorType string `yaml:"selector-type" json:"selector-type" toml:"selector-type"` Security TLSConfig `yaml:"security" json:"security" toml:"security"` } + +func NewNamespace(data []byte) (*Namespace, error) { + var cfg Namespace + if err := toml.Unmarshal(data, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func (cfg *Namespace) ToBytes() ([]byte, error) { + b := new(bytes.Buffer) + err := toml.NewEncoder(b).Encode(cfg) + return b.Bytes(), err +} diff --git a/lib/config/namespace_test.go b/lib/config/namespace_test.go index b7409748..385b9580 100644 --- a/lib/config/namespace_test.go +++ b/lib/config/namespace_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" ) var testNamespaceConfig = Namespace{ @@ -45,9 +44,11 @@ var testNamespaceConfig = Namespace{ } func TestNamespaceConfig(t *testing.T) { - data, err := yaml.Marshal(testNamespaceConfig) + data1, err := testNamespaceConfig.ToBytes() require.NoError(t, err) - var cfg Namespace - require.NoError(t, yaml.Unmarshal(data, &cfg)) - require.Equal(t, testNamespaceConfig, cfg) + cfg, err := NewNamespace(data1) + require.NoError(t, err) + data2, err := cfg.ToBytes() + require.NoError(t, err) + require.Equal(t, data1, data2) } diff --git a/lib/config/proxy.go b/lib/config/proxy.go index fa229da9..a733d9d8 100644 --- a/lib/config/proxy.go +++ b/lib/config/proxy.go @@ -16,11 +16,12 @@ package config import ( + "bytes" "os" "path/filepath" + "github.com/BurntSushi/toml" "github.com/pingcap/TiProxy/lib/util/errors" - "gopkg.in/yaml.v3" ) var ( @@ -116,7 +117,7 @@ func NewConfig(data []byte) (*Config, error) { var cfg Config cfg.Advance.IgnoreWrongNamespace = true cfg.Proxy.RequireBackendTLS = true - if err := yaml.Unmarshal(data, &cfg); err != nil { + if err := toml.Unmarshal(data, &cfg); err != nil { return nil, err } if err := cfg.Check(); err != nil { @@ -143,5 +144,7 @@ func (cfg *Config) Check() error { } func (cfg *Config) ToBytes() ([]byte, error) { - return yaml.Marshal(cfg) + b := new(bytes.Buffer) + err := toml.NewEncoder(b).Encode(cfg) + return b.Bytes(), err } diff --git a/lib/util/security/cert.go b/lib/util/security/cert.go index 132ec38b..48f2b4c3 100644 --- a/lib/util/security/cert.go +++ b/lib/util/security/cert.go @@ -162,7 +162,7 @@ func (ci *CertInfo) buildServerConfig(lg *zap.Logger) (*tls.Config, error) { if ci.cfg.AutoCerts { autoCerts = true } else { - lg.Warn("require certificates to secure clients connections, disable TLS") + lg.Info("require certificates to secure clients connections, disable TLS") return nil, nil } } @@ -195,7 +195,7 @@ func (ci *CertInfo) buildServerConfig(lg *zap.Logger) (*tls.Config, error) { return nil, err } if !ci.cfg.HasCA() { - lg.Warn("no CA, server will not authenticate clients (connection is still secured)") + lg.Info("no CA, server will not authenticate clients (connection is still secured)") return tcfg, nil } else { caPEM, err = os.ReadFile(ci.cfg.CA) @@ -239,7 +239,7 @@ func (ci *CertInfo) buildClientConfig(lg *zap.Logger) (*tls.Config, error) { MinVersion: tls.VersionTLS12, }, nil } - lg.Warn("no CA to verify server connections, disable TLS") + lg.Info("no CA to verify server connections, disable TLS") return nil, nil } @@ -262,7 +262,7 @@ func (ci *CertInfo) buildClientConfig(lg *zap.Logger) (*tls.Config, error) { ci.ca.Store(cas) if !ci.cfg.HasCert() { - lg.Warn("no certificates, server may reject the connection") + lg.Info("no certificates, server may reject the connection") return tcfg, nil } diff --git a/lib/util/security/tls.go b/lib/util/security/tls.go index 29ed1d09..23f313a7 100644 --- a/lib/util/security/tls.go +++ b/lib/util/security/tls.go @@ -42,14 +42,14 @@ func createTLSCertificates(logger *zap.Logger, certpath, keypath, capath string, _, e1 := os.Stat(certpath) _, e2 := os.Stat(keypath) if errors.Is(e1, os.ErrExist) || errors.Is(e2, os.ErrExist) { - logger.Warn("either cert or key exists") + logger.Info("either cert or key exists") return nil } if capath != "" { _, e3 := os.Stat(capath) if errors.Is(e3, os.ErrExist) { - logger.Warn("ca exists") + logger.Info("ca exists") return nil } } @@ -225,7 +225,7 @@ func CreateTLSConfigForTest() (serverTLSConf *tls.Config, clientTLSConf *tls.Con func BuildServerTLSConfig(logger *zap.Logger, cfg config.TLSConfig) (*tls.Config, error) { logger = logger.With(zap.String("tls", "server")) if !cfg.HasCert() { - logger.Warn("require certificates to secure clients connections, disable TLS") + logger.Info("require certificates to secure clients connections, disable TLS") return nil, nil } @@ -239,7 +239,7 @@ func BuildServerTLSConfig(logger *zap.Logger, cfg config.TLSConfig) (*tls.Config tcfg.Certificates = append(tcfg.Certificates, cert) if !cfg.HasCA() { - logger.Warn("no CA, server will not authenticate clients (connection is still secured)") + logger.Info("no CA, server will not authenticate clients (connection is still secured)") return tcfg, nil } @@ -265,7 +265,7 @@ func BuildClientTLSConfig(logger *zap.Logger, cfg config.TLSConfig) (*tls.Config MinVersion: tls.VersionTLS12, }, nil } - logger.Warn("no CA to verify server connections, disable TLS") + logger.Info("no CA to verify server connections, disable TLS") return nil, nil } @@ -282,7 +282,7 @@ func BuildClientTLSConfig(logger *zap.Logger, cfg config.TLSConfig) (*tls.Config } if !cfg.HasCert() { - logger.Warn("no certificates, server may reject the connection") + logger.Info("no certificates, server may reject the connection") return tcfg, nil } cert, err := tls.LoadX509KeyPair(cfg.Cert, cfg.Key) @@ -306,7 +306,7 @@ func BuildEtcdTLSConfig(logger *zap.Logger, server, peer config.TLSConfig) (clie clientInfo.TrustedCAFile = server.CA clientInfo.ClientCertAuth = true } else if !server.SkipCA { - logger.Warn("no CA, proxy will not authenticate etcd clients (connection is still secured)") + logger.Info("no CA, proxy will not authenticate etcd clients (connection is still secured)") } }