diff --git a/cmd/tidb-lightning-ctl/main.go b/cmd/tidb-lightning-ctl/main.go index 203aa43f9..82264904f 100644 --- a/cmd/tidb-lightning-ctl/main.go +++ b/cmd/tidb-lightning-ctl/main.go @@ -17,7 +17,6 @@ import ( "context" "flag" "fmt" - "net/http" "os" "path/filepath" "strconv" @@ -25,10 +24,12 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/import_sstpb" + uuid "github.com/satori/go.uuid" + kv "github.com/pingcap/tidb-lightning/lightning/backend" + "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-lightning/lightning/config" "github.com/pingcap/tidb-lightning/lightning/restore" - uuid "github.com/satori/go.uuid" ) func main() { @@ -70,19 +71,27 @@ func run() error { return err } + tls, err := cfg.ToTLS() + if err != nil { + return err + } + if err = cfg.TiDB.Security.RegisterMySQL(); err != nil { + return err + } + ctx := context.Background() if *compact { - return errors.Trace(compactCluster(ctx, cfg)) + return errors.Trace(compactCluster(ctx, cfg, tls)) } if len(*mode) != 0 { - return errors.Trace(switchMode(ctx, cfg, *mode)) + return errors.Trace(switchMode(ctx, cfg, tls, *mode)) } if len(*flagImportEngine) != 0 { - return errors.Trace(importEngine(ctx, cfg, *flagImportEngine)) + return errors.Trace(importEngine(ctx, cfg, tls, *flagImportEngine)) } if len(*flagCleanupEngine) != 0 { - return errors.Trace(cleanupEngine(ctx, cfg, *flagCleanupEngine)) + return errors.Trace(cleanupEngine(ctx, cfg, tls, *flagCleanupEngine)) } if len(*cpRemove) != 0 { @@ -92,7 +101,7 @@ func run() error { return errors.Trace(checkpointErrorIgnore(ctx, cfg, *cpErrIgnore)) } if len(*cpErrDestroy) != 0 { - return errors.Trace(checkpointErrorDestroy(ctx, cfg, *cpErrDestroy)) + return errors.Trace(checkpointErrorDestroy(ctx, cfg, tls, *cpErrDestroy)) } if len(*cpDump) != 0 { return errors.Trace(checkpointDump(ctx, cfg, *cpDump)) @@ -102,19 +111,18 @@ func run() error { return nil } -func compactCluster(ctx context.Context, cfg *config.Config) error { +func compactCluster(ctx context.Context, cfg *config.Config, tls *common.TLS) error { return kv.ForAllStores( ctx, - &http.Client{}, - cfg.TiDB.PdAddr, + tls.WithHost(cfg.TiDB.PdAddr), kv.StoreStateDisconnected, func(c context.Context, store *kv.Store) error { - return kv.Compact(c, store.Address, restore.FullLevelCompact) + return kv.Compact(c, tls, store.Address, restore.FullLevelCompact) }, ) } -func switchMode(ctx context.Context, cfg *config.Config, mode string) error { +func switchMode(ctx context.Context, cfg *config.Config, tls *common.TLS, mode string) error { var m import_sstpb.SwitchMode switch mode { case config.ImportMode: @@ -127,11 +135,10 @@ func switchMode(ctx context.Context, cfg *config.Config, mode string) error { return kv.ForAllStores( ctx, - &http.Client{}, - cfg.TiDB.PdAddr, + tls.WithHost(cfg.TiDB.PdAddr), kv.StoreStateDisconnected, func(c context.Context, store *kv.Store) error { - return kv.SwitchMode(c, store.Address, m) + return kv.SwitchMode(c, tls, store.Address, m) }, ) } @@ -156,20 +163,20 @@ func checkpointErrorIgnore(ctx context.Context, cfg *config.Config, tableName st return errors.Trace(cpdb.IgnoreErrorCheckpoint(ctx, tableName)) } -func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tableName string) error { +func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common.TLS, tableName string) error { cpdb, err := restore.OpenCheckpointsDB(ctx, cfg) if err != nil { return errors.Trace(err) } defer cpdb.Close() - target, err := restore.NewTiDBManager(cfg.TiDB) + target, err := restore.NewTiDBManager(cfg.TiDB, tls) if err != nil { return errors.Trace(err) } defer target.Close() - importer, err := kv.NewImporter(ctx, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) + importer, err := kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) if err != nil { return errors.Trace(err) } @@ -271,8 +278,8 @@ func unsafeCloseEngine(ctx context.Context, importer kv.Backend, engine string) return ce, errors.Trace(err) } -func importEngine(ctx context.Context, cfg *config.Config, engine string) error { - importer, err := kv.NewImporter(ctx, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) +func importEngine(ctx context.Context, cfg *config.Config, tls *common.TLS, engine string) error { + importer, err := kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) if err != nil { return errors.Trace(err) } @@ -285,8 +292,8 @@ func importEngine(ctx context.Context, cfg *config.Config, engine string) error return errors.Trace(ce.Import(ctx)) } -func cleanupEngine(ctx context.Context, cfg *config.Config, engine string) error { - importer, err := kv.NewImporter(ctx, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) +func cleanupEngine(ctx context.Context, cfg *config.Config, tls *common.TLS, engine string) error { + importer, err := kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) if err != nil { return errors.Trace(err) } diff --git a/lightning/backend/importer.go b/lightning/backend/importer.go index 47cb2bc60..e46930392 100644 --- a/lightning/backend/importer.go +++ b/lightning/backend/importer.go @@ -25,6 +25,7 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" + "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-lightning/lightning/log" ) @@ -42,8 +43,8 @@ type importer struct { // NewImporter creates a new connection to tikv-importer. A single connection // per tidb-lightning instance is enough. -func NewImporter(ctx context.Context, importServerAddr string, pdAddr string) (Backend, error) { - conn, err := grpc.DialContext(ctx, importServerAddr, grpc.WithInsecure()) +func NewImporter(ctx context.Context, tls *common.TLS, importServerAddr string, pdAddr string) (Backend, error) { + conn, err := grpc.DialContext(ctx, importServerAddr, tls.ToGRPCDialOption()) if err != nil { return MakeBackend(nil), errors.Trace(err) } diff --git a/lightning/backend/tidb_test.go b/lightning/backend/tidb_test.go index b9d65355a..6648eefaf 100644 --- a/lightning/backend/tidb_test.go +++ b/lightning/backend/tidb_test.go @@ -70,7 +70,7 @@ func (s *mysqlSuite) TestWriteRowsReplaceOnDup(c *C) { types.NewUintDatum(18446744073709551615), types.NewIntDatum(-9223372036854775808), types.NewUintDatum(0), - types.Datum{}, + {}, types.NewFloat32Datum(7.5), types.NewFloat64Datum(5e-324), types.NewFloat64Datum(1.7976931348623157e+308), diff --git a/lightning/backend/tikv.go b/lightning/backend/tikv.go index 76c23c193..c5b978305 100644 --- a/lightning/backend/tikv.go +++ b/lightning/backend/tikv.go @@ -15,8 +15,6 @@ package backend import ( "context" - "fmt" - "net/http" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/import_sstpb" @@ -76,11 +74,11 @@ type Store struct { State StoreState `json:"state_name"` } -func withTiKVConnection(ctx context.Context, tikvAddr string, action func(import_sstpb.ImportSSTClient) error) error { +func withTiKVConnection(ctx context.Context, tls *common.TLS, tikvAddr string, action func(import_sstpb.ImportSSTClient) error) error { // Connect to the ImportSST service on the given TiKV node. // The connection is needed for executing `action` and will be tear down // when this function exits. - conn, err := grpc.DialContext(ctx, tikvAddr, grpc.WithInsecure()) + conn, err := grpc.DialContext(ctx, tikvAddr, tls.ToGRPCDialOption()) if err != nil { return errors.Trace(err) } @@ -91,7 +89,7 @@ func withTiKVConnection(ctx context.Context, tikvAddr string, action func(import } // ForAllStores executes `action` in parallel for all TiKV stores connected to -// the given PD server. +// a PD server given by the HTTPS client `tls`. // // Returns the first non-nil error returned in all `action` calls. If all // `action` returns nil, this method would return nil as well. @@ -100,22 +98,18 @@ func withTiKVConnection(ctx context.Context, tikvAddr string, action func(import // result (Tombstone < Offline < Down < Disconnected < Up). func ForAllStores( ctx context.Context, - client *http.Client, - pdAddr string, + tls *common.TLS, minState StoreState, action func(c context.Context, store *Store) error, ) error { // Go through the HTTP interface instead of gRPC so we don't need to keep // track of the cluster ID. - url := fmt.Sprintf("http://%s/pd/api/v1/stores", pdAddr) - var stores struct { Stores []struct { Store Store } } - - err := common.GetJSON(client, url, &stores) + err := tls.GetJSON("/pd/api/v1/stores", &stores) if err != nil { return err } @@ -131,9 +125,9 @@ func ForAllStores( } // SwitchMode changes the TiKV node at the given address to a particular mode. -func SwitchMode(ctx context.Context, tikvAddr string, mode import_sstpb.SwitchMode) error { +func SwitchMode(ctx context.Context, tls *common.TLS, tikvAddr string, mode import_sstpb.SwitchMode) error { task := log.With(zap.Stringer("mode", mode)).Begin(zap.DebugLevel, "switch mode") - err := withTiKVConnection(ctx, tikvAddr, func(client import_sstpb.ImportSSTClient) error { + err := withTiKVConnection(ctx, tls, tikvAddr, func(client import_sstpb.ImportSSTClient) error { _, err := client.SwitchMode(ctx, &import_sstpb.SwitchModeRequest{ Mode: mode, }) @@ -144,9 +138,9 @@ func SwitchMode(ctx context.Context, tikvAddr string, mode import_sstpb.SwitchMo } // Compact performs a leveled compaction with the given minimum level. -func Compact(ctx context.Context, tikvAddr string, level int32) error { +func Compact(ctx context.Context, tls *common.TLS, tikvAddr string, level int32) error { task := log.With(zap.Int32("level", level)).Begin(zap.InfoLevel, "compact cluster") - err := withTiKVConnection(ctx, tikvAddr, func(client import_sstpb.ImportSSTClient) error { + err := withTiKVConnection(ctx, tls, tikvAddr, func(client import_sstpb.ImportSSTClient) error { _, err := client.Compact(ctx, &import_sstpb.CompactRequest{ OutputLevel: level, }) diff --git a/lightning/backend/tikv_test.go b/lightning/backend/tikv_test.go index 4b535cb8b..5b43a5546 100644 --- a/lightning/backend/tikv_test.go +++ b/lightning/backend/tikv_test.go @@ -4,13 +4,13 @@ import ( "context" "net/http" "net/http/httptest" - "net/url" "sort" "sync" . "github.com/pingcap/check" kv "github.com/pingcap/tidb-lightning/lightning/backend" + "github.com/pingcap/tidb-lightning/lightning/common" ) type tikvSuite struct{} @@ -18,7 +18,7 @@ type tikvSuite struct{} var _ = Suite(&tikvSuite{}) func (s *tikvSuite) TestForAllStores(c *C) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { w.Write([]byte(` { "count": 5, @@ -74,15 +74,13 @@ func (s *tikvSuite) TestForAllStores(c *C) { })) defer server.Close() - serverURL, err := url.Parse(server.URL) - c.Assert(err, IsNil) - ctx := context.Background() var ( allStoresLock sync.Mutex allStores []*kv.Store ) - err = kv.ForAllStores(ctx, server.Client(), serverURL.Host, kv.StoreStateDown, func(c2 context.Context, store *kv.Store) error { + tls := common.NewTLSFromMockServer(server) + err := kv.ForAllStores(ctx, tls, kv.StoreStateDown, func(c2 context.Context, store *kv.Store) error { allStoresLock.Lock() allStores = append(allStores, store) allStoresLock.Unlock() diff --git a/lightning/checkpoints/checkpoints_test.go b/lightning/checkpoints/checkpoints_test.go index 95e1a6574..017500ac4 100644 --- a/lightning/checkpoints/checkpoints_test.go +++ b/lightning/checkpoints/checkpoints_test.go @@ -302,4 +302,4 @@ func (s *checkpointSuite) TestCheckpointMarshallUnmarshall(c *C) { fileChkp2 := NewFileCheckpointsDB(path) // if not recover empty map explicitly, it will become nil c.Assert(fileChkp2.checkpoints.Checkpoints["a"].Engines, NotNil) -} \ No newline at end of file +} diff --git a/lightning/common/security.go b/lightning/common/security.go new file mode 100644 index 000000000..7f90b61d3 --- /dev/null +++ b/lightning/common/security.go @@ -0,0 +1,143 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + + "github.com/pingcap/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// TLS +type TLS struct { + inner *tls.Config + client *http.Client + url string +} + +// ToTLSConfig constructs a `*tls.Config` from the CA, certification and key +// paths. +// +// If the CA path is empty, returns nil. +func ToTLSConfig(caPath, certPath, keyPath string) (*tls.Config, error) { + if len(caPath) == 0 { + return nil, nil + } + + // Load the client certificates from disk + var certificates []tls.Certificate + if len(certPath) != 0 && len(keyPath) != 0 { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, errors.Annotate(err, "could not load client key pair") + } + certificates = []tls.Certificate{cert} + } + + // Create a certificate pool from CA + certPool := x509.NewCertPool() + ca, err := ioutil.ReadFile(caPath) + if err != nil { + return nil, errors.Annotate(err, "could not read ca certificate") + } + + // Append the certificates from the CA + if !certPool.AppendCertsFromPEM(ca) { + return nil, errors.New("failed to append ca certs") + } + + return &tls.Config{ + Certificates: certificates, + RootCAs: certPool, + NextProtos: []string{"h2", "http/1.1"}, // specify `h2` to let Go use HTTP/2. + }, nil +} + +// NewTLS constructs a new HTTP client with TLS configured with the CA, +// certificate and key paths. +// +// If the CA path is empty, returns an instance where TLS is disabled. +func NewTLS(caPath, certPath, keyPath, host string) (*TLS, error) { + if len(caPath) == 0 { + return &TLS{ + inner: nil, + client: &http.Client{}, + url: "http://" + host, + }, nil + } + inner, err := ToTLSConfig(caPath, certPath, keyPath) + if err != nil { + return nil, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = inner + return &TLS{ + inner: inner, + client: &http.Client{Transport: transport}, + url: "https://" + host, + }, nil +} + +// NewTLSFromMockServer constructs a new TLS instance from the certificates of +// an *httptest.Server. +func NewTLSFromMockServer(server *httptest.Server) *TLS { + return &TLS{ + inner: server.TLS, + client: server.Client(), + url: server.URL, + } +} + +// WithHost creates a new TLS instance with the host replaced. +func (tc *TLS) WithHost(host string) *TLS { + var url string + if tc.inner != nil { + url = "https://" + host + } else { + url = "http://" + host + } + return &TLS{ + inner: tc.inner, + client: tc.client, + url: url, + } +} + +// ToGRPCDialOption constructs a gRPC dial option. +func (tc *TLS) ToGRPCDialOption() grpc.DialOption { + if tc.inner != nil { + return grpc.WithTransportCredentials(credentials.NewTLS(tc.inner)) + } + return grpc.WithInsecure() +} + +// WrapListener places a TLS layer on top of the existing listener. +func (tc *TLS) WrapListener(l net.Listener) net.Listener { + if tc.inner == nil { + return l + } + return tls.NewListener(l, tc.inner) +} + +// GetJSON obtains JSON result with the HTTP GET method. +func (tc *TLS) GetJSON(path string, v interface{}) error { + return GetJSON(tc.client, tc.url+path, v) +} diff --git a/lightning/common/security_test.go b/lightning/common/security_test.go new file mode 100644 index 000000000..b958d339f --- /dev/null +++ b/lightning/common/security_test.go @@ -0,0 +1,96 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package common_test + +import ( + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + + . "github.com/pingcap/check" + + "github.com/pingcap/tidb-lightning/lightning/common" +) + +type securitySuite struct{} + +var _ = Suite(&securitySuite{}) + +func respondPathHandler(w http.ResponseWriter, req *http.Request) { + io.WriteString(w, `{"path":"`) + io.WriteString(w, req.URL.Path) + io.WriteString(w, `"}`) +} + +func (s *securitySuite) TestGetJSONInsecure(c *C) { + mockServer := httptest.NewServer(http.HandlerFunc(respondPathHandler)) + defer mockServer.Close() + + u, err := url.Parse(mockServer.URL) + c.Assert(err, IsNil) + + tls, err := common.NewTLS("", "", "", u.Host) + c.Assert(err, IsNil) + + var result struct{ Path string } + err = tls.GetJSON("/aaa", &result) + c.Assert(err, IsNil) + c.Assert(result.Path, Equals, "/aaa") + err = tls.GetJSON("/bbbb", &result) + c.Assert(err, IsNil) + c.Assert(result.Path, Equals, "/bbbb") +} + +func (s *securitySuite) TestGetJSONSecure(c *C) { + mockServer := httptest.NewTLSServer(http.HandlerFunc(respondPathHandler)) + defer mockServer.Close() + + tls := common.NewTLSFromMockServer(mockServer) + + var result struct{ Path string } + err := tls.GetJSON("/ccc", &result) + c.Assert(err, IsNil) + c.Assert(result.Path, Equals, "/ccc") + err = tls.GetJSON("/dddd", &result) + c.Assert(err, IsNil) + c.Assert(result.Path, Equals, "/dddd") +} + +func (s *securitySuite) TestInvalidTLS(c *C) { + tempDir := c.MkDir() + + caPath := filepath.Join(tempDir, "ca.pem") + _, err := common.NewTLS(caPath, "", "", "localhost") + c.Assert(err, ErrorMatches, "could not read ca certificate:.*") + + err = ioutil.WriteFile(caPath, []byte("invalid ca content"), 0644) + c.Assert(err, IsNil) + _, err = common.NewTLS(caPath, "", "", "localhost") + c.Assert(err, ErrorMatches, "failed to append ca certs") + + certPath := filepath.Join(tempDir, "test.pem") + keyPath := filepath.Join(tempDir, "test.key") + _, err = common.NewTLS(caPath, certPath, keyPath, "localhost") + c.Assert(err, ErrorMatches, "could not load client key pair: open.*") + + err = ioutil.WriteFile(certPath, []byte("invalid cert content"), 0644) + c.Assert(err, IsNil) + err = ioutil.WriteFile(keyPath, []byte("invalid key content"), 0600) + c.Assert(err, IsNil) + _, err = common.NewTLS(caPath, certPath, keyPath, "localhost") + c.Assert(err, ErrorMatches, "could not load client key pair: tls.*") +} diff --git a/lightning/common/util.go b/lightning/common/util.go index ef4ab2d49..581573a14 100644 --- a/lightning/common/util.go +++ b/lightning/common/util.go @@ -46,13 +46,25 @@ const ( defaultMaxRetry = 3 ) -func ToDSN(host string, port int, user string, psw string, sqlMode string, maxAllowedPacket uint64) string { - return fmt.Sprintf("%s:%s@tcp(%s:%d)/?charset=utf8&sql_mode='%s'&maxAllowedPacket=%d", user, psw, host, port, sqlMode, maxAllowedPacket) +// MySQLConnectParam records the parameters needed to connect to a MySQL database. +type MySQLConnectParam struct { + Host string + Port int + User string + Password string + SQLMode string + MaxAllowedPacket uint64 + TLS string } -func ConnectDB(host string, port int, user string, psw string, sqlMode string, maxAllowedPacket uint64) (*sql.DB, error) { - dbDSN := ToDSN(host, port, user, psw, sqlMode, maxAllowedPacket) - db, err := sql.Open("mysql", dbDSN) +func (param *MySQLConnectParam) ToDSN() string { + return fmt.Sprintf("%s:%s@tcp(%s:%d)/?charset=utf8mb4&sql_mode='%s'&maxAllowedPacket=%d&tls=%s", + param.User, param.Password, param.Host, param.Port, + param.SQLMode, param.MaxAllowedPacket, param.TLS) +} + +func (param *MySQLConnectParam) Connect() (*sql.DB, error) { + db, err := sql.Open("mysql", param.ToDSN()) if err != nil { return nil, errors.Trace(err) } diff --git a/lightning/common/util_test.go b/lightning/common/util_test.go index f848f4534..9ac69c827 100644 --- a/lightning/common/util_test.go +++ b/lightning/common/util_test.go @@ -28,10 +28,10 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" tmysql "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb-lightning/lightning/common" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-lightning/lightning/log" ) @@ -121,8 +121,16 @@ func (s *utilSuite) TestIsRetryableError(c *C) { } func (s *utilSuite) TestToDSN(c *C) { - dsn := common.ToDSN("127.0.0.1", 4000, "root", "123456", "strict", 1234) - c.Assert(dsn, Equals, "root:123456@tcp(127.0.0.1:4000)/?charset=utf8&sql_mode='strict'&maxAllowedPacket=1234") + param := common.MySQLConnectParam{ + Host: "127.0.0.1", + Port: 4000, + User: "root", + Password: "123456", + SQLMode: "strict", + MaxAllowedPacket: 1234, + TLS: "cluster", + } + c.Assert(param.ToDSN(), Equals, "root:123456@tcp(127.0.0.1:4000)/?charset=utf8mb4&sql_mode='strict'&maxAllowedPacket=1234&tls=cluster") } func (s *utilSuite) TestIsContextCanceledError(c *C) { diff --git a/lightning/config/config.go b/lightning/config/config.go index 9e020a988..6792b5250 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -16,12 +16,14 @@ package config import ( "encoding/json" "fmt" - "net/http" + "net" "runtime" + "strconv" "strings" "time" "github.com/BurntSushi/toml" + gomysql "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-lightning/lightning/common" @@ -59,13 +61,15 @@ const ( var defaultConfigPaths = []string{"tidb-lightning.toml", "conf/tidb-lightning.toml"} type DBStore struct { - Host string `toml:"host" json:"host"` - Port int `toml:"port" json:"port"` - User string `toml:"user" json:"user"` - Psw string `toml:"password" json:"-"` - StatusPort int `toml:"status-port" json:"status-port"` - PdAddr string `toml:"pd-addr" json:"pd-addr"` - StrSQLMode string `toml:"sql-mode" json:"sql-mode"` + Host string `toml:"host" json:"host"` + Port int `toml:"port" json:"port"` + User string `toml:"user" json:"user"` + Psw string `toml:"password" json:"-"` + StatusPort int `toml:"status-port" json:"status-port"` + PdAddr string `toml:"pd-addr" json:"pd-addr"` + StrSQLMode string `toml:"sql-mode" json:"sql-mode"` + TLS string `toml:"tls" json:"tls"` + Security *Security `toml:"security" json:"security"` SQLMode mysql.SQLMode `toml:"-" json:"-"` MaxAllowedPacket uint64 `toml:"max-allowed-packet" json:"max-allowed-packet"` @@ -89,6 +93,7 @@ type Config struct { PostRestore PostRestore `toml:"post-restore" json:"post-restore"` Cron Cron `toml:"cron" json:"cron"` Routes []*router.TableRule `toml:"routes" json:"routes"` + Security Security `toml:"security" json:"security"` } func (c *Config) String() string { @@ -99,6 +104,11 @@ func (c *Config) String() string { return string(bytes) } +func (c *Config) ToTLS() (*common.TLS, error) { + hostPort := net.JoinHostPort(c.TiDB.Host, strconv.Itoa(c.TiDB.StatusPort)) + return common.NewTLS(c.Security.CAPath, c.Security.CertPath, c.Security.KeyPath, hostPort) +} + type Lightning struct { TableConcurrency int `toml:"table-concurrency" json:"table-concurrency"` IndexConcurrency int `toml:"index-concurrency" json:"index-concurrency"` @@ -155,6 +165,31 @@ type Cron struct { LogProgress Duration `toml:"log-progress" json:"log-progress"` } +type Security struct { + CAPath string `toml:"ca-path" json:"ca-path"` + CertPath string `toml:"cert-path" json:"cert-path"` + KeyPath string `toml:"key-path" json:"key-path"` +} + +// RegistersMySQL registers (or deregisters) the TLS config with name "cluster" +// for use in `sql.Open()`. This method is goroutine-safe. +func (sec *Security) RegisterMySQL() error { + if sec == nil { + return nil + } + tlsConfig, err := common.ToTLSConfig(sec.CAPath, sec.CertPath, sec.KeyPath) + switch { + case err != nil: + return err + case tlsConfig != nil: + // error happens only when the key coincides with the built-in names. + _ = gomysql.RegisterTLSConfig("cluster", tlsConfig) + default: + gomysql.DeregisterTLSConfig("cluster") + } + return nil +} + // A duration which can be deserialized from a TOML string. // Implemented as https://github.com/BurntSushi/toml#using-the-encodingtextunmarshaler-interface type Duration struct { @@ -205,7 +240,7 @@ func NewConfig() *Config { Delimiter: `"`, Header: true, NotNull: false, - Null: `\n`, + Null: `\N`, BackslashEscape: true, TrimLastSep: false, }, @@ -242,6 +277,7 @@ func (cfg *Config) LoadFromGlobal(global *GlobalConfig) error { cfg.PostRestore.Checksum = global.PostRestore.Checksum cfg.PostRestore.Analyze = global.PostRestore.Analyze cfg.App.CheckRequirements = global.App.CheckRequirements + cfg.Security = global.Security return nil } @@ -367,6 +403,27 @@ func (cfg *Config) Adjust() error { return errors.Annotate(err, "invalid config: `mydumper.tidb.sql_mode` must be a valid SQL_MODE") } + if cfg.TiDB.Security == nil { + cfg.TiDB.Security = &cfg.Security + } + + switch cfg.TiDB.TLS { + case "": + if len(cfg.TiDB.Security.CAPath) > 0 { + cfg.TiDB.TLS = "cluster" + } else { + cfg.TiDB.TLS = "false" + } + case "cluster": + if len(cfg.Security.CAPath) == 0 { + return errors.New("invalid config: cannot set `tidb.tls` to 'cluster' without a [security] section") + } + case "false", "skip-verify", "preferred": + break + default: + return errors.Errorf("invalid config: unsupported `tidb.tls` config %s", cfg.TiDB.TLS) + } + cfg.BWList.IgnoreDBs = append(cfg.BWList.IgnoreDBs, "mysql", "information_schema", @@ -385,18 +442,15 @@ func (cfg *Config) Adjust() error { // automatically determine the TiDB port & PD address from TiDB settings if cfg.TiDB.Port <= 0 || len(cfg.TiDB.PdAddr) == 0 { - resp, err := http.Get(fmt.Sprintf("http://%s:%d/settings", cfg.TiDB.Host, cfg.TiDB.StatusPort)) + tls, err := cfg.ToTLS() if err != nil { - return errors.Annotate(err, "cannot fetch settings from TiDB, please manually fill in `tidb.port` and `tidb.pd-addr`") - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return errors.Errorf("TiDB settings returned %s, please manually fill in `tidb.port` and `tidb.pd-addr`", resp.Status) + return err } + var settings tidbcfg.Config - err = json.NewDecoder(resp.Body).Decode(&settings) + err = tls.GetJSON("/settings", &settings) if err != nil { - return errors.Annotate(err, "cannot decode settings from TiDB, please manually fill in `tidb.port` and `tidb.pd-addr`") + return errors.Annotate(err, "cannot fetch settings from TiDB, please manually fill in `tidb.port` and `tidb.pd-addr`") } if cfg.TiDB.Port <= 0 { cfg.TiDB.Port = int(settings.Port) @@ -436,7 +490,16 @@ func (cfg *Config) Adjust() error { if len(cfg.Checkpoint.DSN) == 0 { switch cfg.Checkpoint.Driver { case CheckpointDriverMySQL: - cfg.Checkpoint.DSN = common.ToDSN(cfg.TiDB.Host, cfg.TiDB.Port, cfg.TiDB.User, cfg.TiDB.Psw, mysql.DefaultSQLMode, defaultMaxAllowedPacket) + param := common.MySQLConnectParam{ + Host: cfg.TiDB.Host, + Port: cfg.TiDB.Port, + User: cfg.TiDB.User, + Password: cfg.TiDB.Psw, + SQLMode: mysql.DefaultSQLMode, + MaxAllowedPacket: defaultMaxAllowedPacket, + TLS: cfg.TiDB.TLS, + } + cfg.Checkpoint.DSN = param.ToDSN() case CheckpointDriverFile: cfg.Checkpoint.DSN = "/tmp/" + cfg.Checkpoint.Schema + ".pb" } diff --git a/lightning/config/config_test.go b/lightning/config/config_test.go index 86cc59632..13e75f952 100644 --- a/lightning/config/config_test.go +++ b/lightning/config/config_test.go @@ -102,7 +102,7 @@ func (s *configTestSuite) TestAdjustPageNotFound(c *C) { cfg.TiDB.StatusPort = port err := cfg.Adjust() - c.Assert(err, ErrorMatches, ".*404 Not Found.*") + c.Assert(err, ErrorMatches, "cannot fetch settings from TiDB.*") } func (s *configTestSuite) TestAdjustConnectRefused(c *C) { @@ -134,7 +134,7 @@ func (s *configTestSuite) TestDecodeError(c *C) { cfg.TiDB.StatusPort = port err := cfg.Adjust() - c.Assert(err, ErrorMatches, "cannot decode settings from TiDB.*") + c.Assert(err, ErrorMatches, "cannot fetch settings from TiDB.*") } func (s *configTestSuite) TestInvalidSetting(c *C) { @@ -180,6 +180,87 @@ func (s *configTestSuite) TestAdjustWillBatchImportRatioInvalid(c *C) { c.Assert(cfg.Mydumper.BatchImportRatio, Equals, 0.75) } +func (s *configTestSuite) TestAdjustSecuritySection(c *C) { + testCases := []struct { + input string + expectedCA string + expectedTLS string + }{ + { + input: ``, + expectedCA: "", + expectedTLS: "false", + }, + { + input: ` + [security] + `, + expectedCA: "", + expectedTLS: "false", + }, + { + input: ` + [security] + ca-path = "/path/to/ca.pem" + `, + expectedCA: "/path/to/ca.pem", + expectedTLS: "cluster", + }, + { + input: ` + [security] + ca-path = "/path/to/ca.pem" + [tidb.security] + `, + expectedCA: "", + expectedTLS: "false", + }, + { + input: ` + [security] + ca-path = "/path/to/ca.pem" + [tidb.security] + ca-path = "/path/to/ca2.pem" + `, + expectedCA: "/path/to/ca2.pem", + expectedTLS: "cluster", + }, + { + input: ` + [security] + [tidb.security] + ca-path = "/path/to/ca2.pem" + `, + expectedCA: "/path/to/ca2.pem", + expectedTLS: "cluster", + }, + { + input: ` + [security] + [tidb] + tls = "skip-verify" + [tidb.security] + `, + expectedCA: "", + expectedTLS: "skip-verify", + }, + } + + for _, tc := range testCases { + comment := Commentf("input = %s", tc.input) + + cfg := config.NewConfig() + assignMinimalLegalValue(cfg) + err := cfg.LoadFromTOML([]byte(tc.input)) + c.Assert(err, IsNil, comment) + + err = cfg.Adjust() + c.Assert(err, IsNil, comment) + c.Assert(cfg.TiDB.Security.CAPath, Equals, tc.expectedCA, comment) + c.Assert(cfg.TiDB.TLS, Equals, tc.expectedTLS, comment) + } +} + func (s *configTestSuite) TestInvalidCSV(c *C) { testCases := []struct { input string @@ -400,7 +481,7 @@ func (s *configTestSuite) TestLoadConfig(c *C) { taskCfg.Checkpoint.Driver = config.CheckpointDriverMySQL err = taskCfg.Adjust() c.Assert(err, IsNil) - c.Assert(taskCfg.Checkpoint.DSN, Equals, "guest:12345@tcp(172.16.30.11:4001)/?charset=utf8&sql_mode='"+mysql.DefaultSQLMode+"'&maxAllowedPacket=67108864") + c.Assert(taskCfg.Checkpoint.DSN, Equals, "guest:12345@tcp(172.16.30.11:4001)/?charset=utf8mb4&sql_mode='"+mysql.DefaultSQLMode+"'&maxAllowedPacket=67108864&tls=false") result := taskCfg.String() c.Assert(result, Matches, `.*"pd-addr":"172.16.30.11:2379,172.16.30.12:2379".*`) diff --git a/lightning/config/global.go b/lightning/config/global.go index 90ebce64a..99665c977 100644 --- a/lightning/config/global.go +++ b/lightning/config/global.go @@ -63,6 +63,7 @@ type GlobalConfig struct { Mydumper GlobalMydumper `toml:"mydumper" json:"mydumper"` TikvImporter GlobalImporter `toml:"tikv-importer" json:"tikv-importer"` PostRestore GlobalPostRestore `toml:"post-restore" json:"post-restore"` + Security Security `toml:"security" json:"security"` ConfigFileContent []byte } @@ -144,6 +145,9 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon checksum := fs.Bool("checksum", true, "compare checksum after importing") analyze := fs.Bool("analyze", true, "analyze table after importing") checkRequirements := fs.Bool("check-requirements", true, "check cluster version before starting") + tlsCAPath := fs.String("ca", "", "CA certificate path for TLS connection") + tlsCertPath := fs.String("cert", "", "certificate path for TLS connection") + tlsKeyPath := fs.String("key", "", "private key path for TLS connection") statusAddr := fs.String("status-addr", "", "the Lightning server address") serverMode := fs.Bool("server-mode", false, "start Lightning in server mode, wait for multiple tasks instead of starting immediately") @@ -228,6 +232,15 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon if !*checkRequirements { cfg.App.CheckRequirements = false } + if *tlsCAPath != "" { + cfg.Security.CAPath = *tlsCAPath + } + if *tlsCertPath != "" { + cfg.Security.CertPath = *tlsCertPath + } + if *tlsKeyPath != "" { + cfg.Security.KeyPath = *tlsKeyPath + } if cfg.App.StatusAddr == "" && cfg.App.ServerMode { return nil, errors.New("If server-mode is enabled, the status-addr must be a valid listen address") diff --git a/lightning/lightning.go b/lightning/lightning.go index 1500900e2..bdee5c725 100644 --- a/lightning/lightning.go +++ b/lightning/lightning.go @@ -44,6 +44,7 @@ import ( type Lightning struct { globalCfg *config.GlobalConfig + globalTLS *common.TLS // taskCfgs is the list of task configurations enqueued in the server mode taskCfgs *config.ConfigList ctx context.Context @@ -66,9 +67,15 @@ func New(globalCfg *config.GlobalConfig) *Lightning { os.Exit(1) } + tls, err := common.NewTLS(globalCfg.Security.CAPath, globalCfg.Security.CertPath, globalCfg.Security.KeyPath, globalCfg.App.StatusAddr) + if err != nil { + log.L().Fatal("failed to load TLS certificates", zap.Error(err)) + } + ctx, shutdown := context.WithCancel(context.Background()) return &Lightning{ globalCfg: globalCfg, + globalTLS: tls, ctx: ctx, shutdown: shutdown, } @@ -114,6 +121,7 @@ func (l *Lightning) GoServe() error { } l.serverAddr = listener.Addr() l.server.Handler = mux + listener = l.globalTLS.WrapListener(listener) go func() { err := l.server.Serve(listener) diff --git a/lightning/restore/checkreq_test.go b/lightning/restore/checkreq_test.go index 661402ab1..5e400cbc1 100644 --- a/lightning/restore/checkreq_test.go +++ b/lightning/restore/checkreq_test.go @@ -23,6 +23,7 @@ import ( "github.com/coreos/go-semver/semver" . "github.com/pingcap/check" + "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-lightning/lightning/config" ) @@ -84,7 +85,7 @@ func (s *checkReqSuite) TestCheckVersion(c *C) { func (s *checkReqSuite) TestCheckTiDBVersion(c *C) { var version string - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { c.Assert(req.URL.Path, Equals, "/status") w.WriteHeader(http.StatusOK) err := json.NewEncoder(w).Encode(map[string]interface{}{ @@ -96,7 +97,6 @@ func (s *checkReqSuite) TestCheckTiDBVersion(c *C) { c.Assert(err, IsNil) mockPort, err := strconv.Atoi(mockURL.Port()) c.Assert(err, IsNil) - mockClient := mockServer.Client() rc := &RestoreController{ cfg: &config.Config{ @@ -105,19 +105,20 @@ func (s *checkReqSuite) TestCheckTiDBVersion(c *C) { StatusPort: mockPort, }, }, + tls: common.NewTLSFromMockServer(mockServer), } version = "5.7.25-TiDB-v9999.0.0" - c.Assert(rc.checkTiDBVersion(mockClient), IsNil) + c.Assert(rc.checkTiDBVersion(), IsNil) version = "5.7.25-TiDB-v1.0.0" - c.Assert(rc.checkTiDBVersion(mockClient), ErrorMatches, "TiDB version too old.*") + c.Assert(rc.checkTiDBVersion(), ErrorMatches, "TiDB version too old.*") } func (s *checkReqSuite) TestCheckPDVersion(c *C) { var version string - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { c.Assert(req.URL.Path, Equals, "/pd/api/v1/config/cluster-version") w.WriteHeader(http.StatusOK) err := json.NewEncoder(w).Encode(version) @@ -125,7 +126,6 @@ func (s *checkReqSuite) TestCheckPDVersion(c *C) { })) mockURL, err := url.Parse(mockServer.URL) c.Assert(err, IsNil) - mockClient := mockServer.Client() rc := &RestoreController{ cfg: &config.Config{ @@ -133,19 +133,20 @@ func (s *checkReqSuite) TestCheckPDVersion(c *C) { PdAddr: mockURL.Host, }, }, + tls: common.NewTLSFromMockServer(mockServer), } version = "9999.0.0" - c.Assert(rc.checkPDVersion(mockClient), IsNil) + c.Assert(rc.checkPDVersion(), IsNil) version = "1.0.0" - c.Assert(rc.checkPDVersion(mockClient), ErrorMatches, "PD version too old.*") + c.Assert(rc.checkPDVersion(), ErrorMatches, "PD version too old.*") } func (s *checkReqSuite) TestCheckTiKVVersion(c *C) { var versions []string - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { c.Assert(req.URL.Path, Equals, "/pd/api/v1/stores") w.WriteHeader(http.StatusOK) @@ -166,7 +167,6 @@ func (s *checkReqSuite) TestCheckTiKVVersion(c *C) { })) mockURL, err := url.Parse(mockServer.URL) c.Assert(err, IsNil) - mockClient := mockServer.Client() rc := &RestoreController{ cfg: &config.Config{ @@ -174,11 +174,12 @@ func (s *checkReqSuite) TestCheckTiKVVersion(c *C) { PdAddr: mockURL.Host, }, }, + tls: common.NewTLSFromMockServer(mockServer), } versions = []string{"9999.0.0", "9999.0.0"} - c.Assert(rc.checkTiKVVersion(mockClient), IsNil) + c.Assert(rc.checkTiKVVersion(), IsNil) versions = []string{"9999.0.0", "1.0.0"} - c.Assert(rc.checkTiKVVersion(mockClient), ErrorMatches, `TiKV \(at tikv1\.test:20160\) version too old.*`) + c.Assert(rc.checkTiKVVersion(), ErrorMatches, `TiKV \(at tikv1\.test:20160\) version too old.*`) } diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index db1ef312c..8c5a74979 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -18,7 +18,6 @@ import ( "database/sql" "fmt" "io" - "net/http" "os" "path" "strings" @@ -144,6 +143,7 @@ type RestoreController struct { alterTableLock sync.Mutex compactState int32 rowFormatVer string + tls *common.TLS errorSummaries errorSummaries @@ -159,12 +159,20 @@ func NewRestoreController(ctx context.Context, dbMetas []*mydump.MDDatabaseMeta, } func NewRestoreControllerWithPauser(ctx context.Context, dbMetas []*mydump.MDDatabaseMeta, cfg *config.Config, pauser *common.Pauser) (*RestoreController, error) { + tls, err := cfg.ToTLS() + if err != nil { + return nil, err + } + if err = cfg.TiDB.Security.RegisterMySQL(); err != nil { + return nil, err + } + cpdb, err := OpenCheckpointsDB(ctx, cfg) if err != nil { return nil, errors.Trace(err) } - tidbMgr, err := NewTiDBManager(cfg.TiDB) + tidbMgr, err := NewTiDBManager(cfg.TiDB, tls) if err != nil { return nil, errors.Trace(err) } @@ -173,7 +181,7 @@ func NewRestoreControllerWithPauser(ctx context.Context, dbMetas []*mydump.MDDat switch cfg.TikvImporter.Backend { case config.BackendImporter: var err error - backend, err = kv.NewImporter(ctx, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) + backend, err = kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) if err != nil { return nil, err } @@ -194,6 +202,7 @@ func NewRestoreControllerWithPauser(ctx context.Context, dbMetas []*mydump.MDDat backend: backend, tidbMgr: tidbMgr, rowFormatVer: "1", + tls: tls, errorSummaries: makeErrorSummaries(log.L()), checkpointsDB: cpdb, @@ -278,7 +287,7 @@ outside: } func (rc *RestoreController) restoreSchema(ctx context.Context) error { - tidbMgr, err := NewTiDBManager(rc.cfg.TiDB) + tidbMgr, err := NewTiDBManager(rc.cfg.TiDB, rc.tls) if err != nil { return errors.Trace(err) } @@ -1063,13 +1072,13 @@ func (rc *RestoreController) fullCompact(ctx context.Context) error { } func (rc *RestoreController) doCompact(ctx context.Context, level int32) error { + tls := rc.tls.WithHost(rc.cfg.TiDB.PdAddr) return kv.ForAllStores( ctx, - &http.Client{}, - rc.cfg.TiDB.PdAddr, + tls, kv.StoreStateDisconnected, func(c context.Context, store *kv.Store) error { - return kv.Compact(c, store.Address, level) + return kv.Compact(c, tls, store.Address, level) }, ) } @@ -1094,16 +1103,15 @@ func (rc *RestoreController) switchTiKVMode(ctx context.Context, mode sstpb.Swit } else { minState = kv.StoreStateDisconnected } - + tls := rc.tls.WithHost(rc.cfg.TiDB.PdAddr) // we ignore switch mode failure since it is not fatal. // no need log the error, it is done in kv.SwitchMode already. _ = kv.ForAllStores( ctx, - &http.Client{}, - rc.cfg.TiDB.PdAddr, + tls, minState, func(c context.Context, store *kv.Store) error { - return kv.SwitchMode(c, store.Address, mode) + return kv.SwitchMode(c, tls, store.Address, mode) }, ) } @@ -1114,14 +1122,13 @@ func (rc *RestoreController) checkRequirements(_ context.Context) error { return nil } - client := &http.Client{} - if err := rc.checkTiDBVersion(client); err != nil { + if err := rc.checkTiDBVersion(); err != nil { return errors.Trace(err) } - if err := rc.checkPDVersion(client); err != nil { + if err := rc.checkPDVersion(); err != nil { return errors.Trace(err) } - if err := rc.checkTiKVVersion(client); err != nil { + if err := rc.checkTiKVVersion(); err != nil { return errors.Trace(err) } @@ -1152,10 +1159,9 @@ func extractTiDBVersion(version string) (*semver.Version, error) { return semver.NewVersion(rawVersion) } -func (rc *RestoreController) checkTiDBVersion(client *http.Client) error { - url := fmt.Sprintf("http://%s:%d/status", rc.cfg.TiDB.Host, rc.cfg.TiDB.StatusPort) +func (rc *RestoreController) checkTiDBVersion() error { var status struct{ Version string } - err := common.GetJSON(client, url, &status) + err := rc.tls.GetJSON("/status", &status) if err != nil { return errors.Trace(err) } @@ -1167,10 +1173,9 @@ func (rc *RestoreController) checkTiDBVersion(client *http.Client) error { return checkVersion("TiDB", requiredTiDBVersion, *version) } -func (rc *RestoreController) checkPDVersion(client *http.Client) error { - url := fmt.Sprintf("http://%s/pd/api/v1/config/cluster-version", rc.cfg.TiDB.PdAddr) +func (rc *RestoreController) checkPDVersion() error { var rawVersion string - err := common.GetJSON(client, url, &rawVersion) + err := rc.tls.WithHost(rc.cfg.TiDB.PdAddr).GetJSON("/pd/api/v1/config/cluster-version", &rawVersion) if err != nil { return errors.Trace(err) } @@ -1183,11 +1188,10 @@ func (rc *RestoreController) checkPDVersion(client *http.Client) error { return checkVersion("PD", requiredPDVersion, *version) } -func (rc *RestoreController) checkTiKVVersion(client *http.Client) error { +func (rc *RestoreController) checkTiKVVersion() error { return kv.ForAllStores( context.Background(), - client, - rc.cfg.TiDB.PdAddr, + rc.tls.WithHost(rc.cfg.TiDB.PdAddr), kv.StoreStateDown, func(c context.Context, store *kv.Store) error { component := fmt.Sprintf("TiKV (at %s)", store.Address) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index 1d386af0a..1305bcc41 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -17,8 +17,6 @@ import ( "context" "database/sql" "fmt" - "net/http" - "net/url" "strings" "github.com/pingcap/errors" @@ -37,37 +35,39 @@ import ( ) type TiDBManager struct { - db *sql.DB - client *http.Client - baseURL *url.URL - parser *parser.Parser + db *sql.DB + tls *common.TLS + parser *parser.Parser } -func NewTiDBManager(dsn config.DBStore) (*TiDBManager, error) { - db, err := common.ConnectDB(dsn.Host, dsn.Port, dsn.User, dsn.Psw, dsn.StrSQLMode, dsn.MaxAllowedPacket) +func NewTiDBManager(dsn config.DBStore, tls *common.TLS) (*TiDBManager, error) { + param := common.MySQLConnectParam{ + Host: dsn.Host, + Port: dsn.Port, + User: dsn.User, + Password: dsn.Psw, + SQLMode: dsn.StrSQLMode, + MaxAllowedPacket: dsn.MaxAllowedPacket, + TLS: dsn.TLS, + } + db, err := param.Connect() if err != nil { return nil, errors.Trace(err) } - u, err := url.Parse(fmt.Sprintf("http://%s:%d", dsn.Host, dsn.StatusPort)) - if err != nil { - return nil, errors.Trace(err) - } - - return NewTiDBManagerWithDB(db, u, dsn.SQLMode), nil + return NewTiDBManagerWithDB(db, tls, dsn.SQLMode), nil } // NewTiDBManagerWithDB creates a new TiDB manager with an existing database // connection. -func NewTiDBManagerWithDB(db *sql.DB, baseURL *url.URL, sqlMode mysql.SQLMode) *TiDBManager { +func NewTiDBManagerWithDB(db *sql.DB, tls *common.TLS, sqlMode mysql.SQLMode) *TiDBManager { parser := parser.New() parser.SetSQLMode(sqlMode) return &TiDBManager{ - db: db, - client: &http.Client{}, - baseURL: baseURL, - parser: parser, + db: db, + tls: tls, + parser: parser, } } @@ -145,11 +145,8 @@ func (timgr *TiDBManager) createTableIfNotExistsStmt(createTable, tblName string } func (timgr *TiDBManager) getTables(schema string) ([]*model.TableInfo, error) { - baseURL := *timgr.baseURL - baseURL.Path = fmt.Sprintf("schema/%s", schema) - var tables []*model.TableInfo - err := common.GetJSON(timgr.client, baseURL.String(), &tables) + err := timgr.tls.GetJSON("/schema/"+schema, &tables) if err != nil { return nil, errors.Annotatef(err, "get tables for schema %s", schema) } diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index 75cb3af61..fc2032dc8 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -19,7 +19,6 @@ import ( "errors" "net/http" "net/http/httptest" - "net/url" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -28,10 +27,12 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" tmysql "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb-lightning/lightning/checkpoints" - "github.com/pingcap/tidb-lightning/lightning/mydump" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/util/mock" + + "github.com/pingcap/tidb-lightning/lightning/checkpoints" + "github.com/pingcap/tidb-lightning/lightning/common" + "github.com/pingcap/tidb-lightning/lightning/mydump" ) var _ = Suite(&tidbSuite{}) @@ -52,17 +53,15 @@ func (s *tidbSuite) SetUpTest(c *C) { c.Assert(err, IsNil) s.mockDB = mock - s.mockHTTP = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + s.mockHTTP = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { s.handler.ServeHTTP(w, req) })) defaultSQLMode, err := tmysql.GetSQLMode(tmysql.DefaultSQLMode) c.Assert(err, IsNil) - baseURL, err := url.Parse(s.mockHTTP.URL) - c.Assert(err, IsNil) - - s.timgr = NewTiDBManagerWithDB(db, baseURL, defaultSQLMode) + tls := common.NewTLSFromMockServer(s.mockHTTP) + s.timgr = NewTiDBManagerWithDB(db, tls, defaultSQLMode) } func (s *tidbSuite) TearDownTest(c *C) { diff --git a/lightning/verification/checksum_test.go b/lightning/verification/checksum_test.go index a6740ebda..fe5e95875 100644 --- a/lightning/verification/checksum_test.go +++ b/lightning/verification/checksum_test.go @@ -19,8 +19,8 @@ import ( . "github.com/pingcap/check" - "github.com/pingcap/tidb-lightning/lightning/verification" "github.com/pingcap/tidb-lightning/lightning/common" + "github.com/pingcap/tidb-lightning/lightning/verification" ) type testKVChcksumSuite struct{} diff --git a/tests/README.md b/tests/README.md index e44999c11..b26e71b38 100644 --- a/tests/README.md +++ b/tests/README.md @@ -17,8 +17,8 @@ programs. 2. The following programs must be installed: * `mysql` (the CLI client) - * `python2.7` - * `curl` + * `wget` + * `openssl` 3. The user executing the tests must have permission to create the folder `/tmp/lightning_test_result`. All test artifacts will be written into this folder. diff --git a/tests/_utils/run_curl b/tests/_utils/run_curl new file mode 100755 index 000000000..e87397a75 --- /dev/null +++ b/tests/_utils/run_curl @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: run_curl https://url '{"json":"data"}' + +set -eu +TEST_DIR=/tmp/lightning_test_result + +if [ -z "${2-}" ]; then + POST_ARGS= +else + POST_ARGS="--post-data $2" +fi + +# FIXME: use `wget` instead of `curl` because the latter rejects ECC certs on our CI. +# CentOS is re~~ally old. +wget -q -O - \ + --ca-certificate="$TEST_DIR/tls/ca.pem" \ + --certificate="$TEST_DIR/tls/curl.pem" \ + --private-key="$TEST_DIR/tls/curl.key" \ + $POST_ARGS "$1" diff --git a/tests/_utils/run_lightning b/tests/_utils/run_lightning index 3d106f3f4..60f273848 100755 --- a/tests/_utils/run_lightning +++ b/tests/_utils/run_lightning @@ -15,8 +15,18 @@ set -eu TEST_DIR=/tmp/lightning_test_result -CONFIG="${1-config}" -shift || true -echo "[$(date)] <<<<<< RUNNING TEST FOR: tests/$TEST_NAME/$CONFIG.toml >>>>>>" >> "$TEST_DIR/lightning.log" -bin/tidb-lightning.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out" DEVEL -config "tests/$TEST_NAME/$CONFIG.toml" "$@" +echo "[$(date)] <<<<<< RUNNING TEST FOR: tests/$TEST_NAME $@ >>>>>>" >> "$TEST_DIR/lightning.log" +bin/tidb-lightning.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out" DEVEL \ + --ca "$TEST_DIR/tls/ca.pem" \ + --cert "$TEST_DIR/tls/lightning.pem" \ + --key "$TEST_DIR/tls/lightning.key" \ + --log-file "$TEST_DIR/lightning.log" \ + --tidb-port 4000 \ + --pd-urls '127.0.0.1:2379' \ + --config "tests/$TEST_NAME/config.toml" \ + -d "tests/$TEST_NAME/data" \ + --importer '127.0.0.1:8808' \ + --enable-checkpoint=0 \ + --check-requirements=0 \ + "$@" diff --git a/tests/_utils/run_lightning_ctl b/tests/_utils/run_lightning_ctl index 4bb35ffa7..b972eef22 100755 --- a/tests/_utils/run_lightning_ctl +++ b/tests/_utils/run_lightning_ctl @@ -16,7 +16,15 @@ set -eu TEST_DIR=/tmp/lightning_test_result -CONFIG="$1" -shift - -bin/tidb-lightning-ctl.test -test.coverprofile="$TEST_DIR/cov.ctl.$TEST_NAME.$$.out" DEVEL -config "tests/$TEST_NAME/$CONFIG.toml" "$@" +bin/tidb-lightning-ctl.test -test.coverprofile="$TEST_DIR/cov.ctl.$TEST_NAME.$$.out" DEVEL \ + --ca "$TEST_DIR/tls/ca.pem" \ + --cert "$TEST_DIR/tls/lightning.pem" \ + --key "$TEST_DIR/tls/lightning.key" \ + --log-file "$TEST_DIR/lightning.log" \ + --tidb-port 4000 \ + --pd-urls '127.0.0.1:2379' \ + -d "tests/$TEST_NAME/data" \ + --importer '127.0.0.1:8808' \ + --enable-checkpoint=0 \ + --check-requirements=0 \ + "$@" diff --git a/tests/_utils/run_sql b/tests/_utils/run_sql index 6e83de706..a856a1bd4 100755 --- a/tests/_utils/run_sql +++ b/tests/_utils/run_sql @@ -17,4 +17,8 @@ set -eu TEST_DIR=/tmp/lightning_test_result echo "[$(date)] Executing SQL: $1" > "$TEST_DIR/sql_res.$TEST_NAME.txt" -mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "$1" >> "$TEST_DIR/sql_res.$TEST_NAME.txt" +mysql -uroot -h127.0.0.1 -P4000 \ + --ssl-ca="$TEST_DIR/tls/ca.pem" \ + --ssl-cert="$TEST_DIR/tls/curl.pem" \ + --ssl-key="$TEST_DIR/tls/curl.key" \ + --default-character-set utf8 -E -e "$1" >> "$TEST_DIR/sql_res.$TEST_NAME.txt" diff --git a/tests/black-white-list/even-table-only.toml b/tests/black-white-list/even-table-only.toml index 3344575b3..86c15fbaa 100644 --- a/tests/black-white-list/even-table-only.toml +++ b/tests/black-white-list/even-table-only.toml @@ -1,8 +1,3 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - [[black-white-list.ignore-tables]] db-name = "firstdb" tbl-name = "~." @@ -14,22 +9,3 @@ tbl-name = "second" [[black-white-list.do-tables]] db-name = "seconddb" tbl-name = "fourth" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/black-white-list/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/black-white-list/firstdb-only.toml b/tests/black-white-list/firstdb-only.toml index 999168bca..7dc796a3e 100644 --- a/tests/black-white-list/firstdb-only.toml +++ b/tests/black-white-list/firstdb-only.toml @@ -1,26 +1,2 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - [black-white-list] do-dbs = ["~^f"] - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/black-white-list/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/black-white-list/run.sh b/tests/black-white-list/run.sh index e3cadee6e..be22ef10d 100755 --- a/tests/black-white-list/run.sh +++ b/tests/black-white-list/run.sh @@ -19,7 +19,7 @@ set -eux run_sql 'DROP DATABASE IF EXISTS firstdb;' run_sql 'DROP DATABASE IF EXISTS seconddb;' -run_lightning 'firstdb-only' +run_lightning --config "tests/$TEST_NAME/firstdb-only.toml" run_sql 'SHOW DATABASES;' check_contains 'Database: firstdb' check_not_contains 'Database: seconddb' @@ -31,7 +31,7 @@ check_not_contains 'Tables_in_mysql: testtable' run_sql 'DROP DATABASE IF EXISTS firstdb;' run_sql 'DROP DATABASE IF EXISTS seconddb;' -run_lightning 'even-table-only' +run_lightning --config "tests/$TEST_NAME/even-table-only.toml" run_sql 'SHOW DATABASES;' check_contains 'Database: firstdb' check_contains 'Database: seconddb' diff --git a/tests/character_sets/auto.toml b/tests/character_sets/auto.toml new file mode 100644 index 000000000..4498544d1 --- /dev/null +++ b/tests/character_sets/auto.toml @@ -0,0 +1,5 @@ +[lightning] +table-concurrency = 1 + +[mydumper] +character-set = "auto" diff --git a/tests/character_sets/binary.toml b/tests/character_sets/binary.toml new file mode 100644 index 000000000..b26ede355 --- /dev/null +++ b/tests/character_sets/binary.toml @@ -0,0 +1,5 @@ +[lightning] +table-concurrency = 1 + +[mydumper] +character-set = "binary" diff --git a/tests/character_sets/gb18030-auto.toml b/tests/character_sets/gb18030-auto.toml deleted file mode 100644 index 00315a659..000000000 --- a/tests/character_sets/gb18030-auto.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/gb18030" -character-set = "auto" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/gb18030-binary.toml b/tests/character_sets/gb18030-binary.toml deleted file mode 100644 index 50d469ef5..000000000 --- a/tests/character_sets/gb18030-binary.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/gb18030" -character-set = "binary" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/gb18030-gb18030.toml b/tests/character_sets/gb18030-gb18030.toml deleted file mode 100644 index cd7e230b0..000000000 --- a/tests/character_sets/gb18030-gb18030.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/gb18030" -character-set = "gb18030" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/gb18030-utf8mb4.toml b/tests/character_sets/gb18030-utf8mb4.toml deleted file mode 100644 index 81bcf144a..000000000 --- a/tests/character_sets/gb18030-utf8mb4.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/gb18030" -character-set = "utf8mb4" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/gb18030.toml b/tests/character_sets/gb18030.toml new file mode 100644 index 000000000..87628eb45 --- /dev/null +++ b/tests/character_sets/gb18030.toml @@ -0,0 +1,5 @@ +[lightning] +table-concurrency = 1 + +[mydumper] +character-set = "gb18030" diff --git a/tests/character_sets/mixed-auto.toml b/tests/character_sets/mixed-auto.toml deleted file mode 100644 index 2af6ffe5d..000000000 --- a/tests/character_sets/mixed-auto.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/mixed" -character-set = "auto" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/mixed-binary.toml b/tests/character_sets/mixed-binary.toml deleted file mode 100644 index 6623f0f3b..000000000 --- a/tests/character_sets/mixed-binary.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/mixed" -character-set = "binary" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/mixed-gb18030.toml b/tests/character_sets/mixed-gb18030.toml deleted file mode 100644 index 1545c7f26..000000000 --- a/tests/character_sets/mixed-gb18030.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/mixed" -character-set = "gb18030" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/mixed-utf8mb4.toml b/tests/character_sets/mixed-utf8mb4.toml deleted file mode 100644 index 0b54ecce4..000000000 --- a/tests/character_sets/mixed-utf8mb4.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/mixed" -character-set = "utf8mb4" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/run.sh b/tests/character_sets/run.sh index 5809044d2..36f15a57d 100755 --- a/tests/character_sets/run.sh +++ b/tests/character_sets/run.sh @@ -17,7 +17,7 @@ set -eux run_lightning_expecting_fail() { set +e - run_lightning "$1" + run_lightning "$@" ERRCODE=$? set -e [ "$ERRCODE" != 0 ] @@ -27,50 +27,50 @@ run_sql 'DROP DATABASE IF EXISTS charsets;' # gb18030 -run_lightning 'gb18030-auto' +run_lightning --config "tests/$TEST_NAME/auto.toml" -d "tests/$TEST_NAME/gb18030" run_sql 'SELECT sum(`主键`) AS s FROM charsets.gb18030' check_contains 's: 267' run_sql 'DROP TABLE charsets.gb18030;' -run_lightning 'gb18030-gb18030' +run_lightning --config "tests/$TEST_NAME/gb18030.toml" -d "tests/$TEST_NAME/gb18030" run_sql 'SELECT sum(`主键`) AS s FROM charsets.gb18030' check_contains 's: 267' run_sql 'DROP TABLE charsets.gb18030;' -run_lightning_expecting_fail 'gb18030-utf8mb4' +run_lightning_expecting_fail --config "tests/$TEST_NAME/utf8mb4.toml" -d "tests/$TEST_NAME/gb18030" -run_lightning 'gb18030-binary' +run_lightning --config "tests/$TEST_NAME/binary.toml" -d "tests/$TEST_NAME/gb18030" run_sql 'SELECT sum(`Ö÷¼ü`) AS s FROM charsets.gb18030' check_contains 's: 267' # utf8mb4 -run_lightning 'utf8mb4-auto' +run_lightning --config "tests/$TEST_NAME/auto.toml" -d "tests/$TEST_NAME/utf8mb4" run_sql 'SELECT sum(`主键`) AS s FROM charsets.utf8mb4' check_contains 's: 1119' run_sql 'DROP TABLE charsets.utf8mb4;' -run_lightning 'utf8mb4-gb18030' +run_lightning --config "tests/$TEST_NAME/gb18030.toml" -d "tests/$TEST_NAME/utf8mb4" run_sql 'SELECT sum(`涓婚敭`) AS s FROM charsets.utf8mb4' check_contains 's: 1119' run_sql 'DROP TABLE charsets.utf8mb4;' -run_lightning 'utf8mb4-utf8mb4' +run_lightning --config "tests/$TEST_NAME/utf8mb4.toml" -d "tests/$TEST_NAME/utf8mb4" run_sql 'SELECT sum(`主键`) AS s FROM charsets.utf8mb4' check_contains 's: 1119' run_sql 'DROP TABLE charsets.utf8mb4;' -run_lightning 'utf8mb4-binary' +run_lightning --config "tests/$TEST_NAME/binary.toml" -d "tests/$TEST_NAME/utf8mb4" run_sql 'SELECT sum(`主键`) AS s FROM charsets.utf8mb4' check_contains 's: 1119' # mixed -run_lightning_expecting_fail 'mixed-auto' -run_lightning_expecting_fail 'mixed-gb18030' -run_lightning_expecting_fail 'mixed-utf8mb4' +run_lightning_expecting_fail --config "tests/$TEST_NAME/auto.toml" -d "tests/$TEST_NAME/mixed" +run_lightning_expecting_fail --config "tests/$TEST_NAME/gb18030.toml" -d "tests/$TEST_NAME/mixed" +run_lightning_expecting_fail --config "tests/$TEST_NAME/utf8mb4.toml" -d "tests/$TEST_NAME/mixed" -run_lightning 'mixed-binary' +run_lightning --config "tests/$TEST_NAME/binary.toml" -d "tests/$TEST_NAME/mixed" run_sql 'SELECT sum(`唯一键`) AS s FROM charsets.mixed' check_contains 's: 5291' diff --git a/tests/character_sets/utf8mb4-auto.toml b/tests/character_sets/utf8mb4-auto.toml deleted file mode 100644 index a76cf87f4..000000000 --- a/tests/character_sets/utf8mb4-auto.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/utf8mb4" -character-set = "auto" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/utf8mb4-binary.toml b/tests/character_sets/utf8mb4-binary.toml deleted file mode 100644 index f54d9096b..000000000 --- a/tests/character_sets/utf8mb4-binary.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/utf8mb4" -character-set = "binary" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/utf8mb4-gb18030.toml b/tests/character_sets/utf8mb4-gb18030.toml deleted file mode 100644 index 69309734c..000000000 --- a/tests/character_sets/utf8mb4-gb18030.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/utf8mb4" -character-set = "gb18030" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/utf8mb4-utf8mb4.toml b/tests/character_sets/utf8mb4-utf8mb4.toml deleted file mode 100644 index d4732f198..000000000 --- a/tests/character_sets/utf8mb4-utf8mb4.toml +++ /dev/null @@ -1,28 +0,0 @@ -[lightning] -table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "error" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/character_sets/utf8mb4" -character-set = "utf8mb4" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/character_sets/utf8mb4.toml b/tests/character_sets/utf8mb4.toml new file mode 100644 index 000000000..0bf5ce26e --- /dev/null +++ b/tests/character_sets/utf8mb4.toml @@ -0,0 +1,5 @@ +[lightning] +table-concurrency = 1 + +[mydumper] +character-set = "utf8mb4" diff --git a/tests/check_requirements/config.toml b/tests/check_requirements/config.toml index 9cc8028ac..e69de29bb 100644 --- a/tests/check_requirements/config.toml +++ b/tests/check_requirements/config.toml @@ -1,23 +0,0 @@ -[lightning] -check-requirements = true -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/check_requirements/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = false -compact = false -analyze = false diff --git a/tests/check_requirements/run.sh b/tests/check_requirements/run.sh index 81853e20f..3a9c68638 100755 --- a/tests/check_requirements/run.sh +++ b/tests/check_requirements/run.sh @@ -13,25 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eu +set -eux + +curl_cluster_version() { + run_curl 'https://127.0.0.1:2379/pd/api/v1/config/cluster-version' "$@" +} # should be OK when the version is normal run_sql 'DROP DATABASE IF EXISTS checkreq' -run_lightning +run_lightning --check-requirements=1 -L warning # now try to reduce the version to below 2.1.0 -API='http://127.0.0.1:2379/pd/api/v1/config/cluster-version' -OLD_VERSION=$(curl "$API") +OLD_VERSION=$(curl_cluster_version) reset_cluster_version() { - curl "$API" --data-binary '{"cluster-version":'"$OLD_VERSION"'}' + curl_cluster_version '{"cluster-version":'"$OLD_VERSION"'}' } trap reset_cluster_version EXIT -curl "$API" --data-binary '{"cluster-version":"2.0.0"}' +curl_cluster_version '{"cluster-version":"2.0.0-fake.and.error.expected"}' run_sql 'DROP DATABASE IF EXISTS checkreq' set +e -run_lightning +run_lightning --check-requirements=1 -L warning ERRORCODE=$? set -e diff --git a/tests/checkpoint/config.toml b/tests/checkpoint/config.toml index 00affc67d..7d9a423e5 100644 --- a/tests/checkpoint/config.toml +++ b/tests/checkpoint/config.toml @@ -1,9 +1,5 @@ [lightning] -# pprof-port = 28423 table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" [checkpoint] enable = true @@ -11,22 +7,5 @@ schema = "tidb_lightning_checkpoint_test_cppk" driver = "mysql" keep-after-success = true -[tikv-importer] -addr = "127.0.0.1:8808" - [mydumper] -data-source-dir = "/tmp/lightning_test_result/cppk.mydump" read-block-size = 1 - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint/run.sh b/tests/checkpoint/run.sh index c7a33405f..3e0ecee49 100755 --- a/tests/checkpoint/run.sh +++ b/tests/checkpoint/run.sh @@ -69,7 +69,7 @@ run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cppk.1357924680 set +e for i in $(seq "$TABLE_COUNT"); do echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" - run_lightning 2> /dev/null + run_lightning -d "$DBPATH" --enable-checkpoint=1 2> /dev/null [ $? -ne 0 ] || exit 1 done set -e @@ -78,7 +78,7 @@ export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS" set +e for i in $(seq "$TABLE_COUNT"); do echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" - run_lightning 2> /dev/null + run_lightning -d "$DBPATH" --enable-checkpoint=1 2> /dev/null done set -e @@ -92,7 +92,7 @@ export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb-lightning/lig set +e for i in $(seq "$TABLE_COUNT"); do echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" - run_lightning 2> /dev/null + run_lightning -d "$DBPATH" --enable-checkpoint=1 2> /dev/null [ $? -ne 0 ] || exit 1 done set -e @@ -100,7 +100,7 @@ set -e # After everything is done, there should be no longer new calls to ImportEngine # (and thus `kill_lightning_after_one_import` will spare this final check) echo "******** Verify checkpoint no-op ********" -run_lightning +run_lightning -d "$DBPATH" --enable-checkpoint=1 run_sql "$PARTIAL_IMPORT_QUERY" check_contains "s: $(( (1000 * $CHUNK_COUNT + 1001) * $CHUNK_COUNT * $TABLE_COUNT ))" run_sql 'SELECT count(*) FROM `tidb_lightning_checkpoint_test_cppk.1357924680.bak`.table_v5 WHERE status >= 200' diff --git a/tests/checkpoint_chunks/config.toml b/tests/checkpoint_chunks/config.toml index 00a96ae94..3d0dd64ce 100644 --- a/tests/checkpoint_chunks/config.toml +++ b/tests/checkpoint_chunks/config.toml @@ -1,30 +1,8 @@ [lightning] region-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" [checkpoint] enable = true schema = "tidb_lightning_checkpoint_test_cpch" driver = "mysql" keep-after-success = true - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "/tmp/lightning_test_result/cpch.mydump" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_chunks/file.toml b/tests/checkpoint_chunks/file.toml index beb90c34f..635ee3801 100644 --- a/tests/checkpoint_chunks/file.toml +++ b/tests/checkpoint_chunks/file.toml @@ -1,8 +1,5 @@ [lightning] region-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" [checkpoint] enable = true @@ -10,22 +7,3 @@ schema = "tidb_lightning_checkpoint_test_cpch" driver = "file" dsn = "/tmp/lightning_test_result/cpch.pb" keep-after-success = true - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "/tmp/lightning_test_result/cpch.mydump" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_chunks/run.sh b/tests/checkpoint_chunks/run.sh index 97427e315..0c6468bae 100755 --- a/tests/checkpoint_chunks/run.sh +++ b/tests/checkpoint_chunks/run.sh @@ -20,11 +20,15 @@ DBPATH="$TEST_DIR/cpch.mydump" CHUNK_COUNT=5 ROW_COUNT=1000 +do_run_lightning() { + run_lightning -d "$DBPATH" --enable-checkpoint=1 --config "tests/$TEST_NAME/$1.toml" +} + verify_checkpoint_noop() { # After everything is done, there should be no longer new calls to WriteEngine/CloseAndRecv # (and thus `kill_lightning_after_one_chunk` will spare this final check) echo "******** Verify checkpoint no-op ********" - run_lightning + do_run_lightning config run_sql 'SELECT count(i), sum(i) FROM cpch_tsr.tbl;' check_contains "count(i): $(($ROW_COUNT*$CHUNK_COUNT))" check_contains "sum(i): $(( $ROW_COUNT*$CHUNK_COUNT*(($CHUNK_COUNT+2)*$ROW_COUNT + 1)/2 ))" @@ -57,7 +61,7 @@ run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cpch.1234567890 set +e for i in $(seq "$CHUNK_COUNT"); do echo "******** Importing Chunk Now (step $i/$CHUNK_COUNT) ********" - run_lightning 2> /dev/null + do_run_lightning config 2> /dev/null [ $? -ne 0 ] || exit 1 done set -e @@ -75,7 +79,7 @@ export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/tidb-lightning/light for i in $(seq "$CHUNK_COUNT"); do echo "******** Importing Chunk Now (step $i/$CHUNK_COUNT) ********" - run_lightning + do_run_lightning config done set +e @@ -109,13 +113,13 @@ export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/tidb-lightning/light set +e for i in $(seq "$CHUNK_COUNT"); do echo "******** Importing Chunk using File checkpoint Now (step $i/$CHUNK_COUNT) ********" - run_lightning file 2> /dev/null + do_run_lightning file 2> /dev/null [ $? -ne 0 ] || exit 1 done set -e echo "******** Verify File checkpoint no-op ********" -run_lightning file +do_run_lightning file run_sql 'SELECT count(i), sum(i) FROM cpch_tsr.tbl;' check_contains "count(i): $(($ROW_COUNT*$CHUNK_COUNT))" check_contains "sum(i): $(( $ROW_COUNT*$CHUNK_COUNT*(($CHUNK_COUNT+2)*$ROW_COUNT + 1)/2 ))" diff --git a/tests/checkpoint_engines/config.toml b/tests/checkpoint_engines/config.toml index 1b90360c5..412e5a01f 100644 --- a/tests/checkpoint_engines/config.toml +++ b/tests/checkpoint_engines/config.toml @@ -1,29 +1,9 @@ [lightning] table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning-checkpoint-engines.log" -level = "info" [checkpoint] enable = true driver = "file" -[tikv-importer] -addr = "127.0.0.1:8808" - [mydumper] -data-source-dir = "tests/checkpoint_engines/data" batch-size = 50 # force splitting the data into 4 batches - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_engines/mysql.toml b/tests/checkpoint_engines/mysql.toml index ff00fad82..4beafd480 100644 --- a/tests/checkpoint_engines/mysql.toml +++ b/tests/checkpoint_engines/mysql.toml @@ -1,29 +1,9 @@ [lightning] table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning-checkpoint-engines.log" -level = "info" [checkpoint] enable = true driver = "mysql" -[tikv-importer] -addr = "127.0.0.1:8808" - [mydumper] -data-source-dir = "tests/checkpoint_engines/data" batch-size = 50 # force splitting the data into 4 batches - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_engines/run.sh b/tests/checkpoint_engines/run.sh index 7e9611693..1fedf6b63 100755 --- a/tests/checkpoint_engines/run.sh +++ b/tests/checkpoint_engines/run.sh @@ -15,13 +15,17 @@ set -eux +do_run_lightning() { + run_lightning --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-checkpoint-engines.log" --config "tests/$TEST_NAME/$1.toml" +} + # First, verify that a normal operation is fine. rm -f "$TEST_DIR/lightning-checkpoint-engines.log" rm -f "/tmp/tidb_lightning_checkpoint.pb" run_sql 'DROP DATABASE IF EXISTS cpeng;' -run_lightning +do_run_lightning config # Check that we have indeed opened 6 engines (index + data engine) DATA_ENGINE_COUNT=4 @@ -50,13 +54,13 @@ export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDo set +e for i in $(seq "$ENGINE_COUNT"); do echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" - run_lightning 2> /dev/null + do_run_lightning config 2> /dev/null [ $? -ne 0 ] || exit 1 done set -e echo "******** Verify checkpoint no-op ********" -run_lightning +do_run_lightning config run_sql 'SELECT count(*), sum(c) FROM cpeng.a' check_contains 'count(*): 4' @@ -74,13 +78,13 @@ run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint;' set +e for i in $(seq "$ENGINE_COUNT"); do echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" - run_lightning mysql 2> /dev/null + do_run_lightning mysql 2> /dev/null [ $? -ne 0 ] || exit 1 done set -e echo "******** Verify checkpoint no-op ********" -run_lightning mysql +do_run_lightning mysql run_sql 'SELECT count(*), sum(c) FROM cpeng.a' check_contains 'count(*): 4' diff --git a/tests/checkpoint_error_destroy/bad.toml b/tests/checkpoint_error_destroy/bad.toml deleted file mode 100644 index e1e2175be..000000000 --- a/tests/checkpoint_error_destroy/bad.toml +++ /dev/null @@ -1,27 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - -[checkpoint] -enable = true -driver = "mysql" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/checkpoint_error_destroy/bad-data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_error_destroy/bad_file.toml b/tests/checkpoint_error_destroy/bad_file.toml deleted file mode 100644 index 7d3e2f5f3..000000000 --- a/tests/checkpoint_error_destroy/bad_file.toml +++ /dev/null @@ -1,27 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - -[checkpoint] -enable = true -driver = "file" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/checkpoint_error_destroy/bad-data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_error_destroy/file.toml b/tests/checkpoint_error_destroy/file.toml new file mode 100644 index 000000000..2730c338c --- /dev/null +++ b/tests/checkpoint_error_destroy/file.toml @@ -0,0 +1,3 @@ +[checkpoint] +enable = true +driver = "file" diff --git a/tests/checkpoint_error_destroy/good.toml b/tests/checkpoint_error_destroy/good.toml deleted file mode 100644 index ceedb2ea5..000000000 --- a/tests/checkpoint_error_destroy/good.toml +++ /dev/null @@ -1,27 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - -[checkpoint] -enable = true -driver = "mysql" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/checkpoint_error_destroy/good-data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_error_destroy/good_file.toml b/tests/checkpoint_error_destroy/good_file.toml deleted file mode 100644 index 6cf718ebf..000000000 --- a/tests/checkpoint_error_destroy/good_file.toml +++ /dev/null @@ -1,27 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - -[checkpoint] -enable = true -driver = "file" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/checkpoint_error_destroy/good-data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_error_destroy/mysql.toml b/tests/checkpoint_error_destroy/mysql.toml new file mode 100644 index 000000000..dc4eaf830 --- /dev/null +++ b/tests/checkpoint_error_destroy/mysql.toml @@ -0,0 +1,3 @@ +[checkpoint] +enable = true +driver = "mysql" diff --git a/tests/checkpoint_error_destroy/run.sh b/tests/checkpoint_error_destroy/run.sh index d9e97d1a9..f34eb75fc 100755 --- a/tests/checkpoint_error_destroy/run.sh +++ b/tests/checkpoint_error_destroy/run.sh @@ -13,18 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eu +set -eux # Make sure we won't run out of table concurrency by destroying checkpoints for i in $(seq 8); do + ARGS="--enable-checkpoint=1 --config tests/$TEST_NAME/mysql.toml -d tests/$TEST_NAME/bad-data" set +e - run_lightning bad + run_lightning $ARGS set -e - run_lightning_ctl bad -checkpoint-error-destroy=all + run_lightning_ctl $ARGS -checkpoint-error-destroy=all done -run_lightning good +run_lightning --enable-checkpoint=1 --config "tests/$TEST_NAME/mysql.toml" -d "tests/$TEST_NAME/good-data" run_sql 'SELECT * FROM cped.t' check_contains 'x: 1999-09-09 09:09:09' @@ -33,14 +34,15 @@ check_contains 'x: 1999-09-09 09:09:09' run_sql 'DROP DATABASE cped' for i in $(seq 8); do + ARGS="--enable-checkpoint=1 --config tests/$TEST_NAME/file.toml -d tests/$TEST_NAME/bad-data" set +e - run_lightning bad_file + run_lightning $ARGS set -e ls -la /tmp/lightning_test_result/importer/.temp/ - run_lightning_ctl bad_file -checkpoint-error-destroy=all + run_lightning_ctl $ARGS -checkpoint-error-destroy=all ls -la /tmp/lightning_test_result/importer/.temp/ done -run_lightning good_file +run_lightning --enable-checkpoint=1 --config "tests/$TEST_NAME/file.toml" -d "tests/$TEST_NAME/good-data" run_sql 'SELECT * FROM cped.t' check_contains 'x: 1999-09-09 09:09:09' diff --git a/tests/checkpoint_timestamp/config.toml b/tests/checkpoint_timestamp/config.toml index 89d4a401f..793211e8f 100644 --- a/tests/checkpoint_timestamp/config.toml +++ b/tests/checkpoint_timestamp/config.toml @@ -1,7 +1,5 @@ [lightning] region-concurrency = 1 -file = "/tmp/lightning_test_result/lightning.log" -level = "info" [checkpoint] enable = true @@ -9,22 +7,5 @@ schema = "tidb_lightning_checkpoint_timestamp" driver = "file" dsn = "/tmp/lightning_test_result/cpts.pb" -[tikv-importer] -addr = "127.0.0.1:8808" - [mydumper] -data-source-dir = "tests/checkpoint_timestamp/data" read-block-size = 1 - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_timestamp/mysql.toml b/tests/checkpoint_timestamp/mysql.toml index 121d0bf50..a0d4f5253 100644 --- a/tests/checkpoint_timestamp/mysql.toml +++ b/tests/checkpoint_timestamp/mysql.toml @@ -1,7 +1,5 @@ [lightning] region-concurrency = 1 -file = "/tmp/lightning_test_result/lightning.log" -level = "info" [checkpoint] enable = true @@ -9,22 +7,5 @@ schema = "tidb_lightning_checkpoint_timestamp" driver = "mysql" keep-after-success = true -[tikv-importer] -addr = "127.0.0.1:8808" - [mydumper] -data-source-dir = "tests/checkpoint_timestamp/data" read-block-size = 1 - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/checkpoint_timestamp/run.sh b/tests/checkpoint_timestamp/run.sh index 3a71ce3d3..59d80e31c 100755 --- a/tests/checkpoint_timestamp/run.sh +++ b/tests/checkpoint_timestamp/run.sh @@ -22,7 +22,7 @@ export GO_FAILPOINTS="github.com/pingcap/tidb-lightning/lightning/SetTaskID=retu for i in $(seq 5); do echo "******** Importing Chunk Now (file step $i) ********" - run_lightning 2> /dev/null && break + run_lightning --enable-checkpoint=1 2> /dev/null && break sleep 1 done @@ -36,7 +36,7 @@ run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_timestamp.1234567890 for i in $(seq 5); do echo "******** Importing Chunk Now (mysql step $i) ********" - run_lightning mysql 2> /dev/null && break + run_lightning --enable-checkpoint=1 --config "tests/$TEST_NAME/mysql.toml" 2> /dev/null && break sleep 1 done diff --git a/tests/cmdline_override/run.sh b/tests/cmdline_override/run.sh index 4ceca9a55..cf61ad5eb 100755 --- a/tests/cmdline_override/run.sh +++ b/tests/cmdline_override/run.sh @@ -2,7 +2,7 @@ set -eux -run_lightning config \ +run_lightning \ -L info \ --log-file "$TEST_DIR/lightning.log" \ --tidb-host 127.0.0.1 \ diff --git a/tests/concurrent-restore/config.toml b/tests/concurrent-restore/config.toml index 206810b8a..2899d4221 100644 --- a/tests/concurrent-restore/config.toml +++ b/tests/concurrent-restore/config.toml @@ -1,23 +1,3 @@ [lightning] table-concurrency = 4 index-concurrency = 4 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "/tmp/lightning_test_result/restore_conc.mydump" - -[tidb] -host = "127.0.0.1" -user = "root" -status-port = 10080 -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/concurrent-restore/run.sh b/tests/concurrent-restore/run.sh index 161363e9b..f0bc77fcd 100644 --- a/tests/concurrent-restore/run.sh +++ b/tests/concurrent-restore/run.sh @@ -34,7 +34,7 @@ export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/Increa # Start importing run_sql 'DROP DATABASE IF EXISTS restore_conc' -run_lightning +run_lightning -d "$DBPATH" echo "Import finished" # Verify all data are imported diff --git a/tests/csv/config.toml b/tests/csv/config.toml index 2b21085ed..e61b4516b 100644 --- a/tests/csv/config.toml +++ b/tests/csv/config.toml @@ -1,17 +1,3 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - -[checkpoint] -enable = false - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/csv/data" - [mydumper.csv] separator = ',' delimiter = '"' @@ -20,16 +6,3 @@ not-null = false null = '\N' backslash-escape = true trim-last-separator = false - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/csv/run.sh b/tests/csv/run.sh index 94ab78563..08790ef26 100755 --- a/tests/csv/run.sh +++ b/tests/csv/run.sh @@ -6,7 +6,7 @@ for BACKEND in importer tidb; do run_sql 'DROP DATABASE IF EXISTS csv' -run_lightning config --backend $BACKEND +run_lightning --backend $BACKEND run_sql 'SELECT count(*), sum(PROCESSLIST_TIME), sum(THREAD_OS_ID), count(PROCESSLIST_STATE) FROM csv.threads' check_contains 'count(*): 43' diff --git a/tests/default-columns/config.toml b/tests/default-columns/config.toml index 073074cbe..e69de29bb 100644 --- a/tests/default-columns/config.toml +++ b/tests/default-columns/config.toml @@ -1,18 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/defcol-errors.log" -level = "info" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/default-columns/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" diff --git a/tests/default-columns/run.sh b/tests/default-columns/run.sh index c3f010195..1d8bc4eb1 100755 --- a/tests/default-columns/run.sh +++ b/tests/default-columns/run.sh @@ -17,7 +17,7 @@ set -eu run_sql 'DROP DATABASE IF EXISTS defcol' -run_lightning +run_lightning --log-file "$TEST_DIR/defcol-errors.log" run_sql 'SELECT min(pk), count(pk) FROM defcol.t' check_contains 'min(pk): 1' diff --git a/tests/error_summary/config.toml b/tests/error_summary/config.toml index 1a7f68f2a..5943184e1 100644 --- a/tests/error_summary/config.toml +++ b/tests/error_summary/config.toml @@ -1,28 +1,4 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning-error-summary.log" -level = "info" - [checkpoint] enable = true schema = "tidb_lightning_checkpoint_error_summary" driver = "mysql" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/error_summary/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/error_summary/run.sh b/tests/error_summary/run.sh index ca7eb123f..56f9b2f0e 100755 --- a/tests/error_summary/run.sh +++ b/tests/error_summary/run.sh @@ -28,7 +28,7 @@ run_sql 'INSERT INTO error_summary.a VALUES (2, 4), (6, 8);' run_sql 'INSERT INTO error_summary.c VALUES (3, 9), (27, 81);' set +e -run_lightning +run_lightning --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-error-summary.log" ERRORCODE=$? set -e @@ -49,7 +49,7 @@ grep -Fq '[-] [table=`error_summary`.`c`] [status=checksum] [error="checksum mis # Now check the error log when the checkpoint is not cleaned. set +e -run_lightning +run_lightning --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-error-summary.log" ERRORCODE=$? set -e diff --git a/tests/examples/1.toml b/tests/examples/1.toml index 370739126..1b7d5833c 100644 --- a/tests/examples/1.toml +++ b/tests/examples/1.toml @@ -1,25 +1,6 @@ [lightning] table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" level = "warning" -[tikv-importer] -addr = "127.0.0.1:8808" - [mydumper] -data-source-dir = "lightning/mydump/examples" read-block-size = 1 - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/examples/131072.toml b/tests/examples/131072.toml index 6d1e32322..a03dbacac 100644 --- a/tests/examples/131072.toml +++ b/tests/examples/131072.toml @@ -1,25 +1,6 @@ [lightning] table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" level = "warning" -[tikv-importer] -addr = "127.0.0.1:8808" - [mydumper] -data-source-dir = "lightning/mydump/examples" read-block-size = 131072 - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/examples/512.toml b/tests/examples/512.toml index 993d0dbb3..b25a46f50 100644 --- a/tests/examples/512.toml +++ b/tests/examples/512.toml @@ -1,25 +1,6 @@ [lightning] table-concurrency = 1 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" level = "warning" -[tikv-importer] -addr = "127.0.0.1:8808" - [mydumper] -data-source-dir = "lightning/mydump/examples" read-block-size = 512 - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/examples/run.sh b/tests/examples/run.sh index f710b03a2..f9392fcac 100755 --- a/tests/examples/run.sh +++ b/tests/examples/run.sh @@ -27,9 +27,13 @@ undo_rename() { } trap undo_rename EXIT +do_run_lightning() { + run_lightning -d lightning/mydump/examples --config "tests/$TEST_NAME/$1.toml" +} + # Perform the import run_sql 'DROP DATABASE IF EXISTS mocker_test;' -run_lightning 512 +do_run_lightning 512 # The existing reader_test run_sql 'use mocker_test; select count(distinct ID) cnt from `tbl_autoid`' @@ -78,14 +82,14 @@ check_contains 'count(*): 20' # Rest of the existing reader_test run_sql 'DROP DATABASE mocker_test;' -run_lightning 1 +do_run_lightning 1 run_sql 'use mocker_test; select count(distinct ID) cnt from `tbl_autoid`' check_contains 'cnt: 10000' run_sql 'use mocker_test; select count(distinct Name) cnt from `tbl_multi_index`' check_contains 'cnt: 10000' run_sql 'DROP DATABASE mocker_test;' -run_lightning 131072 +do_run_lightning 131072 run_sql 'use mocker_test; select count(distinct ID) cnt from `tbl_autoid`' check_contains 'cnt: 10000' run_sql 'use mocker_test; select count(distinct Name) cnt from `tbl_multi_index`' diff --git a/tests/exotic_filenames/config.toml b/tests/exotic_filenames/config.toml index b4f9ff873..e69de29bb 100644 --- a/tests/exotic_filenames/config.toml +++ b/tests/exotic_filenames/config.toml @@ -1,23 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "/tmp/lightning_test_result/exotic_filename.mydump" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = true -analyze = true diff --git a/tests/exotic_filenames/run.sh b/tests/exotic_filenames/run.sh index c0a261e2c..eb304b984 100755 --- a/tests/exotic_filenames/run.sh +++ b/tests/exotic_filenames/run.sh @@ -28,7 +28,7 @@ cp "tests/$TEST_NAME/data/xfn.etn.sql" "$DBPATH/"'x`f"n.exotic`table``name.sql' run_sql 'DROP DATABASE IF EXISTS `x``f"n`;' run_sql 'DROP DATABASE IF EXISTS `中文庫`;' -run_lightning +run_lightning -d "$DBPATH" echo 'Import finished' run_sql 'SELECT count(*) FROM `x``f"n`.`exotic``table````name`' diff --git a/tests/no_schema/config.toml b/tests/no_schema/config.toml index 5c1a05432..e69de29bb 100644 --- a/tests/no_schema/config.toml +++ b/tests/no_schema/config.toml @@ -1,24 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/no_schema/data" -no-schema = true - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = true -analyze = true diff --git a/tests/no_schema/run.sh b/tests/no_schema/run.sh index 27e475bd1..d3157405d 100644 --- a/tests/no_schema/run.sh +++ b/tests/no_schema/run.sh @@ -16,7 +16,7 @@ set -eu run_sql "DROP DATABASE IF EXISTS noschema;" -run_lightning schema_config +run_lightning --no-schema=1 -d "tests/$TEST_NAME/schema-data" run_sql "show databases" check_not_contains "noschema" @@ -24,7 +24,7 @@ run_sql "create database noschema;" run_sql "create table noschema.t (x int primary key);" # Starting importing -run_lightning +run_lightning --no-schema=1 run_sql "SELECT sum(x) FROM noschema.t;" check_contains 'sum(x): 120' diff --git a/tests/no_schema/schema_config.toml b/tests/no_schema/schema_config.toml deleted file mode 100644 index 9504b856f..000000000 --- a/tests/no_schema/schema_config.toml +++ /dev/null @@ -1,24 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/no_schema/schema-data" -no-schema = true - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = true -analyze = true diff --git a/tests/partitioned-table/config.toml b/tests/partitioned-table/config.toml index 2833a6666..e69de29bb 100644 --- a/tests/partitioned-table/config.toml +++ b/tests/partitioned-table/config.toml @@ -1,20 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/partitioned-table/data" - -[tidb] -host = "127.0.0.1" -user = "root" -status-port = 10080 -log-level = "error" - -[post-restore] -checksum = true -analyze = true diff --git a/tests/restore/config.toml b/tests/restore/config.toml index 6058e3ed8..0b6c921cd 100644 --- a/tests/restore/config.toml +++ b/tests/restore/config.toml @@ -1,22 +1,2 @@ [lightning] table-concurrency = 4 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "/tmp/lightning_test_result/restore.mydump" - -[tidb] -host = "127.0.0.1" -user = "root" -status-port = 10080 -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/restore/run.sh b/tests/restore/run.sh index 6979fd317..aceda5e23 100755 --- a/tests/restore/run.sh +++ b/tests/restore/run.sh @@ -32,7 +32,7 @@ export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/kv/FailIfEngin # Start importing run_sql 'DROP DATABASE IF EXISTS restore_tsr' -run_lightning +run_lightning -d "$DBPATH" echo "Import finished" # Verify all data are imported diff --git a/tests/routes/config.toml b/tests/routes/config.toml index 0c30b08c6..dbbe7c75b 100644 --- a/tests/routes/config.toml +++ b/tests/routes/config.toml @@ -1,8 +1,3 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "info" - # the complicated routing rules should be tested in tidb-tools repo already # here we're just verifying the basic things do work. [[routes]] @@ -10,22 +5,3 @@ schema-pattern = "routes_a*" table-pattern = "t*" target-schema = "routes_b" target-table = "u" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/routes/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = false -analyze = false diff --git a/tests/row-format-v2/config.toml b/tests/row-format-v2/config.toml index afb440ff5..e69de29bb 100644 --- a/tests/row-format-v2/config.toml +++ b/tests/row-format-v2/config.toml @@ -1,16 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/row-format-v2/data" - -[tidb] -host = "127.0.0.1" -user = "root" -status-port = 10080 -log-level = "error" diff --git a/tests/run.sh b/tests/run.sh index 7238184bc..274613b26 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -16,6 +16,7 @@ set -eu TEST_DIR=/tmp/lightning_test_result +export PATH="tests/_utils:$PATH" stop_services() { killall -9 tikv-server || true @@ -29,52 +30,120 @@ stop_services() { start_services() { stop_services - mkdir -p "$TEST_DIR" + TT="$TEST_DIR/tls" + mkdir -p "$TT" rm -f "$TEST_DIR"/*.log + # Ref: https://docs.microsoft.com/en-us/azure/application-gateway/self-signed-certificates + # gRPC only supports P-256 curves, see https://github.com/grpc/grpc/issues/6722 + echo "Generate TLS keys..." + cat - > "$TT/ipsan.cnf" < /dev/null + for cluster in tidb pd tikv importer lightning curl; do + openssl ecparam -out "$TT/$cluster.key" -name prime256v1 -genkey + openssl req -new -batch -sha256 -subj '/CN=localhost' -key "$TT/$cluster.key" -out "$TT/$cluster.csr" + openssl x509 -req -sha256 -days 1 -extensions EXT -extfile "$TT/ipsan.cnf" -in "$TT/$cluster.csr" -CA "$TT/ca.pem" -CAkey "$TT/ca.key" -CAcreateserial -out "$TT/$cluster.pem" 2> /dev/null + done + + cat - > "$TEST_DIR/pd-config.toml" < "$TEST_DIR/tikv-config.toml" < "$TEST_DIR/tidb-config.toml" < "$TEST_DIR/importer-config.toml" < /dev/null + run_lightning --config "tests/$TEST_NAME/$type.toml" 2> /dev/null ERRORCODE=$? set -e [ "$ERRORCODE" -ne 0 ] @@ -36,14 +36,14 @@ for type in replace ignore error; do if [ $type = 'error' ]; then set +e - run_lightning $type + run_lightning --config "tests/$TEST_NAME/$type.toml" --log-file "$TEST_DIR/lightning-error-on-dup.log" ERRORCODE=$? set -e [ "$ERRORCODE" -ne 0 ] tail -20 "$TEST_DIR/lightning-error-on-dup.log" > "$TEST_DIR/lightning-error-on-dup.tail" grep -Fq 'Duplicate entry' "$TEST_DIR/lightning-error-on-dup.tail" elif [ $type = 'replace' ]; then - run_lightning $type + run_lightning --config "tests/$TEST_NAME/$type.toml" run_sql 'SELECT count(*) FROM dup.dup' check_contains 'count(*): 2' run_sql 'SELECT d FROM dup.dup WHERE pk = 1' @@ -51,7 +51,7 @@ for type in replace ignore error; do run_sql 'SELECT d FROM dup.dup WHERE pk = 2' check_contains 'd: new' elif [ $type = 'ignore' ]; then - run_lightning $type + run_lightning --config "tests/$TEST_NAME/$type.toml" run_sql 'SELECT count(*) FROM dup.dup' check_contains 'count(*): 2' run_sql 'SELECT d FROM dup.dup WHERE pk = 1' diff --git a/tests/tidb_rowid/config.toml b/tests/tidb_rowid/config.toml index 3cc1be23e..e69de29bb 100644 --- a/tests/tidb_rowid/config.toml +++ b/tests/tidb_rowid/config.toml @@ -1,23 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/tidb_rowid/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = true -analyze = true diff --git a/tests/tool_135/config.toml b/tests/tool_135/config.toml index 336013c70..e69de29bb 100644 --- a/tests/tool_135/config.toml +++ b/tests/tool_135/config.toml @@ -1,23 +0,0 @@ -[lightning] -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warn" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/tool_135/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = true -analyze = true diff --git a/tests/tool_1420/config.toml b/tests/tool_1420/config.toml index 4974b2e26..e69de29bb 100644 --- a/tests/tool_1420/config.toml +++ b/tests/tool_1420/config.toml @@ -1,16 +0,0 @@ -[lightning] -file = "/tmp/lightning_test_result/lightning.log" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/tool_1420/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" diff --git a/tests/tool_1472/config.toml b/tests/tool_1472/config.toml index 13021b873..e69de29bb 100644 --- a/tests/tool_1472/config.toml +++ b/tests/tool_1472/config.toml @@ -1,16 +0,0 @@ -[lightning] -file = "/tmp/lightning_test_result/lightning.log" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/tool_1472/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" diff --git a/tests/tool_1472/data/EE1472.notpk-schema.sql b/tests/tool_1472/data/EE1472.notpk-schema.sql index 8cc80840f..713ed4f60 100644 --- a/tests/tool_1472/data/EE1472.notpk-schema.sql +++ b/tests/tool_1472/data/EE1472.notpk-schema.sql @@ -1,5 +1,5 @@ create table `notpk` ( a int primary key, b tinyint auto_increment, - key(a) + key(b) ); diff --git a/tests/tool_1472/run.sh b/tests/tool_1472/run.sh index c0a1a747d..050948edd 100755 --- a/tests/tool_1472/run.sh +++ b/tests/tool_1472/run.sh @@ -14,7 +14,7 @@ # limitations under the License. # This test verifies if TOOL-1420 is fixed. -# It involves column names not in lower-case. +# It involves pre-calculated auto-inc overflowing the tinyint range. set -eu diff --git a/tests/tool_241/config.toml b/tests/tool_241/config.toml index 1b2d3609d..d5e4c96e4 100644 --- a/tests/tool_241/config.toml +++ b/tests/tool_241/config.toml @@ -1,25 +1,5 @@ [lightning] table-concurrency = 3 -check-requirements = false -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/tool_241/data" [tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" sql-mode = '' - -[post-restore] -checksum = true -compact = true -analyze = true diff --git a/tests/unused_config_keys/config.toml b/tests/unused_config_keys/config.toml index 3c202e7f8..7df1d89c1 100644 --- a/tests/unused_config_keys/config.toml +++ b/tests/unused_config_keys/config.toml @@ -1,21 +1,6 @@ [typo-1] [lightning] -file = "/tmp/lightning_test_result/lightning-unused-config-keys.log" typo-2 = "unused test" -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/unused_config_keys/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - [[typo-3]] diff --git a/tests/unused_config_keys/run.sh b/tests/unused_config_keys/run.sh index 8c3805650..f088089ee 100755 --- a/tests/unused_config_keys/run.sh +++ b/tests/unused_config_keys/run.sh @@ -17,7 +17,7 @@ # Lightning should fail to start when finds unused config keys. set +e -run_lightning +run_lightning --log-file "$TEST_DIR/lightning-unused-config-keys.log" ERRORCODE=$? set -e diff --git a/tests/various_types/config.toml b/tests/various_types/config.toml index aac195be1..e69de29bb 100644 --- a/tests/various_types/config.toml +++ b/tests/various_types/config.toml @@ -1,23 +0,0 @@ -[lightning] -check-requirements = true -file = "/tmp/lightning_test_result/lightning.log" -level = "warning" - -[tikv-importer] -addr = "127.0.0.1:8808" - -[mydumper] -data-source-dir = "tests/various_types/data" - -[tidb] -host = "127.0.0.1" -port = 4000 -user = "root" -status-port = 10080 -pd-addr = "127.0.0.1:2379" -log-level = "error" - -[post-restore] -checksum = true -compact = true -analyze = true diff --git a/tests/various_types/run.sh b/tests/various_types/run.sh index 53d2a74ea..da2a0b789 100755 --- a/tests/various_types/run.sh +++ b/tests/various_types/run.sh @@ -20,7 +20,7 @@ set -eu for BACKEND in importer tidb; do run_sql 'DROP DATABASE IF EXISTS vt;' -run_lightning config --backend $BACKEND +run_lightning --backend $BACKEND echo Import using $BACKEND finished run_sql 'SELECT count(pk), bin(min(pk)), bin(max(pk)) FROM vt.bit' diff --git a/tidb-lightning.toml b/tidb-lightning.toml index 665312fe7..77e83a316 100644 --- a/tidb-lightning.toml +++ b/tidb-lightning.toml @@ -40,6 +40,15 @@ max-size = 128 # MB max-days = 28 max-backups = 14 +[security] +# specifies certificates and keys for TLS connections within the cluster. +# public certificate of the CA. Leave empty to disable TLS. +# ca-path = "/path/to/ca.pem" +# public certificate of this service. +# cert-path = "/path/to/lightning.pem" +# private key of this service. +# key-path = "/path/to/lightning.key" + [checkpoint] # Whether to enable checkpoints. # While importing, Lightning will record which tables have been imported, so even if Lightning or other component @@ -137,6 +146,14 @@ log-level = "error" # set this to 0 to automatically fetch the `max_allowed_packet` variable from server on every connection. # max-allowed-packet = 67_108_864 +# whether to use TLS for SQL connections. valid values are: +# * "" - force TLS (same as "cluster") if [tidb.security] section is populated, otherwise same as "false" +# * "false" - disable TLS +# * "cluster" - force TLS and verify the server's certificate with the CA specified in the [tidb.security] section +# * "skip-verify" - force TLS but do not verify the server's certificate (insecure!) +# * "preferred" - same as "skip-verify", but if the server does not support TLS, fallback to unencrypted connection +# tls = "" + # set tidb session variables to speed up checksum/analyze table. # see https://pingcap.com/docs/sql/statistics/#control-analyze-concurrency for the meaning of each setting build-stats-concurrency = 20 @@ -144,6 +161,16 @@ distsql-scan-concurrency = 100 index-serial-scan-concurrency = 20 checksum-table-concurrency = 16 +# specifies certificates and keys for TLS-enabled MySQL connections. +# defaults to a copy of the [security] section. +#[tidb.security] +# public certificate of the CA. Set to empty string to disable TLS. +# ca-path = "/path/to/ca.pem" +# public certificate of this service. Default to copy of `security.cert-path` +# cert-path = "/path/to/lightning.pem" +# private key of this service. Default to copy of `security.key-path` +# key-path = "/path/to/lightning.key" + # post-restore provide some options which will be executed after all kv data has been imported into the tikv cluster. # the execution order are(if set true): checksum -> analyze [post-restore]