From 7bae12ebe2b6ca1c3c712eb35b7f0c39484c86a5 Mon Sep 17 00:00:00 2001 From: kennytm Date: Wed, 3 Apr 2019 22:23:34 +0800 Subject: [PATCH] *: parse the data source directly into data and skip the KV encoder (#145) * *: parse the data source directly into data and skip the KV encoder This skips the more complex pingcap/parser, and speeds up parsing speed by 50%. We have also refactored the KV delivery mechanism to use channels directly, and revamped metrics: - Make the metrics about engines into its own `engines` counter. The `tables` counter is exclusively about tables now. - Removed `block_read_seconds`, `block_read_bytes`, `block_encode_seconds` since the concept of "block" no longer applies. Replaced by the equivalents named `row_***`. - Removed `chunk_parser_read_row_seconds` for being overlapping with `row_read_seconds`. - Changed `block_deliver_bytes` into a histogram vec, with kind=index or kind=data. Introduced `block_deliver_kv_pairs`. * tests,restore: prevent spurious error in checkpoint_chunks test Only kill Lightning if the whole chunk is imported exactly. The chunk checkpoint may be recorded before a chunk is fully written, and this will hit the failpoint more than 5 times. * kv: use composed interface to simplify some types * kv: properly handle the SQL mode * common: disable IsContextCanceledError() when log level = debug This helps debugging some mysterious cancellation where the log is inhibited. Added IsReallyContextCanceledError() for code logic affected by error type. * restore: made some log more detailed * restore: made the SlowDownImport failpoint apply to index engines too * restore: do not open a write stream when there are no KV pairs to send * tests: ensure we drop the checkpoints DB before re-run * mydump: fixed various off-by-one errors in the CSV parser * *: rename `!IsContextCanceledError` to `ShouldLogError` * *: addressed comments * restore: zero the checksums and column permutations on initialization * *: addressed comments * tests: add back a missing license header * tests: improve a comment. --- go.mod | 7 +- go.sum | 44 + lightning/common/once_error.go | 2 +- lightning/common/util.go | 32 +- lightning/config/config.go | 12 +- lightning/kv/allocator.go | 26 +- lightning/kv/importer.go | 15 +- lightning/kv/session.go | 86 + lightning/kv/sql2kv.go | 120 +- lightning/metric/metric.go | 91 +- lightning/mydump/csv_parser.go | 104 +- lightning/mydump/csv_parser.rl | 9 +- lightning/mydump/csv_parser_generated.go | 1836 ++++----- lightning/mydump/csv_parser_test.go | 146 +- lightning/mydump/parser.go | 258 +- lightning/mydump/parser.rl | 93 +- lightning/mydump/parser_generated.go | 3605 +++++++++++------ lightning/mydump/parser_test.go | 51 +- lightning/restore/checkpoints.go | 46 +- lightning/restore/file_checkpoints.pb.go | 449 +- lightning/restore/file_checkpoints.proto | 3 +- lightning/restore/restore.go | 539 +-- lightning/restore/restore_test.go | 53 +- lightning/verification/checksum.go | 9 + tests/checkpoint_chunks/config.toml | 2 +- tests/checkpoint_chunks/file.toml | 2 +- tests/checkpoint_chunks/run.sh | 7 +- tests/checkpoint_engines/run.sh | 3 +- tests/csv/data/csv.empty_strings-schema.sql | 5 + tests/csv/data/csv.empty_strings.csv | 4 + tests/csv/run.sh | 8 + tests/run.sh | 2 +- .../sqlmode/data/sqlmodedb-schema-create.sql | 1 + tests/sqlmode/data/sqlmodedb.t-schema.sql | 7 + tests/sqlmode/data/sqlmodedb.t.1.sql | 6 + tests/sqlmode/off.toml | 27 + tests/sqlmode/on.toml | 26 + tests/sqlmode/run.sh | 55 + tests/tool_241/config.toml | 1 + .../data/vt.empty_strings-schema.sql | 4 + tests/various_types/data/vt.empty_strings.sql | 1 + tests/various_types/run.sh | 9 + 42 files changed, 4715 insertions(+), 3091 deletions(-) create mode 100644 lightning/kv/session.go create mode 100644 tests/csv/data/csv.empty_strings-schema.sql create mode 100644 tests/csv/data/csv.empty_strings.csv create mode 100644 tests/sqlmode/data/sqlmodedb-schema-create.sql create mode 100644 tests/sqlmode/data/sqlmodedb.t-schema.sql create mode 100644 tests/sqlmode/data/sqlmodedb.t.1.sql create mode 100644 tests/sqlmode/off.toml create mode 100644 tests/sqlmode/on.toml create mode 100755 tests/sqlmode/run.sh create mode 100644 tests/various_types/data/vt.empty_strings-schema.sql create mode 100644 tests/various_types/data/vt.empty_strings.sql diff --git a/go.mod b/go.mod index dac646cdf..a797e0f80 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,13 @@ require ( github.com/coreos/go-semver v0.2.0 github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/cznic/golex v0.0.0-20181122101858-9c343928389c // indirect github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7 + github.com/cznic/parser v0.0.0-20181122101858-d773202d5b1f + github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 + github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 + github.com/cznic/y v0.0.0-20181122101901-b05e8c2e8d7b github.com/dustin/go-humanize v1.0.0 // indirect - github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a github.com/go-sql-driver/mysql v1.4.1 github.com/gogo/protobuf v1.2.0 github.com/gorilla/context v1.1.1 // indirect @@ -23,6 +27,7 @@ require ( github.com/pingcap/parser v0.0.0-20190305073013-4f60445a0550 github.com/pingcap/tidb v0.0.0-20190309032432-ea9970968c73 github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible + github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93 github.com/prometheus/client_golang v0.9.2 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 // indirect diff --git a/go.sum b/go.sum index 56d2a3b17..39751491a 100644 --- a/go.sum +++ b/go.sum @@ -8,31 +8,49 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 h1:dzj1/xcivGjNPwwifh/dWTczkwcuqsXXFHY1X/TZMtw= github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 h1:hHWif/4GirK3P5uvCyyj941XSVIQDzuJhbEguCICdPE= github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.2.18+incompatible h1:E8oIF72eom0bDAB74G/H31rAe1JVoiLE5NMn+WUCNLc= github.com/coreos/etcd v3.2.18+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180202092358-40e2722dffea/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cznic/golex v0.0.0-20181122101858-9c343928389c h1:G8zTsaqyVfIHpgMFcGgdbhHSFhlNc77rAKkhVbQ9kQg= +github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= github.com/cznic/mathutil v0.0.0-20160613104831-78ad7f262603/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7 h1:y+DH9ARrWiiNBV+6waYP2IPcsRbxdU1qsnycPfShF4c= github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= +github.com/cznic/parser v0.0.0-20181122101858-d773202d5b1f h1:DUtr2TvhM9rmiHKVJWoLqDY2+MdxljW9hlaS/oYoi1c= +github.com/cznic/parser v0.0.0-20181122101858-d773202d5b1f/go.mod h1:2B43mz36vGZNZEwkWi8ayRSSUXLfjL8OkbzwW4NcPMM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM= +github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= +github.com/cznic/y v0.0.0-20181122101901-b05e8c2e8d7b h1:gvFsf4zJcnW6GRN+HPGTxwuw+7sTwzmoeoBQQCZDEnk= +github.com/cznic/y v0.0.0-20181122101901-b05e8c2e8d7b/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a h1:QNEenQIsGDEEfFNSnN+h6hE1OwnHqTg7Dl9gEk1Cko4= github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= @@ -40,7 +58,9 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -52,34 +72,45 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/google/btree v0.0.0-20161217183710-316fb6d3f031 h1:yAx4v8FikdsGCBPzIaT2F+0WH0J+wcL7cQD9n3UbyOk= github.com/google/btree v0.0.0-20161217183710-316fb6d3f031/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v0.0.0-20170228224354-599cba5e7b61 h1:4lceeSGcX6bSAeeTiqcGX4DX6pMLPG5fKHvGeWAP/5c= github.com/gorilla/mux v0.0.0-20170228224354-599cba5e7b61/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20171020063731-82921fcf811d h1:68J4W1eby3GvUxsMVrCSGZOlHSWWZwP/4fZVwd9iCX0= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20171020063731-82921fcf811d/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20160910222444-6b7015e65d36 h1:cwTrrTEhz13khQS3/UZMLFWwiqlcsdp/2sxFmSjAWeQ= github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20160910222444-6b7015e65d36/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.6.4 h1:xlu6C2WU6gvXt3XLyVpsgweaIL4VCmTjEsEAIt7qFqQ= github.com/grpc-ecosystem/grpc-gateway v1.6.4/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/joho/sqltocsv v0.0.0-20180904231936-b24deec2b806 h1:GmLQK4oeS6/Dvt7uC5pkRw63YTn0pqWcqZYTAfXaK8E= github.com/joho/sqltocsv v0.0.0-20180904231936-b24deec2b806/go.mod h1:mAVCUAYtW9NG31eB30umMSLKcDt6mCUWSjoSn5qBh0k= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -89,6 +120,7 @@ github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPA github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 h1:04yuCf5NMvLU8rB2m4Qs3rynH7EYpMno3lHkewIOdMo= github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030 h1:XJLuW0lsP7vAtQ2iPjZwvXZ14m5urp9No+Qr06ZZcTo= github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= @@ -106,6 +138,7 @@ github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93 h1:gI5bOzLMxjUq6ui+md github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= @@ -129,6 +162,7 @@ github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/spaolacci/murmur3 v0.0.0-20150829172844-0d12bf811670 h1:hKP4ACPoBBCnBbhoiuJXiYlSDhAvC9s4lgzAPmtVdU0= github.com/spaolacci/murmur3 v0.0.0-20150829172844-0d12bf811670/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/twinj/uuid v0.0.0-20150629100731-70cac2bcd273 h1:YqFyfcgqxQqjpRr0SEG0Z555J/3kPqDL/xmRyeAaX/0= github.com/twinj/uuid v0.0.0-20150629100731-70cac2bcd273/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= @@ -136,10 +170,13 @@ github.com/uber/jaeger-client-go v2.8.0+incompatible h1:7DGH8Hqk6PirD+GE+bvCf0cL github.com/uber/jaeger-client-go v2.8.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.1.0 h1:k1oxbz5ToLJtwCGmTlNSmfciXv/SPe1tnmNe+FqTl5w= github.com/uber/jaeger-lib v1.1.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.1 h1:gmervu+jDMvXTbcHQ0pd2wee85nEoE0BsVyEuzkfK8w= github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/unrolled/render v0.0.0-20190117215946-449f39850074 h1:Z2Io8Xv8S7HyOpBqaED5yqn2+NNrzWiS8CrTasc1mxQ= github.com/unrolled/render v0.0.0-20190117215946-449f39850074/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= golang.org/x/crypto v0.0.0-20180503215945-1f94bef427e3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= @@ -152,6 +189,7 @@ golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsq golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -160,11 +198,13 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181105230042-78dc5bac0cac h1:0Nb35Izc6T6Yz1iGmRc4cg14cxRaFjbjD4hWFI6JNJ8= golang.org/x/tools v0.0.0-20181105230042-78dc5bac0cac/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.1.1-0.20180731164958-4216e58b9158 h1:DLu24D8QphjtZaO7ZrMpJgxUV5pldWTLxEiMAYUZd1U= google.golang.org/appengine v1.1.1-0.20180731164958-4216e58b9158/go.mod h1:becAO19CitOrUq7nfYuyxycOBJNZgjTqYPI+mvwLjCs= google.golang.org/genproto v0.0.0-20180427144745-86e600f69ee4/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= @@ -173,12 +213,16 @@ google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE google.golang.org/grpc v1.18.0 h1:IZl7mfBGfbhYx2p2rKRtYgDFw6SBz+kclmxYrCksPPA= google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/lumberjack.v2 v2.0.0-20161104145732-dd45e6a67c53/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/lightning/common/once_error.go b/lightning/common/once_error.go index 83cf5d912..fe117bb47 100644 --- a/lightning/common/once_error.go +++ b/lightning/common/once_error.go @@ -35,7 +35,7 @@ func (oe *OnceError) Set(tag string, e error) { oe.err = e } oe.lock.Unlock() - if !IsContextCanceledError(e) { + if ShouldLogError(e) { AppLogger.Errorf("[%s] error %v", tag, e) } } diff --git a/lightning/common/util.go b/lightning/common/util.go index a8709a87b..8255e885d 100644 --- a/lightning/common/util.go +++ b/lightning/common/util.go @@ -30,6 +30,7 @@ import ( "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" tmysql "github.com/pingcap/parser/mysql" + "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -102,7 +103,7 @@ func TransactWithRetry(ctx context.Context, db *sql.DB, purpose string, action f if IsRetryableError(err) { continue } - if !IsContextCanceledError(err) { + if ShouldLogError(err) { AppLogger.Errorf("transaction %s [error] %v", purpose, err) } return errors.Trace(err) @@ -125,7 +126,7 @@ func transactImpl(ctx context.Context, db *sql.DB, purpose string, action func(c AppLogger.Warnf("transaction %s [error]%v", purpose, err) rerr := txn.Rollback() if rerr != nil { - if !IsContextCanceledError(rerr) { + if ShouldLogError(rerr) { AppLogger.Errorf("transaction %s [error] %v", purpose, rerr) } } @@ -181,9 +182,32 @@ func IsRetryableError(err error) bool { } } +// ShouldLogError returns whether the error should be logged. +// This function should only be used for inhabiting logs related to canceling, +// where the log is usually just noise. +// +// This function returns `false` when: +// +// - the error `IsContextCanceledError` +// - the log level is above "debug" +// +// This function also returns `false` when `err == nil`. +func ShouldLogError(err error) bool { + if err == nil { + return false + } + if AppLogger.IsLevelEnabled(logrus.DebugLevel) { + return true + } + return !IsContextCanceledError(err) +} + // IsContextCanceledError returns whether the error is caused by context -// cancellation. This function returns `false` (not a context-canceled error) if -// `err == nil`. +// cancellation. This function should only be used when the code logic is +// affected by whether the error is canceling or not. Normally, you should +// simply use ShouldLogError. +// +// This function returns `false` (not a context-canceled error) if `err == nil`. func IsContextCanceledError(err error) bool { err = errors.Cause(err) return err == context.Canceled || status.Code(err) == codes.Canceled diff --git a/lightning/config/config.go b/lightning/config/config.go index d24438fe4..e0d929367 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -23,6 +23,7 @@ import ( "github.com/BurntSushi/toml" "github.com/pingcap/errors" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-tools/pkg/filter" ) @@ -41,9 +42,11 @@ type DBStore struct { Psw string `toml:"password" json:"-"` StatusPort int `toml:"status-port" json:"status-port"` PdAddr string `toml:"pd-addr" json:"pd-addr"` - SQLMode string `toml:"sql-mode" json:"sql-mode"` + StrSQLMode string `toml:"sql-mode" json:"sql-mode"` LogLevel string `toml:"log-level" json:"log-level"` + SQLMode mysql.SQLMode `toml:"-" json:"-"` + DistSQLScanConcurrency int `toml:"distsql-scan-concurrency" json:"distsql-scan-concurrency"` BuildStatsConcurrency int `toml:"build-stats-concurrency" json:"build-stats-concurrency"` IndexSerialScanConcurrency int `toml:"index-serial-scan-concurrency" json:"index-serial-scan-concurrency"` @@ -161,7 +164,7 @@ func NewConfig() *Config { CheckRequirements: true, }, TiDB: DBStore{ - SQLMode: "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION", + StrSQLMode: mysql.DefaultSQLMode, BuildStatsConcurrency: 20, DistSQLScanConcurrency: 100, IndexSerialScanConcurrency: 20, @@ -226,6 +229,11 @@ func (cfg *Config) Load() error { return errors.New("invalid config: `mydumper.csv.delimiter` must be one byte long or empty") } + cfg.TiDB.SQLMode, err = mysql.GetSQLMode(cfg.TiDB.StrSQLMode) + if err != nil { + return errors.Annotate(err, "invalid config: `mydumper.tidb.sql_mode` must be a valid SQL_MODE") + } + // handle mydumper if cfg.Mydumper.BatchSize <= 0 { cfg.Mydumper.BatchSize = 100 * _G diff --git a/lightning/kv/allocator.go b/lightning/kv/allocator.go index d17a249a6..4207d0542 100644 --- a/lightning/kv/allocator.go +++ b/lightning/kv/allocator.go @@ -13,25 +13,24 @@ package kv -import "sync/atomic" +import ( + "sync/atomic" + + "github.com/pingcap/tidb/meta/autoid" +) // PanickingAllocator is an ID allocator which panics on all operations except Rebase type PanickingAllocator struct { + autoid.Allocator base int64 } +// NewPanickingAllocator creates a new PanickingAllocator. func NewPanickingAllocator(base int64) *PanickingAllocator { return &PanickingAllocator{base: base} } -func (alloc *PanickingAllocator) Alloc(int64) (int64, error) { - panic("unexpected Alloc() call") -} - -func (alloc *PanickingAllocator) Reset(newBase int64) { - panic("unexpected Reset() call") -} - +// Rebase implements the autoid.Allocator interface func (alloc *PanickingAllocator) Rebase(tableID, newBase int64, allocIDs bool) error { // CAS for { @@ -46,14 +45,7 @@ func (alloc *PanickingAllocator) Rebase(tableID, newBase int64, allocIDs bool) e return nil } +// Base implements the autoid.Allocator interface func (alloc *PanickingAllocator) Base() int64 { return atomic.LoadInt64(&alloc.base) } - -func (alloc *PanickingAllocator) End() int64 { - panic("unexpected End() call") -} - -func (alloc *PanickingAllocator) NextGlobalAutoID(tableID int64) (int64, error) { - panic("unexpected NextGlobalAutoID() call") -} diff --git a/lightning/kv/importer.go b/lightning/kv/importer.go index 24ff34ef5..2f8e26de2 100644 --- a/lightning/kv/importer.go +++ b/lightning/kv/importer.go @@ -178,13 +178,13 @@ func (importer *Importer) OpenEngine( return nil, errors.Trace(err) } - openCounter := metric.EngineCounter.WithLabelValues("open") + openCounter := metric.ImporterEngineCounter.WithLabelValues("open") openCounter.Inc() common.AppLogger.Infof("[%s] open engine %s", tag, engineUUID) // gofail: var FailIfEngineCountExceeds int // { - // closedCounter := metric.EngineCounter.WithLabelValues("closed") + // closedCounter := metric.ImporterEngineCounter.WithLabelValues("closed") // openCount := metric.ReadCounter(openCounter) // closedCount := metric.ReadCounter(closedCounter) // if openCount - closedCount > float64(FailIfEngineCountExceeds) { @@ -276,7 +276,7 @@ func (stream *WriteStream) Put(kvs []kvec.KvPair) error { // Close the write stream. func (stream *WriteStream) Close() error { if _, err := stream.wstream.CloseAndRecv(); err != nil { - if !common.IsContextCanceledError(err) { + if common.ShouldLogError(err) { common.AppLogger.Errorf("[%s] close write stream cause failed : %v", stream.engine.tag, err) } return errors.Trace(err) @@ -303,7 +303,7 @@ func (engine *OpenedEngine) Close(ctx context.Context) (*ClosedEngine, error) { return nil, errors.Trace(err) } common.AppLogger.Infof("[%s] [%s] engine close takes %v", engine.tag, engine.uuid, time.Since(timer)) - metric.EngineCounter.WithLabelValues("closed").Inc() + metric.ImporterEngineCounter.WithLabelValues("closed").Inc() return closedEngine, nil } @@ -350,7 +350,7 @@ func (engine *ClosedEngine) Import(ctx context.Context) error { if !common.IsRetryableError(err) { if err == nil { common.AppLogger.Infof("[%s] [%s] import takes %v", engine.tag, engine.uuid, time.Since(timer)) - } else if !common.IsContextCanceledError(err) { + } else if common.ShouldLogError(err) { common.AppLogger.Errorf("[%s] [%s] import failed and cannot retry, err %v", engine.tag, engine.uuid, err) } return errors.Trace(err) @@ -373,3 +373,8 @@ func (engine *ClosedEngine) Cleanup(ctx context.Context) error { common.AppLogger.Infof("[%s] [%s] cleanup takes %v", engine.tag, engine.uuid, time.Since(timer)) return errors.Trace(err) } + +// Tag gets an identification stirng of this engine for logging. +func (engine *ClosedEngine) Tag() string { + return engine.tag +} diff --git a/lightning/kv/session.go b/lightning/kv/session.go new file mode 100644 index 000000000..c8aa80ea1 --- /dev/null +++ b/lightning/kv/session.go @@ -0,0 +1,86 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kv + +import ( + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/variable" + kvec "github.com/pingcap/tidb/util/kvencoder" +) + +// transaction is a trimmed down Transaction type which only supports adding a +// new KV pair. +type transaction struct { + kv.Transaction + kvPairs []kvec.KvPair +} + +// Set implements the kv.Transaction interface +func (t *transaction) Set(k kv.Key, v []byte) error { + t.kvPairs = append(t.kvPairs, kvec.KvPair{ + Key: k.Clone(), + Val: append([]byte{}, v...), + }) + return nil +} + +// SetOption implements the kv.Transaction interface +func (t *transaction) SetOption(opt kv.Option, val interface{}) {} + +// DelOption implements the kv.Transaction interface +func (t *transaction) DelOption(kv.Option) {} + +// session is a trimmed down Session type which only wraps our own trimmed-down +// transaction type and provides the session variables to the TiDB library +// optimized for Lightning. +type session struct { + sessionctx.Context + txn transaction + vars *variable.SessionVars +} + +func newSession(sqlMode mysql.SQLMode) *session { + vars := variable.NewSessionVars() + vars.LightningMode = true + vars.SkipUTF8Check = true + vars.StmtCtx.InInsertStmt = true + vars.StmtCtx.BadNullAsWarning = !sqlMode.HasStrictMode() + vars.StmtCtx.TruncateAsWarning = !sqlMode.HasStrictMode() + vars.StmtCtx.OverflowAsWarning = !sqlMode.HasStrictMode() + vars.StmtCtx.AllowInvalidDate = sqlMode.HasAllowInvalidDatesMode() + vars.StmtCtx.IgnoreZeroInDate = !sqlMode.HasStrictMode() || sqlMode.HasAllowInvalidDatesMode() + vars.StmtCtx.TimeZone = vars.Location() + return &session{ + txn: transaction{}, + vars: vars, + } +} + +func (se *session) takeKvPairs() []kvec.KvPair { + pairs := se.txn.kvPairs + se.txn.kvPairs = make([]kvec.KvPair, 0, len(pairs)) + return pairs +} + +// Txn implements the sessionctx.Context interface +func (se *session) Txn(active bool) (kv.Transaction, error) { + return &se.txn, nil +} + +// GetSessionVars implements the sessionctx.Context interface +func (se *session) GetSessionVars() *variable.SessionVars { + return se.vars +} diff --git a/lightning/kv/sql2kv.go b/lightning/kv/sql2kv.go index bc9e45076..b73aa0674 100644 --- a/lightning/kv/sql2kv.go +++ b/lightning/kv/sql2kv.go @@ -15,79 +15,89 @@ package kv import ( "github.com/pingcap/errors" - "github.com/pingcap/tidb-lightning/lightning/common" + "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-lightning/lightning/metric" - "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/types" kvec "github.com/pingcap/tidb/util/kvencoder" ) -func InitMembufCap(batchSQLLength int64) { - kv.ImportingTxnMembufCap = int(batchSQLLength) * 4 - // TODO : calculate predicted ratio, bwtween sql and kvs' size, base on specified DDL -} +var extraHandleColumnInfo = model.NewExtraHandleColInfo() type TableKVEncoder struct { - table string - tableID int64 - encoder kvec.KvEncoder - idAllocator autoid.Allocator + tbl table.Table + se *session + recordCache []types.Datum } func NewTableKVEncoder( - dbName string, - table string, tableID int64, - sqlMode string, alloc autoid.Allocator) (*TableKVEncoder, error) { - - encoder, err := kvec.New(dbName, alloc) - if err != nil { - common.AppLogger.Errorf("err %s", errors.ErrorStack(err)) - return nil, errors.Trace(err) - } - err = encoder.SetSystemVariable("tidb_opt_write_row_id", "1") - if err != nil { - encoder.Close() - return nil, errors.Trace(err) - } + tbl table.Table, + sqlMode mysql.SQLMode, +) *TableKVEncoder { + metric.KvEncoderCounter.WithLabelValues("open").Inc() - kvcodec := &TableKVEncoder{ - table: table, - tableID: tableID, - encoder: encoder, - idAllocator: alloc, + return &TableKVEncoder{ + tbl: tbl, + se: newSession(sqlMode), } +} - if err := kvcodec.init(sqlMode); err != nil { - kvcodec.Close() - return nil, errors.Trace(err) - } +func (kvcodec *TableKVEncoder) Close() { + metric.KvEncoderCounter.WithLabelValues("closed").Inc() +} - metric.KvEncoderCounter.WithLabelValues("open").Inc() +// Encode a row of data into KV pairs. +// +// See comments in `(*TableRestore).initializeColumns` for the meaning of the +// `columnPermutation` parameter. +func (kvcodec *TableKVEncoder) Encode( + row []types.Datum, + rowID int64, + columnPermutation []int, +) ([]kvec.KvPair, error) { + cols := kvcodec.tbl.Cols() - return kvcodec, nil -} + var value types.Datum + var err error + var record []types.Datum -func (kvcodec *TableKVEncoder) init(sqlMode string) error { - err := kvcodec.encoder.SetSystemVariable("sql_mode", sqlMode) - if err != nil { - return errors.Trace(err) + if kvcodec.recordCache != nil { + record = kvcodec.recordCache + } else { + record = make([]types.Datum, 0, len(cols)+1) } - common.AppLogger.Debugf("set sql_mode=%s", sqlMode) - return nil -} -func (kvcodec *TableKVEncoder) Close() error { - metric.KvEncoderCounter.WithLabelValues("closed").Inc() - return errors.Trace(kvcodec.encoder.Close()) -} + for i, col := range cols { + if j := columnPermutation[i]; j >= 0 { + value, err = table.CastValue(kvcodec.se, row[j], col.ToInfo()) + if err == nil { + value, err = col.HandleBadNull(value, kvcodec.se.vars.StmtCtx) + } + } else { + value, err = table.GetColOriginDefaultValue(kvcodec.se, col.ToInfo()) + } + if err != nil { + return nil, errors.Trace(err) + } + record = append(record, value) + } -func (kvcodec *TableKVEncoder) SQL2KV(sql string) ([]kvec.KvPair, uint64, error) { - // via sql execution - kvPairs, rowsAffected, err := kvcodec.encoder.Encode(sql, kvcodec.tableID) - if err != nil { - common.AppLogger.Errorf("[sql2kv] sql encode error = %v", err) - return nil, 0, errors.Trace(err) + if !kvcodec.tbl.Meta().PKIsHandle { + if j := columnPermutation[len(cols)]; j >= 0 { + value, err = table.CastValue(kvcodec.se, row[j], extraHandleColumnInfo) + } else { + value, err = types.NewIntDatum(rowID), nil + } + if err != nil { + return nil, errors.Trace(err) + } + record = append(record, value) } - return kvPairs, rowsAffected, nil + _, err = kvcodec.tbl.AddRecord(kvcodec.se, record) + pairs := kvcodec.se.takeKvPairs() + kvcodec.recordCache = record[:0] + + return pairs, errors.Trace(err) } diff --git a/lightning/metric/metric.go b/lightning/metric/metric.go index 2746fbdc8..ebbe85439 100644 --- a/lightning/metric/metric.go +++ b/lightning/metric/metric.go @@ -40,10 +40,13 @@ const ( ChunkStateRunning = "running" ChunkStateFinished = "finished" ChunkStateFailed = "failed" + + BlockDeliverKindIndex = "index" + BlockDeliverKindData = "data" ) var ( - EngineCounter = prometheus.NewCounterVec( + ImporterEngineCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "lightning", Name: "importer_engine", @@ -71,7 +74,12 @@ var ( Name: "tables", Help: "count number of tables processed", }, []string{"state", "result"}) - + ProcessedEngineCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "lightning", + Name: "engines", + Help: "count number of engines processed", + }, []string{"state", "result"}) ChunkCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "lightning", @@ -93,22 +101,6 @@ var ( Buckets: prometheus.ExponentialBuckets(0.125, 2, 6), }, ) - BlockReadSecondsHistogram = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "lightning", - Name: "block_read_seconds", - Help: "time needed to read a block", - Buckets: prometheus.ExponentialBuckets(0.001, 3.1622776601683795, 7), - }, - ) - BlockReadBytesHistogram = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "lightning", - Name: "block_read_bytes", - Help: "number of bytes being read out from data source", - Buckets: prometheus.ExponentialBuckets(1024, 2, 8), - }, - ) ChunkParserReadBlockSecondsHistogram = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "lightning", @@ -117,14 +109,6 @@ var ( Buckets: prometheus.ExponentialBuckets(0.001, 3.1622776601683795, 10), }, ) - ChunkParserReadRowSecondsHistogram = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "lightning", - Name: "chunk_parser_read_row_seconds", - Help: "time needed for chunk parser read a row", - Buckets: prometheus.ExponentialBuckets(0.001, 3.1622776601683795, 10), - }, - ) ApplyWorkerSecondsHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "lightning", @@ -133,11 +117,27 @@ var ( Buckets: prometheus.ExponentialBuckets(0.001, 3.1622776601683795, 10), }, []string{"name"}, ) - BlockEncodeSecondsHistogram = prometheus.NewHistogram( + RowReadSecondsHistogram = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "lightning", + Name: "row_read_seconds", + Help: "time needed to parse a row", + Buckets: prometheus.ExponentialBuckets(0.001, 3.1622776601683795, 7), + }, + ) + RowReadBytesHistogram = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "lightning", - Name: "block_encode_seconds", - Help: "time needed to encode a block", + Name: "row_read_bytes", + Help: "number of bytes being read out from data source", + Buckets: prometheus.ExponentialBuckets(1024, 2, 8), + }, + ) + RowEncodeSecondsHistogram = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "lightning", + Name: "row_encode_seconds", + Help: "time needed to encode a row", Buckets: prometheus.ExponentialBuckets(0.001, 3.1622776601683795, 10), }, ) @@ -149,13 +149,21 @@ var ( Buckets: prometheus.ExponentialBuckets(0.001, 3.1622776601683795, 10), }, ) - BlockDeliverBytesHistogram = prometheus.NewHistogram( + BlockDeliverBytesHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "lightning", Name: "block_deliver_bytes", Help: "number of bytes being sent out to importer", Buckets: prometheus.ExponentialBuckets(512, 2, 10), - }, + }, []string{"kind"}, + ) + BlockDeliverKVPairsHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "lightning", + Name: "block_deliver_kv_pairs", + Help: "number of KV pairs being sent out to importer", + Buckets: prometheus.ExponentialBuckets(1, 2, 10), + }, []string{"kind"}, ) ChecksumSecondsHistogram = prometheus.NewHistogram( prometheus.HistogramOpts{ @@ -169,18 +177,19 @@ var ( func init() { prometheus.MustRegister(IdleWorkersGauge) - prometheus.MustRegister(EngineCounter) + prometheus.MustRegister(ImporterEngineCounter) prometheus.MustRegister(KvEncoderCounter) prometheus.MustRegister(TableCounter) + prometheus.MustRegister(ProcessedEngineCounter) prometheus.MustRegister(ChunkCounter) prometheus.MustRegister(ImportSecondsHistogram) - prometheus.MustRegister(BlockReadSecondsHistogram) - prometheus.MustRegister(BlockReadBytesHistogram) - prometheus.MustRegister(BlockEncodeSecondsHistogram) + prometheus.MustRegister(RowReadSecondsHistogram) + prometheus.MustRegister(RowReadBytesHistogram) + prometheus.MustRegister(RowEncodeSecondsHistogram) prometheus.MustRegister(BlockDeliverSecondsHistogram) prometheus.MustRegister(BlockDeliverBytesHistogram) + prometheus.MustRegister(BlockDeliverKVPairsHistogram) prometheus.MustRegister(ChecksumSecondsHistogram) - prometheus.MustRegister(ChunkParserReadRowSecondsHistogram) prometheus.MustRegister(ChunkParserReadBlockSecondsHistogram) prometheus.MustRegister(ApplyWorkerSecondsHistogram) } @@ -195,6 +204,16 @@ func RecordTableCount(status string, err error) { TableCounter.WithLabelValues(status, result).Inc() } +func RecordEngineCount(status string, err error) { + var result string + if err != nil { + result = TableResultFailure + } else { + result = TableResultSuccess + } + ProcessedEngineCounter.WithLabelValues(status, result).Inc() +} + // ReadCounter reports the current value of the counter. func ReadCounter(counter prometheus.Counter) float64 { var metric dto.Metric diff --git a/lightning/mydump/csv_parser.go b/lightning/mydump/csv_parser.go index 8a4c3772e..77f5266fc 100644 --- a/lightning/mydump/csv_parser.go +++ b/lightning/mydump/csv_parser.go @@ -1,22 +1,19 @@ package mydump import ( - "bytes" "io" + "strings" "github.com/pingcap/errors" "github.com/pingcap/tidb-lightning/lightning/config" "github.com/pingcap/tidb-lightning/lightning/worker" + "github.com/pingcap/tidb/types" ) type CSVParser struct { blockParser - - rowBuf bytes.Buffer - cfg *config.CSVConfig - - delim1 []byte - delim2 []byte + cfg *config.CSVConfig + escFlavor backslashEscapeFlavor } func NewCSVParser( @@ -25,12 +22,19 @@ func NewCSVParser( blockBufSize int64, ioWorkers *worker.Pool, ) *CSVParser { + escFlavor := backslashEscapeFlavorNone + if cfg.BackslashEscape { + escFlavor = backslashEscapeFlavorMySQL + // we need special treatment of the NULL value \N, used by MySQL. + if !cfg.NotNull && cfg.Null == `\N` { + escFlavor = backslashEscapeFlavorMySQLWithNull + } + } + return &CSVParser{ blockParser: makeBlockParser(reader, blockBufSize, ioWorkers), cfg: cfg, - - delim1: []byte(cfg.Delimiter), - delim2: []byte(cfg.Delimiter + cfg.Delimiter), + escFlavor: escFlavor, } } @@ -41,46 +45,61 @@ const ( csvTokSep csvTokNewLine csvTokField - csvTokQuotedField ) func (parser *CSVParser) appendEmptyValues(sepCount int) { - var content string + var datum types.Datum if !parser.cfg.NotNull && parser.cfg.Null == "" { - content = ",NULL" + datum.SetNull() } else { - content = ",''" + datum.SetString("") } - for i := 1; i < sepCount; i++ { - parser.rowBuf.WriteString(content) + for i := 0; i < sepCount; i++ { + parser.lastRow.Row = append(parser.lastRow.Row, datum) } } -func (parser *CSVParser) appendField(content []byte, shouldUnquote bool, quote byte) { - if !parser.cfg.BackslashEscape { - content = bytes.Replace(content, []byte(`\`), []byte(`\\`), -1) - } - if shouldUnquote { - content = bytes.Replace(content[1:len(content)-1], parser.delim2, parser.delim1, -1) +func (parser *CSVParser) appendField(content string) { + input, isNull := parser.unescapeString(content) + + if parser.escFlavor != backslashEscapeFlavorMySQLWithNull { + isNull = !parser.cfg.NotNull && parser.cfg.Null == input } - if !parser.cfg.NotNull && parser.cfg.Null == string(content) { - parser.rowBuf.WriteString(",NULL") - return + + var datum types.Datum + if isNull { + datum.SetNull() + } else { + datum.SetString(input) } + parser.lastRow.Row = append(parser.lastRow.Row, datum) +} - parser.rowBuf.WriteByte(',') - parser.rowBuf.WriteByte(quote) - parser.rowBuf.Write(bytes.Replace(content, []byte{quote}, []byte{quote, quote}, -1)) - parser.rowBuf.WriteByte(quote) +func (parser *CSVParser) unescapeString(input string) (unescaped string, isNull bool) { + delim := parser.cfg.Delimiter + if len(delim) > 0 && len(input) >= 2 && input[0] == delim[0] { + input = input[1 : len(input)-1] + } else { + delim = "" + } + if parser.escFlavor == backslashEscapeFlavorMySQLWithNull && input == `\N` { + return input, true + } + return unescape(input, delim, parser.escFlavor), false } // ReadRow reads a row from the datafile. func (parser *CSVParser) ReadRow() error { - emptySepCount := 0 + emptySepCount := 1 hasField := false + row := &parser.lastRow + row.RowID++ + row.Row = make([]types.Datum, 0, len(row.Row)) + // skip the header first if parser.pos == 0 && parser.cfg.Header { + parser.columns = make([]string, 0, len(row.Row)) outside: for { tok, content, err := parser.lex() @@ -90,16 +109,9 @@ func (parser *CSVParser) ReadRow() error { switch tok { case csvTokSep: case csvTokField: - parser.appendField(content, false, '`') - case csvTokQuotedField: - parser.appendField(content, true, '`') + colName, _ := parser.unescapeString(string(content)) + parser.columns = append(parser.columns, strings.ToLower(colName)) case csvTokNewLine: - len := parser.rowBuf.Len() - parser.columns = make([]byte, len+1) - copy(parser.columns[:len], parser.rowBuf.Bytes()) - parser.columns[0] = '(' - parser.columns[len] = ')' - parser.rowBuf.Reset() break outside } } @@ -126,24 +138,14 @@ func (parser *CSVParser) ReadRow() error { emptySepCount++ case csvTokField: - parser.appendEmptyValues(emptySepCount) - emptySepCount = 0 - parser.appendField(content, false, '\'') - - case csvTokQuotedField: - parser.appendEmptyValues(emptySepCount) + parser.appendEmptyValues(emptySepCount - 1) emptySepCount = 0 - parser.appendField(content, true, '\'') + parser.appendField(string(content)) case csvTokNewLine: if !parser.cfg.TrimLastSep { parser.appendEmptyValues(emptySepCount) } - parser.rowBuf.WriteByte(')') - parser.lastRow.Row = parser.rowBuf.Bytes() - parser.lastRow.Row[0] = '(' - parser.lastRow.RowID++ - parser.rowBuf.Reset() return nil } } diff --git a/lightning/mydump/csv_parser.rl b/lightning/mydump/csv_parser.rl index b005197a0..1fee4e665 100644 --- a/lightning/mydump/csv_parser.rl +++ b/lightning/mydump/csv_parser.rl @@ -32,7 +32,7 @@ machine csv_parser; # q = ^'\n' when { fc == delim }; -bs = '\\' when { parser.cfg.BackslashEscape }; +bs = '\\' when { parser.escFlavor != backslashEscapeFlavorNone }; sep = ^'\n' when { fc == sep }; c = (^'\n' - q - bs - sep) | bs any; @@ -43,12 +43,7 @@ main := |* fbreak; }; - q (c | '\n' | sep | q q)* q => { - consumedToken = csvTokQuotedField - fbreak; - }; - - c+ => { + q (c | '\n' | sep | q q)* q | c+ => { consumedToken = csvTokField fbreak; }; diff --git a/lightning/mydump/csv_parser_generated.go b/lightning/mydump/csv_parser_generated.go index 4a6e27896..86237d31b 100644 --- a/lightning/mydump/csv_parser_generated.go +++ b/lightning/mydump/csv_parser_generated.go @@ -18,7 +18,7 @@ import ( ) -//.... lightning/mydump/csv_parser.rl:63 +//.... lightning/mydump/csv_parser.rl:58 @@ -30,7 +30,7 @@ const csv_parser_error int = 0 const csv_parser_en_main int = 8 -//.... lightning/mydump/csv_parser.rl:66 +//.... lightning/mydump/csv_parser.rl:61 func (parser *CSVParser) lex() (csvToken, []byte, error) { var delim byte @@ -49,7 +49,7 @@ func (parser *CSVParser) lex() (csvToken, []byte, error) { act = 0 } -//.... lightning/mydump/csv_parser.rl:76 +//.... lightning/mydump/csv_parser.rl:71 for { data := parser.buf @@ -78,54 +78,46 @@ func (parser *CSVParser) lex() (csvToken, []byte, error) { goto st_case_10 case 1: goto st_case_1 - case 11: - goto st_case_11 case 2: goto st_case_2 + case 11: + goto st_case_11 case 12: goto st_case_12 - case 13: - goto st_case_13 case 3: goto st_case_3 - case 14: - goto st_case_14 + case 13: + goto st_case_13 case 4: goto st_case_4 + case 14: + goto st_case_14 case 15: goto st_case_15 + case 5: + goto st_case_5 case 16: goto st_case_16 + case 6: + goto st_case_6 case 17: goto st_case_17 + case 7: + goto st_case_7 case 18: goto st_case_18 case 19: goto st_case_19 - case 5: - goto st_case_5 case 20: goto st_case_20 - case 6: - goto st_case_6 case 21: goto st_case_21 - case 7: - goto st_case_7 case 22: goto st_case_22 case 23: goto st_case_23 case 24: goto st_case_24 - case 25: - goto st_case_25 - case 26: - goto st_case_26 - case 27: - goto st_case_27 - case 28: - goto st_case_28 } goto st_out tr0: @@ -140,27 +132,23 @@ tr0: {p++; cs = 8; goto _out } } case 2: - {p = (te) - 1 - - consumedToken = csvTokQuotedField - {p++; cs = 8; goto _out } - } - case 3: {p = (te) - 1 consumedToken = csvTokField {p++; cs = 8; goto _out } } - case 4: - {p = (te) - 1 - - consumedToken = csvTokNewLine - {p++; cs = 8; goto _out } - } } goto st8 -tr20: +tr14: +//.... lightning/mydump/csv_parser.rl:46 +p = (te) - 1 +{ + consumedToken = csvTokField + {p++; cs = 8; goto _out } + } + goto st8 +tr17: //.... lightning/mydump/csv_parser.rl:41 te = p+1 { @@ -168,25 +156,25 @@ te = p+1 {p++; cs = 8; goto _out } } goto st8 -tr28: +tr23: //.... lightning/mydump/csv_parser.rl:51 te = p p-- { - consumedToken = csvTokField + consumedToken = csvTokNewLine {p++; cs = 8; goto _out } } goto st8 -tr29: +tr24: //.... lightning/mydump/csv_parser.rl:46 te = p p-- { - consumedToken = csvTokQuotedField + consumedToken = csvTokField {p++; cs = 8; goto _out } } goto st8 -tr32: +tr25: //.... lightning/mydump/csv_parser.rl:41 te = p p-- @@ -209,11 +197,22 @@ act = 0 //.... NONE:1 ts = p -//.... tmp_parser.go:212 +//.... tmp_parser.go:200 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -239,7 +238,7 @@ ts = p if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -257,15 +256,9 @@ ts = p } switch _widec { case 10: - goto tr16 - case 2829: - goto tr18 - case 3085: - goto tr19 - case 3341: - goto tr21 - case 3597: - goto tr23 + goto st9 + case 13: + goto st9 case 3932: goto tr1 case 4188: @@ -275,33 +268,41 @@ ts = p case 4700: goto st5 case 4956: - goto tr20 + goto tr17 case 5212: - goto tr22 + goto tr18 case 5468: - goto st27 + goto st23 case 5724: - goto st28 + goto st24 } switch { case _widec < 3165: switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto tr1 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto tr1 + } + default: goto tr1 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto st2 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto st2 } - case _widec >= 3072: + default: goto st2 } default: @@ -311,28 +312,37 @@ ts = p switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { - goto tr20 + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto tr17 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { + goto tr17 } - case _widec >= 3328: - goto tr20 + default: + goto tr17 } case _widec > 3583: switch { case _widec < 3595: if 3584 <= _widec && _widec <= 3593 { - goto tr22 + goto tr18 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { - goto tr22 + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr18 + } + case _widec >= 3598: + goto tr18 } default: - goto tr22 + goto tr18 } default: - goto tr20 + goto tr17 } default: goto st2 @@ -342,50 +352,46 @@ st_case_0: st0: cs = 0 goto _out -tr16: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:56 -act = 4; - goto st9 -tr21: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:41 -act = 1; - goto st9 st9: if p++; p == pe { goto _test_eof9 } st_case_9: -//.... tmp_parser.go:364 switch data[p] { case 10: - goto tr16 + goto st9 case 13: - goto tr16 + goto st9 } - goto tr0 + goto tr23 tr1: //.... NONE:1 te = p+1 -//.... lightning/mydump/csv_parser.rl:51 -act = 3; +//.... lightning/mydump/csv_parser.rl:46 +act = 2; goto st10 st10: if p++; p == pe { goto _test_eof10 } st_case_10: -//.... tmp_parser.go:384 +//.... tmp_parser.go:379 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -411,7 +417,7 @@ act = 3; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -444,49 +450,36 @@ act = 3; if 2816 <= _widec && _widec <= 2825 { goto tr1 } - case _widec > 2907: - if 2909 <= _widec && _widec <= 3071 { + case _widec > 2828: + switch { + case _widec > 2907: + if 2909 <= _widec && _widec <= 3071 { + goto tr1 + } + case _widec >= 2830: goto tr1 } default: goto tr1 } - goto tr28 + goto tr24 st1: if p++; p == pe { goto _test_eof1 } st_case_1: goto tr1 -tr18: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:51 -act = 3; - goto st11 - st11: + st2: if p++; p == pe { - goto _test_eof11 + goto _test_eof2 } - st_case_11: -//.... tmp_parser.go:473 + st_case_2: _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { - _widec = 2816 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if data[p] == sep { - _widec += 512 - } - } - case data[p] > 91: + case data[p] < 14: switch { - case data[p] > 92: - if 93 <= data[p] { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -495,71 +488,7 @@ act = 3; _widec += 512 } } - case data[p] >= 92: - _widec = 3840 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if parser.cfg.BackslashEscape { - _widec += 512 - } - if data[p] == sep { - _widec += 1024 - } - } - default: - _widec = 2816 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if data[p] == sep { - _widec += 512 - } - } - switch _widec { - case 10: - goto tr16 - case 2829: - goto tr18 - case 3085: - goto tr16 - case 3341: - goto tr16 - case 3597: - goto tr16 - case 3932: - goto tr1 - case 4444: - goto st1 - case 4700: - goto st1 - case 5468: - goto st1 - case 5724: - goto st1 - } - switch { - case _widec < 2827: - if 2816 <= _widec && _widec <= 2825 { - goto tr1 - } - case _widec > 2907: - if 2909 <= _widec && _widec <= 3071 { - goto tr1 - } - default: - goto tr1 - } - goto tr28 - st2: - if p++; p == pe { - goto _test_eof2 - } - st_case_2: - _widec = int16(data[p]) - switch { - case data[p] < 11: - if data[p] <= 9 { + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -585,7 +514,7 @@ act = 3; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -604,6 +533,8 @@ act = 3; switch _widec { case 10: goto st2 + case 13: + goto st2 case 3932: goto st2 case 4188: @@ -626,20 +557,28 @@ act = 3; switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto st2 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto st2 + } + default: goto st2 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr3 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr3 } - case _widec >= 3072: + default: goto tr3 } default: @@ -649,11 +588,15 @@ act = 3; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { goto st2 } - case _widec >= 3328: + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { + goto st2 + } + default: goto st2 } case _widec > 3583: @@ -662,8 +605,13 @@ act = 3; if 3584 <= _widec && _widec <= 3593 { goto tr4 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr4 + } + case _widec >= 3598: goto tr4 } default: @@ -682,17 +630,24 @@ te = p+1 //.... lightning/mydump/csv_parser.rl:46 act = 2; - goto st12 - st12: + goto st11 + st11: if p++; p == pe { - goto _test_eof12 + goto _test_eof11 } - st_case_12: -//.... tmp_parser.go:691 + st_case_11: +//.... tmp_parser.go:639 _widec = int16(data[p]) switch { - case data[p] > 9: - if 11 <= data[p] { + case data[p] < 11: + if data[p] <= 9 { + _widec = 768 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + } + case data[p] > 12: + if 14 <= data[p] { _widec = 768 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -705,31 +660,46 @@ act = 2; } } switch { - case _widec > 1033: - if 1035 <= _widec && _widec <= 1279 { + case _widec < 1035: + if 1024 <= _widec && _widec <= 1033 { + goto st2 + } + case _widec > 1036: + if 1038 <= _widec && _widec <= 1279 { goto st2 } - case _widec >= 1024: + default: goto st2 } - goto tr29 + goto tr24 tr4: //.... NONE:1 te = p+1 //.... lightning/mydump/csv_parser.rl:46 act = 2; - goto st13 - st13: + goto st12 + st12: if p++; p == pe { - goto _test_eof13 + goto _test_eof12 } - st_case_13: -//.... tmp_parser.go:728 + st_case_12: +//.... tmp_parser.go:687 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -755,7 +725,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -774,6 +744,8 @@ act = 2; switch _widec { case 10: goto st2 + case 13: + goto st2 case 3932: goto st2 case 4188: @@ -796,20 +768,28 @@ act = 2; switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto st2 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto st2 + } + default: goto st2 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr4 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr4 } - case _widec >= 3072: + default: goto tr4 } default: @@ -819,11 +799,15 @@ act = 2; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { goto st2 } - case _widec >= 3328: + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { + goto st2 + } + default: goto st2 } case _widec > 3583: @@ -832,8 +816,13 @@ act = 2; if 3584 <= _widec && _widec <= 3593 { goto tr4 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr4 + } + case _widec >= 3598: goto tr4 } default: @@ -845,7 +834,7 @@ act = 2; default: goto tr4 } - goto tr29 + goto tr24 st3: if p++; p == pe { goto _test_eof3 @@ -858,29 +847,19 @@ te = p+1 //.... lightning/mydump/csv_parser.rl:46 act = 2; - goto st14 - st14: + goto st13 + st13: if p++; p == pe { - goto _test_eof14 + goto _test_eof13 } - st_case_14: -//.... tmp_parser.go:867 + st_case_13: +//.... tmp_parser.go:856 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { - _widec = 2816 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if data[p] == sep { - _widec += 512 - } - } - case data[p] > 91: + case data[p] < 14: switch { - case data[p] > 92: - if 93 <= data[p] { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -889,111 +868,7 @@ act = 2; _widec += 512 } } - case data[p] >= 92: - _widec = 3840 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if parser.cfg.BackslashEscape { - _widec += 512 - } - if data[p] == sep { - _widec += 1024 - } - } - default: - _widec = 2816 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if data[p] == sep { - _widec += 512 - } - } - switch _widec { - case 10: - goto st2 - case 3932: - goto st2 - case 4188: - goto tr4 - case 4444: - goto st4 - case 4700: - goto tr8 - case 4956: - goto st2 - case 5212: - goto tr4 - case 5468: - goto st4 - case 5724: - goto tr8 - } - switch { - case _widec < 3165: - switch { - case _widec < 2909: - switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { - goto st2 - } - case _widec >= 2816: - goto st2 - } - case _widec > 3071: - switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { - goto tr4 - } - case _widec >= 3072: - goto tr4 - } default: - goto st2 - } - case _widec > 3327: - switch { - case _widec < 3421: - switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { - goto st2 - } - case _widec >= 3328: - goto st2 - } - case _widec > 3583: - switch { - case _widec < 3595: - if 3584 <= _widec && _widec <= 3593 { - goto tr4 - } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { - goto tr4 - } - default: - goto tr4 - } - default: - goto st2 - } - default: - goto tr4 - } - goto tr29 - st4: - if p++; p == pe { - goto _test_eof4 - } - st_case_4: - _widec = int16(data[p]) - switch { - case data[p] < 11: - if data[p] <= 9 { _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -1019,7 +894,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -1038,6 +913,8 @@ act = 2; switch _widec { case 10: goto st2 + case 13: + goto st2 case 3932: goto st2 case 4188: @@ -1060,20 +937,28 @@ act = 2; switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto st2 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto st2 + } + default: goto st2 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr4 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr4 } - case _widec >= 3072: + default: goto tr4 } default: @@ -1083,11 +968,15 @@ act = 2; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto st2 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto st2 } - case _widec >= 3328: + default: goto st2 } case _widec > 3583: @@ -1096,8 +985,13 @@ act = 2; if 3584 <= _widec && _widec <= 3593 { goto tr4 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr4 + } + case _widec >= 3598: goto tr4 } default: @@ -1109,87 +1003,18 @@ act = 2; default: goto tr4 } - goto tr0 -tr6: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:46 -act = 2; - goto st15 - st15: - if p++; p == pe { - goto _test_eof15 - } - st_case_15: -//.... tmp_parser.go:1125 - _widec = int16(data[p]) - switch { - case data[p] > 9: - if 11 <= data[p] { - _widec = 768 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - } - default: - _widec = 768 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - } - if _widec == 10 { - goto st2 - } - switch { - case _widec < 779: - if 768 <= _widec && _widec <= 777 { - goto st2 - } - case _widec > 1033: - if 1035 <= _widec && _widec <= 1279 { - goto st2 - } - default: - goto st2 - } - goto tr29 -tr19: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:56 -act = 4; - goto st16 -tr23: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:41 -act = 1; - goto st16 - st16: + goto tr24 + st4: if p++; p == pe { - goto _test_eof16 + goto _test_eof4 } - st_case_16: -//.... tmp_parser.go:1176 + st_case_4: _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { - _widec = 2816 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if data[p] == sep { - _widec += 512 - } - } - case data[p] > 91: + case data[p] < 14: switch { - case data[p] > 92: - if 93 <= data[p] { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -1198,172 +1023,7 @@ act = 1; _widec += 512 } } - case data[p] >= 92: - _widec = 3840 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if parser.cfg.BackslashEscape { - _widec += 512 - } - if data[p] == sep { - _widec += 1024 - } - } - default: - _widec = 2816 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - if data[p] == sep { - _widec += 512 - } - } - switch _widec { - case 10: - goto tr19 - case 2829: - goto tr19 - case 3085: - goto tr30 - case 3341: - goto tr19 - case 3597: - goto tr31 - case 3932: - goto st2 - case 4188: - goto tr3 - case 4444: - goto st3 - case 4700: - goto tr6 - case 4956: - goto st2 - case 5212: - goto tr4 - case 5468: - goto st4 - case 5724: - goto tr8 - } - switch { - case _widec < 3165: - switch { - case _widec < 2909: - switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { - goto st2 - } - case _widec >= 2816: - goto st2 - } - case _widec > 3071: - switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { - goto tr3 - } - case _widec >= 3072: - goto tr3 - } default: - goto st2 - } - case _widec > 3327: - switch { - case _widec < 3421: - switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { - goto st2 - } - case _widec >= 3328: - goto st2 - } - case _widec > 3583: - switch { - case _widec < 3595: - if 3584 <= _widec && _widec <= 3593 { - goto tr4 - } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { - goto tr4 - } - default: - goto tr4 - } - default: - goto st2 - } - default: - goto tr3 - } - goto tr0 -tr30: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:46 -act = 2; - goto st17 - st17: - if p++; p == pe { - goto _test_eof17 - } - st_case_17: -//.... tmp_parser.go:1317 - _widec = int16(data[p]) - switch { - case data[p] > 9: - if 11 <= data[p] { - _widec = 768 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - } - default: - _widec = 768 + (int16(data[p]) - 0) - if data[p] == delim { - _widec += 256 - } - } - switch _widec { - case 10: - goto tr16 - case 781: - goto tr16 - case 1037: - goto tr19 - } - switch { - case _widec > 1033: - if 1035 <= _widec && _widec <= 1279 { - goto st2 - } - case _widec >= 1024: - goto st2 - } - goto tr29 -tr31: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:46 -act = 2; - goto st18 - st18: - if p++; p == pe { - goto _test_eof18 - } - st_case_18: -//.... tmp_parser.go:1362 - _widec = int16(data[p]) - switch { - case data[p] < 11: - if data[p] <= 9 { _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -1389,7 +1049,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -1407,21 +1067,15 @@ act = 2; } switch _widec { case 10: - goto tr19 - case 2829: - goto tr19 - case 3085: - goto tr31 - case 3341: - goto tr19 - case 3597: - goto tr31 + goto st2 + case 13: + goto st2 case 3932: goto st2 case 4188: goto tr4 case 4444: - goto st3 + goto st4 case 4700: goto tr8 case 4956: @@ -1438,20 +1092,28 @@ act = 2; switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto st2 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto st2 + } + default: goto st2 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr4 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr4 } - case _widec >= 3072: + default: goto tr4 } default: @@ -1461,11 +1123,15 @@ act = 2; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto st2 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto st2 } - case _widec >= 3328: + default: goto st2 } case _widec > 3583: @@ -1474,8 +1140,13 @@ act = 2; if 3584 <= _widec && _widec <= 3593 { goto tr4 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr4 + } + case _widec >= 3598: goto tr4 } default: @@ -1487,24 +1158,99 @@ act = 2; default: goto tr4 } - goto tr29 -tr22: + goto tr0 +tr6: //.... NONE:1 te = p+1 -//.... lightning/mydump/csv_parser.rl:41 -act = 1; - goto st19 - st19: +//.... lightning/mydump/csv_parser.rl:46 +act = 2; + goto st14 + st14: if p++; p == pe { - goto _test_eof19 + goto _test_eof14 } - st_case_19: -//.... tmp_parser.go:1503 + st_case_14: +//.... tmp_parser.go:1174 _widec = int16(data[p]) switch { case data[p] < 11: if data[p] <= 9 { + _widec = 768 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + } + case data[p] > 12: + if 14 <= data[p] { + _widec = 768 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + } + default: + _widec = 768 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + } + switch _widec { + case 10: + goto st2 + case 13: + goto st2 + } + switch { + case _widec < 782: + switch { + case _widec > 777: + if 779 <= _widec && _widec <= 780 { + goto st2 + } + case _widec >= 768: + goto st2 + } + case _widec > 1033: + switch { + case _widec > 1036: + if 1038 <= _widec && _widec <= 1279 { + goto st2 + } + case _widec >= 1035: + goto st2 + } + default: + goto st2 + } + goto tr24 +tr18: +//.... NONE:1 +te = p+1 + +//.... lightning/mydump/csv_parser.rl:41 +act = 1; + goto st15 + st15: + if p++; p == pe { + goto _test_eof15 + } + st_case_15: +//.... tmp_parser.go:1238 + _widec = int16(data[p]) + switch { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -1530,7 +1276,7 @@ act = 1; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -1549,6 +1295,8 @@ act = 1; switch _widec { case 10: goto st2 + case 13: + goto st2 case 3932: goto st2 case 4188: @@ -1571,20 +1319,28 @@ act = 1; switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto st2 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto st2 + } + default: goto st2 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr3 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr3 } - case _widec >= 3072: + default: goto tr3 } default: @@ -1594,11 +1350,15 @@ act = 1; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto st2 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto st2 } - case _widec >= 3328: + default: goto st2 } case _widec > 3583: @@ -1607,8 +1367,13 @@ act = 1; if 3584 <= _widec && _widec <= 3593 { goto tr4 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr4 + } + case _widec >= 3598: goto tr4 } default: @@ -1620,7 +1385,7 @@ act = 1; default: goto tr3 } - goto tr32 + goto tr25 st5: if p++; p == pe { goto _test_eof5 @@ -1628,8 +1393,19 @@ act = 1; st_case_5: _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -1655,7 +1431,7 @@ act = 1; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -1674,6 +1450,8 @@ act = 1; switch _widec { case 10: goto tr9 + case 13: + goto tr9 case 3932: goto tr9 case 4188: @@ -1681,35 +1459,43 @@ act = 1; case 4444: goto tr12 case 4700: - goto tr13 + goto tr12 case 4956: goto tr9 case 5212: goto tr11 case 5468: - goto tr14 + goto tr13 case 5724: - goto tr15 + goto tr13 } switch { case _widec < 3165: switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto tr9 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto tr9 + } + default: goto tr9 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { goto tr10 } - case _widec >= 3072: + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { + goto tr10 + } + default: goto tr10 } default: @@ -1719,11 +1505,15 @@ act = 1; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto tr9 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto tr9 } - case _widec >= 3328: + default: goto tr9 } case _widec > 3583: @@ -1732,8 +1522,13 @@ act = 1; if 3584 <= _widec && _widec <= 3593 { goto tr11 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr11 + } + case _widec >= 3598: goto tr11 } default: @@ -1750,19 +1545,30 @@ tr9: //.... NONE:1 te = p+1 -//.... lightning/mydump/csv_parser.rl:51 -act = 3; - goto st20 - st20: +//.... lightning/mydump/csv_parser.rl:46 +act = 2; + goto st16 + st16: if p++; p == pe { - goto _test_eof20 + goto _test_eof16 } - st_case_20: -//.... tmp_parser.go:1761 + st_case_16: +//.... tmp_parser.go:1556 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -1788,7 +1594,7 @@ act = 3; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -1807,6 +1613,8 @@ act = 3; switch _widec { case 10: goto st2 + case 13: + goto st2 case 3932: goto tr9 case 4188: @@ -1814,7 +1622,7 @@ act = 3; case 4444: goto st6 case 4700: - goto st21 + goto st17 case 4956: goto st2 case 5212: @@ -1822,27 +1630,35 @@ act = 3; case 5468: goto st7 case 5724: - goto st23 + goto st19 } switch { case _widec < 3165: switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto tr9 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto tr9 + } + default: goto tr9 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr3 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr3 } - case _widec >= 3072: + default: goto tr3 } default: @@ -1852,11 +1668,15 @@ act = 3; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { goto st2 } - case _widec >= 3328: + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { + goto st2 + } + default: goto st2 } case _widec > 3583: @@ -1865,8 +1685,13 @@ act = 3; if 3584 <= _widec && _widec <= 3593 { goto tr4 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr4 + } + case _widec >= 3598: goto tr4 } default: @@ -1878,22 +1703,29 @@ act = 3; default: goto tr3 } - goto tr28 + goto tr24 st6: if p++; p == pe { goto _test_eof6 } st_case_6: goto tr9 - st21: + st17: if p++; p == pe { - goto _test_eof21 + goto _test_eof17 } - st_case_21: + st_case_17: _widec = int16(data[p]) switch { - case data[p] > 9: - if 11 <= data[p] { + case data[p] < 11: + if data[p] <= 9 { + _widec = 768 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + } + case data[p] > 12: + if 14 <= data[p] { _widec = 768 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -1905,22 +1737,35 @@ act = 3; _widec += 256 } } - if _widec == 10 { + switch _widec { + case 10: + goto tr9 + case 13: goto tr9 } switch { - case _widec < 779: - if 768 <= _widec && _widec <= 777 { + case _widec < 782: + switch { + case _widec > 777: + if 779 <= _widec && _widec <= 780 { + goto tr9 + } + case _widec >= 768: goto tr9 } case _widec > 1033: - if 1035 <= _widec && _widec <= 1279 { + switch { + case _widec > 1036: + if 1038 <= _widec && _widec <= 1279 { + goto tr9 + } + case _widec >= 1035: goto tr9 } default: goto tr9 } - goto tr29 + goto tr24 st7: if p++; p == pe { goto _test_eof7 @@ -1928,8 +1773,19 @@ act = 3; st_case_7: _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -1955,7 +1811,7 @@ act = 3; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -1974,42 +1830,52 @@ act = 3; switch _widec { case 10: goto tr9 + case 13: + goto tr9 case 3932: goto tr9 case 4188: goto tr11 case 4444: - goto tr14 + goto tr13 case 4700: - goto tr15 + goto tr13 case 4956: goto tr9 case 5212: goto tr11 case 5468: - goto tr14 + goto tr13 case 5724: - goto tr15 + goto tr13 } switch { case _widec < 3165: switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto tr9 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto tr9 + } + default: goto tr9 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr11 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr11 } - case _widec >= 3072: + default: goto tr11 } default: @@ -2019,11 +1885,15 @@ act = 3; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto tr9 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto tr9 } - case _widec >= 3328: + default: goto tr9 } case _widec > 3583: @@ -2032,8 +1902,13 @@ act = 3; if 3584 <= _widec && _widec <= 3593 { goto tr11 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr11 + } + case _widec >= 3598: goto tr11 } default: @@ -2045,24 +1920,35 @@ act = 3; default: goto tr11 } - goto tr0 + goto tr14 tr11: //.... NONE:1 te = p+1 //.... lightning/mydump/csv_parser.rl:46 act = 2; - goto st22 - st22: + goto st18 + st18: if p++; p == pe { - goto _test_eof22 + goto _test_eof18 } - st_case_22: -//.... tmp_parser.go:2061 + st_case_18: +//.... tmp_parser.go:1936 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -2088,7 +1974,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -2107,6 +1993,8 @@ act = 2; switch _widec { case 10: goto st2 + case 13: + goto st2 case 3932: goto tr9 case 4188: @@ -2114,7 +2002,7 @@ act = 2; case 4444: goto st6 case 4700: - goto st23 + goto st19 case 4956: goto st2 case 5212: @@ -2122,27 +2010,35 @@ act = 2; case 5468: goto st7 case 5724: - goto st23 + goto st19 } switch { case _widec < 3165: switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto tr9 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto tr9 + } + default: goto tr9 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr4 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr4 } - case _widec >= 3072: + default: goto tr4 } default: @@ -2152,11 +2048,15 @@ act = 2; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto st2 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto st2 } - case _widec >= 3328: + default: goto st2 } case _widec > 3583: @@ -2165,8 +2065,13 @@ act = 2; if 3584 <= _widec && _widec <= 3593 { goto tr4 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr4 + } + case _widec >= 3598: goto tr4 } default: @@ -2178,16 +2083,27 @@ act = 2; default: goto tr4 } - goto tr29 - st23: + goto tr24 + st19: if p++; p == pe { - goto _test_eof23 + goto _test_eof19 } - st_case_23: + st_case_19: _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -2213,7 +2129,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -2232,42 +2148,52 @@ act = 2; switch _widec { case 10: goto tr9 + case 13: + goto tr9 case 3932: goto tr9 case 4188: goto tr11 case 4444: - goto tr14 + goto tr13 case 4700: - goto tr15 + goto tr13 case 4956: goto tr9 case 5212: goto tr11 case 5468: - goto tr14 + goto tr13 case 5724: - goto tr15 + goto tr13 } switch { case _widec < 3165: switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto tr9 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto tr9 + } + default: goto tr9 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { goto tr11 } - case _widec >= 3072: + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { + goto tr11 + } + default: goto tr11 } default: @@ -2277,11 +2203,15 @@ act = 2; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto tr9 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto tr9 } - case _widec >= 3328: + default: goto tr9 } case _widec > 3583: @@ -2290,8 +2220,13 @@ act = 2; if 3584 <= _widec && _widec <= 3593 { goto tr11 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr11 + } + case _widec >= 3598: goto tr11 } default: @@ -2303,31 +2238,35 @@ act = 2; default: goto tr11 } - goto tr29 -tr14: -//.... NONE:1 -te = p+1 - -//.... lightning/mydump/csv_parser.rl:51 -act = 3; - goto st24 -tr15: + goto tr24 +tr13: //.... NONE:1 te = p+1 //.... lightning/mydump/csv_parser.rl:46 act = 2; - goto st24 - st24: + goto st20 + st20: if p++; p == pe { - goto _test_eof24 + goto _test_eof20 } - st_case_24: -//.... tmp_parser.go:2326 + st_case_20: +//.... tmp_parser.go:2254 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -2353,7 +2292,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -2372,6 +2311,8 @@ act = 2; switch _widec { case 10: goto st2 + case 13: + goto st2 case 3932: goto tr9 case 4188: @@ -2379,7 +2320,7 @@ act = 2; case 4444: goto st7 case 4700: - goto st23 + goto st19 case 4956: goto st2 case 5212: @@ -2387,27 +2328,35 @@ act = 2; case 5468: goto st7 case 5724: - goto st23 + goto st19 } switch { case _widec < 3165: switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto tr9 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto tr9 + } + default: goto tr9 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { goto tr4 } - case _widec >= 3072: + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { + goto tr4 + } + default: goto tr4 } default: @@ -2417,11 +2366,15 @@ act = 2; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { + goto st2 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto st2 } - case _widec >= 3328: + default: goto st2 } case _widec > 3583: @@ -2430,8 +2383,13 @@ act = 2; if 3584 <= _widec && _widec <= 3593 { goto tr4 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr4 + } + case _widec >= 3598: goto tr4 } default: @@ -2443,24 +2401,35 @@ act = 2; default: goto tr4 } - goto tr0 + goto tr24 tr10: //.... NONE:1 te = p+1 //.... lightning/mydump/csv_parser.rl:46 act = 2; - goto st25 - st25: + goto st21 + st21: if p++; p == pe { - goto _test_eof25 + goto _test_eof21 } - st_case_25: -//.... tmp_parser.go:2459 + st_case_21: +//.... tmp_parser.go:2417 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -2486,7 +2455,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -2521,8 +2490,13 @@ act = 2; switch { case _widec < 3083: switch { - case _widec < 2827: - if 2816 <= _widec && _widec <= 2825 { + case _widec < 2830: + switch { + case _widec > 2825: + if 2827 <= _widec && _widec <= 2828 { + goto tr1 + } + case _widec >= 2816: goto tr1 } case _widec > 2907: @@ -2537,19 +2511,28 @@ act = 2; default: goto tr1 } - case _widec > 3163: + case _widec > 3084: switch { case _widec < 3584: - if 3165 <= _widec && _widec <= 3327 { + switch { + case _widec > 3163: + if 3165 <= _widec && _widec <= 3327 { + goto st2 + } + case _widec >= 3086: goto st2 } case _widec > 3593: switch { + case _widec < 3598: + if 3595 <= _widec && _widec <= 3596 { + goto st2 + } case _widec > 3675: if 3677 <= _widec && _widec <= 3839 { goto st2 } - case _widec >= 3595: + default: goto st2 } default: @@ -2558,31 +2541,35 @@ act = 2; default: goto st2 } - goto tr29 + goto tr24 tr12: //.... NONE:1 te = p+1 -//.... lightning/mydump/csv_parser.rl:51 -act = 3; - goto st26 -tr13: -//.... NONE:1 -te = p+1 - //.... lightning/mydump/csv_parser.rl:46 act = 2; - goto st26 - st26: + goto st22 + st22: if p++; p == pe { - goto _test_eof26 + goto _test_eof22 } - st_case_26: -//.... tmp_parser.go:2581 + st_case_22: +//.... tmp_parser.go:2557 _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -2608,7 +2595,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -2627,6 +2614,8 @@ act = 2; switch _widec { case 10: goto st2 + case 13: + goto st2 case 3932: goto tr9 case 4188: @@ -2645,20 +2634,29 @@ act = 2; goto st5 } switch { - case _widec < 3083: + case _widec < 3086: switch { - case _widec < 2827: - if 2816 <= _widec && _widec <= 2825 { + case _widec < 2830: + switch { + case _widec > 2825: + if 2827 <= _widec && _widec <= 2828 { + goto tr9 + } + case _widec >= 2816: goto tr9 } case _widec > 2907: switch { - case _widec > 3071: - if 3072 <= _widec && _widec <= 3081 { + case _widec < 3072: + if 2909 <= _widec && _widec <= 3071 { + goto tr9 + } + case _widec > 3081: + if 3083 <= _widec && _widec <= 3084 { goto st2 } - case _widec >= 2909: - goto tr9 + default: + goto st2 } default: goto tr9 @@ -2667,20 +2665,28 @@ act = 2; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3165 <= _widec && _widec <= 3337 { + goto st2 + } + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { goto st2 } - case _widec >= 3165: + default: goto st2 } case _widec > 3593: switch { + case _widec < 3598: + if 3595 <= _widec && _widec <= 3596 { + goto st2 + } case _widec > 3675: if 3677 <= _widec && _widec <= 3839 { goto st2 } - case _widec >= 3595: + default: goto st2 } default: @@ -2689,22 +2695,33 @@ act = 2; default: goto st2 } - goto tr0 - st27: + goto tr24 + st23: if p++; p == pe { - goto _test_eof27 + goto _test_eof23 } - st_case_27: + st_case_23: goto tr1 - st28: + st24: if p++; p == pe { - goto _test_eof28 + goto _test_eof24 } - st_case_28: + st_case_24: _widec = int16(data[p]) switch { - case data[p] < 11: - if data[p] <= 9 { + case data[p] < 14: + switch { + case data[p] > 9: + if 11 <= data[p] && data[p] <= 12 { + _widec = 2816 + (int16(data[p]) - 0) + if data[p] == delim { + _widec += 256 + } + if data[p] == sep { + _widec += 512 + } + } + default: _widec = 2816 + (int16(data[p]) - 0) if data[p] == delim { _widec += 256 @@ -2730,7 +2747,7 @@ act = 2; if data[p] == delim { _widec += 256 } - if parser.cfg.BackslashEscape { + if parser.escFlavor != backslashEscapeFlavorNone { _widec += 512 } if data[p] == sep { @@ -2749,6 +2766,8 @@ act = 2; switch _widec { case 10: goto tr9 + case 13: + goto tr9 case 3932: goto tr9 case 4188: @@ -2756,35 +2775,43 @@ act = 2; case 4444: goto tr12 case 4700: - goto tr13 + goto tr12 case 4956: goto tr9 case 5212: goto tr11 case 5468: - goto tr14 + goto tr13 case 5724: - goto tr15 + goto tr13 } switch { case _widec < 3165: switch { case _widec < 2909: switch { - case _widec > 2825: - if 2827 <= _widec && _widec <= 2907 { + case _widec < 2827: + if 2816 <= _widec && _widec <= 2825 { goto tr9 } - case _widec >= 2816: + case _widec > 2828: + if 2830 <= _widec && _widec <= 2907 { + goto tr9 + } + default: goto tr9 } case _widec > 3071: switch { - case _widec > 3081: - if 3083 <= _widec && _widec <= 3163 { + case _widec < 3083: + if 3072 <= _widec && _widec <= 3081 { + goto tr10 + } + case _widec > 3084: + if 3086 <= _widec && _widec <= 3163 { goto tr10 } - case _widec >= 3072: + default: goto tr10 } default: @@ -2794,11 +2821,15 @@ act = 2; switch { case _widec < 3421: switch { - case _widec > 3337: - if 3339 <= _widec && _widec <= 3419 { + case _widec < 3339: + if 3328 <= _widec && _widec <= 3337 { goto tr9 } - case _widec >= 3328: + case _widec > 3340: + if 3342 <= _widec && _widec <= 3419 { + goto tr9 + } + default: goto tr9 } case _widec > 3583: @@ -2807,8 +2838,13 @@ act = 2; if 3584 <= _widec && _widec <= 3593 { goto tr11 } - case _widec > 3675: - if 3677 <= _widec && _widec <= 3839 { + case _widec > 3596: + switch { + case _widec > 3675: + if 3677 <= _widec && _widec <= 3839 { + goto tr11 + } + case _widec >= 3598: goto tr11 } default: @@ -2820,101 +2856,89 @@ act = 2; default: goto tr10 } - goto tr32 + goto tr25 st_out: _test_eof8: cs = 8; goto _test_eof _test_eof9: cs = 9; goto _test_eof _test_eof10: cs = 10; goto _test_eof _test_eof1: cs = 1; goto _test_eof - _test_eof11: cs = 11; goto _test_eof _test_eof2: cs = 2; goto _test_eof + _test_eof11: cs = 11; goto _test_eof _test_eof12: cs = 12; goto _test_eof - _test_eof13: cs = 13; goto _test_eof _test_eof3: cs = 3; goto _test_eof - _test_eof14: cs = 14; goto _test_eof + _test_eof13: cs = 13; goto _test_eof _test_eof4: cs = 4; goto _test_eof + _test_eof14: cs = 14; goto _test_eof _test_eof15: cs = 15; goto _test_eof + _test_eof5: cs = 5; goto _test_eof _test_eof16: cs = 16; goto _test_eof + _test_eof6: cs = 6; goto _test_eof _test_eof17: cs = 17; goto _test_eof + _test_eof7: cs = 7; goto _test_eof _test_eof18: cs = 18; goto _test_eof _test_eof19: cs = 19; goto _test_eof - _test_eof5: cs = 5; goto _test_eof _test_eof20: cs = 20; goto _test_eof - _test_eof6: cs = 6; goto _test_eof _test_eof21: cs = 21; goto _test_eof - _test_eof7: cs = 7; goto _test_eof _test_eof22: cs = 22; goto _test_eof _test_eof23: cs = 23; goto _test_eof _test_eof24: cs = 24; goto _test_eof - _test_eof25: cs = 25; goto _test_eof - _test_eof26: cs = 26; goto _test_eof - _test_eof27: cs = 27; goto _test_eof - _test_eof28: cs = 28; goto _test_eof _test_eof: {} if p == eof { switch cs { case 9: - goto tr0 + goto tr23 case 10: - goto tr28 + goto tr24 case 1: goto tr0 - case 11: - goto tr28 case 2: goto tr0 + case 11: + goto tr24 case 12: - goto tr29 - case 13: - goto tr29 + goto tr24 case 3: goto tr0 - case 14: - goto tr29 + case 13: + goto tr24 case 4: goto tr0 + case 14: + goto tr24 case 15: - goto tr29 - case 16: + goto tr25 + case 5: goto tr0 + case 16: + goto tr24 + case 6: + goto tr14 case 17: - goto tr29 + goto tr24 + case 7: + goto tr14 case 18: - goto tr29 + goto tr24 case 19: - goto tr32 - case 5: - goto tr0 + goto tr24 case 20: - goto tr28 - case 6: - goto tr0 + goto tr24 case 21: - goto tr29 - case 7: - goto tr0 + goto tr24 case 22: - goto tr29 + goto tr24 case 23: - goto tr29 + goto tr25 case 24: - goto tr0 - case 25: - goto tr29 - case 26: - goto tr0 - case 27: - goto tr32 - case 28: - goto tr32 + goto tr25 } } _out: {} } -//.... lightning/mydump/csv_parser.rl:87 +//.... lightning/mydump/csv_parser.rl:82 if cs == 0 { common.AppLogger.Errorf("Syntax error near byte %d, content is «%s»", parser.pos, string(data)) diff --git a/lightning/mydump/csv_parser_test.go b/lightning/mydump/csv_parser_test.go index 55b1cd536..d717cc75b 100644 --- a/lightning/mydump/csv_parser_test.go +++ b/lightning/mydump/csv_parser_test.go @@ -11,6 +11,7 @@ import ( "github.com/pingcap/tidb-lightning/lightning/config" "github.com/pingcap/tidb-lightning/lightning/mydump" "github.com/pingcap/tidb-lightning/lightning/worker" + "github.com/pingcap/tidb/types" ) var _ = Suite(&testMydumpCSVParserSuite{}) @@ -58,21 +59,51 @@ func (s *testMydumpCSVParserSuite) TestTCPH(c *C) { c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte("('1','goldenrod lavender spring chocolate lace','Manufacturer#1','Brand#13','PROMO BURNISHED COPPER','7','JUMBO PKG','901.00','ly. slyly ironi')"), + Row: []types.Datum{ + types.NewStringDatum("1"), + types.NewStringDatum("goldenrod lavender spring chocolate lace"), + types.NewStringDatum("Manufacturer#1"), + types.NewStringDatum("Brand#13"), + types.NewStringDatum("PROMO BURNISHED COPPER"), + types.NewStringDatum("7"), + types.NewStringDatum("JUMBO PKG"), + types.NewStringDatum("901.00"), + types.NewStringDatum("ly. slyly ironi"), + }, }) c.Assert(parser, posEq, 126, 1) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 2, - Row: []byte("('2','blush thistle blue yellow saddle','Manufacturer#1','Brand#13','LARGE BRUSHED BRASS','1','LG CASE','902.00','lar accounts amo')"), + Row: []types.Datum{ + types.NewStringDatum("2"), + types.NewStringDatum("blush thistle blue yellow saddle"), + types.NewStringDatum("Manufacturer#1"), + types.NewStringDatum("Brand#13"), + types.NewStringDatum("LARGE BRUSHED BRASS"), + types.NewStringDatum("1"), + types.NewStringDatum("LG CASE"), + types.NewStringDatum("902.00"), + types.NewStringDatum("lar accounts amo"), + }, }) c.Assert(parser, posEq, 240, 2) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 3, - Row: []byte("('3','spring green yellow purple cornsilk','Manufacturer#4','Brand#42','STANDARD POLISHED BRASS','21','WRAP CASE','903.00','egular deposits hag')"), + Row: []types.Datum{ + types.NewStringDatum("3"), + types.NewStringDatum("spring green yellow purple cornsilk"), + types.NewStringDatum("Manufacturer#4"), + types.NewStringDatum("Brand#42"), + types.NewStringDatum("STANDARD POLISHED BRASS"), + types.NewStringDatum("21"), + types.NewStringDatum("WRAP CASE"), + types.NewStringDatum("903.00"), + types.NewStringDatum("egular deposits hag"), + }, }) c.Assert(parser, posEq, 367, 3) @@ -92,14 +123,22 @@ func (s *testMydumpCSVParserSuite) TestRFC4180(c *C) { c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte("('aaa','bbb','ccc')"), + Row: []types.Datum{ + types.NewStringDatum("aaa"), + types.NewStringDatum("bbb"), + types.NewStringDatum("ccc"), + }, }) c.Assert(parser, posEq, 12, 1) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 2, - Row: []byte("('zzz','yyy','xxx')"), + Row: []types.Datum{ + types.NewStringDatum("zzz"), + types.NewStringDatum("yyy"), + types.NewStringDatum("xxx"), + }, }) c.Assert(parser, posEq, 24, 2) @@ -112,14 +151,22 @@ func (s *testMydumpCSVParserSuite) TestRFC4180(c *C) { c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte("('aaa','bbb','ccc')"), + Row: []types.Datum{ + types.NewStringDatum("aaa"), + types.NewStringDatum("bbb"), + types.NewStringDatum("ccc"), + }, }) c.Assert(parser, posEq, 12, 1) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 2, - Row: []byte("('zzz','yyy','xxx')"), + Row: []types.Datum{ + types.NewStringDatum("zzz"), + types.NewStringDatum("yyy"), + types.NewStringDatum("xxx"), + }, }) c.Assert(parser, posEq, 23, 2) @@ -132,14 +179,22 @@ func (s *testMydumpCSVParserSuite) TestRFC4180(c *C) { c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte("('aaa','bbb','ccc')"), + Row: []types.Datum{ + types.NewStringDatum("aaa"), + types.NewStringDatum("bbb"), + types.NewStringDatum("ccc"), + }, }) c.Assert(parser, posEq, 18, 1) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 2, - Row: []byte("('zzz','yyy','xxx')"), + Row: []types.Datum{ + types.NewStringDatum("zzz"), + types.NewStringDatum("yyy"), + types.NewStringDatum("xxx"), + }, }) c.Assert(parser, posEq, 29, 2) @@ -154,14 +209,22 @@ zzz,yyy,xxx`), config.ReadBlockSize, s.ioWorkers) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte("('aaa','b\nbb','ccc')"), + Row: []types.Datum{ + types.NewStringDatum("aaa"), + types.NewStringDatum("b\nbb"), + types.NewStringDatum("ccc"), + }, }) c.Assert(parser, posEq, 19, 1) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 2, - Row: []byte("('zzz','yyy','xxx')"), + Row: []types.Datum{ + types.NewStringDatum("zzz"), + types.NewStringDatum("yyy"), + types.NewStringDatum("xxx"), + }, }) c.Assert(parser, posEq, 30, 2) @@ -174,7 +237,11 @@ zzz,yyy,xxx`), config.ReadBlockSize, s.ioWorkers) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte(`('aaa','b"bb','ccc')`), + Row: []types.Datum{ + types.NewStringDatum("aaa"), + types.NewStringDatum("b\"bb"), + types.NewStringDatum("ccc"), + }, }) c.Assert(parser, posEq, 19, 1) @@ -190,23 +257,34 @@ func (s *testMydumpCSVParserSuite) TestMySQL(c *C) { Null: `\N`, } - parser := mydump.NewCSVParser(&cfg, strings.NewReader(`"\"","\\" + parser := mydump.NewCSVParser(&cfg, strings.NewReader(`"\"","\\","\?" "\ -",\N`), config.ReadBlockSize, s.ioWorkers) +",\N,\\N`), config.ReadBlockSize, s.ioWorkers) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte(`('\"','\\')`), + Row: []types.Datum{ + types.NewStringDatum(`"`), + types.NewStringDatum(`\`), + types.NewStringDatum("?"), + }, }) - c.Assert(parser, posEq, 10, 1) + c.Assert(parser, posEq, 15, 1) + + var nullDatum types.Datum + nullDatum.SetNull() c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 2, - Row: []byte("('\\\n',NULL)"), + Row: []types.Datum{ + types.NewStringDatum("\n"), + nullDatum, + types.NewStringDatum(`\N`), + }, }) - c.Assert(parser, posEq, 17, 2) + c.Assert(parser, posEq, 26, 2) c.Assert(errors.Cause(parser.ReadRow()), Equals, io.EOF) } @@ -237,25 +315,49 @@ func (s *testMydumpCSVParserSuite) TestTSV(c *C) { 0 foo 0000-00-00 0 abc def ghi bar 1999-12-31`), config.ReadBlockSize, s.ioWorkers) + var nullDatum types.Datum + nullDatum.SetNull() + c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte(`('0',NULL,NULL,NULL,'foo','0000-00-00')`), + Row: []types.Datum{ + types.NewStringDatum("0"), + nullDatum, + nullDatum, + nullDatum, + types.NewStringDatum("foo"), + types.NewStringDatum("0000-00-00"), + }, }) c.Assert(parser, posEq, 32, 1) - c.Assert(parser.Columns(), DeepEquals, []byte("(`a`,`b`,`c`,`d`,`e`,`f`)")) + c.Assert(parser.Columns(), DeepEquals, []string{"a", "b", "c", "d", "e", "f"}) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 2, - Row: []byte(`('0',NULL,NULL,NULL,'foo','0000-00-00')`), + Row: []types.Datum{ + types.NewStringDatum("0"), + nullDatum, + nullDatum, + nullDatum, + types.NewStringDatum("foo"), + types.NewStringDatum("0000-00-00"), + }, }) c.Assert(parser, posEq, 52, 2) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 3, - Row: []byte(`('0','abc','def','ghi','bar','1999-12-31')`), + Row: []types.Datum{ + types.NewStringDatum("0"), + types.NewStringDatum("abc"), + types.NewStringDatum("def"), + types.NewStringDatum("ghi"), + types.NewStringDatum("bar"), + types.NewStringDatum("1999-12-31"), + }, }) c.Assert(parser, posEq, 80, 3) diff --git a/lightning/mydump/parser.go b/lightning/mydump/parser.go index 20d8cdb24..2dff8e8a6 100644 --- a/lightning/mydump/parser.go +++ b/lightning/mydump/parser.go @@ -16,13 +16,16 @@ package mydump import ( "bytes" "io" + "regexp" + "strings" "time" "github.com/pingcap/errors" - + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-lightning/lightning/config" "github.com/pingcap/tidb-lightning/lightning/metric" "github.com/pingcap/tidb-lightning/lightning/worker" + "github.com/pingcap/tidb/types" ) type blockParser struct { @@ -32,9 +35,8 @@ type blockParser struct { blockBuf []byte isLastChunk bool - // The list of columns in the form `(a, b, c)` in the last INSERT statement. - // Assumed to be constant throughout the entire file. - columns []byte + // The list of column names of the last INSERT statement. + columns []string lastRow Row // Current file offset. @@ -61,9 +63,7 @@ func makeBlockParser(reader io.Reader, blockBufSize int64, ioWorkers *worker.Poo type ChunkParser struct { blockParser - // The (quoted) table name used in the last INSERT statement. Assumed to be - // constant throughout the entire file. - TableName []byte + escFlavor backslashEscapeFlavor } // Chunk represents a portion of the data file. @@ -77,22 +77,41 @@ type Chunk struct { // Row is the content of a row. type Row struct { RowID int64 - Row []byte + Row []types.Datum } +type backslashEscapeFlavor uint8 + +const ( + backslashEscapeFlavorNone backslashEscapeFlavor = iota + backslashEscapeFlavorMySQL + backslashEscapeFlavorMySQLWithNull +) + type Parser interface { Pos() (pos int64, rowID int64) SetPos(pos int64, rowID int64) Close() error ReadRow() error LastRow() Row - Columns() []byte + Columns() []string } // NewChunkParser creates a new parser which can read chunks out of a file. -func NewChunkParser(reader io.Reader, blockBufSize int64, ioWorkers *worker.Pool) *ChunkParser { +func NewChunkParser( + sqlMode mysql.SQLMode, + reader io.Reader, + blockBufSize int64, + ioWorkers *worker.Pool, +) *ChunkParser { + escFlavor := backslashEscapeFlavorMySQL + if sqlMode.HasNoBackslashEscapesMode() { + escFlavor = backslashEscapeFlavorNone + } + return &ChunkParser{ blockParser: makeBlockParser(reader, blockBufSize, ioWorkers), + escFlavor: escFlavor, } } @@ -119,7 +138,7 @@ func (parser *blockParser) Close() error { return errors.New("this parser is not created with a reader that can be closed") } -func (parser *blockParser) Columns() []byte { +func (parser *blockParser) Columns() []string { return parser.columns } @@ -127,9 +146,18 @@ type token byte const ( tokNil token = iota + tokRowBegin + tokRowEnd tokValues - tokRow - tokName + tokNull + tokTrue + tokFalse + tokHexString + tokBinString + tokSingleQuoted + tokDoubleQuoted + tokBackQuoted + tokUnquoted ) func (parser *blockParser) readBlock() error { @@ -160,6 +188,54 @@ func (parser *blockParser) readBlock() error { } } +var unescapeRegexp = regexp.MustCompile(`(?s)\\.`) + +func unescape( + input string, + delim string, + escFlavor backslashEscapeFlavor, +) string { + if len(delim) > 0 { + delim2 := delim + delim + if strings.Index(input, delim2) != -1 { + input = strings.Replace(input, delim2, delim, -1) + } + } + if escFlavor != backslashEscapeFlavorNone && strings.IndexByte(input, '\\') != -1 { + input = unescapeRegexp.ReplaceAllStringFunc(input, func(substr string) string { + switch substr[1] { + case '0': + return "\x00" + case 'b': + return "\b" + case 'n': + return "\n" + case 'r': + return "\r" + case 't': + return "\t" + case 'Z': + return "\x26" + default: + return substr[1:] + } + }) + } + return input +} + +func (parser *ChunkParser) unescapeString(input string) string { + if len(input) >= 2 { + switch input[0] { + case '\'', '"': + return unescape(input[1:len(input)-1], input[:1], parser.escFlavor) + case '`': + return unescape(input[1:len(input)-1], "`", backslashEscapeFlavorNone) + } + } + return input +} + // ReadRow reads a row from the datafile. func (parser *ChunkParser) ReadRow() error { // This parser will recognize contents like: @@ -175,44 +251,150 @@ func (parser *ChunkParser) ReadRow() error { type state byte const ( - // the state after reading "VALUES" - stateRow state = iota - // the state after reading the table name, before "VALUES" + // the state after "INSERT INTO" before the column names or "VALUES" + stateTableName state = iota + + // the state while reading the column names stateColumns + + // the state after reading "VALUES" + stateValues + + // the state while reading row values + stateRow ) + // Dry-run sample of the state machine, first row: + // + // Input Token State + // ~~~~~ ~~~~~ ~~~~~ + // + // stateValues + // INSERT + // INTO + // `tableName` tokBackQuoted + // stateTableName (reset columns) + // ( tokRowBegin + // stateColumns + // `a` tokBackQuoted + // stateColumns (append column) + // , + // `b` tokBackQuoted + // stateColumns (append column) + // ) tokRowEnd + // stateValues + // VALUES + // stateValues (no-op) + // ( tokRowBegin + // stateRow (reset row) + // 1 tokUnquoted + // stateRow (append value) + // , + // 2 tokUnquoted + // stateRow (append value) + // ) tokRowEnd + // return + // + // + // Second row: + // + // Input Token State + // ~~~~~ ~~~~~ ~~~~~ + // + // stateValues + // , + // ( tokRowBegin + // stateRow (reset row) + // 3 tokUnquoted + // stateRow (append value) + // ) tokRowEnd + // return + // + // Third row: + // + // Input Token State + // ~~~~~ ~~~~~ ~~~~~ + // + // ; + // INSERT + // INTO + // `database` tokBackQuoted + // stateTableName (reset columns) + // . + // `tableName` tokBackQuoted + // stateTableName (no-op) + // VALUES + // stateValues + // ( tokRowBegin + // stateRow (reset row) + // 4 tokUnquoted + // stateRow (append value) + // ) tokRowEnd + // return + row := &parser.lastRow - st := stateRow + st := stateValues for { tok, content, err := parser.lex() if err != nil { return errors.Trace(err) } - switch tok { - case tokRow: - switch st { - case stateRow: + switch st { + case stateTableName: + switch tok { + case tokRowBegin: + st = stateColumns + case tokValues: + st = stateValues + } + case stateColumns: + switch tok { + case tokRowEnd: + st = stateValues + case tokUnquoted, tokDoubleQuoted, tokBackQuoted: + columnName := parser.unescapeString(string(content)) + parser.columns = append(parser.columns, columnName) + } + case stateValues: + switch tok { + case tokRowBegin: row.RowID++ - row.Row = content + row.Row = make([]types.Datum, 0, len(row.Row)) + st = stateRow + case tokUnquoted, tokDoubleQuoted, tokBackQuoted: + parser.columns = nil + st = stateTableName + } + case stateRow: + var value types.Datum + switch tok { + case tokRowEnd: return nil - case stateColumns: - parser.columns = content - continue + case tokNull: + value.SetNull() + case tokTrue: + value.SetInt64(1) + case tokFalse: + value.SetInt64(0) + case tokUnquoted, tokSingleQuoted, tokDoubleQuoted: + value.SetString(parser.unescapeString(string(content))) + case tokHexString: + hexLit, err := types.ParseHexStr(string(content)) + if err != nil { + return err + } + value.SetBinaryLiteral(hexLit) + case tokBinString: + binLit, err := types.ParseBitStr(string(content)) + if err != nil { + return err + } + value.SetBinaryLiteral(binLit) + default: + return errors.Errorf("Syntax error at position %d", parser.pos) } - - case tokName: - st = stateColumns - parser.TableName = content - parser.columns = nil - continue - - case tokValues: - st = stateRow - continue - - default: - return errors.Errorf("Syntax error at position %d", parser.pos) + row.Row = append(row.Row, value) } } } diff --git a/lightning/mydump/parser.rl b/lightning/mydump/parser.rl index 5bca1b0c3..29962c4a5 100644 --- a/lightning/mydump/parser.rl +++ b/lightning/mydump/parser.rl @@ -40,50 +40,91 @@ machine chunk_parser; # - Real SQL comments `/* ... */` and `-- ...` # - Whitespace # - Separators `,` and `;` -# - The keywords `INSERT` and `INTO` (suffix `i` means case-insensitive). +# - The keyword `INTO` (suffix `i` means case-insensitive). +# - The parts of the function `CONVERT(` and `USING UTF8MB4)` +# (to strip the unnecessary detail from mydumper JSON output) block_comment = '/*' any* :>> '*/'; line_comment = /--[^\n]*\n/; -comment = block_comment | line_comment | space | [,;] | 'insert'i | 'into'i; +comment = + block_comment | + line_comment | + space | + [,;] | + 'into'i | + 'convert('i | + 'using utf8mb4)'i; # The patterns parse quoted strings. -# They do NOT handle the escape-by-doubling syntax like `'ten o''clock'`, this -# will be handled as two tokens: `'ten o'` and `'clock'`. See the `name` rule -# below for why this doesn't matter. -single_quoted = "'" (^"'" | "\\" any)** "'"; -double_quoted = '"' (^'"' | '\\' any)** '"'; -back_quoted = '`' ^'`'* '`'; -unquoted = ^([,;()'"`] | space)+; - -content = ^[()'"`] | single_quoted | double_quoted | back_quoted; +bs = '\\' when { parser.escFlavor != backslashEscapeFlavorNone }; -# Matches a "row" of the form `( ... )`, where the content doesn't matter. -# Parenthesis can nest for one level. -row = '(' (content | '(' content* ')')* ')'; +single_quoted = "'" (^"'" | bs any | "''")** "'"; +double_quoted = '"' (^'"' | bs any | '""')** '"'; +back_quoted = '`' (^'`' | '``')* '`'; +unquoted = ^([,;()'"`] | space)+; -# Matches a table name, which consists of one or more identifiers. This allows -# us to match a qualified name like `foo.bar`, and also double-backquote like -# ``` `foo``bar` ```. -name = (back_quoted | double_quoted | unquoted)+; +hex_string = /0x[0-9a-fA-F]+/ | "x'"i [0-9a-f]* "'"; +bin_string = /0b[01]+/ | "b'"i [01]* "'"; -# The actual parser only produces 3 kinds of tokens: -# - The keyword VALUES, as a separator between column names and data rows -# - A row (which can be a list of columns or values depending on context) -# - A table name main := |* comment; + '(' => { + consumedToken = tokRowBegin + fbreak; + }; + + ')' => { + consumedToken = tokRowEnd + fbreak; + }; + 'values'i => { consumedToken = tokValues fbreak; }; - row => { - consumedToken = tokRow + 'null'i => { + consumedToken = tokNull + fbreak; + }; + + 'true'i => { + consumedToken = tokTrue + fbreak; + }; + + 'false'i => { + consumedToken = tokFalse + fbreak; + }; + + hex_string => { + consumedToken = tokHexString + fbreak; + }; + + bin_string => { + consumedToken = tokBinString + fbreak; + }; + + single_quoted => { + consumedToken = tokSingleQuoted + fbreak; + }; + + double_quoted => { + consumedToken = tokDoubleQuoted + fbreak; + }; + + back_quoted => { + consumedToken = tokBackQuoted fbreak; }; - name => { - consumedToken = tokName + unquoted => { + consumedToken = tokUnquoted fbreak; }; *|; diff --git a/lightning/mydump/parser_generated.go b/lightning/mydump/parser_generated.go index f3578797c..e87b0b620 100644 --- a/lightning/mydump/parser_generated.go +++ b/lightning/mydump/parser_generated.go @@ -26,33 +26,37 @@ package mydump import ( "io" - "github.com/pingcap/errors" "github.com/pingcap/tidb-lightning/lightning/common" + "github.com/pingcap/errors" ) -//.... lightning/mydump/parser.rl:92 + +//.... lightning/mydump/parser.rl:133 + + //.... tmp_parser.go:38 -const chunk_parser_start int = 27 -const chunk_parser_first_final int = 27 +const chunk_parser_start int = 21 +const chunk_parser_first_final int = 21 const chunk_parser_error int = 0 -const chunk_parser_en_main int = 27 +const chunk_parser_en_main int = 21 + -//.... lightning/mydump/parser.rl:95 +//.... lightning/mydump/parser.rl:136 func (parser *ChunkParser) lex() (token, []byte, error) { var cs, ts, te, act, p int - - //.... tmp_parser.go:51 + +//.... tmp_parser.go:51 { - cs = chunk_parser_start - ts = 0 - te = 0 - act = 0 + cs = chunk_parser_start + ts = 0 + te = 0 + act = 0 } - //.... lightning/mydump/parser.rl:99 +//.... lightning/mydump/parser.rl:140 for { data := parser.buf @@ -63,1434 +67,2337 @@ func (parser *ChunkParser) lex() (token, []byte, error) { eof = pe } - //.... tmp_parser.go:71 - { - if p == pe { - goto _test_eof - } - switch cs { - case 27: - goto st_case_27 - case 28: - goto st_case_28 - case 1: - goto st_case_1 - case 2: - goto st_case_2 - case 3: - goto st_case_3 - case 0: - goto st_case_0 - case 4: - goto st_case_4 - case 5: - goto st_case_5 - case 6: - goto st_case_6 - case 7: - goto st_case_7 - case 8: - goto st_case_8 - case 9: - goto st_case_9 - case 10: - goto st_case_10 - case 11: - goto st_case_11 - case 12: - goto st_case_12 - case 13: - goto st_case_13 - case 14: - goto st_case_14 - case 15: - goto st_case_15 - case 29: - goto st_case_29 - case 30: - goto st_case_30 - case 16: - goto st_case_16 - case 17: - goto st_case_17 - case 31: - goto st_case_31 - case 18: - goto st_case_18 - case 19: - goto st_case_19 - case 32: - goto st_case_32 - case 33: - goto st_case_33 - case 34: - goto st_case_34 - case 20: - goto st_case_20 - case 21: - goto st_case_21 - case 22: - goto st_case_22 - case 23: - goto st_case_23 - case 24: - goto st_case_24 - case 35: - goto st_case_35 - case 25: - goto st_case_25 - case 26: - goto st_case_26 - case 36: - goto st_case_36 - case 37: - goto st_case_37 - case 38: - goto st_case_38 - case 39: - goto st_case_39 - case 40: - goto st_case_40 - case 41: - goto st_case_41 - case 42: - goto st_case_42 - case 43: - goto st_case_43 - case 44: - goto st_case_44 - case 45: - goto st_case_45 - case 46: - goto st_case_46 - } - goto st_out - tr0: - //.... NONE:1 - switch act { - case 0: - { - { - goto st0 - } - } - case 2: - { - p = (te) - 1 + +//.... tmp_parser.go:71 + { + var _widec int16 + if p == pe { + goto _test_eof + } + switch cs { + case 21: + goto st_case_21 + case 22: + goto st_case_22 + case 1: + goto st_case_1 + case 2: + goto st_case_2 + case 23: + goto st_case_23 + case 3: + goto st_case_3 + case 0: + goto st_case_0 + case 4: + goto st_case_4 + case 5: + goto st_case_5 + case 24: + goto st_case_24 + case 6: + goto st_case_6 + case 25: + goto st_case_25 + case 26: + goto st_case_26 + case 7: + goto st_case_7 + case 27: + goto st_case_27 + case 28: + goto st_case_28 + case 8: + goto st_case_8 + case 9: + goto st_case_9 + case 29: + goto st_case_29 + case 30: + goto st_case_30 + case 31: + goto st_case_31 + case 32: + goto st_case_32 + case 33: + goto st_case_33 + case 34: + goto st_case_34 + case 35: + goto st_case_35 + case 10: + goto st_case_10 + case 36: + goto st_case_36 + case 37: + goto st_case_37 + case 38: + goto st_case_38 + case 39: + goto st_case_39 + case 40: + goto st_case_40 + case 41: + goto st_case_41 + case 42: + goto st_case_42 + case 43: + goto st_case_43 + case 44: + goto st_case_44 + case 45: + goto st_case_45 + case 46: + goto st_case_46 + case 47: + goto st_case_47 + case 48: + goto st_case_48 + case 49: + goto st_case_49 + case 50: + goto st_case_50 + case 51: + goto st_case_51 + case 52: + goto st_case_52 + case 53: + goto st_case_53 + case 54: + goto st_case_54 + case 55: + goto st_case_55 + case 56: + goto st_case_56 + case 57: + goto st_case_57 + case 58: + goto st_case_58 + case 59: + goto st_case_59 + case 60: + goto st_case_60 + case 11: + goto st_case_11 + case 12: + goto st_case_12 + case 13: + goto st_case_13 + case 14: + goto st_case_14 + case 15: + goto st_case_15 + case 16: + goto st_case_16 + case 17: + goto st_case_17 + case 18: + goto st_case_18 + case 61: + goto st_case_61 + case 62: + goto st_case_62 + case 63: + goto st_case_63 + case 64: + goto st_case_64 + case 65: + goto st_case_65 + case 66: + goto st_case_66 + case 19: + goto st_case_19 + case 20: + goto st_case_20 + case 67: + goto st_case_67 + } + goto st_out +tr4: +//.... NONE:1 + switch act { + case 0: + {{goto st0 }} + case 4: + {p = (te) - 1 - consumedToken = tokValues - { - p++ - cs = 27 - goto _out - } - } - case 4: - { - p = (te) - 1 + consumedToken = tokValues + {p++; cs = 21; goto _out } + } + case 5: + {p = (te) - 1 - consumedToken = tokName - { - p++ - cs = 27 - goto _out - } - } - default: - { - p = (te) - 1 - } - } + consumedToken = tokNull + {p++; cs = 21; goto _out } + } + case 6: + {p = (te) - 1 - goto st27 - tr9: - //.... lightning/mydump/parser.rl:80 - te = p + 1 - { - consumedToken = tokRow - { - p++ - cs = 27 - goto _out - } - } - goto st27 - tr19: - //.... lightning/mydump/parser.rl:85 - p = (te) - 1 - { - consumedToken = tokName - { - p++ - cs = 27 - goto _out - } - } - goto st27 - tr21: - //.... lightning/mydump/parser.rl:73 - te = p + 1 + consumedToken = tokTrue + {p++; cs = 21; goto _out } + } + case 7: + {p = (te) - 1 - goto st27 - tr40: - //.... lightning/mydump/parser.rl:85 - te = p - p-- - { - consumedToken = tokName - { - p++ - cs = 27 - goto _out - } - } - goto st27 - tr41: - //.... lightning/mydump/parser.rl:73 - te = p - p-- + consumedToken = tokFalse + {p++; cs = 21; goto _out } + } + case 8: + {p = (te) - 1 + + consumedToken = tokHexString + {p++; cs = 21; goto _out } + } + case 9: + {p = (te) - 1 + + consumedToken = tokBinString + {p++; cs = 21; goto _out } + } + case 10: + {p = (te) - 1 + + consumedToken = tokSingleQuoted + {p++; cs = 21; goto _out } + } + case 11: + {p = (te) - 1 + + consumedToken = tokDoubleQuoted + {p++; cs = 21; goto _out } + } + case 12: + {p = (te) - 1 + + consumedToken = tokBackQuoted + {p++; cs = 21; goto _out } + } + case 13: + {p = (te) - 1 + + consumedToken = tokUnquoted + {p++; cs = 21; goto _out } + } + default: + {p = (te) - 1 +} + } + + goto st21 +tr8: +//.... lightning/mydump/parser.rl:126 +p = (te) - 1 +{ + consumedToken = tokUnquoted + {p++; cs = 21; goto _out } + } + goto st21 +tr10: +//.... lightning/mydump/parser.rl:69 +te = p+1 + goto st21 +tr13: +//.... lightning/mydump/parser.rl:106 +te = p+1 +{ + consumedToken = tokBinString + {p++; cs = 21; goto _out } + } + goto st21 +tr22: +//.... lightning/mydump/parser.rl:101 +te = p+1 +{ + consumedToken = tokHexString + {p++; cs = 21; goto _out } + } + goto st21 +tr29: +//.... lightning/mydump/parser.rl:71 +te = p+1 +{ + consumedToken = tokRowBegin + {p++; cs = 21; goto _out } + } + goto st21 +tr30: +//.... lightning/mydump/parser.rl:76 +te = p+1 +{ + consumedToken = tokRowEnd + {p++; cs = 21; goto _out } + } + goto st21 +tr43: +//.... lightning/mydump/parser.rl:116 +te = p +p-- +{ + consumedToken = tokDoubleQuoted + {p++; cs = 21; goto _out } + } + goto st21 +tr44: +//.... lightning/mydump/parser.rl:111 +te = p +p-- +{ + consumedToken = tokSingleQuoted + {p++; cs = 21; goto _out } + } + goto st21 +tr45: +//.... lightning/mydump/parser.rl:126 +te = p +p-- +{ + consumedToken = tokUnquoted + {p++; cs = 21; goto _out } + } + goto st21 +tr84: +//.... lightning/mydump/parser.rl:121 +te = p +p-- +{ + consumedToken = tokBackQuoted + {p++; cs = 21; goto _out } + } + goto st21 + st21: +//.... NONE:1 +ts = 0 + +//.... NONE:1 +act = 0 + + if p++; p == pe { + goto _test_eof21 + } + st_case_21: +//.... NONE:1 +ts = p + +//.... tmp_parser.go:382 + switch data[p] { + case 32: + goto tr10 + case 34: + goto st1 + case 39: + goto st4 + case 40: + goto tr29 + case 41: + goto tr30 + case 44: + goto tr10 + case 45: + goto st25 + case 47: goto st27 - st27: - //.... NONE:1 - ts = 0 + case 48: + goto st30 + case 59: + goto tr10 + case 66: + goto tr34 + case 67: + goto st36 + case 70: + goto st43 + case 73: + goto st47 + case 78: + goto st50 + case 84: + goto st53 + case 85: + goto st56 + case 86: + goto st61 + case 88: + goto tr42 + case 96: + goto st20 + case 98: + goto tr34 + case 99: + goto st36 + case 102: + goto st43 + case 105: + goto st47 + case 110: + goto st50 + case 116: + goto st53 + case 117: + goto st56 + case 118: + goto st61 + case 120: + goto tr42 + } + if 9 <= data[p] && data[p] <= 13 { + goto tr10 + } + goto tr26 +tr26: +//.... NONE:1 +te = p+1 - //.... NONE:1 - act = 0 +//.... lightning/mydump/parser.rl:126 +act = 13; + goto st22 +tr49: +//.... NONE:1 +te = p+1 - if p++; p == pe { - goto _test_eof27 - } - st_case_27: - //.... NONE:1 - ts = p +//.... lightning/mydump/parser.rl:69 +act = 1; + goto st22 +tr53: +//.... NONE:1 +te = p+1 - //.... tmp_parser.go:246 - switch data[p] { - case 32: - goto tr21 - case 34: - goto st1 - case 40: - goto st4 - case 44: - goto tr21 - case 45: - goto tr36 - case 47: - goto tr37 - case 59: - goto tr21 - case 73: - goto tr38 - case 86: - goto tr39 - case 96: - goto st3 - case 105: - goto tr38 - case 118: - goto tr39 - } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto st0 - } - case data[p] >= 9: - goto tr21 - } - goto tr2 - tr2: - //.... NONE:1 - te = p + 1 +//.... lightning/mydump/parser.rl:106 +act = 9; + goto st22 +tr55: +//.... NONE:1 +te = p+1 + +//.... lightning/mydump/parser.rl:101 +act = 8; + goto st22 +tr65: +//.... NONE:1 +te = p+1 - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st28 - tr43: - //.... NONE:1 - te = p + 1 +//.... lightning/mydump/parser.rl:96 +act = 7; + goto st22 +tr70: +//.... NONE:1 +te = p+1 - //.... lightning/mydump/parser.rl:73 - act = 1 - goto st28 - tr53: - //.... NONE:1 - te = p + 1 +//.... lightning/mydump/parser.rl:86 +act = 5; + goto st22 +tr73: +//.... NONE:1 +te = p+1 - //.... lightning/mydump/parser.rl:75 - act = 2 - goto st28 - st28: - if p++; p == pe { - goto _test_eof28 +//.... lightning/mydump/parser.rl:91 +act = 6; + goto st22 +tr83: +//.... NONE:1 +te = p+1 + +//.... lightning/mydump/parser.rl:81 +act = 4; + goto st22 + st22: + if p++; p == pe { + goto _test_eof22 + } + st_case_22: +//.... tmp_parser.go:508 + switch data[p] { + case 32: + goto tr4 + case 34: + goto tr4 + case 44: + goto tr4 + case 59: + goto tr4 + case 96: + goto tr4 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr4 } - st_case_28: - //.... tmp_parser.go:308 - switch data[p] { - case 32: - goto tr0 - case 34: - goto st1 - case 44: - goto tr0 - case 59: - goto tr0 - case 96: - goto st3 + case data[p] >= 9: + goto tr4 + } + goto tr26 + st1: + if p++; p == pe { + goto _test_eof1 + } + st_case_1: + _widec = int16(data[p]) + if 92 <= data[p] && data[p] <= 92 { + _widec = 256 + (int16(data[p]) - 0) + if parser.escFlavor != backslashEscapeFlavorNone { + _widec += 256 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr0 - } - case data[p] >= 9: - goto tr0 + } + switch _widec { + case 34: + goto tr1 + case 348: + goto st2 + case 604: + goto st3 + } + switch { + case _widec > 91: + if 93 <= _widec { + goto st2 } - goto tr2 - st1: - if p++; p == pe { - goto _test_eof1 + default: + goto st2 + } + goto st0 + st2: + if p++; p == pe { + goto _test_eof2 + } + st_case_2: + _widec = int16(data[p]) + if 92 <= data[p] && data[p] <= 92 { + _widec = 256 + (int16(data[p]) - 0) + if parser.escFlavor != backslashEscapeFlavorNone { + _widec += 256 } - st_case_1: - switch data[p] { - case 34: - goto tr2 - case 92: + } + switch _widec { + case 34: + goto tr1 + case 348: + goto st2 + case 604: + goto st3 + } + switch { + case _widec > 91: + if 93 <= _widec { goto st2 } - goto st1 - st2: - if p++; p == pe { - goto _test_eof2 - } - st_case_2: - goto st1 - st3: - if p++; p == pe { - goto _test_eof3 + default: + goto st2 + } + goto tr4 +tr1: +//.... NONE:1 +te = p+1 + +//.... lightning/mydump/parser.rl:116 +act = 11; + goto st23 + st23: + if p++; p == pe { + goto _test_eof23 + } + st_case_23: +//.... tmp_parser.go:600 + if data[p] == 34 { + goto st2 + } + goto tr43 + st3: + if p++; p == pe { + goto _test_eof3 + } + st_case_3: + _widec = int16(data[p]) + if 92 <= data[p] && data[p] <= 92 { + _widec = 256 + (int16(data[p]) - 0) + if parser.escFlavor != backslashEscapeFlavorNone { + _widec += 256 } - st_case_3: - if data[p] == 96 { - goto tr2 + } + switch _widec { + case 348: + goto st2 + case 604: + goto st2 + } + switch { + case _widec > 91: + if 93 <= _widec { + goto st2 } - goto st3 - st_case_0: - st0: - cs = 0 - goto _out - st4: - if p++; p == pe { - goto _test_eof4 + default: + goto st2 + } + goto tr4 +st_case_0: + st0: + cs = 0 + goto _out + st4: + if p++; p == pe { + goto _test_eof4 + } + st_case_4: + _widec = int16(data[p]) + if 92 <= data[p] && data[p] <= 92 { + _widec = 256 + (int16(data[p]) - 0) + if parser.escFlavor != backslashEscapeFlavorNone { + _widec += 256 } - st_case_4: - switch data[p] { - case 34: + } + switch _widec { + case 39: + goto tr6 + case 348: + goto st5 + case 604: + goto st6 + } + switch { + case _widec > 91: + if 93 <= _widec { goto st5 - case 39: - goto st7 - case 40: - goto st9 - case 41: - goto tr9 - case 96: - goto st15 } - goto st4 - st5: - if p++; p == pe { - goto _test_eof5 + default: + goto st5 + } + goto st0 + st5: + if p++; p == pe { + goto _test_eof5 + } + st_case_5: + _widec = int16(data[p]) + if 92 <= data[p] && data[p] <= 92 { + _widec = 256 + (int16(data[p]) - 0) + if parser.escFlavor != backslashEscapeFlavorNone { + _widec += 256 } - st_case_5: - switch data[p] { - case 34: - goto st4 - case 92: - goto st6 + } + switch _widec { + case 39: + goto tr6 + case 348: + goto st5 + case 604: + goto st6 + } + switch { + case _widec > 91: + if 93 <= _widec { + goto st5 } + default: + goto st5 + } + goto tr4 +tr6: +//.... NONE:1 +te = p+1 + +//.... lightning/mydump/parser.rl:111 +act = 10; + goto st24 + st24: + if p++; p == pe { + goto _test_eof24 + } + st_case_24: +//.... tmp_parser.go:706 + if data[p] == 39 { goto st5 - st6: - if p++; p == pe { - goto _test_eof6 + } + goto tr44 + st6: + if p++; p == pe { + goto _test_eof6 + } + st_case_6: + _widec = int16(data[p]) + if 92 <= data[p] && data[p] <= 92 { + _widec = 256 + (int16(data[p]) - 0) + if parser.escFlavor != backslashEscapeFlavorNone { + _widec += 256 } - st_case_6: + } + switch _widec { + case 348: goto st5 - st7: - if p++; p == pe { - goto _test_eof7 + case 604: + goto st5 + } + switch { + case _widec > 91: + if 93 <= _widec { + goto st5 } - st_case_7: - switch data[p] { - case 39: - goto st4 - case 92: - goto st8 + default: + goto st5 + } + goto tr4 + st25: + if p++; p == pe { + goto _test_eof25 + } + st_case_25: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 45: + goto tr46 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } + case data[p] >= 9: + goto tr45 + } + goto tr26 +tr46: +//.... NONE:1 +te = p+1 + + goto st26 + st26: + if p++; p == pe { + goto _test_eof26 + } + st_case_26: +//.... tmp_parser.go:776 + switch data[p] { + case 10: + goto tr10 + case 32: goto st7 - st8: - if p++; p == pe { - goto _test_eof8 + case 34: + goto st7 + case 44: + goto st7 + case 59: + goto st7 + case 96: + goto st7 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto st7 } - st_case_8: + case data[p] >= 9: goto st7 - st9: - if p++; p == pe { - goto _test_eof9 + } + goto tr46 + st7: + if p++; p == pe { + goto _test_eof7 + } + st_case_7: + if data[p] == 10 { + goto tr10 + } + goto st7 + st27: + if p++; p == pe { + goto _test_eof27 + } + st_case_27: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 42: + goto tr47 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_9: - switch data[p] { - case 34: - goto st10 - case 39: - goto st12 - case 40: - goto st0 - case 41: - goto st4 - case 96: - goto st14 + case data[p] >= 9: + goto tr45 + } + goto tr26 +tr47: +//.... NONE:1 +te = p+1 + + goto st28 + st28: + if p++; p == pe { + goto _test_eof28 + } + st_case_28: +//.... tmp_parser.go:847 + switch data[p] { + case 32: + goto st8 + case 34: + goto st8 + case 42: + goto tr48 + case 44: + goto st8 + case 59: + goto st8 + case 96: + goto st8 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto st8 } + case data[p] >= 9: + goto st8 + } + goto tr47 + st8: + if p++; p == pe { + goto _test_eof8 + } + st_case_8: + if data[p] == 42 { goto st9 - st10: - if p++; p == pe { - goto _test_eof10 - } - st_case_10: - switch data[p] { - case 34: - goto st9 - case 92: - goto st11 - } - goto st10 - st11: - if p++; p == pe { - goto _test_eof11 - } - st_case_11: - goto st10 - st12: - if p++; p == pe { - goto _test_eof12 - } - st_case_12: - switch data[p] { - case 39: - goto st9 - case 92: - goto st13 - } - goto st12 - st13: - if p++; p == pe { - goto _test_eof13 - } - st_case_13: - goto st12 - st14: - if p++; p == pe { - goto _test_eof14 - } - st_case_14: - if data[p] == 96 { - goto st9 + } + goto st8 + st9: + if p++; p == pe { + goto _test_eof9 + } + st_case_9: + switch data[p] { + case 42: + goto st9 + case 47: + goto tr10 + } + goto st8 +tr48: +//.... NONE:1 +te = p+1 + + goto st29 + st29: + if p++; p == pe { + goto _test_eof29 + } + st_case_29: +//.... tmp_parser.go:902 + switch data[p] { + case 32: + goto st8 + case 34: + goto st8 + case 42: + goto tr48 + case 44: + goto st8 + case 47: + goto tr49 + case 59: + goto st8 + case 96: + goto st8 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto st8 } - goto st14 - st15: - if p++; p == pe { - goto _test_eof15 + case data[p] >= 9: + goto st8 + } + goto tr47 + st30: + if p++; p == pe { + goto _test_eof30 + } + st_case_30: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + case 98: + goto st31 + case 120: + goto st33 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_15: - if data[p] == 96 { - goto st4 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st31: + if p++; p == pe { + goto _test_eof31 + } + st_case_31: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] < 39: + if 9 <= data[p] && data[p] <= 13 { + goto tr45 } - goto st15 - tr36: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st29 - st29: - if p++; p == pe { - goto _test_eof29 + case data[p] > 41: + if 48 <= data[p] && data[p] <= 49 { + goto st32 } - st_case_29: - //.... tmp_parser.go:499 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 45: - goto tr24 - case 59: - goto tr40 - case 96: - goto st3 + default: + goto tr45 + } + goto tr26 + st32: + if p++; p == pe { + goto _test_eof32 + } + st_case_32: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 43: + goto tr53 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } + case data[p] >= 9: + goto tr45 + } + goto tr26 + st33: + if p++; p == pe { + goto _test_eof33 + } + st_case_33: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] < 48: switch { case data[p] > 13: if 39 <= data[p] && data[p] <= 41 { - goto tr40 + goto tr45 } case data[p] >= 9: - goto tr40 - } - goto tr2 - tr24: - //.... NONE:1 - te = p + 1 - - goto st30 - st30: - if p++; p == pe { - goto _test_eof30 - } - st_case_30: - //.... tmp_parser.go:533 - switch data[p] { - case 10: - goto tr21 - case 32: - goto st16 - case 34: - goto st17 - case 44: - goto st16 - case 59: - goto st16 - case 96: - goto st19 + goto tr45 } + case data[p] > 57: switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto st16 + case data[p] > 70: + if 97 <= data[p] && data[p] <= 102 { + goto st34 } - case data[p] >= 9: - goto st16 - } - goto tr24 - st16: - if p++; p == pe { - goto _test_eof16 + case data[p] >= 65: + goto st34 } - st_case_16: - if data[p] == 10 { - goto tr21 - } - goto st16 - st17: - if p++; p == pe { - goto _test_eof17 - } - st_case_17: - switch data[p] { - case 10: - goto tr23 - case 34: - goto tr24 - case 92: - goto st18 + default: + goto st34 + } + goto tr26 + st34: + if p++; p == pe { + goto _test_eof34 + } + st_case_34: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 43: + goto tr55 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto st17 - tr23: - //.... NONE:1 - te = p + 1 + case data[p] >= 9: + goto tr45 + } + goto tr26 +tr34: +//.... NONE:1 +te = p+1 - //.... lightning/mydump/parser.rl:73 - act = 1 - goto st31 - st31: - if p++; p == pe { - goto _test_eof31 - } - st_case_31: - //.... tmp_parser.go:592 - switch data[p] { - case 34: - goto tr2 - case 92: - goto st2 - } - goto st1 - st18: - if p++; p == pe { - goto _test_eof18 + goto st35 + st35: + if p++; p == pe { + goto _test_eof35 + } + st_case_35: +//.... tmp_parser.go:1094 + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 39: + goto st10 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] > 13: + if 40 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_18: - if data[p] == 10 { - goto tr23 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st10: + if p++; p == pe { + goto _test_eof10 + } + st_case_10: + if data[p] == 39 { + goto tr13 + } + if 48 <= data[p] && data[p] <= 49 { + goto st10 + } + goto tr8 + st36: + if p++; p == pe { + goto _test_eof36 + } + st_case_36: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 79: + goto st37 + case 96: + goto tr45 + case 111: + goto st37 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto st17 - st19: - if p++; p == pe { - goto _test_eof19 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st37: + if p++; p == pe { + goto _test_eof37 + } + st_case_37: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 78: + goto st38 + case 96: + goto tr45 + case 110: + goto st38 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_19: - switch data[p] { - case 10: - goto tr27 - case 96: - goto tr24 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st38: + if p++; p == pe { + goto _test_eof38 + } + st_case_38: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 86: + goto st39 + case 96: + goto tr45 + case 118: + goto st39 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto st19 - tr27: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:73 - act = 1 - goto st32 - st32: - if p++; p == pe { - goto _test_eof32 - } - st_case_32: - //.... tmp_parser.go:633 - if data[p] == 96 { - goto tr2 - } - goto st3 - tr37: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st33 - st33: - if p++; p == pe { - goto _test_eof33 - } - st_case_33: - //.... tmp_parser.go:650 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 42: - goto tr31 - case 44: - goto tr40 - case 59: - goto tr40 - case 96: - goto st3 - } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 - } - goto tr2 - tr31: - //.... NONE:1 - te = p + 1 - - goto st34 - st34: - if p++; p == pe { - goto _test_eof34 - } - st_case_34: - //.... tmp_parser.go:684 - switch data[p] { - case 32: - goto st20 - case 34: - goto st22 - case 42: - goto tr42 - case 44: - goto st20 - case 59: - goto st20 - case 96: - goto st25 - } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto st20 - } - case data[p] >= 9: - goto st20 - } - goto tr31 - st20: - if p++; p == pe { - goto _test_eof20 - } - st_case_20: - if data[p] == 42 { - goto st21 - } - goto st20 - st21: - if p++; p == pe { - goto _test_eof21 - } - st_case_21: - switch data[p] { - case 42: - goto st21 - case 47: - goto tr21 - } - goto st20 - st22: - if p++; p == pe { - goto _test_eof22 - } - st_case_22: - switch data[p] { - case 34: - goto tr31 - case 42: - goto st23 - case 92: - goto st24 - } - goto st22 - st23: - if p++; p == pe { - goto _test_eof23 - } - st_case_23: - switch data[p] { - case 34: - goto tr31 - case 42: - goto st23 - case 47: - goto tr23 - case 92: - goto st24 - } - goto st22 - st24: - if p++; p == pe { - goto _test_eof24 - } - st_case_24: - if data[p] == 42 { - goto st23 - } - goto st22 - tr42: - //.... NONE:1 - te = p + 1 - - goto st35 - st35: - if p++; p == pe { - goto _test_eof35 - } - st_case_35: - //.... tmp_parser.go:778 - switch data[p] { - case 32: - goto st20 - case 34: - goto st22 - case 42: - goto tr42 - case 44: - goto st20 - case 47: - goto tr43 - case 59: - goto st20 - case 96: - goto st25 - } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto st20 - } - case data[p] >= 9: - goto st20 - } - goto tr31 - st25: - if p++; p == pe { - goto _test_eof25 - } - st_case_25: - switch data[p] { - case 42: - goto st26 - case 96: - goto tr31 - } - goto st25 - st26: - if p++; p == pe { - goto _test_eof26 - } - st_case_26: - switch data[p] { - case 42: - goto st26 - case 47: - goto tr27 - case 96: - goto tr31 - } - goto st25 - tr38: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st36 - st36: - if p++; p == pe { - goto _test_eof36 - } - st_case_36: - //.... tmp_parser.go:842 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 78: - goto tr44 - case 96: - goto st3 - case 110: - goto tr44 - } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 - } - goto tr2 - tr44: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st37 - st37: - if p++; p == pe { - goto _test_eof37 - } - st_case_37: - //.... tmp_parser.go:880 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 83: + case data[p] >= 9: + goto tr45 + } + goto tr26 + st39: + if p++; p == pe { + goto _test_eof39 + } + st_case_39: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 69: + goto st40 + case 96: + goto tr45 + case 101: + goto st40 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { goto tr45 - case 84: - goto tr46 - case 96: - goto st3 - case 115: + } + case data[p] >= 9: + goto tr45 + } + goto tr26 + st40: + if p++; p == pe { + goto _test_eof40 + } + st_case_40: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 82: + goto st41 + case 96: + goto tr45 + case 114: + goto st41 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { goto tr45 - case 116: - goto tr46 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st41: + if p++; p == pe { + goto _test_eof41 + } + st_case_41: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 84: + goto st42 + case 96: + goto tr45 + case 116: + goto st42 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr45: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st38 - st38: - if p++; p == pe { - goto _test_eof38 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st42: + if p++; p == pe { + goto _test_eof42 + } + st_case_42: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 40: + goto tr10 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_38: - //.... tmp_parser.go:922 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 69: - goto tr47 - case 96: - goto st3 - case 101: - goto tr47 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st43: + if p++; p == pe { + goto _test_eof43 + } + st_case_43: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 65: + goto st44 + case 96: + goto tr45 + case 97: + goto st44 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st44: + if p++; p == pe { + goto _test_eof44 + } + st_case_44: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 76: + goto st45 + case 96: + goto tr45 + case 108: + goto st45 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr47: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st39 - st39: - if p++; p == pe { - goto _test_eof39 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st45: + if p++; p == pe { + goto _test_eof45 + } + st_case_45: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 83: + goto st46 + case 96: + goto tr45 + case 115: + goto st46 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_39: - //.... tmp_parser.go:960 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 82: - goto tr48 - case 96: - goto st3 - case 114: - goto tr48 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st46: + if p++; p == pe { + goto _test_eof46 + } + st_case_46: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 69: + goto tr65 + case 96: + goto tr45 + case 101: + goto tr65 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st47: + if p++; p == pe { + goto _test_eof47 + } + st_case_47: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 78: + goto st48 + case 96: + goto tr45 + case 110: + goto st48 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr48: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st40 - st40: - if p++; p == pe { - goto _test_eof40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st48: + if p++; p == pe { + goto _test_eof48 + } + st_case_48: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 84: + goto st49 + case 96: + goto tr45 + case 116: + goto st49 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_40: - //.... tmp_parser.go:998 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 84: - goto tr43 - case 96: - goto st3 - case 116: - goto tr43 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st49: + if p++; p == pe { + goto _test_eof49 + } + st_case_49: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 79: + goto tr49 + case 96: + goto tr45 + case 111: + goto tr49 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st50: + if p++; p == pe { + goto _test_eof50 + } + st_case_50: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 85: + goto st51 + case 96: + goto tr45 + case 117: + goto st51 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr46: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st41 - st41: - if p++; p == pe { - goto _test_eof41 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st51: + if p++; p == pe { + goto _test_eof51 + } + st_case_51: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 76: + goto st52 + case 96: + goto tr45 + case 108: + goto st52 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_41: - //.... tmp_parser.go:1036 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 79: - goto tr43 - case 96: - goto st3 - case 111: - goto tr43 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st52: + if p++; p == pe { + goto _test_eof52 + } + st_case_52: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 76: + goto tr70 + case 96: + goto tr45 + case 108: + goto tr70 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st53: + if p++; p == pe { + goto _test_eof53 + } + st_case_53: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 82: + goto st54 + case 96: + goto tr45 + case 114: + goto st54 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr39: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st42 - st42: - if p++; p == pe { - goto _test_eof42 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st54: + if p++; p == pe { + goto _test_eof54 + } + st_case_54: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 85: + goto st55 + case 96: + goto tr45 + case 117: + goto st55 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_42: - //.... tmp_parser.go:1074 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 65: - goto tr49 - case 96: - goto st3 - case 97: - goto tr49 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st55: + if p++; p == pe { + goto _test_eof55 + } + st_case_55: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 69: + goto tr73 + case 96: + goto tr45 + case 101: + goto tr73 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st56: + if p++; p == pe { + goto _test_eof56 + } + st_case_56: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 83: + goto st57 + case 96: + goto tr45 + case 115: + goto st57 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr49: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st43 - st43: - if p++; p == pe { - goto _test_eof43 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st57: + if p++; p == pe { + goto _test_eof57 + } + st_case_57: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 73: + goto st58 + case 96: + goto tr45 + case 105: + goto st58 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_43: - //.... tmp_parser.go:1112 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 76: - goto tr50 - case 96: - goto st3 - case 108: - goto tr50 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st58: + if p++; p == pe { + goto _test_eof58 + } + st_case_58: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 78: + goto st59 + case 96: + goto tr45 + case 110: + goto st59 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st59: + if p++; p == pe { + goto _test_eof59 + } + st_case_59: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 71: + goto tr77 + case 96: + goto tr45 + case 103: + goto tr77 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr50: - //.... NONE:1 - te = p + 1 + case data[p] >= 9: + goto tr45 + } + goto tr26 +tr77: +//.... NONE:1 +te = p+1 - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st44 - st44: - if p++; p == pe { - goto _test_eof44 + goto st60 + st60: + if p++; p == pe { + goto _test_eof60 + } + st_case_60: +//.... tmp_parser.go:1858 + switch data[p] { + case 32: + goto st11 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_44: - //.... tmp_parser.go:1150 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 85: - goto tr51 - case 96: - goto st3 - case 117: - goto tr51 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st11: + if p++; p == pe { + goto _test_eof11 + } + st_case_11: + switch data[p] { + case 85: + goto st12 + case 117: + goto st12 + } + goto tr8 + st12: + if p++; p == pe { + goto _test_eof12 + } + st_case_12: + switch data[p] { + case 84: + goto st13 + case 116: + goto st13 + } + goto tr8 + st13: + if p++; p == pe { + goto _test_eof13 + } + st_case_13: + switch data[p] { + case 70: + goto st14 + case 102: + goto st14 + } + goto tr8 + st14: + if p++; p == pe { + goto _test_eof14 + } + st_case_14: + if data[p] == 56 { + goto st15 + } + goto tr8 + st15: + if p++; p == pe { + goto _test_eof15 + } + st_case_15: + switch data[p] { + case 77: + goto st16 + case 109: + goto st16 + } + goto tr8 + st16: + if p++; p == pe { + goto _test_eof16 + } + st_case_16: + switch data[p] { + case 66: + goto st17 + case 98: + goto st17 + } + goto tr8 + st17: + if p++; p == pe { + goto _test_eof17 + } + st_case_17: + if data[p] == 52 { + goto st18 + } + goto tr8 + st18: + if p++; p == pe { + goto _test_eof18 + } + st_case_18: + if data[p] == 41 { + goto tr10 + } + goto tr8 + st61: + if p++; p == pe { + goto _test_eof61 + } + st_case_61: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 65: + goto st62 + case 96: + goto tr45 + case 97: + goto st62 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st62: + if p++; p == pe { + goto _test_eof62 + } + st_case_62: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 76: + goto st63 + case 96: + goto tr45 + case 108: + goto st63 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr51: - //.... NONE:1 - te = p + 1 - - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st45 - st45: - if p++; p == pe { - goto _test_eof45 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st63: + if p++; p == pe { + goto _test_eof63 + } + st_case_63: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 85: + goto st64 + case 96: + goto tr45 + case 117: + goto st64 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - st_case_45: - //.... tmp_parser.go:1188 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 69: - goto tr52 - case 96: - goto st3 - case 101: - goto tr52 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st64: + if p++; p == pe { + goto _test_eof64 + } + st_case_64: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 69: + goto st65 + case 96: + goto tr45 + case 101: + goto st65 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st65: + if p++; p == pe { + goto _test_eof65 + } + st_case_65: + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 44: + goto tr45 + case 59: + goto tr45 + case 83: + goto tr83 + case 96: + goto tr45 + case 115: + goto tr83 + } + switch { + case data[p] > 13: + if 39 <= data[p] && data[p] <= 41 { + goto tr45 } - goto tr2 - tr52: - //.... NONE:1 - te = p + 1 + case data[p] >= 9: + goto tr45 + } + goto tr26 +tr42: +//.... NONE:1 +te = p+1 - //.... lightning/mydump/parser.rl:85 - act = 4 - goto st46 - st46: - if p++; p == pe { - goto _test_eof46 - } - st_case_46: - //.... tmp_parser.go:1226 - switch data[p] { - case 32: - goto tr40 - case 34: - goto st1 - case 44: - goto tr40 - case 59: - goto tr40 - case 83: - goto tr53 - case 96: - goto st3 - case 115: - goto tr53 + goto st66 + st66: + if p++; p == pe { + goto _test_eof66 + } + st_case_66: +//.... tmp_parser.go:2127 + switch data[p] { + case 32: + goto tr45 + case 34: + goto tr45 + case 39: + goto st19 + case 44: + goto tr45 + case 59: + goto tr45 + case 96: + goto tr45 + } + switch { + case data[p] > 13: + if 40 <= data[p] && data[p] <= 41 { + goto tr45 } - switch { - case data[p] > 13: - if 39 <= data[p] && data[p] <= 41 { - goto tr40 - } - case data[p] >= 9: - goto tr40 + case data[p] >= 9: + goto tr45 + } + goto tr26 + st19: + if p++; p == pe { + goto _test_eof19 + } + st_case_19: + if data[p] == 39 { + goto tr22 + } + switch { + case data[p] > 57: + if 97 <= data[p] && data[p] <= 102 { + goto st19 } - goto tr2 - st_out: - _test_eof27: - cs = 27 - goto _test_eof - _test_eof28: - cs = 28 - goto _test_eof - _test_eof1: - cs = 1 - goto _test_eof - _test_eof2: - cs = 2 - goto _test_eof - _test_eof3: - cs = 3 - goto _test_eof - _test_eof4: - cs = 4 - goto _test_eof - _test_eof5: - cs = 5 - goto _test_eof - _test_eof6: - cs = 6 - goto _test_eof - _test_eof7: - cs = 7 - goto _test_eof - _test_eof8: - cs = 8 - goto _test_eof - _test_eof9: - cs = 9 - goto _test_eof - _test_eof10: - cs = 10 - goto _test_eof - _test_eof11: - cs = 11 - goto _test_eof - _test_eof12: - cs = 12 - goto _test_eof - _test_eof13: - cs = 13 - goto _test_eof - _test_eof14: - cs = 14 - goto _test_eof - _test_eof15: - cs = 15 - goto _test_eof - _test_eof29: - cs = 29 - goto _test_eof - _test_eof30: - cs = 30 - goto _test_eof - _test_eof16: - cs = 16 - goto _test_eof - _test_eof17: - cs = 17 - goto _test_eof - _test_eof31: - cs = 31 - goto _test_eof - _test_eof18: - cs = 18 - goto _test_eof - _test_eof19: - cs = 19 - goto _test_eof - _test_eof32: - cs = 32 - goto _test_eof - _test_eof33: - cs = 33 - goto _test_eof - _test_eof34: - cs = 34 - goto _test_eof - _test_eof20: - cs = 20 - goto _test_eof - _test_eof21: - cs = 21 - goto _test_eof - _test_eof22: - cs = 22 - goto _test_eof - _test_eof23: - cs = 23 - goto _test_eof - _test_eof24: - cs = 24 - goto _test_eof - _test_eof35: - cs = 35 - goto _test_eof - _test_eof25: - cs = 25 - goto _test_eof - _test_eof26: - cs = 26 - goto _test_eof - _test_eof36: - cs = 36 - goto _test_eof - _test_eof37: - cs = 37 - goto _test_eof - _test_eof38: - cs = 38 - goto _test_eof - _test_eof39: - cs = 39 - goto _test_eof - _test_eof40: - cs = 40 - goto _test_eof - _test_eof41: - cs = 41 - goto _test_eof - _test_eof42: - cs = 42 - goto _test_eof - _test_eof43: - cs = 43 - goto _test_eof - _test_eof44: - cs = 44 - goto _test_eof - _test_eof45: - cs = 45 - goto _test_eof - _test_eof46: - cs = 46 - goto _test_eof + case data[p] >= 48: + goto st19 + } + goto tr8 + st20: + if p++; p == pe { + goto _test_eof20 + } + st_case_20: + if data[p] == 96 { + goto tr25 + } + goto st20 +tr25: +//.... NONE:1 +te = p+1 - _test_eof: - { - } - if p == eof { - switch cs { - case 28: - goto tr0 - case 1: - goto tr0 - case 2: - goto tr0 - case 3: - goto tr0 - case 29: - goto tr40 - case 30: - goto tr40 - case 16: - goto tr19 - case 17: - goto tr19 - case 31: - goto tr41 - case 18: - goto tr19 - case 19: - goto tr19 - case 32: - goto tr41 - case 33: - goto tr40 - case 34: - goto tr40 - case 20: - goto tr19 - case 21: - goto tr19 - case 22: - goto tr19 - case 23: - goto tr19 - case 24: - goto tr19 - case 35: - goto tr40 - case 25: - goto tr19 - case 26: - goto tr19 - case 36: - goto tr40 - case 37: - goto tr40 - case 38: - goto tr40 - case 39: - goto tr40 - case 40: - goto tr40 - case 41: - goto tr40 - case 42: - goto tr40 - case 43: - goto tr40 - case 44: - goto tr40 - case 45: - goto tr40 - case 46: - goto tr40 - } - } +//.... lightning/mydump/parser.rl:121 +act = 12; + goto st67 + st67: + if p++; p == pe { + goto _test_eof67 + } + st_case_67: +//.... tmp_parser.go:2189 + if data[p] == 96 { + goto st20 + } + goto tr84 + st_out: + _test_eof21: cs = 21; goto _test_eof + _test_eof22: cs = 22; goto _test_eof + _test_eof1: cs = 1; goto _test_eof + _test_eof2: cs = 2; goto _test_eof + _test_eof23: cs = 23; goto _test_eof + _test_eof3: cs = 3; goto _test_eof + _test_eof4: cs = 4; goto _test_eof + _test_eof5: cs = 5; goto _test_eof + _test_eof24: cs = 24; goto _test_eof + _test_eof6: cs = 6; goto _test_eof + _test_eof25: cs = 25; goto _test_eof + _test_eof26: cs = 26; goto _test_eof + _test_eof7: cs = 7; goto _test_eof + _test_eof27: cs = 27; goto _test_eof + _test_eof28: cs = 28; goto _test_eof + _test_eof8: cs = 8; goto _test_eof + _test_eof9: cs = 9; goto _test_eof + _test_eof29: cs = 29; goto _test_eof + _test_eof30: cs = 30; goto _test_eof + _test_eof31: cs = 31; goto _test_eof + _test_eof32: cs = 32; goto _test_eof + _test_eof33: cs = 33; goto _test_eof + _test_eof34: cs = 34; goto _test_eof + _test_eof35: cs = 35; goto _test_eof + _test_eof10: cs = 10; goto _test_eof + _test_eof36: cs = 36; goto _test_eof + _test_eof37: cs = 37; goto _test_eof + _test_eof38: cs = 38; goto _test_eof + _test_eof39: cs = 39; goto _test_eof + _test_eof40: cs = 40; goto _test_eof + _test_eof41: cs = 41; goto _test_eof + _test_eof42: cs = 42; goto _test_eof + _test_eof43: cs = 43; goto _test_eof + _test_eof44: cs = 44; goto _test_eof + _test_eof45: cs = 45; goto _test_eof + _test_eof46: cs = 46; goto _test_eof + _test_eof47: cs = 47; goto _test_eof + _test_eof48: cs = 48; goto _test_eof + _test_eof49: cs = 49; goto _test_eof + _test_eof50: cs = 50; goto _test_eof + _test_eof51: cs = 51; goto _test_eof + _test_eof52: cs = 52; goto _test_eof + _test_eof53: cs = 53; goto _test_eof + _test_eof54: cs = 54; goto _test_eof + _test_eof55: cs = 55; goto _test_eof + _test_eof56: cs = 56; goto _test_eof + _test_eof57: cs = 57; goto _test_eof + _test_eof58: cs = 58; goto _test_eof + _test_eof59: cs = 59; goto _test_eof + _test_eof60: cs = 60; goto _test_eof + _test_eof11: cs = 11; goto _test_eof + _test_eof12: cs = 12; goto _test_eof + _test_eof13: cs = 13; goto _test_eof + _test_eof14: cs = 14; goto _test_eof + _test_eof15: cs = 15; goto _test_eof + _test_eof16: cs = 16; goto _test_eof + _test_eof17: cs = 17; goto _test_eof + _test_eof18: cs = 18; goto _test_eof + _test_eof61: cs = 61; goto _test_eof + _test_eof62: cs = 62; goto _test_eof + _test_eof63: cs = 63; goto _test_eof + _test_eof64: cs = 64; goto _test_eof + _test_eof65: cs = 65; goto _test_eof + _test_eof66: cs = 66; goto _test_eof + _test_eof19: cs = 19; goto _test_eof + _test_eof20: cs = 20; goto _test_eof + _test_eof67: cs = 67; goto _test_eof - _out: - { - } + _test_eof: {} + if p == eof { + switch cs { + case 22: + goto tr4 + case 2: + goto tr4 + case 23: + goto tr43 + case 3: + goto tr4 + case 5: + goto tr4 + case 24: + goto tr44 + case 6: + goto tr4 + case 25: + goto tr45 + case 26: + goto tr45 + case 7: + goto tr8 + case 27: + goto tr45 + case 28: + goto tr45 + case 8: + goto tr8 + case 9: + goto tr8 + case 29: + goto tr45 + case 30: + goto tr45 + case 31: + goto tr45 + case 32: + goto tr45 + case 33: + goto tr45 + case 34: + goto tr45 + case 35: + goto tr45 + case 10: + goto tr8 + case 36: + goto tr45 + case 37: + goto tr45 + case 38: + goto tr45 + case 39: + goto tr45 + case 40: + goto tr45 + case 41: + goto tr45 + case 42: + goto tr45 + case 43: + goto tr45 + case 44: + goto tr45 + case 45: + goto tr45 + case 46: + goto tr45 + case 47: + goto tr45 + case 48: + goto tr45 + case 49: + goto tr45 + case 50: + goto tr45 + case 51: + goto tr45 + case 52: + goto tr45 + case 53: + goto tr45 + case 54: + goto tr45 + case 55: + goto tr45 + case 56: + goto tr45 + case 57: + goto tr45 + case 58: + goto tr45 + case 59: + goto tr45 + case 60: + goto tr45 + case 11: + goto tr8 + case 12: + goto tr8 + case 13: + goto tr8 + case 14: + goto tr8 + case 15: + goto tr8 + case 16: + goto tr8 + case 17: + goto tr8 + case 18: + goto tr8 + case 61: + goto tr45 + case 62: + goto tr45 + case 63: + goto tr45 + case 64: + goto tr45 + case 65: + goto tr45 + case 66: + goto tr45 + case 19: + goto tr8 + case 20: + goto tr4 + case 67: + goto tr84 } + } + + _out: {} + } - //.... lightning/mydump/parser.rl:110 +//.... lightning/mydump/parser.rl:151 if cs == 0 { common.AppLogger.Errorf("Syntax error near byte %d, content is «%s»", parser.pos, string(data)) diff --git a/lightning/mydump/parser_test.go b/lightning/mydump/parser_test.go index e5ae1d215..3304aa925 100644 --- a/lightning/mydump/parser_test.go +++ b/lightning/mydump/parser_test.go @@ -19,11 +19,12 @@ import ( "strings" . "github.com/pingcap/check" + "github.com/pingcap/errors" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-lightning/lightning/config" "github.com/pingcap/tidb-lightning/lightning/mydump" "github.com/pingcap/tidb-lightning/lightning/worker" - - "github.com/pingcap/errors" + "github.com/pingcap/tidb/types" ) var _ = Suite(&testMydumpParserSuite{}) @@ -42,15 +43,18 @@ func (s *testMydumpParserSuite) TestReadRow(c *C) { ) ioWorkers := worker.NewPool(context.Background(), 5, "test") - parser := mydump.NewChunkParser(reader, config.ReadBlockSize, ioWorkers) + parser := mydump.NewChunkParser(mysql.ModeNone, reader, config.ReadBlockSize, ioWorkers) c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 1, - Row: []byte("(1, 2, 3)"), + Row: []types.Datum{ + types.NewStringDatum("1"), + types.NewStringDatum("2"), + types.NewStringDatum("3"), + }, }) - c.Assert(parser.TableName, DeepEquals, []byte("`namespaced`.`table`")) - c.Assert(parser.Columns(), DeepEquals, []byte("(columns, more, columns)")) + c.Assert(parser.Columns(), DeepEquals, []string{"columns", "more", "columns"}) offset, rowID := parser.Pos() c.Assert(offset, Equals, int64(97)) c.Assert(rowID, Equals, int64(1)) @@ -58,10 +62,12 @@ func (s *testMydumpParserSuite) TestReadRow(c *C) { c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 2, - Row: []byte("(4, 5, 6)"), - }) - c.Assert(parser.TableName, DeepEquals, []byte("`namespaced`.`table`")) - c.Assert(parser.Columns(), DeepEquals, []byte("(columns, more, columns)")) + Row: []types.Datum{ + types.NewStringDatum("4"), + types.NewStringDatum("5"), + types.NewStringDatum("6"), + }}) + c.Assert(parser.Columns(), DeepEquals, []string{"columns", "more", "columns"}) offset, rowID = parser.Pos() c.Assert(offset, Equals, int64(108)) c.Assert(rowID, Equals, int64(2)) @@ -69,10 +75,13 @@ func (s *testMydumpParserSuite) TestReadRow(c *C) { c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 3, - Row: []byte("(7,8,9)"), + Row: []types.Datum{ + types.NewStringDatum("7"), + types.NewStringDatum("8"), + types.NewStringDatum("9"), + }, }) - c.Assert(parser.TableName, DeepEquals, []byte("`namespaced`.`table`")) - c.Assert(parser.Columns(), DeepEquals, []byte("(x,y,z)")) + c.Assert(parser.Columns(), DeepEquals, []string{"x", "y", "z"}) offset, rowID = parser.Pos() c.Assert(offset, Equals, int64(159)) c.Assert(rowID, Equals, int64(3)) @@ -80,9 +89,15 @@ func (s *testMydumpParserSuite) TestReadRow(c *C) { c.Assert(parser.ReadRow(), IsNil) c.Assert(parser.LastRow(), DeepEquals, mydump.Row{ RowID: 4, - Row: []byte("(10, 11, 12, '(13)', '(', 14, ')')"), - }) - c.Assert(parser.TableName, DeepEquals, []byte("another_table")) + Row: []types.Datum{ + types.NewStringDatum("10"), + types.NewStringDatum("11"), + types.NewStringDatum("12"), + types.NewStringDatum("(13)"), + types.NewStringDatum("("), + types.NewStringDatum("14"), + types.NewStringDatum(")"), + }}) c.Assert(parser.Columns(), IsNil) offset, rowID = parser.Pos() c.Assert(offset, Equals, int64(222)) @@ -99,7 +114,7 @@ func (s *testMydumpParserSuite) TestReadChunks(c *C) { `) ioWorkers := worker.NewPool(context.Background(), 5, "test") - parser := mydump.NewChunkParser(reader, config.ReadBlockSize, ioWorkers) + parser := mydump.NewChunkParser(mysql.ModeNone, reader, config.ReadBlockSize, ioWorkers) chunks, err := mydump.ReadChunks(parser, 32) c.Assert(err, IsNil) @@ -146,7 +161,7 @@ func (s *testMydumpParserSuite) TestNestedRow(c *C) { `) ioWorkers := worker.NewPool(context.Background(), 5, "test") - parser := mydump.NewChunkParser(reader, config.ReadBlockSize, ioWorkers) + parser := mydump.NewChunkParser(mysql.ModeNone, reader, config.ReadBlockSize, ioWorkers) chunks, err := mydump.ReadChunks(parser, 96) c.Assert(err, IsNil) diff --git a/lightning/restore/checkpoints.go b/lightning/restore/checkpoints.go index 65af26e39..c4aadf65b 100644 --- a/lightning/restore/checkpoints.go +++ b/lightning/restore/checkpoints.go @@ -16,6 +16,7 @@ package restore import ( "context" "database/sql" + "encoding/json" "fmt" "io" "io/ioutil" @@ -69,6 +70,8 @@ func (status CheckpointStatus) MetricName() string { return "closed" case CheckpointStatusImported: return "imported" + case CheckpointStatusIndexImported: + return "index_imported" case CheckpointStatusAlteredAutoInc: return "altered_auto_inc" case CheckpointStatusChecksummed, CheckpointStatusChecksumSkipped: @@ -101,11 +104,10 @@ func (key *ChunkCheckpointKey) less(other *ChunkCheckpointKey) bool { } type ChunkCheckpoint struct { - Key ChunkCheckpointKey - Columns []byte - ShouldIncludeRowID bool - Chunk mydump.Chunk - Checksum verify.KVChecksum + Key ChunkCheckpointKey + ColumnPermutation []int + Chunk mydump.Chunk + Checksum verify.KVChecksum } type EngineCheckpoint struct { @@ -244,6 +246,9 @@ type CheckpointsDB interface { Initialize(ctx context.Context, dbInfo map[string]*TidbDBInfo) error Get(ctx context.Context, tableName string) (*TableCheckpoint, error) Close() error + // InsertEngineCheckpoints initializes the checkpoints related to a table. + // It assumes the entire table has not been imported before and will fill in + // default values for the column permutations and checksums. InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints map[int32]*EngineCheckpoint) error Update(checkpointDiffs map[string]*TableCheckpointDiff) @@ -448,7 +453,7 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab chunkQuery := fmt.Sprintf(` SELECT - engine_id, path, offset, columns, should_include_row_id, + engine_id, path, offset, columns, pos, end_offset, prev_rowid_max, rowid_max, kvc_bytes, kvc_kvs, kvc_checksum FROM %s.%s WHERE table_name = ? @@ -462,19 +467,23 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab for chunkRows.Next() { var ( value = new(ChunkCheckpoint) + colPerm []byte engineID int32 kvcBytes uint64 kvcKVs uint64 kvcChecksum uint64 ) if err := chunkRows.Scan( - &engineID, &value.Key.Path, &value.Key.Offset, &value.Columns, &value.ShouldIncludeRowID, + &engineID, &value.Key.Path, &value.Key.Offset, &colPerm, &value.Chunk.Offset, &value.Chunk.EndOffset, &value.Chunk.PrevRowIDMax, &value.Chunk.RowIDMax, &kvcBytes, &kvcKVs, &kvcChecksum, ); err != nil { return errors.Trace(err) } value.Checksum = verify.MakeKVChecksum(kvcBytes, kvcKVs, kvcChecksum) + if err := json.Unmarshal(colPerm, &value.ColumnPermutation); err != nil { + return errors.Trace(err) + } cp.Engines[engineID].Chunks = append(cp.Engines[engineID].Chunks, value) } if err := chunkRows.Err(); err != nil { @@ -520,9 +529,9 @@ func (cpdb *MySQLCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tab kvc_bytes, kvc_kvs, kvc_checksum ) VALUES ( ?, ?, + ?, ?, '[]', FALSE, ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ? + 0, 0, 0 ); `, cpdb.schema, checkpointTableNameChunk)) if err != nil { @@ -538,9 +547,8 @@ func (cpdb *MySQLCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tab for _, value := range engine.Chunks { _, err = chunkStmt.ExecContext( c, tableName, engineID, - value.Key.Path, value.Key.Offset, value.Columns, value.ShouldIncludeRowID, + value.Key.Path, value.Key.Offset, value.Chunk.Offset, value.Chunk.EndOffset, value.Chunk.PrevRowIDMax, value.Chunk.RowIDMax, - value.Checksum.SumSize(), value.Checksum.SumKVS(), value.Checksum.Sum(), ) if err != nil { return errors.Trace(err) @@ -714,13 +722,16 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC } for _, chunkModel := range engineModel.Chunks { + colPerm := make([]int, 0, len(chunkModel.ColumnPermutation)) + for _, c := range chunkModel.ColumnPermutation { + colPerm = append(colPerm, int(c)) + } engine.Chunks = append(engine.Chunks, &ChunkCheckpoint{ Key: ChunkCheckpointKey{ Path: chunkModel.Path, Offset: chunkModel.Offset, }, - Columns: chunkModel.Columns, - ShouldIncludeRowID: chunkModel.ShouldIncludeRowId, + ColumnPermutation: colPerm, Chunk: mydump.Chunk{ Offset: chunkModel.Pos, EndOffset: chunkModel.EndOffset, @@ -756,10 +767,8 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN chunk, ok := engineModel.Chunks[key] if !ok { chunk = &ChunkCheckpointModel{ - Path: value.Key.Path, - Offset: value.Key.Offset, - Columns: value.Columns, - ShouldIncludeRowId: value.ShouldIncludeRowID, + Path: value.Key.Path, + Offset: value.Key.Offset, } engineModel.Chunks[key] = chunk } @@ -767,9 +776,6 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN chunk.EndOffset = value.Chunk.EndOffset chunk.PrevRowidMax = value.Chunk.PrevRowIDMax chunk.RowidMax = value.Chunk.RowIDMax - chunk.KvcBytes = value.Checksum.SumSize() - chunk.KvcKvs = value.Checksum.SumKVS() - chunk.KvcChecksum = value.Checksum.Sum() } tableModel.Engines[engineID] = engineModel } diff --git a/lightning/restore/file_checkpoints.pb.go b/lightning/restore/file_checkpoints.pb.go index b3b38bd41..06bcfc9fd 100644 --- a/lightning/restore/file_checkpoints.pb.go +++ b/lightning/restore/file_checkpoints.pb.go @@ -3,14 +3,14 @@ package restore -import ( - encoding_binary "encoding/binary" - fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" -) +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" + +import encoding_binary "encoding/binary" + +import io "io" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -25,14 +25,16 @@ const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package type CheckpointsModel struct { // key is table_name - Checkpoints map[string]*TableCheckpointModel `protobuf:"bytes,1,rep,name=checkpoints,proto3" json:"checkpoints,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Checkpoints map[string]*TableCheckpointModel `protobuf:"bytes,1,rep,name=checkpoints" json:"checkpoints,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *CheckpointsModel) Reset() { *m = CheckpointsModel{} } func (m *CheckpointsModel) String() string { return proto.CompactTextString(m) } func (*CheckpointsModel) ProtoMessage() {} func (*CheckpointsModel) Descriptor() ([]byte, []int) { - return fileDescriptor_c47ec4f2f281cd62, []int{0} + return fileDescriptor_file_checkpoints_55cd1127f9b6b03a, []int{0} } func (m *CheckpointsModel) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -49,8 +51,8 @@ func (m *CheckpointsModel) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return b[:n], nil } } -func (m *CheckpointsModel) XXX_Merge(src proto.Message) { - xxx_messageInfo_CheckpointsModel.Merge(m, src) +func (dst *CheckpointsModel) XXX_Merge(src proto.Message) { + xxx_messageInfo_CheckpointsModel.Merge(dst, src) } func (m *CheckpointsModel) XXX_Size() int { return m.Size() @@ -62,17 +64,19 @@ func (m *CheckpointsModel) XXX_DiscardUnknown() { var xxx_messageInfo_CheckpointsModel proto.InternalMessageInfo type TableCheckpointModel struct { - Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` - Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` - AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` - Engines map[int32]*EngineCheckpointModel `protobuf:"bytes,8,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"zigzag32,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` + Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` + AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` + Engines map[int32]*EngineCheckpointModel `protobuf:"bytes,8,rep,name=engines" json:"engines,omitempty" protobuf_key:"zigzag32,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *TableCheckpointModel) Reset() { *m = TableCheckpointModel{} } func (m *TableCheckpointModel) String() string { return proto.CompactTextString(m) } func (*TableCheckpointModel) ProtoMessage() {} func (*TableCheckpointModel) Descriptor() ([]byte, []int) { - return fileDescriptor_c47ec4f2f281cd62, []int{1} + return fileDescriptor_file_checkpoints_55cd1127f9b6b03a, []int{1} } func (m *TableCheckpointModel) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -89,8 +93,8 @@ func (m *TableCheckpointModel) XXX_Marshal(b []byte, deterministic bool) ([]byte return b[:n], nil } } -func (m *TableCheckpointModel) XXX_Merge(src proto.Message) { - xxx_messageInfo_TableCheckpointModel.Merge(m, src) +func (dst *TableCheckpointModel) XXX_Merge(src proto.Message) { + xxx_messageInfo_TableCheckpointModel.Merge(dst, src) } func (m *TableCheckpointModel) XXX_Size() int { return m.Size() @@ -104,14 +108,16 @@ var xxx_messageInfo_TableCheckpointModel proto.InternalMessageInfo type EngineCheckpointModel struct { Status uint32 `protobuf:"varint,1,opt,name=status,proto3" json:"status,omitempty"` // key is "$path:$offset" - Chunks map[string]*ChunkCheckpointModel `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Chunks map[string]*ChunkCheckpointModel `protobuf:"bytes,2,rep,name=chunks" json:"chunks,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *EngineCheckpointModel) Reset() { *m = EngineCheckpointModel{} } func (m *EngineCheckpointModel) String() string { return proto.CompactTextString(m) } func (*EngineCheckpointModel) ProtoMessage() {} func (*EngineCheckpointModel) Descriptor() ([]byte, []int) { - return fileDescriptor_c47ec4f2f281cd62, []int{2} + return fileDescriptor_file_checkpoints_55cd1127f9b6b03a, []int{2} } func (m *EngineCheckpointModel) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -128,8 +134,8 @@ func (m *EngineCheckpointModel) XXX_Marshal(b []byte, deterministic bool) ([]byt return b[:n], nil } } -func (m *EngineCheckpointModel) XXX_Merge(src proto.Message) { - xxx_messageInfo_EngineCheckpointModel.Merge(m, src) +func (dst *EngineCheckpointModel) XXX_Merge(src proto.Message) { + xxx_messageInfo_EngineCheckpointModel.Merge(dst, src) } func (m *EngineCheckpointModel) XXX_Size() int { return m.Size() @@ -141,24 +147,25 @@ func (m *EngineCheckpointModel) XXX_DiscardUnknown() { var xxx_messageInfo_EngineCheckpointModel proto.InternalMessageInfo type ChunkCheckpointModel struct { - Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` - Offset int64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` - Columns []byte `protobuf:"bytes,3,opt,name=columns,proto3" json:"columns,omitempty"` - ShouldIncludeRowId bool `protobuf:"varint,4,opt,name=should_include_row_id,json=shouldIncludeRowId,proto3" json:"should_include_row_id,omitempty"` - EndOffset int64 `protobuf:"varint,5,opt,name=end_offset,json=endOffset,proto3" json:"end_offset,omitempty"` - Pos int64 `protobuf:"varint,6,opt,name=pos,proto3" json:"pos,omitempty"` - PrevRowidMax int64 `protobuf:"varint,7,opt,name=prev_rowid_max,json=prevRowidMax,proto3" json:"prev_rowid_max,omitempty"` - RowidMax int64 `protobuf:"varint,8,opt,name=rowid_max,json=rowidMax,proto3" json:"rowid_max,omitempty"` - KvcBytes uint64 `protobuf:"varint,9,opt,name=kvc_bytes,json=kvcBytes,proto3" json:"kvc_bytes,omitempty"` - KvcKvs uint64 `protobuf:"varint,10,opt,name=kvc_kvs,json=kvcKvs,proto3" json:"kvc_kvs,omitempty"` - KvcChecksum uint64 `protobuf:"fixed64,11,opt,name=kvc_checksum,json=kvcChecksum,proto3" json:"kvc_checksum,omitempty"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Offset int64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` + ColumnPermutation []int32 `protobuf:"varint,12,rep,packed,name=column_permutation,json=columnPermutation" json:"column_permutation,omitempty"` + EndOffset int64 `protobuf:"varint,5,opt,name=end_offset,json=endOffset,proto3" json:"end_offset,omitempty"` + Pos int64 `protobuf:"varint,6,opt,name=pos,proto3" json:"pos,omitempty"` + PrevRowidMax int64 `protobuf:"varint,7,opt,name=prev_rowid_max,json=prevRowidMax,proto3" json:"prev_rowid_max,omitempty"` + RowidMax int64 `protobuf:"varint,8,opt,name=rowid_max,json=rowidMax,proto3" json:"rowid_max,omitempty"` + KvcBytes uint64 `protobuf:"varint,9,opt,name=kvc_bytes,json=kvcBytes,proto3" json:"kvc_bytes,omitempty"` + KvcKvs uint64 `protobuf:"varint,10,opt,name=kvc_kvs,json=kvcKvs,proto3" json:"kvc_kvs,omitempty"` + KvcChecksum uint64 `protobuf:"fixed64,11,opt,name=kvc_checksum,json=kvcChecksum,proto3" json:"kvc_checksum,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ChunkCheckpointModel) Reset() { *m = ChunkCheckpointModel{} } func (m *ChunkCheckpointModel) String() string { return proto.CompactTextString(m) } func (*ChunkCheckpointModel) ProtoMessage() {} func (*ChunkCheckpointModel) Descriptor() ([]byte, []int) { - return fileDescriptor_c47ec4f2f281cd62, []int{3} + return fileDescriptor_file_checkpoints_55cd1127f9b6b03a, []int{3} } func (m *ChunkCheckpointModel) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -175,8 +182,8 @@ func (m *ChunkCheckpointModel) XXX_Marshal(b []byte, deterministic bool) ([]byte return b[:n], nil } } -func (m *ChunkCheckpointModel) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChunkCheckpointModel.Merge(m, src) +func (dst *ChunkCheckpointModel) XXX_Merge(src proto.Message) { + xxx_messageInfo_ChunkCheckpointModel.Merge(dst, src) } func (m *ChunkCheckpointModel) XXX_Size() int { return m.Size() @@ -196,52 +203,6 @@ func init() { proto.RegisterMapType((map[string]*ChunkCheckpointModel)(nil), "EngineCheckpointModel.ChunksEntry") proto.RegisterType((*ChunkCheckpointModel)(nil), "ChunkCheckpointModel") } - -func init() { - proto.RegisterFile("lightning/restore/file_checkpoints.proto", fileDescriptor_c47ec4f2f281cd62) -} - -var fileDescriptor_c47ec4f2f281cd62 = []byte{ - // 579 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xce, 0xc6, 0xad, 0x93, 0x8c, 0x03, 0x2a, 0xab, 0xb6, 0xac, 0x82, 0xb0, 0x4c, 0x05, 0x92, - 0x25, 0xc0, 0x11, 0xe5, 0x82, 0x2a, 0x4e, 0x0d, 0x3d, 0x54, 0xa8, 0x02, 0xad, 0xe0, 0xc2, 0xc5, - 0xf2, 0xcf, 0xc6, 0xb6, 0xec, 0x78, 0x23, 0xaf, 0xed, 0x36, 0x6f, 0xc1, 0x9b, 0xf0, 0x04, 0xdc, - 0x7b, 0xec, 0x91, 0x03, 0x07, 0x48, 0x1e, 0x81, 0x17, 0x40, 0x5e, 0xbb, 0x8a, 0x83, 0xa2, 0x8a, - 0xdb, 0xcc, 0xf7, 0x7d, 0xf3, 0x4d, 0x3e, 0x8d, 0x37, 0x60, 0x26, 0x51, 0x10, 0xe6, 0x69, 0x94, - 0x06, 0xe3, 0x8c, 0x89, 0x9c, 0x67, 0x6c, 0x3c, 0x8d, 0x12, 0x66, 0x7b, 0x21, 0xf3, 0xe2, 0x39, - 0x8f, 0xd2, 0x5c, 0x58, 0xf3, 0x8c, 0xe7, 0x7c, 0xf4, 0x32, 0x88, 0xf2, 0xb0, 0x70, 0x2d, 0x8f, - 0xcf, 0xc6, 0x01, 0x0f, 0xf8, 0x58, 0xc2, 0x6e, 0x31, 0x95, 0x9d, 0x6c, 0x64, 0x55, 0xcb, 0x8f, - 0xbe, 0x21, 0xd8, 0x9b, 0xac, 0x4d, 0x2e, 0xb8, 0xcf, 0x12, 0xfc, 0x0e, 0xb4, 0x96, 0x31, 0x41, - 0x86, 0x62, 0x6a, 0xc7, 0x47, 0xd6, 0xbf, 0xba, 0x36, 0x70, 0x96, 0xe6, 0xd9, 0x82, 0xb6, 0xc7, - 0x46, 0x9f, 0x37, 0x9c, 0xa5, 0x00, 0xef, 0x81, 0x12, 0xb3, 0x05, 0x41, 0x06, 0x32, 0x07, 0xb4, - 0x2a, 0xf1, 0x73, 0xd8, 0x2d, 0x9d, 0xa4, 0x60, 0xa4, 0x6b, 0x20, 0x53, 0x3b, 0x3e, 0xb0, 0x3e, - 0x39, 0x6e, 0xc2, 0xd6, 0x83, 0x72, 0x13, 0xad, 0x35, 0x27, 0xdd, 0x37, 0xe8, 0xe8, 0x0f, 0x82, - 0xfd, 0x6d, 0x1a, 0x8c, 0x61, 0x27, 0x74, 0x44, 0x28, 0xcd, 0x87, 0x54, 0xd6, 0xf8, 0x10, 0x54, - 0x91, 0x3b, 0x79, 0x21, 0x88, 0x62, 0x20, 0xf3, 0x1e, 0x6d, 0x3a, 0xfc, 0x18, 0xc0, 0x49, 0x12, - 0xee, 0xd9, 0xae, 0x23, 0x18, 0xd9, 0x31, 0x90, 0xa9, 0xd0, 0x81, 0x44, 0x4e, 0x1d, 0xc1, 0xf0, - 0x5b, 0xe8, 0xb1, 0x34, 0x88, 0x52, 0x26, 0x48, 0xbf, 0x09, 0xbf, 0x6d, 0xa5, 0x75, 0x56, 0x8b, - 0xea, 0xf0, 0xb7, 0x23, 0x23, 0x0a, 0xc3, 0x36, 0xd1, 0x0e, 0xfd, 0xa0, 0x0e, 0xfd, 0x62, 0x33, - 0xf4, 0x61, 0x63, 0x74, 0x47, 0xea, 0xef, 0x08, 0x0e, 0xb6, 0x8a, 0x5a, 0x11, 0xd1, 0x46, 0xc4, - 0x13, 0x50, 0xbd, 0xb0, 0x48, 0x63, 0x41, 0xba, 0x4d, 0x84, 0xad, 0xf3, 0xd6, 0x44, 0x8a, 0xea, - 0x08, 0xcd, 0xc4, 0xe8, 0x23, 0x68, 0x2d, 0xf8, 0x7f, 0xae, 0x26, 0xe5, 0x77, 0xfc, 0xfe, 0x9f, - 0x5d, 0xd8, 0xdf, 0xa6, 0xa9, 0xae, 0x36, 0x77, 0xf2, 0xb0, 0x31, 0x97, 0x75, 0x15, 0x89, 0x4f, - 0xa7, 0x82, 0xe5, 0xd2, 0x5e, 0xa1, 0x4d, 0x87, 0x09, 0xf4, 0x3c, 0x9e, 0x14, 0xb3, 0xb4, 0x3e, - 0xe7, 0x90, 0xde, 0xb6, 0xf8, 0x15, 0x1c, 0x88, 0x90, 0x17, 0x89, 0x6f, 0x47, 0xa9, 0x97, 0x14, - 0x3e, 0xb3, 0x33, 0x7e, 0x69, 0x47, 0xbe, 0x3c, 0x6d, 0x9f, 0xe2, 0x9a, 0x3c, 0xaf, 0x39, 0xca, - 0x2f, 0xcf, 0xfd, 0xea, 0x13, 0x60, 0xa9, 0x6f, 0x37, 0x8b, 0x76, 0xeb, 0x4f, 0x80, 0xa5, 0xfe, - 0x87, 0x7a, 0xd7, 0x1e, 0x28, 0x73, 0x2e, 0x88, 0x2a, 0xf1, 0xaa, 0xc4, 0x4f, 0xe1, 0xfe, 0x3c, - 0x63, 0x65, 0xe5, 0x1c, 0xf9, 0xf6, 0xcc, 0xb9, 0x22, 0x3d, 0x49, 0x0e, 0x2b, 0x94, 0x56, 0xe0, - 0x85, 0x73, 0x85, 0x1f, 0xc1, 0x60, 0x2d, 0xe8, 0x4b, 0x41, 0x3f, 0x6b, 0x91, 0x71, 0xe9, 0xd9, - 0xee, 0x22, 0x67, 0x82, 0x0c, 0x0c, 0x64, 0xee, 0xd0, 0x7e, 0x5c, 0x7a, 0xa7, 0x55, 0x8f, 0x1f, - 0x42, 0xaf, 0x22, 0xe3, 0x52, 0x10, 0x90, 0x94, 0x1a, 0x97, 0xde, 0xfb, 0x52, 0xe0, 0x27, 0x30, - 0xac, 0x08, 0xf9, 0xb6, 0x44, 0x31, 0x23, 0x9a, 0x81, 0x4c, 0x95, 0x6a, 0x71, 0xe9, 0x4d, 0x1a, - 0xe8, 0xf4, 0xd9, 0xf5, 0x6f, 0xbd, 0x73, 0xbd, 0xd4, 0xd1, 0xcd, 0x52, 0x47, 0xbf, 0x96, 0x3a, - 0xfa, 0xba, 0xd2, 0x3b, 0x37, 0x2b, 0xbd, 0xf3, 0x63, 0xa5, 0x77, 0xbe, 0xf4, 0x9a, 0xff, 0x0c, - 0x57, 0x95, 0x8f, 0xfe, 0xf5, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd3, 0x90, 0x6a, 0x0b, 0x4f, - 0x04, 0x00, 0x00, -} - func (m *CheckpointsModel) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -426,22 +387,6 @@ func (m *ChunkCheckpointModel) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintFileCheckpoints(dAtA, i, uint64(m.Offset)) } - if len(m.Columns) > 0 { - dAtA[i] = 0x1a - i++ - i = encodeVarintFileCheckpoints(dAtA, i, uint64(len(m.Columns))) - i += copy(dAtA[i:], m.Columns) - } - if m.ShouldIncludeRowId { - dAtA[i] = 0x20 - i++ - if m.ShouldIncludeRowId { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } if m.EndOffset != 0 { dAtA[i] = 0x28 i++ @@ -478,6 +423,24 @@ func (m *ChunkCheckpointModel) MarshalTo(dAtA []byte) (int, error) { encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(m.KvcChecksum)) i += 8 } + if len(m.ColumnPermutation) > 0 { + dAtA5 := make([]byte, len(m.ColumnPermutation)*10) + var j4 int + for _, num1 := range m.ColumnPermutation { + num := uint64(num1) + for num >= 1<<7 { + dAtA5[j4] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j4++ + } + dAtA5[j4] = uint8(num) + j4++ + } + dAtA[i] = 0x62 + i++ + i = encodeVarintFileCheckpoints(dAtA, i, uint64(j4)) + i += copy(dAtA[i:], dAtA5[:j4]) + } return i, nil } @@ -491,9 +454,6 @@ func encodeVarintFileCheckpoints(dAtA []byte, offset int, v uint64) int { return offset + 1 } func (m *CheckpointsModel) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Checkpoints) > 0 { @@ -513,9 +473,6 @@ func (m *CheckpointsModel) Size() (n int) { } func (m *TableCheckpointModel) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Hash) @@ -545,9 +502,6 @@ func (m *TableCheckpointModel) Size() (n int) { } func (m *EngineCheckpointModel) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Status != 0 { @@ -570,9 +524,6 @@ func (m *EngineCheckpointModel) Size() (n int) { } func (m *ChunkCheckpointModel) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Path) @@ -582,13 +533,6 @@ func (m *ChunkCheckpointModel) Size() (n int) { if m.Offset != 0 { n += 1 + sovFileCheckpoints(uint64(m.Offset)) } - l = len(m.Columns) - if l > 0 { - n += 1 + l + sovFileCheckpoints(uint64(l)) - } - if m.ShouldIncludeRowId { - n += 2 - } if m.EndOffset != 0 { n += 1 + sovFileCheckpoints(uint64(m.EndOffset)) } @@ -610,6 +554,13 @@ func (m *ChunkCheckpointModel) Size() (n int) { if m.KvcChecksum != 0 { n += 9 } + if len(m.ColumnPermutation) > 0 { + l = 0 + for _, e := range m.ColumnPermutation { + l += sovFileCheckpoints(uint64(e)) + } + n += 1 + sovFileCheckpoints(uint64(l)) + l + } return n } @@ -641,7 +592,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -669,7 +620,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -678,9 +629,6 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthFileCheckpoints - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -701,7 +649,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -718,7 +666,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLenmapkey |= uint64(b&0x7F) << shift + stringLenmapkey |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -728,9 +676,6 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey < 0 { - return ErrInvalidLengthFileCheckpoints - } if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } @@ -747,7 +692,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapmsglen |= int(b&0x7F) << shift + mapmsglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -756,7 +701,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postmsgIndex := iNdEx + mapmsglen - if postmsgIndex < 0 { + if mapmsglen < 0 { return ErrInvalidLengthFileCheckpoints } if postmsgIndex > l { @@ -793,9 +738,6 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthFileCheckpoints } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthFileCheckpoints - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -823,7 +765,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -851,7 +793,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -860,9 +802,6 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthFileCheckpoints - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -885,7 +824,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Status |= uint32(b&0x7F) << shift + m.Status |= (uint32(b) & 0x7F) << shift if b < 0x80 { break } @@ -904,7 +843,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.AllocBase |= int64(b&0x7F) << shift + m.AllocBase |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -923,7 +862,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -932,9 +871,6 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthFileCheckpoints - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -955,7 +891,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -972,7 +908,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapkeytemp |= int32(b&0x7F) << shift + mapkeytemp |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -990,7 +926,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapmsglen |= int(b&0x7F) << shift + mapmsglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -999,7 +935,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postmsgIndex := iNdEx + mapmsglen - if postmsgIndex < 0 { + if mapmsglen < 0 { return ErrInvalidLengthFileCheckpoints } if postmsgIndex > l { @@ -1036,9 +972,6 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthFileCheckpoints } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthFileCheckpoints - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -1066,7 +999,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1094,7 +1027,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Status |= uint32(b&0x7F) << shift + m.Status |= (uint32(b) & 0x7F) << shift if b < 0x80 { break } @@ -1113,7 +1046,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -1122,9 +1055,6 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthFileCheckpoints - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -1145,7 +1075,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1162,7 +1092,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLenmapkey |= uint64(b&0x7F) << shift + stringLenmapkey |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1172,9 +1102,6 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey < 0 { - return ErrInvalidLengthFileCheckpoints - } if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } @@ -1191,7 +1118,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapmsglen |= int(b&0x7F) << shift + mapmsglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } @@ -1200,7 +1127,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postmsgIndex := iNdEx + mapmsglen - if postmsgIndex < 0 { + if mapmsglen < 0 { return ErrInvalidLengthFileCheckpoints } if postmsgIndex > l { @@ -1237,9 +1164,6 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthFileCheckpoints } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthFileCheckpoints - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -1267,7 +1191,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= uint64(b&0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1295,7 +1219,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1305,9 +1229,6 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthFileCheckpoints - } if postIndex > l { return io.ErrUnexpectedEOF } @@ -1327,65 +1248,11 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Offset |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Columns", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFileCheckpoints - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthFileCheckpoints - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthFileCheckpoints - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Columns = append(m.Columns[:0], dAtA[iNdEx:postIndex]...) - if m.Columns == nil { - m.Columns = []byte{} - } - iNdEx = postIndex - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ShouldIncludeRowId", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFileCheckpoints - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift + m.Offset |= (int64(b) & 0x7F) << shift if b < 0x80 { break } } - m.ShouldIncludeRowId = bool(v != 0) case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field EndOffset", wireType) @@ -1400,7 +1267,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.EndOffset |= int64(b&0x7F) << shift + m.EndOffset |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1419,7 +1286,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Pos |= int64(b&0x7F) << shift + m.Pos |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1438,7 +1305,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.PrevRowidMax |= int64(b&0x7F) << shift + m.PrevRowidMax |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1457,7 +1324,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.RowidMax |= int64(b&0x7F) << shift + m.RowidMax |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1476,7 +1343,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.KvcBytes |= uint64(b&0x7F) << shift + m.KvcBytes |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1495,7 +1362,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.KvcKvs |= uint64(b&0x7F) << shift + m.KvcKvs |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1510,6 +1377,68 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } m.KvcChecksum = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 + case 12: + if wireType == 0 { + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ColumnPermutation = append(m.ColumnPermutation, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthFileCheckpoints + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ColumnPermutation = append(m.ColumnPermutation, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnPermutation", wireType) + } default: iNdEx = preIndex skippy, err := skipFileCheckpoints(dAtA[iNdEx:]) @@ -1519,9 +1448,6 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthFileCheckpoints } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthFileCheckpoints - } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -1588,11 +1514,8 @@ func skipFileCheckpoints(dAtA []byte) (n int, err error) { break } } - if length < 0 { - return 0, ErrInvalidLengthFileCheckpoints - } iNdEx += length - if iNdEx < 0 { + if length < 0 { return 0, ErrInvalidLengthFileCheckpoints } return iNdEx, nil @@ -1623,9 +1546,6 @@ func skipFileCheckpoints(dAtA []byte) (n int, err error) { return 0, err } iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthFileCheckpoints - } } return iNdEx, nil case 4: @@ -1644,3 +1564,46 @@ var ( ErrInvalidLengthFileCheckpoints = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowFileCheckpoints = fmt.Errorf("proto: integer overflow") ) + +func init() { + proto.RegisterFile("lightning/restore/file_checkpoints.proto", fileDescriptor_file_checkpoints_55cd1127f9b6b03a) +} + +var fileDescriptor_file_checkpoints_55cd1127f9b6b03a = []byte{ + // 552 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xc1, 0x6e, 0xd3, 0x4a, + 0x14, 0xed, 0xc4, 0xad, 0x93, 0x5c, 0xe7, 0x3d, 0xa5, 0xa3, 0xb6, 0x8c, 0x82, 0x1a, 0x99, 0x88, + 0x85, 0x25, 0xa8, 0x23, 0x95, 0x0d, 0xaa, 0x58, 0x25, 0x74, 0x85, 0x2a, 0xaa, 0x11, 0x6c, 0xd8, + 0x58, 0xb6, 0x33, 0xb1, 0x2d, 0x3b, 0x1e, 0xcb, 0x33, 0x36, 0xcd, 0x5f, 0xf0, 0x27, 0x7c, 0x01, + 0xac, 0xbb, 0xe4, 0x13, 0x20, 0x7c, 0x02, 0x3f, 0x80, 0x3c, 0x36, 0x8a, 0x83, 0xa2, 0x8a, 0xdd, + 0xbd, 0xe7, 0x9c, 0x7b, 0xc6, 0x47, 0x57, 0xd7, 0x60, 0x25, 0x51, 0x10, 0xca, 0x34, 0x4a, 0x83, + 0x69, 0xce, 0x84, 0xe4, 0x39, 0x9b, 0x2e, 0xa3, 0x84, 0x39, 0x7e, 0xc8, 0xfc, 0x38, 0xe3, 0x51, + 0x2a, 0x85, 0x9d, 0xe5, 0x5c, 0xf2, 0xd1, 0x45, 0x10, 0xc9, 0xb0, 0xf0, 0x6c, 0x9f, 0xaf, 0xa6, + 0x01, 0x0f, 0xf8, 0x54, 0xc1, 0x5e, 0xb1, 0x54, 0x9d, 0x6a, 0x54, 0x55, 0xcb, 0x27, 0x9f, 0x11, + 0x0c, 0xe7, 0x5b, 0x93, 0x1b, 0xbe, 0x60, 0x09, 0x7e, 0x0d, 0x46, 0xcb, 0x98, 0x20, 0x53, 0xb3, + 0x8c, 0xcb, 0x89, 0xfd, 0xb7, 0xae, 0x0d, 0x5c, 0xa7, 0x32, 0x5f, 0xd3, 0xf6, 0xd8, 0xe8, 0xfd, + 0x8e, 0xb3, 0x12, 0xe0, 0x21, 0x68, 0x31, 0x5b, 0x13, 0x64, 0x22, 0xab, 0x4f, 0xab, 0x12, 0x3f, + 0x83, 0xa3, 0xd2, 0x4d, 0x0a, 0x46, 0x3a, 0x26, 0xb2, 0x8c, 0xcb, 0x53, 0xfb, 0x9d, 0xeb, 0x25, + 0x6c, 0x3b, 0xa8, 0x5e, 0xa2, 0xb5, 0xe6, 0xaa, 0xf3, 0x12, 0x4d, 0x7e, 0x21, 0x38, 0xd9, 0xa7, + 0xc1, 0x18, 0x0e, 0x43, 0x57, 0x84, 0xca, 0x7c, 0x40, 0x55, 0x8d, 0xcf, 0x40, 0x17, 0xd2, 0x95, + 0x85, 0x20, 0x9a, 0x89, 0xac, 0xff, 0x68, 0xd3, 0xe1, 0x73, 0x00, 0x37, 0x49, 0xb8, 0xef, 0x78, + 0xae, 0x60, 0xe4, 0xd0, 0x44, 0x96, 0x46, 0xfb, 0x0a, 0x99, 0xb9, 0x82, 0xe1, 0x57, 0xd0, 0x65, + 0x69, 0x10, 0xa5, 0x4c, 0x90, 0x5e, 0x13, 0x7e, 0xdf, 0x93, 0xf6, 0x75, 0x2d, 0xaa, 0xc3, 0xff, + 0x19, 0x19, 0x51, 0x18, 0xb4, 0x89, 0x76, 0xe8, 0xe3, 0x3a, 0xf4, 0xf3, 0xdd, 0xd0, 0x67, 0x8d, + 0xd1, 0x03, 0xa9, 0xbf, 0x20, 0x38, 0xdd, 0x2b, 0x6a, 0x45, 0x44, 0x3b, 0x11, 0xaf, 0x40, 0xf7, + 0xc3, 0x22, 0x8d, 0x05, 0xe9, 0x34, 0x11, 0xf6, 0xce, 0xdb, 0x73, 0x25, 0xaa, 0x23, 0x34, 0x13, + 0xa3, 0x5b, 0x30, 0x5a, 0xf0, 0xbf, 0x6c, 0x4d, 0xc9, 0x1f, 0xf8, 0xfe, 0xaf, 0x1d, 0x38, 0xd9, + 0xa7, 0xa9, 0xb6, 0x96, 0xb9, 0x32, 0x6c, 0xcc, 0x55, 0x5d, 0x45, 0xe2, 0xcb, 0xa5, 0x60, 0x52, + 0xd9, 0x6b, 0xb4, 0xe9, 0xaa, 0xad, 0xb1, 0x74, 0xe1, 0x34, 0xdc, 0x51, 0xbd, 0x35, 0x96, 0x2e, + 0xde, 0xd6, 0xf4, 0x10, 0xb4, 0x8c, 0x0b, 0xa2, 0x2b, 0xbc, 0x2a, 0xf1, 0x53, 0xf8, 0x3f, 0xcb, + 0x59, 0xe9, 0xe4, 0xfc, 0x63, 0xb4, 0x70, 0x56, 0xee, 0x1d, 0xe9, 0x2a, 0x72, 0x50, 0xa1, 0xb4, + 0x02, 0x6f, 0xdc, 0x3b, 0xfc, 0x18, 0xfa, 0x5b, 0x41, 0x4f, 0x09, 0x7a, 0x79, 0x8b, 0x8c, 0x4b, + 0xdf, 0xf1, 0xd6, 0x92, 0x09, 0xd2, 0x37, 0x91, 0x75, 0x48, 0x7b, 0x71, 0xe9, 0xcf, 0xaa, 0x1e, + 0x3f, 0x82, 0x6e, 0x45, 0xc6, 0xa5, 0x20, 0xa0, 0x28, 0x3d, 0x2e, 0xfd, 0x37, 0xa5, 0xc0, 0x4f, + 0x60, 0x50, 0x11, 0xea, 0x1c, 0x44, 0xb1, 0x22, 0x86, 0x89, 0x2c, 0x9d, 0x1a, 0x71, 0xe9, 0xcf, + 0x1b, 0x08, 0x5f, 0x00, 0xf6, 0x79, 0x52, 0xac, 0x52, 0x27, 0x63, 0xf9, 0xaa, 0x90, 0xae, 0x8c, + 0x78, 0x4a, 0x06, 0xa6, 0x66, 0x1d, 0xd1, 0xe3, 0x9a, 0xb9, 0xdd, 0x12, 0xb3, 0xf3, 0xfb, 0x1f, + 0xe3, 0x83, 0xfb, 0xcd, 0x18, 0x7d, 0xdb, 0x8c, 0xd1, 0xf7, 0xcd, 0x18, 0x7d, 0xfa, 0x39, 0x3e, + 0xf8, 0xd0, 0x6d, 0xfe, 0x06, 0x9e, 0xae, 0xce, 0xf9, 0xc5, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x03, 0xe5, 0xe0, 0x2f, 0x29, 0x04, 0x00, 0x00, +} diff --git a/lightning/restore/file_checkpoints.proto b/lightning/restore/file_checkpoints.proto index 3980e5ac0..2ee0bff87 100644 --- a/lightning/restore/file_checkpoints.proto +++ b/lightning/restore/file_checkpoints.proto @@ -39,8 +39,7 @@ message EngineCheckpointModel { message ChunkCheckpointModel { string path = 1; int64 offset = 2; - bytes columns = 3; - bool should_include_row_id = 4; + repeated int32 column_permutation = 12; int64 end_offset = 5; int64 pos = 6; int64 prev_rowid_max = 7; diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 124d4003c..068777b8d 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -14,7 +14,6 @@ package restore import ( - "bytes" "context" "database/sql" "fmt" @@ -23,7 +22,6 @@ import ( "net/http" "os" "path" - "regexp" "strings" "sync" "sync/atomic" @@ -36,6 +34,8 @@ import ( "github.com/pingcap/parser/model" tidbcfg "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/kvencoder" @@ -76,8 +76,6 @@ var ( func init() { cfg := tidbcfg.GetGlobalConfig() cfg.Log.SlowThreshold = 3000 - - kv.InitMembufCap(defReadBlockSize) } type saveCp struct { @@ -223,16 +221,16 @@ func (rc *RestoreController) Run(ctx context.Context) error { var err error outside: - for _, process := range opts { + for i, process := range opts { err = process(ctx) switch { case err == nil: - case common.IsContextCanceledError(err): - common.AppLogger.Infof("user terminated : %v", err) + case !common.ShouldLogError(err): + common.AppLogger.Infof("[step %d] user terminated : %v", i, err) err = nil break outside default: - common.AppLogger.Errorf("run cause error : %v", err) + common.AppLogger.Errorf("[step %d] run cause error : %v", i, err) fmt.Fprintf(os.Stderr, "Error: %s\n", err) break outside // ps : not continue } @@ -253,6 +251,8 @@ func (rc *RestoreController) restoreSchema(ctx context.Context) error { defer tidbMgr.Close() if !rc.cfg.Mydumper.NoSchema { + tidbMgr.db.ExecContext(ctx, "SET SQL_MODE = ?", rc.cfg.TiDB.StrSQLMode) + for _, dbMeta := range rc.dbMetas { timer := time.Now() common.AppLogger.Infof("restore table schema for `%s`", dbMeta.Name) @@ -309,7 +309,12 @@ func (rc *RestoreController) saveStatusCheckpoint(tableName string, engineID int return } - metric.RecordTableCount(statusIfSucceed.MetricName(), err) + if engineID == wholeTableEngineID { + metric.RecordTableCount(statusIfSucceed.MetricName(), err) + } else { + metric.RecordEngineCount(statusIfSucceed.MetricName(), err) + } + rc.saveCpCh <- saveCp{tableName: tableName, merger: merger} } @@ -352,8 +357,8 @@ func (rc *RestoreController) listenCheckpointUpdates() { lock.Unlock() - // gofail: var FailIfImportedChunk struct{} - // if _, ok := scp.merger.(*ChunkCheckpointMerger); ok { + // gofail: var FailIfImportedChunk int + // if merger, ok := scp.merger.(*ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(FailIfImportedChunk) { // rc.checkpointsWg.Done() // rc.checkpointsWg.Wait() // panic("forcing failure due to FailIfImportedChunk") @@ -425,7 +430,7 @@ func (rc *RestoreController) runPeriodicActions(ctx context.Context, stop <-chan finished := metric.ReadCounter(metric.ChunkCounter.WithLabelValues(metric.ChunkStateFinished)) totalTables := metric.ReadCounter(metric.TableCounter.WithLabelValues(metric.TableStatePending, metric.TableResultSuccess)) completedTables := metric.ReadCounter(metric.TableCounter.WithLabelValues(metric.TableStateCompleted, metric.TableResultSuccess)) - bytesRead := metric.ReadHistogramSum(metric.BlockReadBytesHistogram) + bytesRead := metric.ReadHistogramSum(metric.RowReadBytesHistogram) var remaining string if finished >= estimated { @@ -716,7 +721,7 @@ func (t *TableRestore) restoreEngine( // 3. load kvs data (into kv deliver server) // 4. flush kvs data (into tikv node) - cr, err := newChunkRestore(chunkIndex, &rc.cfg.Mydumper, chunk, rc.cfg.Mydumper.ReadBlockSize, rc.ioWorkers) + cr, err := newChunkRestore(chunkIndex, rc.cfg, chunk, rc.ioWorkers) if err != nil { return nil, nil, errors.Trace(err) } @@ -790,7 +795,6 @@ func (t *TableRestore) importEngine( // the lock ensures the import() step will not be concurrent. rc.postProcessLock.Lock() err := t.importKV(ctx, closedEngine) - // gofail: var SlowDownImport struct{} rc.postProcessLock.Unlock() rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusImported) if err != nil { @@ -1046,11 +1050,12 @@ type chunkRestore struct { func newChunkRestore( index int, - cfg *config.MydumperRuntime, + cfg *config.Config, chunk *ChunkCheckpoint, - blockBufSize int64, ioWorkers *worker.Pool, ) (*chunkRestore, error) { + blockBufSize := cfg.Mydumper.ReadBlockSize + reader, err := os.Open(chunk.Key.Path) if err != nil { return nil, errors.Trace(err) @@ -1059,9 +1064,9 @@ func newChunkRestore( var parser mydump.Parser switch path.Ext(strings.ToLower(chunk.Key.Path)) { case ".csv": - parser = mydump.NewCSVParser(&cfg.CSV, reader, blockBufSize, ioWorkers) + parser = mydump.NewCSVParser(&cfg.Mydumper.CSV, reader, blockBufSize, ioWorkers) default: - parser = mydump.NewChunkParser(reader, blockBufSize, ioWorkers) + parser = mydump.NewChunkParser(cfg.TiDB.SQLMode, reader, blockBufSize, ioWorkers) } reader.Seek(chunk.Chunk.Offset, io.SeekStart) @@ -1084,7 +1089,7 @@ type TableRestore struct { dbInfo *TidbDBInfo tableInfo *TidbTableInfo tableMeta *mydump.MDTableMeta - encoder kvenc.KvEncoder + encTable table.Table alloc autoid.Allocator } @@ -1096,14 +1101,9 @@ func NewTableRestore( cp *TableCheckpoint, ) (*TableRestore, error) { idAlloc := kv.NewPanickingAllocator(cp.AllocBase) - encoder, err := kvenc.New(dbInfo.Name, idAlloc) - if err != nil { - return nil, errors.Annotatef(err, "failed to kvenc.New %s", tableName) - } - // create table in encoder. - err = encoder.ExecDDLSQL(tableInfo.CreateTableStmt) + tbl, err := tables.TableFromMeta(idAlloc, tableInfo.core) if err != nil { - return nil, errors.Annotatef(err, "failed to ExecDDLSQL %s", tableName) + return nil, errors.Annotatef(err, "failed to tables.TableFromMeta %s", tableName) } return &TableRestore{ @@ -1111,18 +1111,16 @@ func NewTableRestore( dbInfo: dbInfo, tableInfo: tableInfo, tableMeta: tableMeta, - encoder: encoder, + encTable: tbl, alloc: idAlloc, }, nil } func (tr *TableRestore) Close() { - tr.encoder.Close() + tr.encTable = nil common.AppLogger.Infof("[%s] restore done", tr.tableName) } -var tidbRowIDColumnRegex = regexp.MustCompile(fmt.Sprintf("`%[1]s`|(?i:\\b%[1]s\\b)", model.ExtraHandleName)) - func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) error { common.AppLogger.Infof("[%s] load chunks", t.tableName) timer := time.Now() @@ -1145,8 +1143,8 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e Path: chunk.File, Offset: chunk.Chunk.Offset, }, - Columns: nil, - Chunk: chunk.Chunk, + ColumnPermutation: nil, + Chunk: chunk.Chunk, }) } @@ -1157,30 +1155,49 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e return nil } -func (t *TableRestore) initializeColumns(columns []byte, ccp *ChunkCheckpoint) { - shouldIncludeRowID := !t.tableInfo.core.PKIsHandle && !tidbRowIDColumnRegex.Match(columns) - if shouldIncludeRowID { - // we need to inject the _tidb_rowid column - if len(columns) != 0 { - // column listing already exists, just append the new column. - columns = append(columns[:len(columns)-1], (",`" + model.ExtraHandleName.String() + "`)")...) - } else { - // we need to recreate the columns - var buf bytes.Buffer - buf.WriteString("(`") - for _, columnInfo := range t.tableInfo.core.Columns { - buf.WriteString(columnInfo.Name.String()) - buf.WriteString("`,`") +// initializeColumns computes the "column permutation" for an INSERT INTO +// statement. Suppose a table has columns (a, b, c, d) in canonical order, and +// we execute `INSERT INTO (d, b, a) VALUES ...`, we will need to remap the +// columns as: +// +// - column `a` is at position 2 +// - column `b` is at position 1 +// - column `c` is missing +// - column `d` is at position 0 +// +// The column permutation of (d, b, a) is set to be [2, 1, -1, 0]. +func (t *TableRestore) initializeColumns(columns []string, ccp *ChunkCheckpoint) { + colPerm := make([]int, 0, len(t.tableInfo.core.Columns)+1) + shouldIncludeRowID := !t.tableInfo.core.PKIsHandle + + if len(columns) == 0 { + // no provided columns, so use identity permutation. + for i := range t.tableInfo.core.Columns { + colPerm = append(colPerm, i) + } + if shouldIncludeRowID { + colPerm = append(colPerm, -1) + } + } else { + columnMap := make(map[string]int) + for i, column := range columns { + columnMap[column] = i + } + for _, colInfo := range t.tableInfo.core.Columns { + if i, ok := columnMap[colInfo.Name.L]; ok { + colPerm = append(colPerm, i) + } else { + colPerm = append(colPerm, -1) } - buf.WriteString(model.ExtraHandleName.String()) - buf.WriteString("`)") - columns = buf.Bytes() } - } else if columns == nil { - columns = []byte{} + if i, ok := columnMap[model.ExtraHandleName.L]; ok { + colPerm = append(colPerm, i) + } else if shouldIncludeRowID { + colPerm = append(colPerm, -1) + } } - ccp.Columns = columns - ccp.ShouldIncludeRowID = shouldIncludeRowID + + ccp.ColumnPermutation = colPerm } func (tr *TableRestore) restoreTableMeta(ctx context.Context, db *sql.DB) error { @@ -1195,12 +1212,12 @@ func (tr *TableRestore) restoreTableMeta(ctx context.Context, db *sql.DB) error } func (tr *TableRestore) importKV(ctx context.Context, closedEngine *kv.ClosedEngine) error { - common.AppLogger.Infof("[%s] flush kv deliver ...", tr.tableName) + common.AppLogger.Infof("[%s] flush kv deliver ...", closedEngine.Tag()) start := time.Now() if err := closedEngine.Import(ctx); err != nil { - if !common.IsContextCanceledError(err) { - common.AppLogger.Errorf("[%s] failed to flush kvs : %s", tr.tableName, err.Error()) + if common.ShouldLogError(err) { + common.AppLogger.Errorf("[%s] failed to flush kvs : %s", closedEngine.Tag(), err.Error()) } return errors.Trace(err) } @@ -1208,7 +1225,9 @@ func (tr *TableRestore) importKV(ctx context.Context, closedEngine *kv.ClosedEng dur := time.Since(start) metric.ImportSecondsHistogram.Observe(dur.Seconds()) - common.AppLogger.Infof("[%s] kv deliver all flushed, takes %v", tr.tableName, dur) + common.AppLogger.Infof("[%s] kv deliver all flushed, takes %v", closedEngine.Tag(), dur) + + // gofail: var SlowDownImport struct{} return nil } @@ -1286,7 +1305,7 @@ func DoChecksum(ctx context.Context, db *sql.DB, table string) (*RemoteChecksum, // set it back finally defer func() { err = UpdateGCLifeTime(ctx, db, ori) - if err != nil && !common.IsContextCanceledError(err) { + if err != nil && common.ShouldLogError(err) { common.AppLogger.Errorf("[%s] update tikv_gc_life_time error %v", table, errors.ErrorStack(err)) } }() @@ -1345,8 +1364,9 @@ func increaseGCLifeTime(ctx context.Context, db *sql.DB) (oriGCLifeTime string, //////////////////////////////////////////////////////////////// const ( - maxKVQueueSize = 128 + maxKVQueueSize = 128 // Cache at most this number of rows before blocking the encode loop maxDeliverBytes = 31 << 20 // 31 MB. hardcoded by importer, so do we + minDeliverBytes = 65536 // 64 KB. batch at least this amount of bytes to reduce number of messages ) func splitIntoDeliveryStreams(totalKVs []kvenc.KvPair, splitSize int) [][]kvenc.KvPair { @@ -1367,6 +1387,142 @@ func splitIntoDeliveryStreams(totalKVs []kvenc.KvPair, splitSize int) [][]kvenc. return append(res, totalKVs[i:]) } +func writeToEngine(ctx context.Context, engine *kv.OpenedEngine, totalKVs []kvenc.KvPair) error { + if len(totalKVs) == 0 { + return nil + } + + stream, err := engine.NewWriteStream(ctx) + if err != nil { + return errors.Trace(err) + } + + var putError error + for _, kvs := range splitIntoDeliveryStreams(totalKVs, maxDeliverBytes) { + putError = stream.Put(kvs) + if putError != nil { + break + } + } + + if err := stream.Close(); err != nil { + return errors.Trace(err) + } + return errors.Trace(putError) +} + +type deliveredKVs struct { + kvs []kvenc.KvPair // if kvs is empty, this indicated we've got the last message. + offset int64 + rowID int64 +} + +type deliverResult struct { + totalDur time.Duration + err error +} + +func (cr *chunkRestore) deliverLoop( + ctx context.Context, + kvsCh <-chan deliveredKVs, + t *TableRestore, + engineID int32, + dataEngine, indexEngine *kv.OpenedEngine, + rc *RestoreController, +) (deliverTotalDur time.Duration, err error) { + var channelClosed bool + var dataKVs, indexKVs []kvenc.KvPair + + for !channelClosed { + var dataChecksum, indexChecksum verify.KVChecksum + var offset, rowID int64 + + // Fetch enough KV pairs from the source. + populate: + for dataChecksum.SumSize()+indexChecksum.SumSize() < minDeliverBytes { + select { + case d := <-kvsCh: + if len(d.kvs) == 0 { + channelClosed = true + break populate + } + + for _, kv := range d.kvs { + if kv.Key[tablecodec.TableSplitKeyLen+1] == 'r' { + dataKVs = append(dataKVs, kv) + dataChecksum.UpdateOne(kv) + } else { + indexKVs = append(indexKVs, kv) + indexChecksum.UpdateOne(kv) + } + } + offset = d.offset + rowID = d.rowID + case <-ctx.Done(): + err = ctx.Err() + return + } + } + + // Write KVs into the engine + start := time.Now() + + if err = writeToEngine(ctx, dataEngine, dataKVs); err != nil { + return + } + if err = writeToEngine(ctx, indexEngine, indexKVs); err != nil { + return + } + + deliverDur := time.Since(start) + deliverTotalDur += deliverDur + metric.BlockDeliverSecondsHistogram.Observe(deliverDur.Seconds()) + metric.BlockDeliverBytesHistogram.WithLabelValues(metric.BlockDeliverKindData).Observe(float64(dataChecksum.SumSize())) + metric.BlockDeliverBytesHistogram.WithLabelValues(metric.BlockDeliverKindIndex).Observe(float64(indexChecksum.SumSize())) + metric.BlockDeliverKVPairsHistogram.WithLabelValues(metric.BlockDeliverKindData).Observe(float64(dataChecksum.SumKVS())) + metric.BlockDeliverKVPairsHistogram.WithLabelValues(metric.BlockDeliverKindIndex).Observe(float64(indexChecksum.SumKVS())) + + dataKVs = dataKVs[:0] + indexKVs = indexKVs[:0] + + // Update the table, and save a checkpoint. + // (the write to the importer is effective immediately, thus update these here) + // No need to apply a lock since this is the only thread updating these variables. + cr.chunk.Checksum.Add(&dataChecksum) + cr.chunk.Checksum.Add(&indexChecksum) + cr.chunk.Chunk.Offset = offset + cr.chunk.Chunk.PrevRowIDMax = rowID + if dataChecksum.SumKVS() != 0 || indexChecksum.SumKVS() != 0 { + // No need to save checkpoint if nothing was delivered. + cr.saveCheckpoint(t, engineID, rc) + } + } + + return +} + +func (cr *chunkRestore) saveCheckpoint(t *TableRestore, engineID int32, rc *RestoreController) { + // We need to update the AllocBase every time we've finished a file. + // The AllocBase is determined by the maximum of the "handle" (_tidb_rowid + // or integer primary key), which can only be obtained by reading all data. + rc.saveCpCh <- saveCp{ + tableName: t.tableName, + merger: &RebaseCheckpointMerger{ + AllocBase: t.alloc.Base() + 1, + }, + } + rc.saveCpCh <- saveCp{ + tableName: t.tableName, + merger: &ChunkCheckpointMerger{ + EngineID: engineID, + Key: cr.chunk.Key, + Checksum: cr.chunk.Checksum, + Pos: cr.chunk.Chunk.Offset, + RowID: cr.chunk.Chunk.PrevRowIDMax, + }, + } +} + func (cr *chunkRestore) restore( ctx context.Context, t *TableRestore, @@ -1375,260 +1531,109 @@ func (cr *chunkRestore) restore( rc *RestoreController, ) error { // Create the encoder. - kvEncoder, err := kv.NewTableKVEncoder( - t.dbInfo.Name, - t.tableInfo.Name, - t.tableInfo.ID, + kvEncoder := kv.NewTableKVEncoder( + t.encTable, rc.cfg.TiDB.SQLMode, - t.alloc, ) - if err != nil { - return errors.Trace(err) - } defer func() { - closeErr := kvEncoder.Close() + kvEncoder.Close() kvEncoder = nil - if closeErr != nil { - common.AppLogger.Errorf("restore chunk task err %v", errors.ErrorStack(closeErr)) - } }() timer := time.Now() readTotalDur := time.Duration(0) encodeTotalDur := time.Duration(0) - deliverTotalDur := time.Duration(0) - var block struct { - cond *sync.Cond - encodeCompleted bool - totalKVs []kvenc.KvPair - localChecksum verify.KVChecksum - chunkOffset int64 - chunkRowID int64 - } - block.cond = sync.NewCond(new(sync.Mutex)) - deliverCompleteCh := make(chan error, 1) + kvsCh := make(chan deliveredKVs, maxKVQueueSize) + deliverCompleteCh := make(chan deliverResult) go func() { - var dataKVs, indexKVs []kvenc.KvPair - for { - block.cond.L.Lock() - for !block.encodeCompleted && len(block.totalKVs) == 0 { - block.cond.Wait() - } - b := block - block.totalKVs = nil - block.localChecksum = verify.MakeKVChecksum(0, 0, 0) - block.cond.L.Unlock() - - if b.encodeCompleted && len(b.totalKVs) == 0 { - deliverCompleteCh <- nil - return - } - - // kv -> deliver ( -> tikv ) - start := time.Now() - dataStream, err := dataEngine.NewWriteStream(ctx) - if err != nil { - deliverCompleteCh <- errors.Trace(err) - return - } - indexStream, err := indexEngine.NewWriteStream(ctx) - if err != nil { - deliverCompleteCh <- errors.Trace(err) - return - } - // class kvs - for _, k := range b.totalKVs { - if k.Key[tablecodec.TableSplitKeyLen+1] == 'r' { - dataKVs = append(dataKVs, k) - } else { - indexKVs = append(indexKVs, k) - } - } - b.totalKVs = nil - - for _, kvs := range splitIntoDeliveryStreams(dataKVs, maxDeliverBytes) { - if e := dataStream.Put(kvs); e != nil { - if err != nil { - common.AppLogger.Warnf("failed to put write stream: %s", e.Error()) - } else { - err = e - } - } - } - for _, kvs := range splitIntoDeliveryStreams(indexKVs, maxDeliverBytes) { - if e := indexStream.Put(kvs); e != nil { - if err != nil { - common.AppLogger.Warnf("failed to put write stream: %s", e.Error()) - } else { - err = e - } - } - } - dataKVs = dataKVs[:0] - indexKVs = indexKVs[:0] - - block.cond.Signal() - if e := dataStream.Close(); e != nil { - if err != nil { - common.AppLogger.Warnf("[%s:%d] failed to close write stream: %s", t.tableName, engineID, e.Error()) - } else { - err = e - } - } - if e := indexStream.Close(); e != nil { - if err != nil { - common.AppLogger.Warnf("[%s:%d] failed to close write stream: %s", t.tableName, engineID, e.Error()) - } else { - err = e - } - } - deliverDur := time.Since(start) - deliverTotalDur += deliverDur - metric.BlockDeliverSecondsHistogram.Observe(deliverDur.Seconds()) - metric.BlockDeliverBytesHistogram.Observe(float64(b.localChecksum.SumSize())) - - if err != nil { - if !common.IsContextCanceledError(err) { - common.AppLogger.Errorf("[%s:%d] kv deliver failed = %v", t.tableName, engineID, err) - } - // TODO : retry ~ - deliverCompleteCh <- errors.Trace(err) - return - } - - // Update the table, and save a checkpoint. - // (the write to the importer is effective immediately, thus update these here) - cr.chunk.Checksum.Add(&b.localChecksum) - cr.chunk.Chunk.Offset = b.chunkOffset - cr.chunk.Chunk.PrevRowIDMax = b.chunkRowID - rc.saveCpCh <- saveCp{ - tableName: t.tableName, - merger: &RebaseCheckpointMerger{ - AllocBase: t.alloc.Base() + 1, - }, - } - rc.saveCpCh <- saveCp{ - tableName: t.tableName, - merger: &ChunkCheckpointMerger{ - EngineID: engineID, - Key: cr.chunk.Key, - Checksum: cr.chunk.Checksum, - Pos: cr.chunk.Chunk.Offset, - RowID: cr.chunk.Chunk.PrevRowIDMax, - }, - } - } + dur, err := cr.deliverLoop(ctx, kvsCh, t, engineID, dataEngine, indexEngine, rc) + deliverCompleteCh <- deliverResult{dur, err} }() - var buffer bytes.Buffer - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } + defer close(kvsCh) + initializedColumns := false +outside: + for { offset, _ := cr.parser.Pos() - endOffset := mathutil.MinInt64(cr.chunk.Chunk.EndOffset, offset+rc.cfg.Mydumper.ReadBlockSize) - if offset >= endOffset { + if offset >= cr.chunk.Chunk.EndOffset { break } - buffer.Reset() start := time.Now() - - var sep byte = ' ' - readLoop: - for { - offset, _ := cr.parser.Pos() - if offset >= endOffset { - break - } - readRowStartTime := time.Now() - err := cr.parser.ReadRow() - switch errors.Cause(err) { - case nil: - buffer.WriteByte(sep) - if sep == ' ' { - buffer.WriteString("INSERT INTO ") - buffer.WriteString(t.tableName) - if cr.chunk.Columns == nil { - t.initializeColumns(cr.parser.Columns(), cr.chunk) - } - buffer.Write(cr.chunk.Columns) - buffer.WriteString(" VALUES ") - sep = ',' - } - metric.ChunkParserReadRowSecondsHistogram.Observe(time.Since(readRowStartTime).Seconds()) - lastRow := cr.parser.LastRow() - if cr.chunk.ShouldIncludeRowID { - buffer.Write(lastRow.Row[:len(lastRow.Row)-1]) - fmt.Fprintf(&buffer, ",%d)", lastRow.RowID) - } else { - buffer.Write(lastRow.Row) + err := cr.parser.ReadRow() + newOffset, rowID := cr.parser.Pos() + switch errors.Cause(err) { + case nil: + if !initializedColumns { + if len(cr.chunk.ColumnPermutation) == 0 { + t.initializeColumns(cr.parser.Columns(), cr.chunk) } - case io.EOF: - cr.chunk.Chunk.EndOffset, cr.chunk.Chunk.RowIDMax = cr.parser.Pos() - break readLoop - default: - return errors.Trace(err) + initializedColumns = true } + case io.EOF: + break outside + default: + return errors.Trace(err) } - if sep != ',' { // quick and dirty way to check if `buffer` actually contained any values - continue - } - buffer.WriteByte(';') readDur := time.Since(start) readTotalDur += readDur - metric.BlockReadSecondsHistogram.Observe(readDur.Seconds()) - metric.BlockReadBytesHistogram.Observe(float64(buffer.Len())) + metric.RowReadSecondsHistogram.Observe(readDur.Seconds()) + metric.RowReadBytesHistogram.Observe(float64(newOffset - offset)) // sql -> kv start = time.Now() - kvs, _, err := kvEncoder.SQL2KV(buffer.String()) + lastRow := cr.parser.LastRow() + kvs, err := kvEncoder.Encode(lastRow.Row, lastRow.RowID, cr.chunk.ColumnPermutation) encodeDur := time.Since(start) encodeTotalDur += encodeDur - metric.BlockEncodeSecondsHistogram.Observe(encodeDur.Seconds()) + metric.RowEncodeSecondsHistogram.Observe(encodeDur.Seconds()) - common.AppLogger.Debugf("len(kvs) %d, len(sql) %d", len(kvs), buffer.Len()) if err != nil { common.AppLogger.Errorf("kv encode failed = %s\n", err.Error()) return errors.Trace(err) } - block.cond.L.Lock() - for len(block.totalKVs) > len(kvs)*maxKVQueueSize { - // ^ hack to create a back-pressure preventing sending too many KV pairs at once - // this happens when delivery is slower than encoding. - // note that the KV pairs will retain the memory buffer backing the KV encoder - // and thus blow up the memory usage and will easily cause lightning to go OOM. - block.cond.Wait() + select { + case kvsCh <- deliveredKVs{kvs: kvs, offset: newOffset, rowID: rowID}: + continue + case <-ctx.Done(): + return ctx.Err() + case deliverResult := <-deliverCompleteCh: + if deliverResult.err == nil { + panic("unexpected: deliverCompleteCh prematurely fulfilled with no error") + } + return errors.Trace(deliverResult.err) } - block.totalKVs = append(block.totalKVs, kvs...) - block.localChecksum.Update(kvs) - block.chunkOffset, block.chunkRowID = cr.parser.Pos() - block.cond.Signal() - block.cond.L.Unlock() } - block.cond.L.Lock() - block.encodeCompleted = true - block.cond.Signal() - block.cond.L.Unlock() + lastOffset, lastRowID := cr.parser.Pos() + select { + case kvsCh <- deliveredKVs{kvs: nil, offset: lastOffset, rowID: lastRowID}: + break + case <-ctx.Done(): + return ctx.Err() + case deliverResult := <-deliverCompleteCh: + if deliverResult.err == nil { + panic("unexpected: deliverCompleteCh prematurely fulfilled with no error") + } + return errors.Trace(deliverResult.err) + } select { - case err := <-deliverCompleteCh: - if err == nil { + case deliverResult := <-deliverCompleteCh: + if deliverResult.err == nil { common.AppLogger.Infof( - "[%s:%d] restore chunk #%d (%s) takes %v (read: %v, encode: %v, deliver: %v)", + "[%s:%d] restore chunk #%d (%s) takes %v (read: %v, encode: %v, deliver: %v, size: %d, kvs: %d)", t.tableName, engineID, cr.index, &cr.chunk.Key, time.Since(timer), - readTotalDur, encodeTotalDur, deliverTotalDur, + readTotalDur, encodeTotalDur, deliverResult.totalDur, + cr.chunk.Checksum.SumSize(), cr.chunk.Checksum.SumKVS(), ) } - return errors.Trace(err) + return errors.Trace(deliverResult.err) case <-ctx.Done(): return ctx.Err() } diff --git a/lightning/restore/restore_test.go b/lightning/restore/restore_test.go index 5f161e117..635d7804a 100644 --- a/lightning/restore/restore_test.go +++ b/lightning/restore/restore_test.go @@ -15,7 +15,12 @@ package restore import ( . "github.com/pingcap/check" + "github.com/pingcap/parser" + "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/model" "github.com/pingcap/tidb-lightning/lightning/common" + "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/util/mock" ) var _ = Suite(&restoreSuite{}) @@ -26,18 +31,27 @@ func (s *restoreSuite) TestNewTableRestore(c *C) { testCases := []struct { name string createStmt string - errRegexp string }{ - {"t1", "CREATE TABLE `t1` (`c1` varchar(5) NOT NULL)", ""}, - {"t2", "CREATE TABLE `t2` (`c1` varchar(30000) NOT NULL)", "failed to ExecDDLSQL `mockdb`.`t2`:.*"}, - {"t3", "CREATE TABLE `t3-a` (`c1-a` varchar(5) NOT NULL)", ""}, + {"t1", "CREATE TABLE `t1` (`c1` varchar(5) NOT NULL)"}, + // {"t2", "CREATE TABLE `t2` (`c1` varchar(30000) NOT NULL)"}, // no longer able to create this kind of table. + {"t3", "CREATE TABLE `t3-a` (`c1-a` varchar(5) NOT NULL)"}, } + p := parser.New() + se := mock.NewContext() + dbInfo := &TidbDBInfo{Name: "mockdb", Tables: map[string]*TidbTableInfo{}} - for _, c := range testCases { - dbInfo.Tables[c.name] = &TidbTableInfo{ - Name: c.name, - CreateTableStmt: c.createStmt, + for i, tc := range testCases { + node, err := p.ParseOneStmt(tc.createStmt, "utf8mb4", "utf8mb4_bin") + c.Assert(err, IsNil) + tableInfo, err := ddl.MockTableInfo(se, node.(*ast.CreateTableStmt), int64(i+1)) + c.Assert(err, IsNil) + tableInfo.State = model.StatePublic + + dbInfo.Tables[tc.name] = &TidbTableInfo{ + Name: tc.name, + CreateTableStmt: tc.createStmt, + core: tableInfo, } } @@ -45,11 +59,22 @@ func (s *restoreSuite) TestNewTableRestore(c *C) { tableInfo := dbInfo.Tables[tc.name] tableName := common.UniqueTable("mockdb", tableInfo.Name) tr, err := NewTableRestore(tableName, nil, dbInfo, tableInfo, &TableCheckpoint{}) - if tc.errRegexp != "" { - c.Assert(err, ErrorMatches, tc.errRegexp) - } else { - c.Assert(tr, NotNil) - c.Assert(err, IsNil) - } + c.Assert(tr, NotNil) + c.Assert(err, IsNil) } } + +func (s *restoreSuite) TestNewTableRestoreFailure(c *C) { + tableInfo := &TidbTableInfo{ + Name: "failure", + CreateTableStmt: "N/A", + core: &model.TableInfo{}, + } + dbInfo := &TidbDBInfo{Name: "mockdb", Tables: map[string]*TidbTableInfo{ + "failure": tableInfo, + }} + tableName := common.UniqueTable("mockdb", "failure") + + _, err := NewTableRestore(tableName, nil, dbInfo, tableInfo, &TableCheckpoint{}) + c.Assert(err, ErrorMatches, `failed to tables\.TableFromMeta.*`) +} diff --git a/lightning/verification/checksum.go b/lightning/verification/checksum.go index 7e3268620..a73097f32 100644 --- a/lightning/verification/checksum.go +++ b/lightning/verification/checksum.go @@ -41,6 +41,15 @@ func MakeKVChecksum(bytes uint64, kvs uint64, checksum uint64) KVChecksum { } } +func (c *KVChecksum) UpdateOne(kv kvec.KvPair) { + sum := crc64.Update(0, ecmaTable, kv.Key) + sum = crc64.Update(sum, ecmaTable, kv.Val) + + c.bytes += uint64(len(kv.Key) + len(kv.Val)) + c.kvs++ + c.checksum ^= sum +} + func (c *KVChecksum) Update(kvs []kvec.KvPair) { var ( checksum uint64 diff --git a/tests/checkpoint_chunks/config.toml b/tests/checkpoint_chunks/config.toml index 074ae3af3..00a96ae94 100644 --- a/tests/checkpoint_chunks/config.toml +++ b/tests/checkpoint_chunks/config.toml @@ -2,7 +2,7 @@ region-concurrency = 1 check-requirements = false file = "/tmp/lightning_test_result/lightning.log" -level = "error" +level = "info" [checkpoint] enable = true diff --git a/tests/checkpoint_chunks/file.toml b/tests/checkpoint_chunks/file.toml index 434094170..4489cdb10 100644 --- a/tests/checkpoint_chunks/file.toml +++ b/tests/checkpoint_chunks/file.toml @@ -2,7 +2,7 @@ region-concurrency = 1 check-requirements = false file = "/tmp/lightning_test_result/lightning.log" -level = "error" +level = "info" [checkpoint] enable = true diff --git a/tests/checkpoint_chunks/run.sh b/tests/checkpoint_chunks/run.sh index 7ad535611..435581c6f 100755 --- a/tests/checkpoint_chunks/run.sh +++ b/tests/checkpoint_chunks/run.sh @@ -43,9 +43,10 @@ for i in $(seq "$CHUNK_COUNT"); do done done -# Set the failpoint to kill the lightning instance as soon as one chunk is imported -# If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings. -export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/FailIfImportedChunk=return' +# Set the failpoint to kill the lightning instance as soon as +# one file (after writing totally $ROW_COUNT rows) is imported. +# If checkpoint does work, this should kill exactly $CHUNK_COUNT instances of lightnings. +export GOFAIL_FAILPOINTS="github.com/pingcap/tidb-lightning/lightning/restore/FailIfImportedChunk=return($ROW_COUNT)" # Start importing the tables. run_sql 'DROP DATABASE IF EXISTS cpch_tsr' diff --git a/tests/checkpoint_engines/run.sh b/tests/checkpoint_engines/run.sh index 672d779bd..70b91d8e1 100755 --- a/tests/checkpoint_engines/run.sh +++ b/tests/checkpoint_engines/run.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eu +set -eux # First, verify that a normal operation is fine. @@ -69,6 +69,7 @@ check_contains 'sum(c): 46' # Now, try again with MySQL checkpoints run_sql 'DROP DATABASE cpeng;' +run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint;' set +e for i in $(seq "$ENGINE_COUNT"); do diff --git a/tests/csv/data/csv.empty_strings-schema.sql b/tests/csv/data/csv.empty_strings-schema.sql new file mode 100644 index 000000000..49354b652 --- /dev/null +++ b/tests/csv/data/csv.empty_strings-schema.sql @@ -0,0 +1,5 @@ +create table empty_strings ( + id int, + a varchar(20), + b varchar(20) +); diff --git a/tests/csv/data/csv.empty_strings.csv b/tests/csv/data/csv.empty_strings.csv new file mode 100644 index 000000000..5edd50b59 --- /dev/null +++ b/tests/csv/data/csv.empty_strings.csv @@ -0,0 +1,4 @@ +1,,"" +2,, +3,"""", +4,"","" diff --git a/tests/csv/run.sh b/tests/csv/run.sh index 2cb33c216..9072b4f31 100755 --- a/tests/csv/run.sh +++ b/tests/csv/run.sh @@ -12,6 +12,9 @@ check_contains 'sum(PROCESSLIST_TIME): 322253' check_contains 'sum(THREAD_OS_ID): 303775702' check_contains 'count(PROCESSLIST_STATE): 3' +run_sql 'SELECT count(*) FROM csv.threads WHERE PROCESSLIST_TIME IS NOT NULL' +check_contains 'count(*): 12' + run_sql 'SELECT hex(t), j, hex(b) FROM csv.escapes WHERE i = 1' check_contains 'hex(t): 5C' check_contains 'j: {"?": []}' @@ -26,3 +29,8 @@ run_sql 'SELECT hex(t), j, hex(b) FROM csv.escapes WHERE i = 3' check_contains 'hex(t): 0A' check_contains 'j: [",,,"]' check_contains 'hex(b): 5C2C5C2C' + +run_sql 'SELECT id FROM csv.empty_strings WHERE a = """"' +check_contains 'id: 3' +run_sql 'SELECT id FROM csv.empty_strings WHERE b <> ""' +check_not_contains 'id:' diff --git a/tests/run.sh b/tests/run.sh index edbd25964..7238184bc 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -93,7 +93,7 @@ if [ "${1-}" = '--debug' ]; then fi for script in tests/*/run.sh; do - echo "Running test $script..." + echo "\x1b[32;1m@@@@@@@ Running test $script...\x1b[0m" TEST_DIR="$TEST_DIR" \ PATH="tests/_utils:$PATH" \ TEST_NAME="$(basename "$(dirname "$script")")" \ diff --git a/tests/sqlmode/data/sqlmodedb-schema-create.sql b/tests/sqlmode/data/sqlmodedb-schema-create.sql new file mode 100644 index 000000000..f39e114f8 --- /dev/null +++ b/tests/sqlmode/data/sqlmodedb-schema-create.sql @@ -0,0 +1 @@ +create database sqlmodedb; diff --git a/tests/sqlmode/data/sqlmodedb.t-schema.sql b/tests/sqlmode/data/sqlmodedb.t-schema.sql new file mode 100644 index 000000000..9317ba2ec --- /dev/null +++ b/tests/sqlmode/data/sqlmodedb.t-schema.sql @@ -0,0 +1,7 @@ +create table t ( + id INT PRIMARY KEY, + a TIMESTAMP NOT NULL, + b TINYINT NOT NULL, + c VARCHAR(1) CHARSET latin1 NOT NULL, + d SET('x','y') NOT NULL +); diff --git a/tests/sqlmode/data/sqlmodedb.t.1.sql b/tests/sqlmode/data/sqlmodedb.t.1.sql new file mode 100644 index 000000000..99b995081 --- /dev/null +++ b/tests/sqlmode/data/sqlmodedb.t.1.sql @@ -0,0 +1,6 @@ +insert into t values +(1, 9, 128, 'too long', 'x,y,z'), +(2, '2000-00-00 00:00:00', -99999, '🤩', 3), +(3, '9999-12-31 23:59:59', 'NaN', x'99', 'x+y'), +(4, '2000-01-01 00:00:00', 99.999, '', 'x,y,x,y,x,y,x,y'), +(5, NULL, NULL, NULL, NULL); diff --git a/tests/sqlmode/off.toml b/tests/sqlmode/off.toml new file mode 100644 index 000000000..3ddbfb3fa --- /dev/null +++ b/tests/sqlmode/off.toml @@ -0,0 +1,27 @@ +[lightning] +check-requirements = false +file = "/tmp/lightning_test_result/lightning.log" +level = "info" + +[checkpoint] +enable = false + +[tikv-importer] +addr = "127.0.0.1:8808" + +[mydumper] +data-source-dir = "tests/sqlmode/data" + +[tidb] +host = "127.0.0.1" +port = 4000 +user = "root" +status-port = 10080 +pd-addr = "127.0.0.1:2379" +log-level = "error" +sql-mode = 'ALLOW_INVALID_DATES' + +[post-restore] +checksum = true +compact = false +analyze = false diff --git a/tests/sqlmode/on.toml b/tests/sqlmode/on.toml new file mode 100644 index 000000000..212567841 --- /dev/null +++ b/tests/sqlmode/on.toml @@ -0,0 +1,26 @@ +[lightning] +check-requirements = false +file = "/tmp/lightning_test_result/lightning.log" +level = "info" + +[checkpoint] +enable = false + +[tikv-importer] +addr = "127.0.0.1:8808" + +[mydumper] +data-source-dir = "tests/sqlmode/data" + +[tidb] +host = "127.0.0.1" +port = 4000 +user = "root" +status-port = 10080 +pd-addr = "127.0.0.1:2379" +log-level = "error" + +[post-restore] +checksum = true +compact = false +analyze = false diff --git a/tests/sqlmode/run.sh b/tests/sqlmode/run.sh new file mode 100755 index 000000000..04bc6cd85 --- /dev/null +++ b/tests/sqlmode/run.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +run_sql 'DROP DATABASE IF EXISTS sqlmodedb' + +run_lightning off + +run_sql 'SELECT a, b, hex(c), d FROM sqlmodedb.t WHERE id = 1' +check_contains 'a: 0000-00-00 00:00:00' +check_contains 'b: 127' +check_contains 'hex(c): 74' +check_contains 'd: ' + +run_sql 'SELECT a, b, hex(c), d FROM sqlmodedb.t WHERE id = 2' +check_contains 'a: 0000-00-00 00:00:00' +check_contains 'b: -128' +check_contains 'hex(c): F0' +check_contains 'd: x,y' + +run_sql 'SELECT a, b, hex(c), d FROM sqlmodedb.t WHERE id = 3' +check_contains 'a: 0000-00-00 00:00:00' +check_contains 'b: 0' +check_contains 'hex(c): 99' +check_contains 'd: ' + +run_sql 'SELECT a, b, hex(c), d FROM sqlmodedb.t WHERE id = 4' +check_contains 'a: 2000-01-01 00:00:00' +check_contains 'b: 100' +check_contains 'hex(c): ' +check_contains 'd: x,y' + +run_sql 'SELECT a, b, hex(c), d FROM sqlmodedb.t WHERE id = 5' +check_contains 'a: 0000-00-00 00:00:00' +check_contains 'b: 0' +check_contains 'hex(c): ' +check_contains 'd: ' + +run_sql 'DROP DATABASE IF EXISTS sqlmodedb' + +set +e +run_lightning on +[ $? -ne 0 ] || exit 1 +set -e diff --git a/tests/tool_241/config.toml b/tests/tool_241/config.toml index f7f058aaf..1b2d3609d 100644 --- a/tests/tool_241/config.toml +++ b/tests/tool_241/config.toml @@ -17,6 +17,7 @@ user = "root" status-port = 10080 pd-addr = "127.0.0.1:2379" log-level = "error" +sql-mode = '' [post-restore] checksum = true diff --git a/tests/various_types/data/vt.empty_strings-schema.sql b/tests/various_types/data/vt.empty_strings-schema.sql new file mode 100644 index 000000000..a0b6e2a55 --- /dev/null +++ b/tests/various_types/data/vt.empty_strings-schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE `empty_strings` ( + `pk` INT NOT NULL, + `a` VARCHAR(20) NOT NULL +); diff --git a/tests/various_types/data/vt.empty_strings.sql b/tests/various_types/data/vt.empty_strings.sql new file mode 100644 index 000000000..786a3c1e4 --- /dev/null +++ b/tests/various_types/data/vt.empty_strings.sql @@ -0,0 +1 @@ +INSERT INTO `empty_strings` VALUES (1, ""), (2, """"), (4, ''), (8, ''''), (16, "\""), (32, '\''); diff --git a/tests/various_types/run.sh b/tests/various_types/run.sh index a922b7607..b25fa9cbb 100755 --- a/tests/various_types/run.sh +++ b/tests/various_types/run.sh @@ -87,3 +87,12 @@ run_sql 'SELECT count(*) FROM vt.`enum-set` WHERE find_in_set("x50", `set`) > 0' check_contains 'count(*): 10' run_sql 'SELECT `set` FROM vt.`enum-set` WHERE `enum` = "gcc"' check_contains 'set: x00,x06,x07,x09,x17,x20,x23,x24,x27,x37,x44,x46,x49,x54,x55,x58,x61,x62' + +run_sql 'SELECT count(*) FROM vt.empty_strings' +check_contains 'count(*): 6' +run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = ""' +check_contains 'sum(pk): 5' +run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = 0x22' +check_contains 'sum(pk): 18' +run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = 0x27' +check_contains 'sum(pk): 40'