Skip to content

Commit

Permalink
fix ut
Browse files Browse the repository at this point in the history
  • Loading branch information
buchuitoudegou committed Jan 10, 2023
1 parent cd6e96e commit dc59781
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 10 deletions.
7 changes: 6 additions & 1 deletion br/pkg/lightning/mydump/csv_parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,13 @@ var (
errUnterminatedQuotedField = errors.NewNoStackError("syntax error: unterminated quoted field")
errDanglingBackslash = errors.NewNoStackError("syntax error: no character after backslash")
errUnexpectedQuoteField = errors.NewNoStackError("syntax error: cannot have consecutive fields without separator")
LargestEntryLimit int
)

func init() {
LargestEntryLimit = tidbconfig.MaxTxnEntrySizeLimit
}

// CSVParser is basically a copy of encoding/csv, but special-cased for MySQL-like input.
type CSVParser struct {
blockParser
Expand Down Expand Up @@ -337,7 +342,7 @@ func (parser *CSVParser) readUntil(chars *byteSet) ([]byte, byte, error) {
var buf []byte
for {
buf = append(buf, parser.buf...)
if len(buf) > tidbconfig.MaxTxnEntrySizeLimit {
if len(buf) > LargestEntryLimit {
return buf, 0, errors.New("size of row cannot exceed the max value of txn-entry-size-limit")
}
parser.buf = nil
Expand Down
12 changes: 3 additions & 9 deletions br/pkg/lightning/mydump/csv_parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/br/pkg/lightning/mydump"
"github.com/pingcap/tidb/br/pkg/lightning/worker"
tidbconfig "github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -26,12 +25,6 @@ import (

var ioWorkers = worker.NewPool(context.Background(), 5, "test_csv")

var largestEntryLimit int

func init() {
largestEntryLimit = tidbconfig.MaxTxnEntrySizeLimit
}

func assertPosEqual(t *testing.T, parser mydump.Parser, expectPos, expectRowID int64) {
pos, rowID := parser.Pos()
require.Equal(t, expectPos, pos)
Expand Down Expand Up @@ -697,8 +690,9 @@ func TestTooLargeRow(t *testing.T) {
}
var testCase bytes.Buffer
testCase.WriteString("a,b,c,d")
// WARN: will take up 120MB memory here.
for i := 0; i < largestEntryLimit; i++ {
// WARN: will take up 10KB memory here.
mydump.LargestEntryLimit = 10 * 1024
for i := 0; i < mydump.LargestEntryLimit; i++ {
testCase.WriteByte('d')
}
charsetConvertor, err := mydump.NewCharsetConvertor(cfg.DataCharacterSet, cfg.DataInvalidCharReplace)
Expand Down

0 comments on commit dc59781

Please sign in to comment.