encoding/japanese: make EUCJP replace with FFFD on error

Updates golang/go#18898 Change-Id: Ic5fb77af67656889a387fa75a3e6efc9b9975817 Reviewed-on: https://go-review.googlesource.com/37316 Run-TryBot: Marcel van Lohuizen <mpvl@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Nigel Tao <nigeltao@golang.org>
golang · Feb 23, 2017 · afcdff5 · afcdff5
1 parent 767daa1
commit afcdff5
Show file tree

Hide file tree

Showing 2 changed files with 66 additions and 29 deletions.
diff --git a/encoding/japanese/all_test.go b/encoding/japanese/all_test.go
@@ -5,6 +5,7 @@
 package japanese_test
 
 import (
+	"strings"
 	"testing"
 
 	"golang.org/x/text/encoding"
@@ -22,6 +23,8 @@ func enc(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
 }
 
 func TestNonRepertoire(t *testing.T) {
+	// Pick n to cause the destination buffer in transform.String to overflow.
+	const n = 10000
 	testCases := []struct {
 		init      func(e encoding.Encoding) (string, transform.Transformer, error)
 		e         encoding.Encoding
@@ -42,6 +45,26 @@ func TestNonRepertoire(t *testing.T) {
 		{enc, japanese.ShiftJIS, "갂", ""},
 		{enc, japanese.ShiftJIS, "a갂", "a"},
 		{enc, japanese.ShiftJIS, "\u2190갂", "\x81\xa9"},
+
+		// Continue correctly after errors
+		{dec, japanese.EUCJP, "\x8e\xa0", "\ufffd\ufffd"},
+		{dec, japanese.EUCJP, "\x8e\xe0", "\ufffd"},
+		{dec, japanese.EUCJP, "\x8e\xff", "\ufffd\ufffd"},
+		{dec, japanese.EUCJP, "\x8ea", "\ufffda"},
+		{dec, japanese.EUCJP, "\x8f\xa0", "\ufffd\ufffd"},
+		{dec, japanese.EUCJP, "\x8f\xa1a", "\ufffda"},
+		{dec, japanese.EUCJP, "\x8f\xa1a", "\ufffda"},
+		{dec, japanese.EUCJP, "\x8f\xa1\xa0", "\ufffd\ufffd"},
+		{dec, japanese.EUCJP, "\x8f\xa1a", "\ufffda"},
+		{dec, japanese.EUCJP, "\x8f\xa2\xa2", "\ufffd"},
+		{dec, japanese.EUCJP, "\xfe", "\ufffd"},
+		{dec, japanese.EUCJP, "\xfe\xff", "\ufffd\ufffd"},
+		// Correct handling of end of source
+		{dec, japanese.EUCJP, strings.Repeat("\x8e", n), strings.Repeat("\ufffd", n)},
+		{dec, japanese.EUCJP, strings.Repeat("\x8f", n), strings.Repeat("\ufffd", n)},
+		{dec, japanese.EUCJP, strings.Repeat("\x8f\xa0", n), strings.Repeat("\ufffd", 2*n)},
+		{dec, japanese.EUCJP, "a" + strings.Repeat("\x8f\xa1", n), "a" + strings.Repeat("\ufffd", n)},
+		{dec, japanese.EUCJP, "a" + strings.Repeat("\x8f\xa1\xff", n), "a" + strings.Repeat("\ufffd", 2*n)},
 	}
 	for _, tc := range testCases {
 		dir, tr, wantErr := tc.init(tc.e)

diff --git a/encoding/japanese/eucjp.go b/encoding/japanese/eucjp.go
@@ -5,7 +5,6 @@
 package japanese
 
 import (
-	"errors"
 	"unicode/utf8"
 
 	"golang.org/x/text/encoding"
@@ -23,10 +22,9 @@ var eucJP = internal.Encoding{
 	identifier.EUCPkdFmtJapanese,
 }
 
-var errInvalidEUCJP = errors.New("japanese: invalid EUC-JP encoding")
-
 type eucJPDecoder struct{ transform.NopResetter }
 
+// See https://encoding.spec.whatwg.org/#euc-jp-decoder.
 func (eucJPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	r, size := rune(0), 0
 loop:
@@ -37,60 +35,79 @@ loop:
 
 		case c0 == 0x8e:
 			if nSrc+1 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				break
 			}
 			c1 := src[nSrc+1]
-			if c1 < 0xa1 || 0xdf < c1 {
-				err = errInvalidEUCJP
-				break loop
+			switch {
+			case c1 < 0xa1:
+				r, size = utf8.RuneError, 1
+			case c1 > 0xdf:
+				r, size = utf8.RuneError, 2
+				if c1 == 0xff {
+					size = 1
+				}
+			default:
+				r, size = rune(c1)+(0xff61-0xa1), 2
 			}
-			r, size = rune(c1)+(0xff61-0xa1), 2
-
 		case c0 == 0x8f:
 			if nSrc+2 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				if p := nSrc + 1; p < len(src) && 0xa1 <= src[p] && src[p] < 0xfe {
+					size = 2
+				}
+				break
 			}
 			c1 := src[nSrc+1]
 			if c1 < 0xa1 || 0xfe < c1 {
-				err = errInvalidEUCJP
-				break loop
+				r, size = utf8.RuneError, 1
+				break
 			}
 			c2 := src[nSrc+2]
 			if c2 < 0xa1 || 0xfe < c2 {
-				err = errInvalidEUCJP
-				break loop
+				r, size = utf8.RuneError, 2
+				break
 			}
-			r, size = '\ufffd', 3
+			r, size = utf8.RuneError, 3
 			if i := int(c1-0xa1)*94 + int(c2-0xa1); i < len(jis0212Decode) {
 				r = rune(jis0212Decode[i])
 				if r == 0 {
-					r = '\ufffd'
+					r = utf8.RuneError
 				}
 			}
 
 		case 0xa1 <= c0 && c0 <= 0xfe:
 			if nSrc+1 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				break
 			}
 			c1 := src[nSrc+1]
 			if c1 < 0xa1 || 0xfe < c1 {
-				err = errInvalidEUCJP
-				break loop
+				r, size = utf8.RuneError, 1
+				break
 			}
-			r, size = '\ufffd', 2
+			r, size = utf8.RuneError, 2
 			if i := int(c0-0xa1)*94 + int(c1-0xa1); i < len(jis0208Decode) {
 				r = rune(jis0208Decode[i])
 				if r == 0 {
-					r = '\ufffd'
+					r = utf8.RuneError
 				}
 			}
 
 		default:
-			err = errInvalidEUCJP
-			break loop
+			r, size = utf8.RuneError, 1
 		}
 
 		if nDst+utf8.RuneLen(r) > len(dst) {
@@ -99,9 +116,6 @@ loop:
 		}
 		nDst += utf8.EncodeRune(dst[nDst:], r)
 	}
-	if atEOF && err == transform.ErrShortSrc {
-		err = errInvalidEUCJP
-	}
 	return nDst, nSrc, err
 }