Skip to content

Commit

Permalink
encoding/japanese: make iso2022jp replace with FFFD on error
Browse files Browse the repository at this point in the history 
This CL also fixes a bug in the JIS 212 mode.

Updates golang/go#18898

Change-Id: Idb10e375bbc5d278db4e8550f26986a5d2bb6caa
Reviewed-on: https://go-review.googlesource.com/37595
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
  • Loading branch information
@mpvl
mpvl committed Mar 3, 2017
1 parent d680ca3 commit f28f367
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 45 deletions.
85 changes: 69 additions & 16 deletions encoding/japanese/all_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package japanese

import (
"fmt"
"strings"
"testing"

Expand All @@ -23,16 +24,13 @@ func enc(e encoding.Encoding) (dir string, t transform.Transformer, err error) {

func TestNonRepertoire(t *testing.T) {
// Pick n to cause the destination buffer in transform.String to overflow.
const n = 10000
const n = 100
long := strings.Repeat(".", n)
testCases := []struct {
init func(e encoding.Encoding) (string, transform.Transformer, error)
e encoding.Encoding
src, want string
}{
{dec, EUCJP, "\xfe\xfc", "\ufffd"},
{dec, ISO2022JP, "\x1b$B\x7e\x7e", "\ufffd"},
{dec, ShiftJIS, "\xef\xfc", "\ufffd"},

{enc, EUCJP, "갂", ""},
{enc, EUCJP, "a갂", "a"},
{enc, EUCJP, "丌갂", "\x8f\xb0\xa4"},
Expand All @@ -51,12 +49,13 @@ func TestNonRepertoire(t *testing.T) {
{dec, EUCJP, "\x8e\xff", "\ufffd\ufffd"},
{dec, EUCJP, "\x8ea", "\ufffda"},
{dec, EUCJP, "\x8f\xa0", "\ufffd\ufffd"},
{dec, EUCJP, "\x8f\xa1\xa0", "\ufffd\ufffd"},
{dec, EUCJP, "\x8f\xa1a", "\ufffda"},
{dec, EUCJP, "\x8f\xa1a", "\ufffda"},
{dec, EUCJP, "\x8f\xa1\xa0", "\ufffd\ufffd"},
{dec, EUCJP, "\x8f\xa1a", "\ufffda"},
{dec, EUCJP, "\x8f\xa2\xa2", "\ufffd"},
{dec, EUCJP, "\xfe", "\ufffd"},
{dec, EUCJP, "\xfe\xfc", "\ufffd"},
{dec, EUCJP, "\xfe\xff", "\ufffd\ufffd"},
// Correct handling of end of source
{dec, EUCJP, strings.Repeat("\x8e", n), strings.Repeat("\ufffd", n)},
Expand All @@ -68,29 +67,83 @@ func TestNonRepertoire(t *testing.T) {
// Continue correctly after errors
{dec, ShiftJIS, "\x80", "\u0080"}, // It's what the spec says.
{dec, ShiftJIS, "\x81", "\ufffd"},
{dec, ShiftJIS, "\x81\x7f", "\ufffd\u007f"},
{dec, ShiftJIS, "\xe0", "\ufffd"},
{dec, ShiftJIS, "\xe0\x39", "\ufffd\u0039"},
{dec, ShiftJIS, "\x81\x7f", "\ufffd\u007f"},
{dec, ShiftJIS, "\xe0\xfd", "\ufffd"},
{dec, ShiftJIS, "\xe0\x9f", "燹"},
{dec, ShiftJIS, "\xe0\xfd", "\ufffd"},
{dec, ShiftJIS, "\xef\xfc", "\ufffd"},
{dec, ShiftJIS, "\xfc\xfc", "\ufffd"},
{dec, ShiftJIS, "\xfc\xfd", "\ufffd"},
{dec, ShiftJIS, "\xfdaa", "\ufffdaa"},

{dec, ShiftJIS, strings.Repeat("\x81\x81", n), strings.Repeat("＝", n)},
{dec, ShiftJIS, strings.Repeat("\xe0\xfd", n), strings.Repeat("\ufffd", n)},
{dec, ShiftJIS, "a" + strings.Repeat("\xe0\xfd", n), "a" + strings.Repeat("\ufffd", n)},

{dec, ISO2022JP, "\x1b$", "\ufffd$"},
{dec, ISO2022JP, "\x1b(", "\ufffd("},
{dec, ISO2022JP, "\x1b@", "\ufffd@"},
{dec, ISO2022JP, "\x1bZ", "\ufffdZ"},
// incomplete escapes
{dec, ISO2022JP, "\x1b$", "\ufffd$"},
{dec, ISO2022JP, "\x1b$J.", "\ufffd$J."}, // illegal
{dec, ISO2022JP, "\x1b$B.", "\ufffd"}, // JIS208
{dec, ISO2022JP, "\x1b$(", "\ufffd$("}, // JIS212
{dec, ISO2022JP, "\x1b$(..", "\ufffd$(.."}, // JIS212
{dec, ISO2022JP, "\x1b$(" + long, "\ufffd$(" + long}, // JIS212
{dec, ISO2022JP, "\x1b$(D.", "\ufffd"}, // JIS212
{dec, ISO2022JP, "\x1b$(D..", "\ufffd"}, // JIS212
{dec, ISO2022JP, "\x1b$(D...", "\ufffd\ufffd"}, // JIS212
{dec, ISO2022JP, "\x1b(B.", "."}, // ascii
{dec, ISO2022JP, "\x1b(B..", ".."}, // ascii
{dec, ISO2022JP, "\x1b(J.", "."}, // roman
{dec, ISO2022JP, "\x1b(J..", ".."}, // roman
{dec, ISO2022JP, "\x1b(I\x20", "\ufffd"}, // katakana
{dec, ISO2022JP, "\x1b(I\x20\x20", "\ufffd\ufffd"}, // katakana
// recover to same state
{dec, ISO2022JP, "\x1b(B\x1b.", "\ufffd."},
{dec, ISO2022JP, "\x1b(I\x1b.", "\ufffdｮ"},
{dec, ISO2022JP, "\x1b(I\x1b$.", "\ufffd､ｮ"},
{dec, ISO2022JP, "\x1b(I\x1b(.", "\ufffdｨｮ"},
{dec, ISO2022JP, "\x1b$B\x7e\x7e", "\ufffd"},
{dec, ISO2022JP, "\x1b$@\x0a.", "\x0a."},
{dec, ISO2022JP, "\x1b$B\x0a.", "\x0a."},
{dec, ISO2022JP, "\x1b$(D\x0a.", "\x0a."},
{dec, ISO2022JP, "\x1b$(D\x7e\x7e", "\ufffd"},
{dec, ISO2022JP, "\x80", "\ufffd"},

// TODO: according to https://encoding.spec.whatwg.org/#iso-2022-jp,
// these should all be correct.
// {dec, ISO2022JP, "\x1b(B\x0E", "\ufffd"},
// {dec, ISO2022JP, "\x1b(B\x0F", "\ufffd"},
{dec, ISO2022JP, "\x1b(B\x5C", "\u005C"},
{dec, ISO2022JP, "\x1b(B\x7E", "\u007E"},
// {dec, ISO2022JP, "\x1b(J\x0E", "\ufffd"},
// {dec, ISO2022JP, "\x1b(J\x0F", "\ufffd"},
// {dec, ISO2022JP, "\x1b(J\x5C", "\u00A5"},
// {dec, ISO2022JP, "\x1b(J\x7E", "\u203E"},
}
for _, tc := range testCases {
dir, tr, wantErr := tc.init(tc.e)

dst, _, err := transform.String(tr, tc.src)
if err != wantErr {
t.Errorf("%s %v(%q): got %v; want %v", dir, tc.e, tc.src, err, wantErr)
}
if got := string(dst); got != tc.want {
t.Errorf("%s %v(%q):\ngot %q\nwant %q", dir, tc.e, tc.src, got, tc.want)
}
t.Run(fmt.Sprintf("%s/%v/%q", dir, tc.e, tc.src), func(t *testing.T) {
dst := make([]byte, 100000)
src := []byte(tc.src)
for i := 0; i <= len(tc.src); i++ {
nDst, nSrc, err := tr.Transform(dst, src[:i], false)
if err != nil && err != transform.ErrShortSrc && err != wantErr {
t.Fatalf("error on first call to Transform: %v", err)
}
n, _, err := tr.Transform(dst[nDst:], src[nSrc:], true)
nDst += n
if err != wantErr {
t.Fatalf("(%q|%q): got %v; want %v", tc.src[:i], tc.src[i:], err, wantErr)
}
if got := string(dst[:nDst]); got != tc.want {
t.Errorf("(%q|%q):\ngot %q\nwant %q", tc.src[:i], tc.src[i:], got, tc.want)
}
}
})
}
}

Expand Down
59 changes: 31 additions & 28 deletions encoding/japanese/iso2022jp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
package japanese

import (
"errors"
"unicode/utf8"

"golang.org/x/text/encoding"
Expand All @@ -31,8 +30,6 @@ func iso2022JPNewEncoder() transform.Transformer {
return new(iso2022JPEncoder)
}

var errInvalidISO2022JP = errors.New("japanese: invalid ISO-2022-JP encoding")

const (
asciiState = iota
katakanaState
Expand All @@ -50,45 +47,51 @@ func (d *iso2022JPDecoder) Reset() {

func (d *iso2022JPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
r, size := rune(0), 0
loop:
for ; nSrc < len(src); nSrc += size {
c0 := src[nSrc]
if c0 >= utf8.RuneSelf {
err = errInvalidISO2022JP
break loop
r, size = '\ufffd', 1
goto write
}

if c0 == asciiEsc {
if nSrc+2 >= len(src) {
err = transform.ErrShortSrc
break loop
if !atEOF {
return nDst, nSrc, transform.ErrShortSrc
}
// TODO: is it correct to only skip 1??
r, size = '\ufffd', 1
goto write
}
size = 3
c1 := src[nSrc+1]
c2 := src[nSrc+2]
switch {
case c1 == '$' && (c2 == '@' || c2 == 'B'):
case c1 == '$' && (c2 == '@' || c2 == 'B'): // 0x24 {0x40, 0x42}
*d = jis0208State
continue
case c1 == '$' && c2 == '(':
case c1 == '$' && c2 == '(': // 0x24 0x28
if nSrc+3 >= len(src) {
err = transform.ErrShortSrc
break loop
if !atEOF {
return nDst, nSrc, transform.ErrShortSrc
}
r, size = '\ufffd', 1
goto write
}
size = 4
if src[nSrc]+3 == 'D' {
if src[nSrc+3] == 'D' {
*d = jis0212State
continue
}
case c1 == '(' && (c2 == 'B' || c2 == 'J'):
case c1 == '(' && (c2 == 'B' || c2 == 'J'): // 0x28 {0x42, 0x4A}
*d = asciiState
continue
case c1 == '(' && c2 == 'I':
case c1 == '(' && c2 == 'I': // 0x28 0x49
*d = katakanaState
continue
}
err = errInvalidISO2022JP
break loop
r, size = '\ufffd', 1
goto write
}

switch *d {
Expand All @@ -97,20 +100,23 @@ loop:

case katakanaState:
if c0 < 0x21 || 0x60 <= c0 {
err = errInvalidISO2022JP
break loop
r, size = '\ufffd', 1
goto write
}
r, size = rune(c0)+(0xff61-0x21), 1

default:
if c0 == 0x0a {
*d = asciiState
r, size = rune(c0), 1
break
goto write
}
if nSrc+1 >= len(src) {
err = transform.ErrShortSrc
break loop
if !atEOF {
return nDst, nSrc, transform.ErrShortSrc
}
r, size = '\ufffd', 1
goto write
}
size = 2
c1 := src[nSrc+1]
Expand All @@ -121,22 +127,19 @@ loop:
r = rune(jis0212Decode[i])
} else {
r = '\ufffd'
break
goto write
}
if r == 0 {
r = '\ufffd'
}
}

write:
if nDst+utf8.RuneLen(r) > len(dst) {
err = transform.ErrShortDst
break loop
return nDst, nSrc, transform.ErrShortDst
}
nDst += utf8.EncodeRune(dst[nDst:], r)
}
if atEOF && err == transform.ErrShortSrc {
err = errInvalidISO2022JP
}
return nDst, nSrc, err
}

Expand Down
2 changes: 1 addition & 1 deletion encoding/japanese/shiftjis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ loop:
c0 = 2*c0 - 0x21

if nSrc+1 >= len(src) {
err = transform.ErrShortSrc
if !atEOF {
err = transform.ErrShortSrc
break loop
}
r, size = '\ufffd', 1
Expand Down

0 comments on commit f28f367

Please sign in to comment.