diff --git a/internal/decoder/stream.go b/internal/decoder/stream.go index a383f725..6c665b21 100644 --- a/internal/decoder/stream.go +++ b/internal/decoder/stream.go @@ -190,7 +190,19 @@ func (s *Stream) reset() { func (s *Stream) readBuf() []byte { if s.filledBuffer { + // s.buf may have grown beyond bufSize (e.g. when invalid/multi-byte + // characters are expanded to RuneError in place), so resync bufSize to + // the real buffer length before growing. Keep doubling until the new + // buffer can hold everything already read, otherwise the copy below + // truncates the buffer while s.length still refers to the old size, + // causing an out-of-range panic when the buffer is scanned. + if int64(len(s.buf)) > s.bufSize { + s.bufSize = int64(len(s.buf)) + } s.bufSize *= 2 + for s.bufSize < s.length { + s.bufSize *= 2 + } remainBuf := s.buf s.buf = make([]byte, s.bufSize) copy(s.buf, remainBuf) diff --git a/stream_test.go b/stream_test.go index 7b32df86..917f21a2 100644 --- a/stream_test.go +++ b/stream_test.go @@ -526,6 +526,24 @@ func TestLongUTF8(t *testing.T) { } } +func TestIssue424(t *testing.T) { + // A JSON string containing a run of incomplete multi-byte UTF-8 lead bytes + // used to overflow the stream decode buffer and panic with + // "index out of range" (#424). It must now return an error cleanly. + var input []byte + input = append(input, '"') + input = append(input, bytes.Repeat([]byte{'0'}, 252)...) + input = append(input, bytes.Repeat([]byte{0xE2}, 257)...) + input = append(input, '0', '0') + + dec := json.NewDecoder(bytes.NewReader(input)) + for { + if _, err := dec.Token(); err != nil { + break + } + } +} + func TestIssue278(t *testing.T) { a := `{"嗷嗷":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\u55f7"}` r := strings.NewReader(a)