parser/gotest: Add support for parsing lines longer than 64K

The gotest parser used a bufio.Scanner to read its input, which
prevented us from reading lines larger than 64K.

In order to support reading larger lines, bufio.Scanner has been
replaced with bufio.Reader. The maximum line size has been increased to
4MiB and instead of returning an error when reading lines that exceed
the maximum size, we truncate that line and continue parsing.

Fixes #135

Author: jstemmer

parser/gotest/gotest.go

@@ -3,6 +3,7 @@ package gotest
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"regexp"
@@ -101,6 +102,12 @@ func SetSubtestMode(mode SubtestMode) Option {
 	}
 }
 
+const (
+	// maxLineSize is the maximum amount of bytes we'll read for a single line.
+	// Lines longer than maxLineSize will be truncated.
+	maxLineSize = 4 * 1024 * 1024
+)
+
 // Parser is a Go test output Parser.
 type Parser struct {
 	packageName string
@@ -124,11 +131,59 @@ func NewParser(options ...Option) *Parser {
 // gtr.Report.
 func (p *Parser) Parse(r io.Reader) (gtr.Report, error) {
 	p.events = nil
-	s := bufio.NewScanner(r)
-	for s.Scan() {
-		p.parseLine(s.Text())
+	s := bufio.NewReader(r)
+	for {
+		line, isPrefix, err := s.ReadLine()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return gtr.Report{}, err
+		}
+
+		if !isPrefix {
+			p.parseLine(string(line))
+			continue
+		}
+
+		// Line is incomplete, keep reading until we reach the end of the line.
+		var buf bytes.Buffer
+		buf.Write(line) // ignore err, always nil
+		for isPrefix {
+			line, isPrefix, err = s.ReadLine()
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				return gtr.Report{}, err
+			}
+
+			if buf.Len() >= maxLineSize {
+				// Stop writing to buf if we exceed maxLineSize. We continue
+				// reading however to make sure we consume the entire line.
+				continue
+			}
+
+			buf.Write(line) // ignore err, always nil
+		}
+
+		if buf.Len() > maxLineSize {
+			buf.Truncate(maxLineSize)
+		}
+
+		// Lines that exceed bufio.MaxScanTokenSize are not expected to contain
+		// any relevant test infrastructure output, so instead of parsing them
+		// we treat them as regular output to increase performance.
+		//
+		// Parser used a bufio.Scanner in the past, which only supported
+		// reading lines up to bufio.MaxScanTokenSize in length. Since this
+		// turned out to be fine in almost all cases, it seemed an appropriate
+		// value to use to decide whether or not to attempt parsing this line.
+		if buf.Len() > bufio.MaxScanTokenSize {
+			p.output(buf.String())
+		} else {
+			p.parseLine(buf.String())
+		}
 	}
-	return p.report(p.events), s.Err()
+	return p.report(p.events), nil
 }
 
 // report generates a gtr.Report from the given list of events.

parser/gotest/gotest_test.go

@@ -1,6 +1,8 @@
 package gotest
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"strings"
 	"testing"
@@ -216,6 +218,58 @@ func TestParseLine(t *testing.T) {
 	}
 }
 
+func TestParseLargeLine(t *testing.T) {
+	tests := []struct {
+		desc      string
+		inputSize int
+	}{
+		{"small size", 128},
+		{"under buf size", 4095},
+		{"buf size", 4096},
+		{"multiple of buf size ", 4096 * 2},
+		{"not multiple of buf size", 10 * 1024},
+		{"bufio.MaxScanTokenSize", bufio.MaxScanTokenSize},
+		{"over bufio.MaxScanTokenSize", bufio.MaxScanTokenSize + 1},
+		{"under limit", maxLineSize - 1},
+		{"at limit", maxLineSize},
+		{"just over limit", maxLineSize + 1},
+		{"over limit", maxLineSize + 128},
+	}
+
+	createInput := func(lines ...string) *bytes.Buffer {
+		buf := &bytes.Buffer{}
+		buf.WriteString("=== RUN TestOne\n--- PASS: TestOne (0.00s)\n")
+		buf.WriteString(strings.Join(lines, "\n"))
+		return buf
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			line1 := string(make([]byte, test.inputSize))
+			line2 := "other line"
+			report, err := NewParser().Parse(createInput(line1, line2))
+			if err != nil {
+				t.Fatalf("Parse() returned error %v", err)
+			} else if len(report.Packages) != 1 {
+				t.Fatalf("Parse() returned unexpected number of packages, got %d want 1.", len(report.Packages))
+			} else if len(report.Packages[0].Output) != 2 {
+				t.Fatalf("Parse() returned unexpected number of output lines, got %d want 1.", len(report.Packages[0].Output))
+			}
+
+			want := line1
+			if len(want) > maxLineSize {
+				want = want[:maxLineSize]
+			}
+			if got := report.Packages[0].Output[0]; got != want {
+				t.Fatalf("Parse() output line1 mismatch, got len %d want len %d", len(got), len(want))
+			}
+			if report.Packages[0].Output[1] != line2 {
+				t.Fatalf("Parse() output line2 mismatch, got %v want %v", report.Packages[0].Output[1], line2)
+			}
+		})
+	}
+}
+
 func TestReport(t *testing.T) {
 	events := []Event{
 		{Type: "run_test", Name: "TestOne"},