added option to specify size of decoder

go-logfmt · Jan 31, 2023 · 5a3c9dc · 5a3c9dc
1 parent ff8ea8b
commit 5a3c9dc
Show file tree

Hide file tree

Showing 2 changed files with 172 additions and 37 deletions.
diff --git a/decode.go b/decode.go
@@ -29,6 +29,23 @@ func NewDecoder(r io.Reader) *Decoder {
 	return dec
 }
 
+// NewDecoderSize returns a new decoder that reads from r.
+//
+// The decoder introduces its own buffering and may read data from r beyond
+// the logfmt records requested.
+// The size argument specifies the size of the initial buffer that the
+// Decoder will use to read records from r.
+// If a log line is longer than the size argument, the Decoder will return
+// a bufio.ErrTooLong error.
+func NewDecoderSize(r io.Reader, size int) *Decoder {
+	scanner := bufio.NewScanner(r)
+	scanner.Buffer(make([]byte, 0, size), size)
+	dec := &Decoder{
+		s: scanner,
+	}
+	return dec
+}
+
 // ScanRecord advances the Decoder to the next record, which can then be
 // parsed with the ScanKeyval method. It returns false when decoding stops,
 // either by reaching the end of the input or an error. After ScanRecord

diff --git a/decode_test.go b/decode_test.go
@@ -1,6 +1,7 @@
 package logfmt
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"reflect"
@@ -17,21 +18,56 @@ func (s kv) String() string {
 }
 
 func TestDecoder_scan(t *testing.T) {
+	defaultDecoder := func(s string) *Decoder { return NewDecoder(strings.NewReader(s)) }
 	tests := []struct {
 		data string
+		dec  func(string) *Decoder
 		want [][]kv
 	}{
-		{"", nil},
-		{"\n\n", [][]kv{nil, nil}},
-		{`x= `, [][]kv{{{[]byte("x"), nil}}}},
-		{`y=`, [][]kv{{{[]byte("y"), nil}}}},
-		{`y`, [][]kv{{{[]byte("y"), nil}}}},
-		{`y=f`, [][]kv{{{[]byte("y"), []byte("f")}}}},
-		{"y=\"\\tf\"", [][]kv{{{[]byte("y"), []byte("\tf")}}}},
-		{"a=1\n", [][]kv{{{[]byte("a"), []byte("1")}}}},
-		{
-			`a=1 b="bar" ƒ=2h3s r="esc\t" d x=sf   `,
-			[][]kv{{
+		{
+			data: "",
+			dec:  defaultDecoder,
+			want: nil,
+		},
+		{
+			data: "\n\n",
+			dec:  defaultDecoder,
+			want: [][]kv{nil, nil},
+		},
+		{
+			data: `x= `,
+			dec:  defaultDecoder,
+			want: [][]kv{{{[]byte("x"), nil}}},
+		},
+		{
+			data: `y=`,
+			dec:  defaultDecoder,
+			want: [][]kv{{{[]byte("y"), nil}}},
+		},
+		{
+			data: `y`,
+			dec:  defaultDecoder,
+			want: [][]kv{{{[]byte("y"), nil}}},
+		},
+		{
+			data: `y=f`,
+			dec:  defaultDecoder,
+			want: [][]kv{{{[]byte("y"), []byte("f")}}},
+		},
+		{
+			data: "y=\"\\tf\"",
+			dec:  defaultDecoder,
+			want: [][]kv{{{[]byte("y"), []byte("\tf")}}},
+		},
+		{
+			data: "a=1\n",
+			dec:  defaultDecoder,
+			want: [][]kv{{{[]byte("a"), []byte("1")}}},
+		},
+		{
+			data: `a=1 b="bar" ƒ=2h3s r="esc\t" d x=sf   `,
+			dec:  defaultDecoder,
+			want: [][]kv{{
 				{[]byte("a"), []byte("1")},
 				{[]byte("b"), []byte("bar")},
 				{[]byte("ƒ"), []byte("2h3s")},
@@ -41,46 +77,62 @@ func TestDecoder_scan(t *testing.T) {
 			}},
 		},
 		{
-			"y=f\ny=g",
-			[][]kv{
+			data: "y=f\ny=g",
+			dec:  defaultDecoder,
+			want: [][]kv{
 				{{[]byte("y"), []byte("f")}},
 				{{[]byte("y"), []byte("g")}},
 			},
 		},
 		{
-			"y=f  \n\x1e y=g",
-			[][]kv{
+			data: "y=f  \n\x1e y=g",
+			dec:  defaultDecoder,
+			want: [][]kv{
 				{{[]byte("y"), []byte("f")}},
 				{{[]byte("y"), []byte("g")}},
 			},
 		},
 		{
-			"y= d y=g",
-			[][]kv{{
+			data: "y= d y=g",
+			dec:  defaultDecoder,
+			want: [][]kv{{
 				{[]byte("y"), nil},
 				{[]byte("d"), nil},
 				{[]byte("y"), []byte("g")},
 			}},
 		},
 		{
-			"y=\"f\"\ny=g",
-			[][]kv{
+			data: "y=\"f\"\ny=g",
+			dec:  defaultDecoder,
+			want: [][]kv{
 				{{[]byte("y"), []byte("f")}},
 				{{[]byte("y"), []byte("g")}},
 			},
 		},
 		{
-			"y=\"f\\n\"y=g",
-			[][]kv{{
+			data: "y=\"f\\n\"y=g",
+			dec:  defaultDecoder,
+			want: [][]kv{{
 				{[]byte("y"), []byte("f\n")},
 				{[]byte("y"), []byte("g")},
 			}},
 		},
+		{
+			data: strings.Repeat(`y=f `, 5),
+			dec:  func(s string) *Decoder { return NewDecoderSize(strings.NewReader(s), 21) },
+			want: [][]kv{{
+				{[]byte("y"), []byte("f")},
+				{[]byte("y"), []byte("f")},
+				{[]byte("y"), []byte("f")},
+				{[]byte("y"), []byte("f")},
+				{[]byte("y"), []byte("f")},
+			}},
+		},
 	}
 
 	for _, test := range tests {
 		var got [][]kv
-		dec := NewDecoder(strings.NewReader(test.data))
+		dec := test.dec(test.data)
 
 		for dec.ScanRecord() {
 			var kvs []kv
@@ -103,28 +155,94 @@ func TestDecoder_scan(t *testing.T) {
 }
 
 func TestDecoder_errors(t *testing.T) {
+	defaultDecoder := func(s string) *Decoder { return NewDecoder(strings.NewReader(s)) }
 	tests := []struct {
 		data string
+		dec  func(string) *Decoder
 		want error
 	}{
-		{"a=1\n=bar", &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 1}},
-		{"a=1\n\"k\"=bar", &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 1}},
-		{"a=1\nk\"ey=bar", &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 2}},
-		{"a=1\nk=b\"ar", &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 4}},
-		{"a=1\nk=b =ar", &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 5}},
-		{"a==", &SyntaxError{Msg: "unexpected '='", Line: 1, Pos: 3}},
-		{"a=1\nk=b=ar", &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 4}},
-		{"a=\"1", &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 5}},
-		{"a=\"1\\", &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 6}},
-		{"a=\"\\t1", &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 7}},
-		{"a=\"\\u1\"", &SyntaxError{Msg: "invalid quoted value", Line: 1, Pos: 8}},
-		{"a\ufffd=bar", &SyntaxError{Msg: "invalid key", Line: 1, Pos: 5}},
-		{"\x80=bar", &SyntaxError{Msg: "invalid key", Line: 1, Pos: 2}},
-		{"\x80", &SyntaxError{Msg: "invalid key", Line: 1, Pos: 2}},
+		{
+			data: "a=1\n=bar",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 1},
+		},
+		{
+			data: "a=1\n\"k\"=bar",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 1},
+		},
+		{
+			data: "a=1\nk\"ey=bar",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 2},
+		},
+		{
+			data: "a=1\nk=b\"ar",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 4},
+		},
+		{
+			data: "a=1\nk=b =ar",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 5},
+		},
+		{
+			data: "a==",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unexpected '='", Line: 1, Pos: 3},
+		},
+		{
+			data: "a=1\nk=b=ar",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 4},
+		},
+		{
+			data: "a=\"1",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 5},
+		},
+		{
+			data: "a=\"1\\",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 6},
+		},
+		{
+			data: "a=\"\\t1",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 7},
+		},
+		{
+			data: "a=\"\\u1\"",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "invalid quoted value", Line: 1, Pos: 8},
+		},
+		{
+			data: "a\ufffd=bar",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "invalid key", Line: 1, Pos: 5},
+		},
+		{
+			data: "\x80=bar",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "invalid key", Line: 1, Pos: 2},
+		},
+		{
+			data: "\x80",
+			dec:  defaultDecoder,
+			want: &SyntaxError{Msg: "invalid key", Line: 1, Pos: 2},
+		},
+		{
+			data: "a=1\nb=2",
+			dec: func(s string) *Decoder {
+				dec := NewDecoderSize(strings.NewReader(s), 1)
+				return dec
+			},
+			want: bufio.ErrTooLong,
+		},
 	}
 
 	for _, test := range tests {
-		dec := NewDecoder(strings.NewReader(test.data))
+		dec := test.dec(test.data)
 
 		for dec.ScanRecord() {
 			for dec.ScanKeyval() {