Reduce memory consumption when parsing large objects and arrays.

parser.go

@@ -2,10 +2,11 @@ package fastjson
 
 import (
 	"fmt"
-	"github.com/valyala/fastjson/fastfloat"
 	"strconv"
 	"strings"
 	"unicode/utf16"
+
+	"github.com/valyala/fastjson/fastfloat"
 )
 
 // Parser parses JSON.
@@ -54,20 +55,55 @@ func (p *Parser) ParseBytes(b []byte) (*Value, error) {
 
 type cache struct {
 	vs []Value
+	nx *cache // next
+	lt *cache // last
 }
 
 func (c *cache) reset() {
 	c.vs = c.vs[:0]
+	c.lt = nil
+	if c.nx != nil {
+		c.nx.reset()
+	}
 }
 
+const (
+	preAllocatedCacheSize = 341   // 32kb class size
+	maxAllocatedCacheSize = 10922 // 1MB
+)
+
 func (c *cache) getValue() *Value {
-	if cap(c.vs) > len(c.vs) {
-		c.vs = c.vs[:len(c.vs)+1]
-	} else {
-		c.vs = append(c.vs, Value{})
+	if c == nil {
+		return &Value{}
+	}
+	readSrc := c
+	if readSrc.lt != nil {
+		readSrc = readSrc.lt
+	}
+	switch {
+	case cap(readSrc.vs) == 0:
+		// initial state
+		readSrc.vs = make([]Value, 1, preAllocatedCacheSize)
+
+	case cap(readSrc.vs) > len(readSrc.vs):
+		readSrc.vs = readSrc.vs[:len(readSrc.vs)+1]
+
+	default:
+		if readSrc.nx == nil {
+			nextLen := len(readSrc.vs) * 2
+			if nextLen > maxAllocatedCacheSize {
+				nextLen = maxAllocatedCacheSize
+			}
+			readSrc.nx = &cache{
+				vs: make([]Value, 0, nextLen),
+			}
+		}
+		c.lt = readSrc.nx
+		readSrc = readSrc.nx
+		readSrc.vs = readSrc.vs[:len(readSrc.vs)+1]
 	}
 	// Do not reset the value, since the caller must properly init it.
-	return &c.vs[len(c.vs)-1]
+	return &readSrc.vs[len(readSrc.vs)-1]
 }
 
 func skipWS(s string) string {
@@ -445,36 +481,44 @@ func parseRawNumber(s string) (string, string, error) {
 	return s, "", nil
 }
 
-// Object represents JSON object.
-//
-// Object cannot be used from concurrent goroutines.
-// Use per-goroutine parsers or ParserPool instead.
 type Object struct {
 	kvs           []kv
 	keysUnescaped bool
+	nx            *Object
+	lt            *Object
 }
 
 func (o *Object) reset() {
 	o.kvs = o.kvs[:0]
 	o.keysUnescaped = false
+	o.lt = nil
+	if o.nx != nil {
+		o.nx.reset()
+	}
 }
 
 // MarshalTo appends marshaled o to dst and returns the result.
 func (o *Object) MarshalTo(dst []byte) []byte {
 	dst = append(dst, '{')
-	for i, kv := range o.kvs {
-		if o.keysUnescaped {
-			dst = escapeString(dst, kv.k)
-		} else {
-			dst = append(dst, '"')
-			dst = append(dst, kv.k...)
-			dst = append(dst, '"')
-		}
-		dst = append(dst, ':')
-		dst = kv.v.MarshalTo(dst)
-		if i != len(o.kvs)-1 {
-			dst = append(dst, ',')
+	srcKV := o
+	lastN := o.Len()
+	n := 0
+	for srcKV != nil {
+		for _, kv := range srcKV.kvs {
+			if srcKV.keysUnescaped {
+				dst = escapeString(dst, kv.k)
+			} else {
+				dst = append(dst, '"')
+				dst = append(dst, kv.k...)
+				dst = append(dst, '"')
+			}
+			dst = append(dst, ':')
+			dst = kv.v.MarshalTo(dst)
+			if n++; n != lastN {
+				dst = append(dst, ',')
+			}
 		}
+		srcKV = srcKV.nx
 	}
 	dst = append(dst, '}')
 	return dst
@@ -491,13 +535,45 @@ func (o *Object) String() string {
 	return b2s(b)
 }
 
+const (
+	preAllocatedObjectKVs = 170   // 8kb class
+	maxAllocatedObjectKVS = 21845 // 1MB class
+)
+
 func (o *Object) getKV() *kv {
-	if cap(o.kvs) > len(o.kvs) {
-		o.kvs = o.kvs[:len(o.kvs)+1]
-	} else {
-		o.kvs = append(o.kvs, kv{})
+	kvSrc := o
+	if kvSrc.lt != nil {
+		kvSrc = kvSrc.lt
 	}
-	return &o.kvs[len(o.kvs)-1]
+	switch {
+	case cap(kvSrc.kvs) == 0:
+		// initial state
+		kvSrc.kvs = append(kvSrc.kvs, kv{})
+
+	case cap(kvSrc.kvs) > len(kvSrc.kvs):
+		kvSrc.kvs = kvSrc.kvs[:len(kvSrc.kvs)+1]
+
+	default:
+		if cap(kvSrc.kvs) < preAllocatedObjectKVs {
+			kvSrc.kvs = append(kvSrc.kvs, kv{})
+			break
+		}
+		// new chain
+		if kvSrc.nx == nil {
+			nextLen := len(kvSrc.kvs) * 2
+			if nextLen > maxAllocatedObjectKVS {
+				nextLen = maxAllocatedObjectKVS
+			}
+			kvSrc.nx = &Object{
+				kvs: make([]kv, 0, nextLen),
+			}
+		}
+		kvSrc = kvSrc.nx
+		o.lt = kvSrc
+		kvSrc.kvs = kvSrc.kvs[:len(kvSrc.kvs)+1]
+	}
+
+	return &kvSrc.kvs[len(kvSrc.kvs)-1]
 }
 
 func (o *Object) unescapeKeys() {
@@ -509,12 +585,18 @@ func (o *Object) unescapeKeys() {
 		kv := &kvs[i]
 		kv.k = unescapeStringBestEffort(kv.k)
 	}
+	if o.nx != nil {
+		o.nx.unescapeKeys()
+	}
 	o.keysUnescaped = true
 }
 
 // Len returns the number of items in the o.
 func (o *Object) Len() int {
-	return len(o.kvs)
+	if o.nx == nil {
+		return len(o.kvs)
+	}
+	return len(o.kvs) + o.nx.Len()
 }
 
 // Get returns the value for the given key in the o.
@@ -530,6 +612,11 @@ func (o *Object) Get(key string) *Value {
 				return kv.v
 			}
 		}
+		if o.nx != nil {
+			if v := o.nx.Get(key); v != nil {
+				return v
+			}
+		}
 	}
 
 	// Slow path - unescape object keys.
@@ -540,6 +627,11 @@ func (o *Object) Get(key string) *Value {
 			return kv.v
 		}
 	}
+	if o.nx != nil {
+		if v := o.nx.Get(key); v != nil {
+			return v
+		}
+	}
 	return nil
 }
 
@@ -557,6 +649,9 @@ func (o *Object) Visit(f func(key []byte, v *Value)) {
 	for _, kv := range o.kvs {
 		f(s2b(kv.k), kv.v)
 	}
+	if o.nx != nil {
+		o.nx.Visit(f)
+	}
 }
 
 // Value represents any JSON value.

parser_test.go

@@ -144,7 +144,6 @@ func TestParseRawString(t *testing.T) {
 		f(`"x\\y"tail`, `x\\y`, "tail")
 		f(`"\\\"й\n\"я"tail`, `\\\"й\n\"я`, "tail")
 		f(`"\\\\\\\\"tail`, `\\\\\\\\`, "tail")
-
 	})
 
 	t.Run("error", func(t *testing.T) {
@@ -1197,7 +1196,6 @@ func TestParserParse(t *testing.T) {
 		if ss != s {
 			t.Fatalf("unexpected string representation for object; got\n%q; want\n%q", ss, s)
 		}
-
 	})
 }
 
@@ -1275,3 +1273,27 @@ func testParseGetSerial(s string) error {
 	}
 	return nil
 }
+
+func TestMarshalTo(t *testing.T) {
+	fileData := getFromFile("testdata/bunchFields.json")
+	var p Parser
+	v, err := p.Parse(fileData)
+	if err != nil {
+		t.Fatalf("cannot parse json: %s", err)
+	}
+	data := make([]byte, 0, len(fileData))
+	data = v.MarshalTo(data)
+	// check
+	var p2 Parser
+	v, err = p2.ParseBytes(data)
+	if err != nil {
+		t.Fatalf("cannot parse json: %s", err)
+	}
+	o, err := v.Object()
+	if err != nil {
+		t.Fatalf("expected object, got: %s", o.String())
+	}
+	if o.Len() != 871 {
+		t.Fatalf("expected 871 fields, got %d", o.Len())
+	}
+}