Describe this repository

_assign and _modify create an allocator on every call, so the value they
are given is never one that allocator created, and updateObject and
updateArrayIndex always take the copying branch. Accumulating into an
object or an array with a reduce expression therefore copies the whole
accumulated value on every iteration, which is quadratic in the number
of elements. The query of #241,

    [range(20000)] | reduce .[] as $item ({}; .[($item|tostring)] = {})

took between 10 and 21 seconds here over three runs, against 0.03
seconds for jq.

Create the allocator once per reduce expression and pass it to the
assignment as a third argument, so that the accumulated value is
allocated on the first iteration and modified in place afterwards. The
query above now takes 0.03 seconds, and the memory allocated by an
accumulating reduction grows linearly instead of quadratically: 2.1 MB
instead of 545 MB for 4000 elements.

This is only correct while the accumulated value, and the values the
allocator created inside it, are reachable through the reduction alone.
Query#inPlaceUpdate therefore requires the update query to be a single
assignment, and its right-hand side to neither observe the input value,
which would let the assigned value alias the value it is assigned into,
nor yield more than one value, which would run the assignment again
against a value it already modified. Everything else keeps copying. A
foreach expression is left alone, because it yields every intermediate
accumulated value.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

README.md

@@ -0,0 +1,10 @@
+# Contributions to itchyny/gojq
+
+Pure Go implementation of jq
+
+This repository holds only my commits to the upstream project, with their original author, date and diff.
+Each of my commits is preceded by a snapshot of the upstream files it touched, so every diff shows exactly what I changed.
+
+## Merged pull requests
+
+- [fix: release the allocator when an updated value escapes to a query](https://github.com/itchyny/gojq/pull/306) (2026-09-14)

cli/test.yaml

@@ -2407,6 +2407,29 @@
     "/a,b/c/d/e,f/"
     ",/ /a/,/b/,/ /c/,/ /d/,/ /e/,/f/,/ "
 
+- name: split/1 function against empty string
+  args:
+    - -c
+    - '.[] as $t | .[] as $s | $s | split($t)'
+  input: '["abc", "a", "b", ""]'
+  expected: |
+    ["",""]
+    ["a"]
+    ["b"]
+    []
+    ["","bc"]
+    ["",""]
+    ["b"]
+    []
+    ["a","c"]
+    ["a"]
+    ["",""]
+    []
+    ["a","b","c"]
+    ["a"]
+    ["b"]
+    []
+
 - name: join function
   args:
     - 'join(",")'
@@ -3585,6 +3608,16 @@
   expected: |
     8
 
+- name: reduce with assignment of multiple paths in update
+  args:
+    - -c
+    - 'reduce (1,2) as $i (.; (.[0], .[.[0]]) = $i),
+       reduce (1,2) as $i (.; (.[0], .[.[0]]) |= $i)'
+  input: '[0,0,0]'
+  expected: |
+    [2,2,0]
+    [2,2,0]
+
 - name: reduce with query in start
   args:
     - 'reduce range(5) as $x (range(3); . + $x)'
@@ -4989,6 +5022,46 @@
   expected: |
     {"a":[{},{"b":[1,2,3]}]}
 
+- name: getpath function against string
+  args:
+    - -c
+    - '[getpath([{"start":1,"end":3}]), getpath([{"start":-2,"end":null}]), getpath([1]), getpath([])]'
+  input: '"abcde"'
+  expected: |
+    ["bc","de","b","abcde"]
+
+- name: getpath function against string in a container
+  args:
+    - -c
+    - 'getpath(["a",{"start":0,"end":2}]), getpath(["a",1])'
+  input: '{"a":"abc"}'
+  expected: |
+    "ab"
+    "b"
+
+- name: getpath function against string round trip with path function
+  args:
+    - -c
+    - '[getpath(path(.[1:3])), getpath(path(.[1]))]'
+  input: '"abcde"'
+  expected: |
+    ["bc","b"]
+
+- name: getpath function against string round trip in a container
+  args:
+    - -c
+    - '. as $d | [path(.a[0:2]), path(.b[1])] | map(. as $p | $d | getpath($p))'
+  input: '{"a":"abc","b":"xyz"}'
+  expected: |
+    ["ab","y"]
+
+- name: getpath function against string with object key
+  args:
+    - 'getpath(["k"])'
+  input: '"abc"'
+  error: |
+    getpath(["k"]) cannot be applied to "abc": expected an object but got: string ("abc")
+
 - name: getpath function error
   args:
     - 'getpath(["a",1,"b","c"])'
@@ -5021,9 +5094,9 @@
   expected: |
     {"a":[{"b":5}]}
     {"a":[{"b":5}],"b":0}
-    "getpath([\"a\",0,\"b\"]) cannot be applied to: object ({\"a\":0})"
+    "getpath([\"a\",0,\"b\"]) cannot be applied to {\"a\":0}: expected an array but got: number (0)"
     {"a":[{"b":5}]}
-    "getpath([\"a\",0,\"b\"]) cannot be applied to: object ({\"a\":[0,1]})"
+    "getpath([\"a\",0,\"b\"]) cannot be applied to {\"a\":[0,1]}: expected an object but got: number (0)"
     "getpath([\"a\",0,\"b\"]) cannot be applied to {\"a\":{\"b\":1}}: expected an array but got: object ({\"b\":1})"
     {"a":[{"b":5}]}
     {"a":[{"b":5,"c":3}]}
@@ -5437,6 +5510,14 @@
   expected: |
     {"a":{"p":{"b":{"p":1,"q":1},"z":{"p":null,"q":null}},"q":{"b":{"p":1,"q":1}}}}
 
+- name: update-assignment operator against nested object with overlapping paths
+  args:
+    - -c
+    - '(.a.b.c,.a,.a.p.b.z) |= {p: ., q: .}'
+  input: '{"a":{"b":{"c":1}}}'
+  expected: |
+    {"a":{"p":{"b":{"c":{"p":1,"q":1},"z":{"p":null,"q":null}}},"q":{"b":{"c":{"p":1,"q":1}}}}}
+
 - name: update-assignment operator with optional operator
   args:
     - -c

compiler.go

@@ -132,18 +132,13 @@ func (c *compiler) compile(q *Query) error {
 }
 
 func (c *compiler) compileImport(i *Import) error {
-	var path, alias string
-	var err error
-	if i.ImportPath != "" {
-		path, alias = i.ImportPath, i.ImportAlias
-	} else {
-		path = i.IncludePath
-	}
+	path, alias := cmp.Or(i.ImportPath, i.IncludePath), i.ImportAlias
 	if c.moduleLoader == nil {
 		return fmt.Errorf("cannot load module: %q", path)
 	}
 	if strings.HasPrefix(alias, "$") {
 		var vals any
+		var err error
 		if moduleLoader, ok := c.moduleLoader.(interface {
 			LoadJSONWithMeta(string, map[string]any) (any, error)
 		}); ok {
@@ -166,6 +161,7 @@ func (c *compiler) compileImport(i *Import) error {
 		return nil
 	}
 	var q *Query
+	var err error
 	if moduleLoader, ok := c.moduleLoader.(interface {
 		LoadModuleWithMeta(string, map[string]any) (*Query, error)
 	}); ok {
@@ -1274,11 +1270,7 @@ func listModuleDeps(q *Query) []any {
 		if v == nil {
 			v = make(map[string]any)
 		}
-		relpath := i.ImportPath
-		if relpath == "" {
-			relpath = i.IncludePath
-		}
-		v["relpath"] = relpath
+		v["relpath"] = cmp.Or(i.ImportPath, i.IncludePath)
 		if i.ImportAlias != "" {
 			v["as"] = strings.TrimPrefix(i.ImportAlias, "$")
 		}

compiler_test.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"reflect"
+	"runtime"
 	"sync"
 	"testing"
 	"time"
@@ -374,6 +375,59 @@ func TestCodeRun_RaceRegexp(t *testing.T) {
 	wg.Wait()
 }
 
+// An assignment forming the whole update query of a reduce expression modifies
+// the accumulated value in place, so building an object or an array of n
+// elements allocates memory proportional to n, not to n squared.
+func TestCodeRun_ReduceAssignmentAllocation(t *testing.T) {
+	for _, src := range []string{
+		`reduce .[] as $x ({}; .[$x|tostring] = $x)`,
+		`reduce .[] as $x ({}; .[$x|tostring] |= $x)`,
+		`reduce .[] as $x ({}; .[$x|tostring] += 1)`,
+		`reduce .[] as $x ([]; .[$x] = $x)`,
+	} {
+		t.Run(src, func(t *testing.T) {
+			query, err := gojq.Parse(src)
+			if err != nil {
+				t.Fatal(err)
+			}
+			code, err := gojq.Compile(query)
+			if err != nil {
+				t.Fatal(err)
+			}
+			small, large := allocatedBytes(t, code, 1000), allocatedBytes(t, code, 4000)
+			// Copying the accumulated value on every iteration allocates about
+			// sixteen times as much memory for four times as many elements.
+			if expected := small * 8; large > expected {
+				t.Errorf("expected at most %v bytes for 4000 elements, got %v (%v bytes for 1000 elements)",
+					expected, large, small)
+			}
+		})
+	}
+}
+
+func allocatedBytes(t *testing.T, code *gojq.Code, n int) uint64 {
+	t.Helper()
+	input := make([]any, n)
+	for i := range input {
+		input[i] = i
+	}
+	var before, after runtime.MemStats
+	runtime.GC()
+	runtime.ReadMemStats(&before)
+	iter := code.Run(input)
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			break
+		}
+		if err, ok := v.(error); ok {
+			t.Fatal(err)
+		}
+	}
+	runtime.ReadMemStats(&after)
+	return after.TotalAlloc - before.TotalAlloc
+}
+
 func BenchmarkCompile(b *testing.B) {
 	cnt, err := os.ReadFile("builtin.jq")
 	if err != nil {

func.go

@@ -14,7 +14,6 @@ import (
 	"reflect"
 	"regexp"
 	"slices"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -438,9 +437,7 @@ func add(xs iter.Seq[any]) any {
 		case []any:
 			switch w := v.(type) {
 			case nil:
-				s := make([]any, len(x))
-				copy(s, x)
-				v = s
+				v = slices.Clone(x)
 				continue
 			case []any:
 				v = append(w, x...)
@@ -769,6 +766,13 @@ func funcSplit(v, x any) any {
 	if !ok {
 		return &func0TypeError{"split", x}
 	}
+	return splitString(s, t)
+}
+
+func splitString(s, t string) []any {
+	if s == "" {
+		return []any{}
+	}
 	ss := strings.Split(s, t)
 	xs := make([]any, len(ss))
 	for i, s := range ss {
@@ -1484,10 +1488,8 @@ func (a allocator) allocated(v any) bool {
 	return ok
 }
 
-// release forgets every allocated address when the value is one of them. It is
-// called wherever a value the allocator created is handed to a query, which
-// may keep any number of references to it; from that point on the values are
-// copied on update again, as if they had been allocated elsewhere.
+// release forgets every allocated address when the value is one of them, so
+// that a value handed to a query is copied on update again.
 func (a allocator) release(v any) {
 	switch v.(type) {
 	case map[string]any, []any:
@@ -1523,10 +1525,9 @@ func funcSetpathWithAllocator(v any, args []any) any {
 	return setpath(v, args[0], args[1], args[2].(allocator))
 }
 
-// Used in compiler#compileModify. The value at the path is passed to the
-// update function, which may store it in any number of places, so it must not
-// be modified in place afterwards. It is registered under the name of getpath
-// to keep tracking paths through the call.
+// Used in compiler#compileModify: the update function may store the value
+// anywhere, so it must not be modified in place afterwards. Registered as
+// getpath to keep tracking paths through the call.
 func funcGetpathWithAllocator(v any, args []any) any {
 	w := funcGetpath(v, args[0])
 	args[1].(allocator).release(w)
@@ -1610,9 +1611,6 @@ func update(v any, path []any, n any, a allocator) (any, error) {
 			return nil, &expectedArrayError{v}
 		}
 	case map[string]any:
-		// A path element is an object only when slicing an array, and it stays
-		// reachable through the path, so it must not be modified in place.
-		a.release(p)
 		switch v := v.(type) {
 		case nil:
 			return updateArraySlice(nil, p, path[1:], n, a)
@@ -1805,14 +1803,9 @@ func funcGetpath(v, p any) any {
 	}
 	u := v
 	for _, x := range path {
-		switch v.(type) {
-		case nil, []any, map[string]any:
-			v = funcIndex2(nil, v, x)
-			if err, ok := v.(error); ok {
-				return &func1WrapError{"getpath", u, p, err}
-			}
-		default:
-			return &func1TypeError{"getpath", u, p}
+		v = funcIndex2(nil, v, x)
+		if err, ok := v.(error); ok {
+			return &func1WrapError{"getpath", u, p, err}
 		}
 	}
 	return v
@@ -1856,10 +1849,8 @@ func funcBsearch(v, t any) any {
 	if !ok {
 		return &func1TypeError{"bsearch", v, t}
 	}
-	i := sort.Search(len(vs), func(i int) bool {
-		return Compare(vs[i], t) >= 0
-	})
-	if i < len(vs) && Compare(vs[i], t) == 0 {
+	i, ok := slices.BinarySearchFunc(vs, t, Compare)
+	if ok {
 		return i
 	}
 	return -i - 1

inplace.go

@@ -0,0 +1,252 @@
+package gojq
+
+// Static analysis deciding whether the accumulated value of a reduce
+// expression may be updated in place; see compiler#compileReduce. The checks
+// are conservative: failing them only keeps the previous copying behavior.
+
+// inPlaceUpdate reports whether the query is a single assignment applied to
+// the input value, and returns its operands.
+func (e *Query) inPlaceUpdate() (l, r *Query, op Operator, ok bool) {
+	if e.Term != nil || len(e.FuncDefs) > 0 {
+		return nil, nil, 0, false
+	}
+	// Every path is evaluated against the input value, which the setpath calls
+	// for earlier paths would already have modified in place.
+	if !e.Left.pathsIgnoreInput() {
+		return nil, nil, 0, false
+	}
+	switch e.Op {
+	case OpModify:
+		// _modify releases the allocator before calling the update function.
+		return e.Left, e.Right, e.Op, true
+	case OpAssign:
+		// A constant path compiles to setpath, which uses no allocator.
+		if e.Left.toIndices(nil) != nil {
+			return nil, nil, 0, false
+		}
+		fallthrough
+	case OpUpdateAdd, OpUpdateSub, OpUpdateMul,
+		OpUpdateDiv, OpUpdateMod, OpUpdateAlt:
+		// The right-hand side must neither observe the input value, which it
+		// could otherwise alias, nor yield multiple values, which would run
+		// the assignment against an already modified input.
+		if !e.Right.readsInput() && e.Right.singleValued() {
+			return e.Left, e.Right, e.Op, true
+		}
+	}
+	return nil, nil, 0, false
+}
+
+// pathsIgnoreInput reports whether the paths the query denotes do not depend on
+// the input value, such as .a.b, .[$k] or (.a, .b). It answers false unless it
+// can prove otherwise.
+func (e *Query) pathsIgnoreInput() bool {
+	if e == nil || len(e.FuncDefs) > 0 {
+		return false
+	}
+	if e.Term != nil {
+		return e.Term.pathsIgnoreInput()
+	}
+	switch e.Op {
+	case OpComma:
+		return e.Left.pathsIgnoreInput() && e.Right.pathsIgnoreInput()
+	case OpPipe:
+		return len(e.Patterns) == 0 &&
+			e.Left.pathsIgnoreInput() && e.Right.pathsIgnoreInput()
+	default:
+		return false
+	}
+}
+
+func (e *Term) pathsIgnoreInput() bool {
+	switch e.Type {
+	case TermTypeIdentity:
+	case TermTypeIndex:
+		if e.Index.readsInput() {
+			return false
+		}
+	case TermTypeQuery:
+		if !e.Query.pathsIgnoreInput() {
+			return false
+		}
+	default:
+		return false
+	}
+	for _, s := range e.SuffixList {
+		if s.Iter || s.Index == nil || s.Index.readsInput() {
+			return false
+		}
+	}
+	return true
+}
+
+// readsInput reports whether evaluating the query may observe its input value.
+// It answers true unless it can prove otherwise.
+func (e *Query) readsInput() bool {
+	if len(e.FuncDefs) > 0 {
+		return true
+	}
+	if e.Term != nil {
+		return e.Term.readsInput()
+	}
+	switch e.Op {
+	case OpPipe:
+		// Binding variables also exposes the input to the right-hand side.
+		if len(e.Patterns) > 0 {
+			return e.Left.readsInput() || e.Right.readsInput()
+		}
+		return e.Left.readsInput()
+	case OpComma, OpAdd, OpSub, OpMul, OpDiv, OpMod, OpEq, OpNe,
+		OpGt, OpLt, OpGe, OpLe, OpAnd, OpOr, OpAlt:
+		return e.Left.readsInput() || e.Right.readsInput()
+	default:
+		return true
+	}
+}
+
+// singleValued reports whether the query yields at most one value.
+// It answers false unless it can prove otherwise.
+func (e *Query) singleValued() bool {
+	if len(e.FuncDefs) > 0 {
+		return false
+	}
+	if e.Term != nil {
+		return e.Term.singleValued()
+	}
+	switch e.Op {
+	case OpPipe:
+		return len(e.Patterns) == 0 &&
+			e.Left.singleValued() && e.Right.singleValued()
+	case OpAdd, OpSub, OpMul, OpDiv, OpMod, OpEq, OpNe,
+		OpGt, OpLt, OpGe, OpLe, OpAnd, OpOr:
+		return e.Left.singleValued() && e.Right.singleValued()
+	default:
+		return false
+	}
+}
+
+func (e *Term) readsInput() bool {
+	for _, s := range e.SuffixList {
+		if s.readsInput() {
+			return true
+		}
+	}
+	switch e.Type {
+	case TermTypeNull, TermTypeTrue, TermTypeFalse, TermTypeNumber:
+		return false
+	case TermTypeString:
+		return e.Str.readsInput()
+	case TermTypeObject:
+		for _, kv := range e.Object.KeyVals {
+			if kv.readsInput() {
+				return true
+			}
+		}
+		return false
+	case TermTypeArray:
+		return e.Array.Query != nil && e.Array.Query.readsInput()
+	case TermTypeUnary:
+		return e.Unary.Term.readsInput()
+	case TermTypeFunc:
+		// Any function may be redefined, so only $x is known not to read it.
+		return e.Func.Name[0] != '$' || len(e.Func.Args) > 0
+	case TermTypeQuery:
+		return e.Query.readsInput()
+	default:
+		return true
+	}
+}
+
+func (e *Term) singleValued() bool {
+	for _, s := range e.SuffixList {
+		if !s.singleValued() {
+			return false
+		}
+	}
+	switch e.Type {
+	case TermTypeNull, TermTypeTrue, TermTypeFalse, TermTypeNumber:
+		return true
+	case TermTypeString:
+		return e.Str.singleValued()
+	case TermTypeObject:
+		for _, kv := range e.Object.KeyVals {
+			if !kv.singleValued() {
+				return false
+			}
+		}
+		return true
+	case TermTypeArray:
+		return true
+	case TermTypeUnary:
+		return e.Unary.Term.singleValued()
+	case TermTypeFunc:
+		return e.Func.Name[0] == '$' && len(e.Func.Args) == 0
+	case TermTypeQuery:
+		return e.Query.singleValued()
+	default:
+		return false
+	}
+}
+
+func (e *Suffix) readsInput() bool {
+	// Only an index is evaluated against the input value.
+	return e.Index != nil && e.Index.readsInput()
+}
+
+func (e *Suffix) singleValued() bool {
+	return !e.Iter && (e.Index == nil || e.Index.singleValued())
+}
+
+func (e *Index) readsInput() bool {
+	if e.Str != nil {
+		return e.Str.readsInput()
+	}
+	return e.Start != nil && e.Start.readsInput() ||
+		e.End != nil && e.End.readsInput()
+}
+
+func (e *Index) singleValued() bool {
+	if e.Str != nil {
+		return e.Str.singleValued()
+	}
+	return (e.Start == nil || e.Start.singleValued()) &&
+		(e.End == nil || e.End.singleValued())
+}
+
+func (e *String) readsInput() bool {
+	for _, q := range e.Queries {
+		if q.readsInput() {
+			return true
+		}
+	}
+	return false
+}
+
+func (e *String) singleValued() bool {
+	for _, q := range e.Queries {
+		if !q.singleValued() {
+			return false
+		}
+	}
+	return true
+}
+
+func (e *ObjectKeyVal) readsInput() bool {
+	if e.Val == nil {
+		// {foo} reads the input value, {$foo} does not.
+		return e.Key == "" || e.Key[0] != '$'
+	}
+	if e.KeyString != nil && e.KeyString.readsInput() ||
+		e.KeyQuery != nil && e.KeyQuery.readsInput() {
+		return true
+	}
+	return e.Val.readsInput()
+}
+
+func (e *ObjectKeyVal) singleValued() bool {
+	if e.KeyString != nil && !e.KeyString.singleValued() ||
+		e.KeyQuery != nil && !e.KeyQuery.singleValued() {
+		return false
+	}
+	return e.Val == nil || e.Val.singleValued()
+}