perf: update the accumulated value of a reduce expression in place
_assign and _modify create an allocator on every call, so the value they are given is never one that allocator created, and updateObject and updateArrayIndex always take the copying branch. Accumulating into an object or an array with a reduce expression therefore copies the whole accumulated value on every iteration, which is quadratic in the number of elements. The query of #241, [range(20000)] | reduce .[] as $item ({}; .[($item|tostring)] = {}) took between 10 and 21 seconds here over three runs, against 0.03 seconds for jq. Create the allocator once per reduce expression and pass it to the assignment as a third argument, so that the accumulated value is allocated on the first iteration and modified in place afterwards. The query above now takes 0.03 seconds, and the memory allocated by an accumulating reduction grows linearly instead of quadratically: 2.1 MB instead of 545 MB for 4000 elements. This is only correct while the accumulated value, and the values the allocator created inside it, are reachable through the reduction alone. Query#inPlaceUpdate therefore requires the update query to be a single assignment, and its right-hand side to neither observe the input value, which would let the assigned value alias the value it is assigned into, nor yield more than one value, which would run the assignment again against a value it already modified. Everything else keeps copying. A foreach expression is left alone, because it yields every intermediate accumulated value. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
dualfrozandClaude Opus 5 committed
1 parent
c5274c6af4
commit
70b3f7d698
4 files changed
+493
-43
No files matched your search
+225
@@ -0,0 +1,225 @@
|
||||
package gojq
|
||||
|
||||
// This file implements the static analysis deciding whether the accumulated
|
||||
// value of a reduce expression may be updated in place; see
|
||||
// compiler#compileReduce for how the result is used, and the comment on
|
||||
// compiler#compileAssign for why an allocator makes assignments cheap.
|
||||
//
|
||||
// Reusing an allocator across the iterations of a reduce expression is only
|
||||
// correct while the accumulated value, and the values the allocator created
|
||||
// inside it, are reachable through the accumulated value alone. The checks
|
||||
// below are deliberately conservative; failing them only means the assignment
|
||||
// keeps copying the accumulated value, which is the previous behavior.
|
||||
|
||||
// inPlaceUpdate reports whether the query is a single assignment applied to
|
||||
// the input value, and returns its operands. Such an assignment receives the
|
||||
// accumulated value of the enclosing reduce expression, and passes it on as
|
||||
// the next accumulated value, without any other part of the query observing
|
||||
// it in between.
|
||||
func (e *Query) inPlaceUpdate() (l, r *Query, op Operator, ok bool) {
|
||||
if e.Term != nil || len(e.FuncDefs) > 0 {
|
||||
return nil, nil, 0, false
|
||||
}
|
||||
switch e.Op {
|
||||
case OpModify:
|
||||
// The update function is applied to the values found at the target
|
||||
// paths, so it never observes the input value itself. When a target
|
||||
// path points to a value the allocator created, _modify releases the
|
||||
// allocator before calling the update function.
|
||||
return e.Left, e.Right, e.Op, true
|
||||
case OpAssign:
|
||||
// An assignment to a constant path is compiled to a setpath call,
|
||||
// which does not use an allocator; see compiler#compileQueryUpdate.
|
||||
if e.Left.toIndices(nil) != nil {
|
||||
return nil, nil, 0, false
|
||||
}
|
||||
fallthrough
|
||||
case OpUpdateAdd, OpUpdateSub, OpUpdateMul,
|
||||
OpUpdateDiv, OpUpdateMod, OpUpdateAlt:
|
||||
// The right-hand side is evaluated against the input value, so it must
|
||||
// not observe it; the assigned value would otherwise be able to alias
|
||||
// the value it is assigned into. It must not yield multiple values
|
||||
// either, because the assignment then runs once per value, each time
|
||||
// against an input value the previous run already modified.
|
||||
if !e.Right.readsInput() && e.Right.singleValued() {
|
||||
return e.Left, e.Right, e.Op, true
|
||||
}
|
||||
}
|
||||
return nil, nil, 0, false
|
||||
}
|
||||
|
||||
// readsInput reports whether evaluating the query may observe its input value.
|
||||
// It answers true unless it can prove otherwise.
|
||||
func (e *Query) readsInput() bool {
|
||||
if len(e.FuncDefs) > 0 {
|
||||
return true
|
||||
}
|
||||
if e.Term != nil {
|
||||
return e.Term.readsInput()
|
||||
}
|
||||
switch e.Op {
|
||||
case OpPipe:
|
||||
// The right-hand side is evaluated against the output of the
|
||||
// left-hand side, unless the query binds variables, in which case it
|
||||
// is evaluated against the input value as well.
|
||||
if len(e.Patterns) > 0 {
|
||||
return e.Left.readsInput() || e.Right.readsInput()
|
||||
}
|
||||
return e.Left.readsInput()
|
||||
case OpComma, OpAdd, OpSub, OpMul, OpDiv, OpMod, OpEq, OpNe,
|
||||
OpGt, OpLt, OpGe, OpLe, OpAnd, OpOr, OpAlt:
|
||||
return e.Left.readsInput() || e.Right.readsInput()
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// singleValued reports whether the query yields at most one value.
|
||||
// It answers false unless it can prove otherwise.
|
||||
func (e *Query) singleValued() bool {
|
||||
if len(e.FuncDefs) > 0 {
|
||||
return false
|
||||
}
|
||||
if e.Term != nil {
|
||||
return e.Term.singleValued()
|
||||
}
|
||||
switch e.Op {
|
||||
case OpPipe:
|
||||
return len(e.Patterns) == 0 &&
|
||||
e.Left.singleValued() && e.Right.singleValued()
|
||||
case OpAdd, OpSub, OpMul, OpDiv, OpMod, OpEq, OpNe,
|
||||
OpGt, OpLt, OpGe, OpLe, OpAnd, OpOr:
|
||||
return e.Left.singleValued() && e.Right.singleValued()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Term) readsInput() bool {
|
||||
for _, s := range e.SuffixList {
|
||||
if s.readsInput() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
switch e.Type {
|
||||
case TermTypeNull, TermTypeTrue, TermTypeFalse, TermTypeNumber:
|
||||
return false
|
||||
case TermTypeString:
|
||||
return e.Str.readsInput()
|
||||
case TermTypeObject:
|
||||
for _, kv := range e.Object.KeyVals {
|
||||
if kv.readsInput() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
case TermTypeArray:
|
||||
return e.Array.Query != nil && e.Array.Query.readsInput()
|
||||
case TermTypeUnary:
|
||||
return e.Unary.Term.readsInput()
|
||||
case TermTypeFunc:
|
||||
// Only a variable reference is known not to read the input value; any
|
||||
// function may be redefined, so its name tells us nothing.
|
||||
return e.Func.Name[0] != '$' || len(e.Func.Args) > 0
|
||||
case TermTypeQuery:
|
||||
return e.Query.readsInput()
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Term) singleValued() bool {
|
||||
for _, s := range e.SuffixList {
|
||||
if !s.singleValued() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
switch e.Type {
|
||||
case TermTypeNull, TermTypeTrue, TermTypeFalse, TermTypeNumber:
|
||||
return true
|
||||
case TermTypeString:
|
||||
return e.Str.singleValued()
|
||||
case TermTypeObject:
|
||||
for _, kv := range e.Object.KeyVals {
|
||||
if !kv.singleValued() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
case TermTypeArray:
|
||||
return true
|
||||
case TermTypeUnary:
|
||||
return e.Unary.Term.singleValued()
|
||||
case TermTypeFunc:
|
||||
return e.Func.Name[0] == '$' && len(e.Func.Args) == 0
|
||||
case TermTypeQuery:
|
||||
return e.Query.singleValued()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Suffix) readsInput() bool {
|
||||
// Both iterating and the optional operator work on the value of the term,
|
||||
// while an index is evaluated against the input value.
|
||||
return e.Index != nil && e.Index.readsInput()
|
||||
}
|
||||
|
||||
func (e *Suffix) singleValued() bool {
|
||||
return !e.Iter && (e.Index == nil || e.Index.singleValued())
|
||||
}
|
||||
|
||||
func (e *Index) readsInput() bool {
|
||||
if e.Str != nil {
|
||||
return e.Str.readsInput()
|
||||
}
|
||||
return e.Start != nil && e.Start.readsInput() ||
|
||||
e.End != nil && e.End.readsInput()
|
||||
}
|
||||
|
||||
func (e *Index) singleValued() bool {
|
||||
if e.Str != nil {
|
||||
return e.Str.singleValued()
|
||||
}
|
||||
return (e.Start == nil || e.Start.singleValued()) &&
|
||||
(e.End == nil || e.End.singleValued())
|
||||
}
|
||||
|
||||
func (e *String) readsInput() bool {
|
||||
for _, q := range e.Queries {
|
||||
if q.readsInput() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *String) singleValued() bool {
|
||||
for _, q := range e.Queries {
|
||||
if !q.singleValued() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *ObjectKeyVal) readsInput() bool {
|
||||
if e.Val == nil {
|
||||
// The shorthand {foo} and {"foo"} read the input value, while {$foo}
|
||||
// does not.
|
||||
return e.Key == "" || e.Key[0] != '$'
|
||||
}
|
||||
if e.KeyString != nil && e.KeyString.readsInput() ||
|
||||
e.KeyQuery != nil && e.KeyQuery.readsInput() {
|
||||
return true
|
||||
}
|
||||
return e.Val.readsInput()
|
||||
}
|
||||
|
||||
func (e *ObjectKeyVal) singleValued() bool {
|
||||
if e.KeyString != nil && !e.KeyString.singleValued() ||
|
||||
e.KeyQuery != nil && !e.KeyQuery.singleValued() {
|
||||
return false
|
||||
}
|
||||
return e.Val == nil || e.Val.singleValued()
|
||||
}
|
||||
Reference in new issue
Block a user