dualfrozandClaude Opus 5
70b3f7d698
_assign and _modify create an allocator on every call, so the value they are given is never one that allocator created, and updateObject and updateArrayIndex always take the copying branch. Accumulating into an object or an array with a reduce expression therefore copies the whole accumulated value on every iteration, which is quadratic in the number of elements. The query of #241, [range(20000)] | reduce .[] as $item ({}; .[($item|tostring)] = {}) took between 10 and 21 seconds here over three runs, against 0.03 seconds for jq. Create the allocator once per reduce expression and pass it to the assignment as a third argument, so that the accumulated value is allocated on the first iteration and modified in place afterwards. The query above now takes 0.03 seconds, and the memory allocated by an accumulating reduction grows linearly instead of quadratically: 2.1 MB instead of 545 MB for 4000 elements. This is only correct while the accumulated value, and the values the allocator created inside it, are reachable through the reduction alone. Query#inPlaceUpdate therefore requires the update query to be a single assignment, and its right-hand side to neither observe the input value, which would let the assigned value alias the value it is assigned into, nor yield more than one value, which would run the assignment again against a value it already modified. Everything else keeps copying. A foreach expression is left alone, because it yields every intermediate accumulated value. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
450 lines
9.5 KiB
Go
450 lines
9.5 KiB
Go
package gojq_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"reflect"
|
|
"runtime"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
"unsafe"
|
|
|
|
"github.com/itchyny/gojq"
|
|
)
|
|
|
|
func ExampleCompile() {
|
|
query, err := gojq.Parse(".[] | .foo")
|
|
if err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
iter := code.Run([]any{
|
|
nil,
|
|
"string",
|
|
42,
|
|
[]any{"foo"},
|
|
map[string]any{"foo": 42},
|
|
})
|
|
for {
|
|
v, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if err, ok := v.(error); ok {
|
|
fmt.Println(err)
|
|
continue
|
|
}
|
|
fmt.Printf("%#v\n", v)
|
|
}
|
|
|
|
// Output:
|
|
// <nil>
|
|
// expected an object but got: string ("string")
|
|
// expected an object but got: number (42)
|
|
// expected an object but got: array (["foo"])
|
|
// 42
|
|
}
|
|
|
|
func ExampleCode_Run() {
|
|
query, err := gojq.Parse(".foo")
|
|
if err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
input := map[string]any{"foo": 42}
|
|
iter := code.Run(input)
|
|
for {
|
|
v, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if err, ok := v.(error); ok {
|
|
log.Fatalln(err)
|
|
}
|
|
fmt.Printf("%#v\n", v)
|
|
}
|
|
|
|
// Output:
|
|
// 42
|
|
}
|
|
|
|
func ExampleCode_RunWithContext() {
|
|
query, err := gojq.Parse("def f: f; f, f")
|
|
if err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
log.Fatalln(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
defer cancel()
|
|
iter := code.RunWithContext(ctx, nil)
|
|
for {
|
|
v, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if err, ok := v.(error); ok {
|
|
fmt.Println(err)
|
|
continue
|
|
}
|
|
_ = v
|
|
}
|
|
|
|
// Output:
|
|
// context deadline exceeded
|
|
}
|
|
|
|
func TestCodeCompile_OptimizeConstants(t *testing.T) {
|
|
query, err := gojq.Parse(`[1,{foo:2,"bar":+3},[-4]]`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
codes := reflect.ValueOf(code).Elem().FieldByName("codes")
|
|
if got, expected := codes.Len(), 3; expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
iter := code.Run(nil)
|
|
for {
|
|
got, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if expected := []any{
|
|
1, map[string]any{"foo": 2, "bar": 3}, []any{-4},
|
|
}; !reflect.DeepEqual(got, expected) {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCodeCompile_OptimizeIndexSlice(t *testing.T) {
|
|
query, err := gojq.Parse(`.foo."bar".["baz"].[-1]."".[0:1]`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
codes := reflect.ValueOf(code).Elem().FieldByName("codes")
|
|
if got, expected := codes.Len(), 8; expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
iter := code.Run(nil)
|
|
for {
|
|
got, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if got != nil {
|
|
t.Errorf("expected: %v, got: %v", nil, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCodeCompile_OptimizeIndexSliceAssign(t *testing.T) {
|
|
query, err := gojq.Parse(`.foo."bar".["baz"].[0]."".[0:1] = [0]`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
codes := reflect.ValueOf(code).Elem().FieldByName("codes")
|
|
if got, expected := codes.Len(), 8; expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
iter := code.Run(nil)
|
|
for {
|
|
got, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if expected := map[string]any{
|
|
"foo": map[string]any{
|
|
"bar": map[string]any{
|
|
"baz": []any{map[string]any{"": []any{0}}},
|
|
},
|
|
},
|
|
}; !reflect.DeepEqual(got, expected) {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCodeCompile_OptimizeTailRec_While(t *testing.T) {
|
|
query, err := gojq.Parse("0 | while(. < 10; . + 1)")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
codes := reflect.ValueOf(code).Elem().FieldByName("codes")
|
|
if got, expected := codes.Len(), 48; expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
op1 := codes.Index(2).Elem().FieldByName("op")
|
|
op2 := codes.Index(21).Elem().FieldByName("op") // test jump of call _while
|
|
if got, expected := *(*int)(unsafe.Pointer(op2.UnsafeAddr())),
|
|
*(*int)(unsafe.Pointer(op1.UnsafeAddr())); expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
iter := code.Run(nil)
|
|
n := 0
|
|
for {
|
|
got, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if !reflect.DeepEqual(got, n) {
|
|
t.Errorf("expected: %v, got: %v", n, got)
|
|
}
|
|
n++
|
|
}
|
|
if expected := 10; n != expected {
|
|
t.Errorf("expected: %v, got: %v", expected, n)
|
|
}
|
|
}
|
|
|
|
func TestCodeCompile_OptimizeTailRec_CallRec(t *testing.T) {
|
|
query, err := gojq.Parse(`
|
|
def f: . as $x | $x, (if $x < 3 then $x + 1 | f else empty end), $x; f
|
|
`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
codes := reflect.ValueOf(code).Elem().FieldByName("codes")
|
|
if got, expected := codes.Len(), 48; expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
op1 := codes.Index(39).Elem().FieldByName("op") // callrec f
|
|
op2 := codes.Index(38).Elem().FieldByName("op") // call _add/2
|
|
if got, expected := *(*int)(unsafe.Pointer(op2.UnsafeAddr()))+1,
|
|
*(*int)(unsafe.Pointer(op1.UnsafeAddr())); expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
}
|
|
|
|
func TestCodeCompile_OptimizeJumps(t *testing.T) {
|
|
query, err := gojq.Parse("def f: 1; def g: 2; def h: 3; f")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
codes := reflect.ValueOf(code).Elem().FieldByName("codes")
|
|
if got, expected := codes.Len(), 15; expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
v := codes.Index(1).Elem().FieldByName("v")
|
|
if got, expected := *(*any)(unsafe.Pointer(v.UnsafeAddr())), 13; expected != got {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
iter := code.Run(nil)
|
|
for {
|
|
got, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if expected := 1; !reflect.DeepEqual(got, expected) {
|
|
t.Errorf("expected: %v, got: %v", expected, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseErrorTokenOffset(t *testing.T) {
|
|
testCases := []struct {
|
|
src string
|
|
offset int
|
|
}{
|
|
{src: "^", offset: 1},
|
|
{src: " ^", offset: 2},
|
|
{src: " ^ ", offset: 2},
|
|
{src: "👍", offset: 4},
|
|
{src: " 👍", offset: 5},
|
|
{src: " 👍 ", offset: 5},
|
|
{src: "test👍", offset: 8},
|
|
}
|
|
for _, tc := range testCases {
|
|
t.Run(tc.src, func(t *testing.T) {
|
|
_, err := gojq.Parse(tc.src)
|
|
if err == nil {
|
|
t.Fatal("expected: error")
|
|
}
|
|
var pe *gojq.ParseError
|
|
if !errors.As(err, &pe) {
|
|
t.Fatalf("expected: *gojq.ParseError, got %v", err)
|
|
}
|
|
if pe.Offset != tc.offset {
|
|
t.Fatalf("expected: %v, got %v", tc.offset, pe.Offset)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCodeRun_Race(t *testing.T) {
|
|
query, err := gojq.Parse("range(10)")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var wg sync.WaitGroup
|
|
for range 10 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
iter := code.Run(nil)
|
|
n := 0
|
|
for {
|
|
got, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if got != n {
|
|
t.Errorf("expected: %v, got: %v", n, got)
|
|
}
|
|
n++
|
|
}
|
|
if expected := 10; n != expected {
|
|
t.Errorf("expected: %v, got: %v", expected, n)
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
func TestCodeRun_RaceRegexp(t *testing.T) {
|
|
query, err := gojq.Parse(`
|
|
"abcdefg" | test("cd"), split("e") | . == true or length > 0
|
|
`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var wg sync.WaitGroup
|
|
for range 10 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
iter := code.Run(nil)
|
|
for {
|
|
v, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if err, ok := v.(error); ok {
|
|
t.Errorf("unexpected error: %v", err)
|
|
}
|
|
if v != true {
|
|
t.Errorf("expected: true, got: %v", v)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
// An assignment forming the whole update query of a reduce expression modifies
|
|
// the accumulated value in place, so building an object or an array of n
|
|
// elements allocates memory proportional to n, not to n squared.
|
|
func TestCodeRun_ReduceAssignmentAllocation(t *testing.T) {
|
|
for _, src := range []string{
|
|
`reduce .[] as $x ({}; .[$x|tostring] = $x)`,
|
|
`reduce .[] as $x ({}; .[$x|tostring] |= $x)`,
|
|
`reduce .[] as $x ({}; .[$x|tostring] += 1)`,
|
|
`reduce .[] as $x ([]; .[$x] = $x)`,
|
|
} {
|
|
t.Run(src, func(t *testing.T) {
|
|
query, err := gojq.Parse(src)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, err := gojq.Compile(query)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
small, large := allocatedBytes(t, code, 1000), allocatedBytes(t, code, 4000)
|
|
// Copying the accumulated value on every iteration allocates about
|
|
// sixteen times as much memory for four times as many elements.
|
|
if expected := small * 8; large > expected {
|
|
t.Errorf("expected at most %v bytes for 4000 elements, got %v (%v bytes for 1000 elements)",
|
|
expected, large, small)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func allocatedBytes(t *testing.T, code *gojq.Code, n int) uint64 {
|
|
t.Helper()
|
|
input := make([]any, n)
|
|
for i := range input {
|
|
input[i] = i
|
|
}
|
|
var before, after runtime.MemStats
|
|
runtime.GC()
|
|
runtime.ReadMemStats(&before)
|
|
iter := code.Run(input)
|
|
for {
|
|
v, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
if err, ok := v.(error); ok {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
runtime.ReadMemStats(&after)
|
|
return after.TotalAlloc - before.TotalAlloc
|
|
}
|
|
|
|
func BenchmarkCompile(b *testing.B) {
|
|
cnt, err := os.ReadFile("builtin.jq")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
query, err := gojq.Parse(string(cnt) + ".")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
for b.Loop() {
|
|
_, err := gojq.Compile(
|
|
query,
|
|
gojq.WithInputIter(gojq.NewIter[any]()),
|
|
)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|