dualfroz/itchyny-gojq
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upstream itchyny/gojq state of the touched files before a0fd150fb8

Browse Source
This commit is contained in:
upstream committed 2026-09-04 08:21:35 +02:00
1 parent af32cf4c58
commit 33d02f2fdb
4 files changed
+82 -899
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cli/test.yaml
+3 -397
View File
@@ -710,27 +710,6 @@
error: |
expected a number for indexing an array but got: boolean (true)
- name: indexing number literal error
args:
- '-1[0]'
input: 'null'
error: |
expected an array but got: number (1)
- name: slicing number literal error
args:
- '1 - -1[1:]'
input: 'null'
error: |
expected an array but got: number (1)
- name: indexing by indexed number literal error
args:
- '.[1[0]]'
input: '[0, 1]'
error: |
expected an array but got: number (1)
- name: array slicing start error
args:
- '.[[]:0]'
@@ -1346,33 +1325,6 @@
expected: |
9223372036854775808
- name: abs function against non-numbers
args:
- -c
- 'abs'
input: |
""
"-42"
[]
[1,-2]
{}
{"a":-1}
expected: |
""
"-42"
[]
[1,-2]
{}
{"a":-1}
- name: abs function error
args:
- 'abs'
input: 'null true'
error: |
cannot negate: null
cannot negate: boolean (true)
- name: length function
args:
- 'length'
@@ -2478,17 +2430,6 @@
["b"]
[]
- name: split/1 function error
args:
- -c
- 'try split(",") catch ., try split(1) catch .'
input: '1 "a"'
expected: |
"split(\",\") cannot be applied to: number (1)"
"split(1) cannot be applied to: number (1)"
["a"]
"split(1) cannot be applied to: string (\"a\")"
- name: join function
args:
- 'join(",")'
@@ -3408,13 +3349,6 @@
expected: |
[5,5,false,3,3,[null,false],null,false]
- name: optional operator with alternative operator
args:
- '.a ?// 1'
input: 'null'
expected: |
1
- name: condition
args:
- 'if . then . else [.] end'
@@ -3630,29 +3564,6 @@
expected: |
5
- name: reduce with destructuring alternative operator
args:
- 'reduce .[] as [$i,$j] ?// {$i,$j} (0; . + $i * $j)'
input: '[[1,2], {"i":3,"j":4}, [5,6]]'
expected: |
44
- name: reduce with destructuring alternative operator resetting variables
args:
- -c
- 'reduce .[] as [$a] ?// {$b} ?// $c ([]; . + [[$a, $b, $c]])'
input: '[{"b":1}, [2], 3, [4]]'
expected: |
[[null,1,null],[2,null,null],[null,null,3],[4,null,null]]
- name: reduce with destructuring alternative operator and error backtrack
args:
- -c
- 'reduce .[] as [$a] ?// $a ([]; . + [if $a == 2 then error else $a end])'
input: '[[1], [2], [3]]'
expected: |
[1,[2],3]
- name: reduce null
args:
- 'reduce . as $n (.; .)'
@@ -3731,68 +3642,6 @@
expected: |
[[1,1],[1]]
- name: reduce with assignment in update
args:
- -c
- 'reduce range(3) as $i ({}; .[$i|tostring] = {}),
reduce range(3) as $i ({}; .a[$i|tostring] = $i),
reduce range(3) as $i ([]; .[$i] = $i),
reduce range(3) as $i ({}; .[$i|tostring] += 1)'
input: 'null'
expected: |
{"0":{},"1":{},"2":{}}
{"a":{"0":0,"1":1,"2":2}}
[0,1,2]
{"0":1,"1":1,"2":1}
- name: reduce with assignment of the accumulated value
args:
- -c
- 'reduce range(3) as $i ({}; .a = .),
reduce range(3) as $i ({}; .a = {v: .}),
reduce range(3) as $i ({}; . as $p | .a = $i | .b = $p)'
input: 'null'
expected: |
{"a":{"a":{"a":{}}}}
{"a":{"v":{"a":{"v":{"a":{"v":{}}}}}}}
{"a":2,"b":{"a":1,"b":{"a":0,"b":{}}}}
- name: reduce with assignment of multiple values
args:
- -c
- 'reduce range(2) as $i ({"k":"k"}; .[.k] = ("x","y")),
reduce range(2) as $i ({}; (.a = 1), (.b = 2))'
input: 'null'
expected: |
{"k":"y","y":"y"}
{"b":2}
- name: reduce with assignment of a shared value
args:
- -c
- '{"a":1} as $x | reduce range(2) as $i ({}; .[$i|tostring] = $x) |
[(.["0"].a = 9), $x]'
input: 'null'
expected: |
[{"0":{"a":9},"1":{"a":1}},{"a":1}]
- name: reduce with assignment result reused
args:
- -c
- 'reduce range(3) as $i ({}; .[$i|tostring] = $i) as $r |
[($r | .["0"] = 9), $r]'
input: 'null'
expected: |
[{"0":9,"1":1,"2":2},{"0":0,"1":1,"2":2}]
- name: foreach with assignment in update
args:
- -c
- '[foreach range(3) as $i ({}; .[$i|tostring] = $i)]'
input: 'null'
expected: |
[{"0":0},{"0":0,"1":1},{"0":0,"1":1,"2":2}]
- name: foreach iterator
args:
- 'foreach range(5) as $item (0; $item)'
@@ -3822,37 +3671,6 @@
-1
-4
- name: foreach with destructuring alternative operator
args:
- 'foreach .[] as [$i,$j] ?// {$i,$j} (0; . + $i * $j)'
input: '[[1,2], {"i":3,"j":4}, [5,6]]'
expected: |
2
14
44
- name: foreach with destructuring alternative operator resetting variables
args:
- -c
- 'foreach .[] as [$a] ?// {$b} ?// $c (null; [$a, $b, $c])'
input: '[{"b":1}, [2], 3, [4]]'
expected: |
[null,1,null]
[2,null,null]
[null,null,3]
[4,null,null]
- name: foreach with destructuring alternative operator and error backtrack in extract
args:
- -c
- 'foreach .[] as {$a} ?// [$a] ?// $a (null; $a; if 1 < . and . < 4 then error else . end)'
input: '[[1], [2], {"a":3}, 4]'
expected: |
1
[2]
{"a":3}
4
- name: foreach with iterator in update
args:
- 'foreach .[] as $i (1; ., . + $i, range(.))'
@@ -4677,17 +4495,6 @@
[-3,-3,-2,-1,-1,-0,-0,0,0,0,0,1,1,2,3,3]
[3.7,3.3,2.5,1.7,1,0.7,0.3,0,0.3,0.5,0.7,1,1.7,2.5,3.3,3.7]
- name: math functions against large number literals
args:
- -c
- '[floor, ceil, sqrt, exp10]'
input: |
1e1000
-1e1000
expected: |
[1.7976931348623157e+308,1.7976931348623157e+308,1.7976931348623157e+308,1.7976931348623157e+308]
[-1.7976931348623157e+308,-1.7976931348623157e+308,null,0]
- name: round, nearbyint, rint functions
args:
- -c
@@ -6068,7 +5875,7 @@
args:
- -c
- 'gmtime, localtime | map(. * 1000000 | floor / 1000000)'
input: '1500000000 1567890123.456 1600000000.111 -1 -0.5 0.5'
input: '1500000000 1567890123.456 1600000000.111'
expected: | # UTC-7 (TZ=Etc/GMT+7)
[2017,6,14,2,40,0,5,194]
[2017,6,13,19,40,0,4,193]
@@ -6076,12 +5883,6 @@
[2019,8,7,14,2,3.456,6,249]
[2020,8,13,12,26,40.111,0,256]
[2020,8,13,5,26,40.111,0,256]
[1969,11,31,23,59,59,3,364]
[1969,11,31,16,59,59,3,364]
[1969,11,31,23,59,59.5,3,364]
[1969,11,31,16,59,59.5,3,364]
[1970,0,1,0,0,0.5,4,0]
[1969,11,31,17,0,0.5,3,364]
- name: mktime, strftime, strflocaltime, todate functions
args:
@@ -6093,7 +5894,6 @@
[2020,8,13,12,26,40.111,0,256]
[2024,7,20]
[2025]
[1969,11,31,23,59,59.5,3,364]
expected: |
[2017,6,14,2,40,0,5,194]
2017-07-14T02:40:00Z
@@ -6130,105 +5930,15 @@
2024-12-31T00:00:00Z
2024-12-30T17:00:00-0700
2024-12-31T00:00:00Z
[1969,11,31,23,59,59.5,3,364]
1969-12-31T23:59:59Z
1969-12-31T23:59:59-0700
-0.5
1969-12-31T23:59:59Z
1969-12-31T16:59:59-0700
1969-12-31T23:59:59Z
- name: date functions with fractional seconds
args:
- -r
- 'mktime, strftime("%Y-%m-%dT%H:%M:%S.%fZ"), strflocaltime("%Y-%m-%dT%H:%M:%S.%f%z")'
input: |
[1970,0,1,0,0,-0.5]
[1970,0,1,0,0,-1.25]
[1970,0,1,0,0,-60.25]
[1970,0,1,0,0,-1]
[1970,0,1,0,0,0]
[1970,0,1,0,0,0.5]
[1970,0,1,0,0,60.25]
expected: | # UTC-7 (TZ=Etc/GMT+7)
-0.5
1969-12-31T23:59:59.500000Z
1969-12-31T23:59:59.500000-0700
-1.25
1969-12-31T23:59:58.750000Z
1969-12-31T23:59:58.750000-0700
-60.25
1969-12-31T23:58:59.750000Z
1969-12-31T23:58:59.750000-0700
-1
1969-12-31T23:59:59.000000Z
1969-12-31T23:59:59.000000-0700
0
1970-01-01T00:00:00.000000Z
1970-01-01T00:00:00.000000-0700
0.5
1970-01-01T00:00:00.500000Z
1970-01-01T00:00:00.500000-0700
60.25
1970-01-01T00:01:00.250000Z
1970-01-01T00:01:00.250000-0700
- name: mktime after subtracting fractional seconds
args:
- 'gmtime | .[5] -= 0.5 | mktime'
input: '0 60 86400 -60'
expected: |
-0.5
59.5
86399.5
-60.5
- name: date functions with non-finite or out of range epoch timestamps
args:
- -n
- '[infinite, -infinite, nan, 1e19, -1e19, 9223372036854775807, -9223372036854775808] | .[] | try gmtime catch .'
expected: |
"gmtime cannot be applied to 1.7976931348623157e+308: number of seconds is out of range"
"gmtime cannot be applied to -1.7976931348623157e+308: number of seconds is out of range"
"gmtime cannot be applied to null: number of seconds is out of range"
"gmtime cannot be applied to 10000000000000000000: number of seconds is out of range"
"gmtime cannot be applied to -10000000000000000000: number of seconds is out of range"
"gmtime cannot be applied to 9223372036854775807: number of seconds is out of range"
"gmtime cannot be applied to -9223372036854775808: number of seconds is out of range"
- name: date functions with out of range numbers in the array
args:
- -n
- '[[infinite,0,1], [nan,0,1], [1e19,0,1], [0,1e19,1], [2001,0,1,0,0,infinite], [2001,0,1,0,0,nan], [2001,0,1,0,0,1e19], [9223372036854775807,0,1], ["x",0,1]] | .[] | try mktime catch .'
expected: |
"mktime cannot be applied to [1.7976931348623157e+308,0,1]: number is out of range"
"mktime cannot be applied to [null,0,1]: number is out of range"
"mktime cannot be applied to [10000000000000000000,0,1]: number is out of range"
"mktime cannot be applied to [0,10000000000000000000,1]: number is out of range"
"mktime cannot be applied to [2001,0,1,0,0,1.797693134 ...]: number is out of range"
"mktime cannot be applied to [2001,0,1,0,0,null]: number is out of range"
"mktime cannot be applied to [2001,0,1,0,0,10000000000 ...]: number is out of range"
"mktime cannot be applied to [9223372036854775807,0,1]: number is out of range"
"mktime cannot be applied to [\"x\",0,1]: expected an array of 8 numbers"
- name: strftime function with out of range datetime
args:
- -n
- 'try (infinite | todate) catch ., try ([1e19,0,1] | strftime("%Y")) catch .'
expected: |
"strftime(\"%Y-%m-%dT%H:%M:%SZ\") cannot be applied to 1.7976931348623157e+308: number of seconds is out of range"
"strftime(\"%Y\") cannot be applied to [10000000000000000000,0,1]: number is out of range"
- name: strptime, fromdate functions
args:
- -c
- 'strptime("%Y-%m-%dT%H:%M:%S%z")? // strptime("%Y-%m-%dT%H:%M:%S.%f%z"), fromdate?'
- 'strptime("%Y-%m-%dT%H:%M:%S%z"), fromdate'
input: |
"2017-07-14T02:40:00Z"
"2019-09-08T06:02:03+0900"
"2020-09-13T07:26:40-05:00"
"1969-12-31T23:59:59.5+0000"
"2345-01-02T03:04:05.678901+0000"
expected: |
[2017,6,14,2,40,0,5,194]
1500000000
@@ -6236,8 +5946,6 @@
1567890123
[2020,8,13,12,26,40,0,256]
1600000000
[1969,11,31,23,59,59.5,3,364]
[2345,0,2,3,4,5.678901,2,1]
- name: strftime, strptime functions
args:
@@ -6953,49 +6661,6 @@
"aabcABC☆★☆ABCabc"
"abcABC☆★☆ABCabc"
- name: match function with empty matches
args:
- -c
- '[match("a*"; "g").offset], [scan("[a-z]*")], gsub("[0-9]*"; "#"), split("-*"; ""), [splits("\\s*")]'
input: '"a1b-c d"'
expected: |
[0,1,2,3,4,5,6,7]
["a","","b","","c","","d",""]
"#a##b#-#c# #d#"
["","a","1","b","","c"," ","d",""]
["","a","1","b","-","c","","d",""]
- name: match function with empty matches and assertions
args:
- -c
- 'gsub("(?m)^a*"; "#"), gsub("\\ba*"; "#"), gsub("(^|-)a*"; "#"), gsub("[a-z]*\\b"; "#")'
input: '"a-a b"'
expected: |
"#-a b"
"##-## #b#"
"## b"
"##-## ##"
- name: match function with empty matches and captures
args:
- -c
- '[scan("(x)?a*")], [match("(a|)"; "g") | [.offset, .captures[0].offset]], [scan("[^a]*"; "i")]'
input: '"1a-A"'
expected: |
[[null],[null],[null],[null],[null]]
[[0,0],[1,1],[2,2],[3,3],[4,4]]
["1","","-","",""]
- name: match function with empty matches against multibyte characters
args:
- -c
- '[scan("[☆★]*")], [match("[☆★]*"; "g").offset], gsub("[☆★]*"; "#")'
input: '"☆★1☆★"'
expected: |
["☆★","","☆★",""]
[0,2,3,5]
"##1##"
- name: INDEX function
args:
- -c
@@ -7293,17 +6958,6 @@
{"a":3}
{"a":4}
- name: destructuring alternative operator with break in try
args:
- -c
- 'first((. as {$a} ?// [$a] ?// $a | {$a})?)'
input: '[1] [2] {"a":3} 4'
expected: |
{"a":1}
{"a":2}
{"a":3}
{"a":4}
- name: destructuring alternative operator with halt
args:
- -c
@@ -7323,28 +6977,6 @@
[1]
exit_code: 5
- name: destructuring alternative operator with halt_error in try
args:
- -c
- 'try (. as {$a} ?// [$a] ?// $a | {$a}, halt_error)'
input: '[1]'
expected: |
{"a":1}
error: |
[1]
exit_code: 5
- name: destructuring alternative operator resets variables of later alternatives
args:
- -c
- '.[] as [$a] ?// {$b} ?// $c | [$a, $b, $c]'
input: '[{"b":1}, [2], 3, [4]]'
expected: |
[null,1,null]
[2,null,null]
[null,null,3]
[4,null,null]
- name: destructuring alternative operator with multiple variables
args:
- -c
@@ -8008,7 +7640,7 @@
[["c"],3]
[["c"]]
- name: stream option with unterminated input after a comma
- name: stream option with unterminated input
args:
- -c
- --stream
@@ -8021,21 +7653,6 @@
{"a":1,
^ unexpected EOF
- name: stream option with unterminated input after a value
args:
- -c
- --stream
- '.'
input: '[1,[2,3'
expected: |
[[0],1]
[[1,0],2]
[[1,1],3]
error: |
invalid json: <stdin>
[1,[2,3
^ unexpected EOF
- name: yaml input option
args:
- --yaml-input
@@ -8205,17 +7822,6 @@
2 | bar: ]
^ did not find expected node content
- name: yaml input option error with multibyte characters
args:
- --yaml-input
- '.'
input: |
{１２３: 4, １２３: 5}
error: |
invalid yaml: <stdin>:1
1 | {１２３: 4, １２３: 5}
^ mapping key "１２３" already defined at line 1
- name: yaml input option multiple errors
args:
- --yaml-input
compiler.go
+79 -208
View File
@@ -17,7 +17,6 @@ type compiler struct {
variables []string
customFuncs map[string]function
inputIter Iter
inputMu sync.Mutex
codes []*code
codeinfos []codeinfo
builtinScope *scopeinfo
@@ -394,7 +393,7 @@ func (c *compiler) compileQuery(e *Query) error {
return c.compileAlt(e.Left, e.Right)
case OpAssign, OpModify, OpUpdateAdd, OpUpdateSub,
OpUpdateMul, OpUpdateDiv, OpUpdateMod, OpUpdateAlt:
return c.compileQueryUpdate(e.Left, e.Right, e.Op, nil)
return c.compileQueryUpdate(e.Left, e.Right, e.Op)
case OpOr:
return c.compileIf(
&If{
@@ -468,10 +467,7 @@ func (c *compiler) compileAlt(l, r *Query) error {
return c.compileQuery(r)
}
// compileQueryUpdate compiles an assignment operator. alloc is nil unless the
// assignment may update the input value in place, in which case it yields the
// allocator of the enclosing reduce expression.
func (c *compiler) compileQueryUpdate(l, r *Query, op Operator, alloc *Query) error {
func (c *compiler) compileQueryUpdate(l, r *Query, op Operator) error {
switch op {
case OpAssign:
// optimize assignment operator with constant indexing and slicing
@@ -491,7 +487,12 @@ func (c *compiler) compileQueryUpdate(l, r *Query, op Operator, alloc *Query) er
}
fallthrough
case OpModify:
return c.compileUpdateFunc(op.getFunc(), []*Query{l, r}, alloc)
return c.compileFunc(
&Func{
Name: op.getFunc(),
Args: []*Query{l, r},
},
)
default:
name := "$%0"
c.append(&code{op: opdup})
@@ -499,53 +500,27 @@ func (c *compiler) compileQueryUpdate(l, r *Query, op Operator, alloc *Query) er
return err
}
c.append(&code{op: opstore, v: c.pushVariable(name)})
return c.compileUpdateFunc(
"_modify",
[]*Query{
l,
{Term: &Term{
Type: TermTypeFunc,
Func: &Func{
Name: op.getFunc(),
Args: []*Query{
{Term: &Term{Type: TermTypeIdentity}},
{Term: &Term{Type: TermTypeFunc, Func: &Func{Name: name}}},
return c.compileFunc(
&Func{
Name: "_modify",
Args: []*Query{
l,
{Term: &Term{
Type: TermTypeFunc,
Func: &Func{
Name: op.getFunc(),
Args: []*Query{
{Term: &Term{Type: TermTypeIdentity}},
{Term: &Term{Type: TermTypeFunc, Func: &Func{Name: name}}},
},
},
},
}},
}},
},
},
alloc,
)
}
}
const (
// Both names are deliberately invalid identifiers, so that a query cannot
// reach the allocator variants directly.
allocatorArgSuffix = "/3"
allocatorName = "$%allocator"
)
// compileUpdateFunc calls _assign or _modify, or the variant taking an
// allocator as its third argument.
func (c *compiler) compileUpdateFunc(name string, args []*Query, alloc *Query) error {
if alloc == nil {
return c.compileFunc(&Func{Name: name, Args: args})
}
args = append(args, alloc)
fn := c.lookupBuiltin(name+allocatorArgSuffix, len(args))
if fn == nil {
switch name {
case "_assign":
c.compileAssign(true)
case "_modify":
c.compileModify(true)
}
fn = c.lookupBuiltin(name+allocatorArgSuffix, len(args))
}
return c.compileCallPc(fn, args)
}
func (c *compiler) compileBind(l, r *Query, patterns []*Pattern) error {
defer c.newScopeDepth()()
c.append(&code{op: opdup})
@@ -553,36 +528,23 @@ func (c *compiler) compileBind(l, r *Query, patterns []*Pattern) error {
if err := c.compileQuery(l); err != nil {
return err
}
return c.compilePatterns(patterns, func() error {
if len(patterns) == 1 && c.codes[len(c.codes)-2].op == opexpbegin {
c.codes[len(c.codes)-2].op = opnop
} else {
c.append(&code{op: opexpend})
}
return c.compileQuery(r)
})
}
func (c *compiler) compilePatterns(patterns []*Pattern, body func() error) error {
var pc int
var vs [][2]int
if len(patterns) > 1 {
for _, p := range patterns {
vs = c.pushPatternVariables(vs, p)
}
}
for i, p := range patterns {
var pcc int
var err error
if i < len(patterns)-1 {
defer c.lazy(func() *code {
return &code{op: opforkalt, v: pcc}
})()
}
for _, v := range vs {
c.append(&code{op: oppush, v: nil})
c.append(&code{op: opstore, v: v})
if 0 < i {
for _, v := range vs {
c.append(&code{op: oppush, v: nil})
c.append(&code{op: opstore, v: v})
}
}
if _, err := c.compilePattern(nil, p); err != nil {
if vs, err = c.compilePattern(vs[:0], p); err != nil {
return err
}
if i < len(patterns)-1 {
@@ -595,29 +557,12 @@ func (c *compiler) compilePatterns(patterns []*Pattern, body func() error) error
if len(patterns) > 1 {
pc = len(c.codes)
}
return body()
}
func (c *compiler) pushPatternVariables(vs [][2]int, p *Pattern) [][2]int {
if p.Name != "" {
if v := c.pushVariable(p.Name); !slices.Contains(vs, v) {
vs = append(vs, v)
}
} else if len(p.Array) > 0 {
for _, p := range p.Array {
vs = c.pushPatternVariables(vs, p)
}
} else if len(p.Object) > 0 {
for _, kv := range p.Object {
if kv.Key != "" && kv.Key[0] == '$' {
vs = c.pushPatternVariables(vs, &Pattern{Name: kv.Key})
}
if kv.Val != nil {
vs = c.pushPatternVariables(vs, kv.Val)
}
}
if len(patterns) == 1 && c.codes[len(c.codes)-2].op == opexpbegin {
c.codes[len(c.codes)-2].op = opnop
} else {
c.append(&code{op: opexpend})
}
return vs
return c.compileQuery(r)
}
func (c *compiler) compilePattern(vs [][2]int, p *Pattern) ([][2]int, error) {
@@ -771,46 +716,23 @@ func (c *compiler) compileReduce(e *Reduce) error {
}
f()
c.append(&code{op: opstore, v: v})
// An assignment forming the whole update query is the only thing that
// observes the accumulated value, so it may modify it in place; see
// Query#inPlaceUpdate. The allocator lives across the whole reduction.
l, r, op, inPlace := e.Update.inPlaceUpdate()
var alloc *Query
if inPlace {
c.appends(
&code{op: oppush, v: nil},
&code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}},
&code{op: opstore, v: c.pushVariable(allocatorName)},
)
alloc = &Query{Term: &Term{
Type: TermTypeFunc, Func: &Func{Name: allocatorName},
}}
}
setfork := c.lazy(func() *code {
return &code{op: opfork, v: len(c.codes)}
})
if err := c.compileQuery(e.Query); err != nil {
return err
}
if err := c.compilePatterns(e.Patterns, func() error {
c.append(&code{op: opload, v: v})
f := c.newScopeDepth()
var err error
if inPlace {
err = c.compileQueryUpdate(l, r, op, alloc)
} else {
err = c.compileQuery(e.Update)
}
if err != nil {
return err
}
f()
c.append(&code{op: opstore, v: v})
c.append(&code{op: opbacktrack})
return nil
}); err != nil {
if _, err := c.compilePattern(nil, e.Pattern); err != nil {
return err
}
c.append(&code{op: opload, v: v})
f = c.newScopeDepth()
if err := c.compileQuery(e.Update); err != nil {
return err
}
f()
c.append(&code{op: opstore, v: v})
c.append(&code{op: opbacktrack})
setfork()
c.append(&code{op: oppop})
c.append(&code{op: opload, v: v})
@@ -831,21 +753,22 @@ func (c *compiler) compileForeach(e *Foreach) error {
if err := c.compileQuery(e.Query); err != nil {
return err
}
return c.compilePatterns(e.Patterns, func() error {
c.append(&code{op: opload, v: v})
f := c.newScopeDepth()
if err := c.compileQuery(e.Update); err != nil {
return err
}
f()
c.append(&code{op: opdup})
c.append(&code{op: opstore, v: v})
if e.Extract != nil {
defer c.newScopeDepth()()
return c.compileQuery(e.Extract)
}
return nil
})
if _, err := c.compilePattern(nil, e.Pattern); err != nil {
return err
}
c.append(&code{op: opload, v: v})
f = c.newScopeDepth()
if err := c.compileQuery(e.Update); err != nil {
return err
}
f()
c.append(&code{op: opdup})
c.append(&code{op: opstore, v: v})
if e.Extract != nil {
defer c.newScopeDepth()()
return c.compileQuery(e.Extract)
}
return nil
}
func (c *compiler) compileLabel(e *Label) error {
@@ -1009,9 +932,9 @@ func (c *compiler) compileFunc(e *Func) error {
if !compiled {
switch e.Name {
case "_assign":
c.compileAssign(false)
c.compileAssign()
case "_modify":
c.compileModify(false)
c.compileModify()
case "_last":
c.compileLast()
}
@@ -1107,54 +1030,29 @@ func (c *compiler) compileFunc(e *Func) error {
//
// To overcome the difficulty of reducing allocations on `setpath`, we use the
// `allocator` type and track the allocated addresses during the reduction.
//
// The shared variant reuses the allocator of the enclosing reduce expression
// instead of creating a new one, modifying the value in place.
func (c *compiler) compileAssign(shared bool) {
name, argcnt := "_assign", 2
if shared {
name, argcnt = name+allocatorArgSuffix, 3
}
defer c.appendBuiltin(name, argcnt)()
func (c *compiler) compileAssign() {
defer c.appendBuiltin("_assign", 2)()
scope := c.newScope()
v, p := [2]int{scope.id, 0}, [2]int{scope.id, 1}
x, a := [2]int{scope.id, 2}, [2]int{scope.id, 3}
// Cannot reuse v, p due to backtracking in x.
w, q := [2]int{scope.id, 4}, [2]int{scope.id, 5}
c.appends(
&code{op: opscope, v: [3]int{scope.id, 6, argcnt}},
&code{op: opscope, v: [3]int{scope.id, 6, 2}},
&code{op: opstore, v: v}, // def _assign(p; $x):
&code{op: opstore, v: p},
&code{op: opstore, v: x},
)
if shared {
c.append(&code{op: opstore, v: a}) // the allocator argument
}
c.appends(
&code{op: opload, v: v},
&code{op: opexpbegin},
&code{op: opload, v: x},
&code{op: opcallpc},
&code{op: opstore, v: x},
&code{op: opexpend},
)
if shared {
c.appends(
&code{op: opload, v: v},
&code{op: opload, v: a},
&code{op: opcallpc},
&code{op: opstore, v: a},
)
} else {
c.appends(
&code{op: oppush, v: nil},
&code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}},
&code{op: opstore, v: a},
)
}
c.appends(
&code{op: oppush, v: nil},
&code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}},
&code{op: opstore, v: a},
&code{op: opload, v: v},
&code{op: opfork, v: len(c.codes) + 17}, // reduce [L1]
&code{op: opfork, v: len(c.codes) + 30}, // reduce [L1]
&code{op: opdup},
&code{op: opstore, v: w},
&code{op: oppathbegin}, // path(p)
@@ -1178,49 +1076,24 @@ func (c *compiler) compileAssign(shared bool) {
// Appends the compiled code for the update-assignment operator (`|=`) to
// maximize performance. We use the `allocator` type, just like `_assign/2`.
//
// The shared variant reuses the allocator of the enclosing reduce expression;
// see compiler#compileAssign.
func (c *compiler) compileModify(shared bool) {
name, argcnt := "_modify", 2
if shared {
name, argcnt = name+allocatorArgSuffix, 3
}
defer c.appendBuiltin(name, argcnt)()
func (c *compiler) compileModify() {
defer c.appendBuiltin("_modify", 2)()
scope := c.newScope()
v, p := [2]int{scope.id, 0}, [2]int{scope.id, 1}
f, d := [2]int{scope.id, 2}, [2]int{scope.id, 3}
a, l := [2]int{scope.id, 4}, [2]int{scope.id, 5}
c.appends(
&code{op: opscope, v: [3]int{scope.id, 6, argcnt}},
&code{op: opscope, v: [3]int{scope.id, 6, 2}},
&code{op: opstore, v: v}, // def _modify(p; f):
&code{op: opstore, v: p},
&code{op: opstore, v: f},
)
if shared {
c.append(&code{op: opstore, v: a}) // the allocator argument
}
c.appends(
&code{op: oppush, v: []any{}},
&code{op: opstore, v: d},
)
if shared {
c.appends(
&code{op: opload, v: v},
&code{op: opload, v: a},
&code{op: opcallpc},
&code{op: opstore, v: a},
)
} else {
c.appends(
&code{op: oppush, v: nil},
&code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}},
&code{op: opstore, v: a},
)
}
c.appends(
&code{op: oppush, v: nil},
&code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}},
&code{op: opstore, v: a},
&code{op: opload, v: v},
&code{op: opfork, v: len(c.codes) + 31}, // reduce [L1]
&code{op: opfork, v: len(c.codes) + 40}, // reduce [L1]
&code{op: oppathbegin}, // path(p)
&code{op: opload, v: p},
&code{op: opcallpc},
@@ -1229,7 +1102,7 @@ func (c *compiler) compileModify(shared bool) {
&code{op: opstore, v: p}, // as $p (.;
&code{op: opforklabel, v: l}, // label $l |
&code{op: opload, v: v}, //
&code{op: opfork, v: len(c.codes) + 28}, // [L2]
&code{op: opfork, v: len(c.codes) + 37}, // [L2]
&code{op: oppop}, // (getpath($p) |
&code{op: opload, v: a},
&code{op: opload, v: a},
@@ -1243,8 +1116,8 @@ func (c *compiler) compileModify(shared bool) {
&code{op: opcall, v: [3]any{funcSetpathWithAllocator, 3, "_setpath"}},
&code{op: opstore, v: v},
&code{op: opload, v: v}, // ., break $l
&code{op: opfork, v: len(c.codes) + 26}, // [L4]
&code{op: opjump, v: len(c.codes) + 30}, // [L3]
&code{op: opfork, v: len(c.codes) + 35}, // [L4]
&code{op: opjump, v: len(c.codes) + 39}, // [L3]
&code{op: opload, v: l}, // [L4]
&code{op: opcall, v: [3]any{funcBreak(""), 0, "_break"}},
&code{op: opload, v: p}, // append $p to $d [L2]
@@ -1330,8 +1203,6 @@ func (c *compiler) funcBuiltins(any, []any) any {
}
func (c *compiler) funcInput(any, []any) any {
c.inputMu.Lock()
defer c.inputMu.Unlock()
v, ok := c.inputIter.Next()
if !ok {
return errors.New("break")
compiler_test.go
-90
View File
@@ -7,7 +7,6 @@ import (
"log"
"os"
"reflect"
"runtime"
"sync"
"testing"
"time"
@@ -375,95 +374,6 @@ func TestCodeRun_RaceRegexp(t *testing.T) {
wg.Wait()
}
func TestCodeRun_RaceInputIter(t *testing.T) {
const n = 10
values := make([]int, n)
for i := range values {
values[i] = i
}
query, err := gojq.Parse("input")
if err != nil {
t.Fatal(err)
}
code, err := gojq.Compile(query, gojq.WithInputIter(gojq.NewIter(values...)))
if err != nil {
t.Fatal(err)
}
var seen sync.Map
var wg sync.WaitGroup
for range n {
wg.Add(1)
go func() {
defer wg.Done()
v, ok := code.Run(nil).Next()
if !ok {
t.Error("expected a value")
return
}
if _, loaded := seen.LoadOrStore(v, struct{}{}); loaded {
t.Errorf("input %v was consumed more than once", v)
}
}()
}
wg.Wait()
for i := range n {
if _, ok := seen.Load(i); !ok {
t.Errorf("expected input %d to be consumed", i)
}
}
}
// Building an object or an array of n elements must allocate memory
// proportional to n, not to n squared.
func TestCodeRun_ReduceAssignmentAllocation(t *testing.T) {
for _, src := range []string{
`reduce .[] as $x ({}; .[$x|tostring] = $x)`,
`reduce .[] as $x ({}; .[$x|tostring] |= $x)`,
`reduce .[] as $x ({}; .[$x|tostring] += 1)`,
`reduce .[] as $x ([]; .[$x] = $x)`,
} {
t.Run(src, func(t *testing.T) {
query, err := gojq.Parse(src)
if err != nil {
t.Fatal(err)
}
code, err := gojq.Compile(query)
if err != nil {
t.Fatal(err)
}
small, large := allocatedBytes(t, code, 1000), allocatedBytes(t, code, 4000)
// Quadratic growth would be about sixteen times as much.
if expected := small * 8; large > expected {
t.Errorf("expected at most %v bytes for 4000 elements, got %v (%v bytes for 1000 elements)",
expected, large, small)
}
})
}
}
func allocatedBytes(t *testing.T, code *gojq.Code, n int) uint64 {
t.Helper()
input := make([]any, n)
for i := range input {
input[i] = i
}
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
iter := code.Run(input)
for {
v, ok := iter.Next()
if !ok {
break
}
if err, ok := v.(error); ok {
t.Fatal(err)
}
}
runtime.ReadMemStats(&after)
return after.TotalAlloc - before.TotalAlloc
}
func BenchmarkCompile(b *testing.B) {
cnt, err := os.ReadFile("builtin.jq")
if err != nil {
inplace.go
-204
View File
@@ -1,204 +0,0 @@
package gojq
// Static analysis deciding whether the accumulated value of a reduce
// expression may be updated in place; see compiler#compileReduce. The checks
// are conservative: failing them only keeps the previous copying behavior.
// inPlaceUpdate reports whether the query is a single assignment applied to
// the input value, and returns its operands.
func (e *Query) inPlaceUpdate() (l, r *Query, op Operator, ok bool) {
if e.Term != nil || len(e.FuncDefs) > 0 {
return nil, nil, 0, false
}
switch e.Op {
case OpModify:
// _modify releases the allocator before calling the update function.
return e.Left, e.Right, e.Op, true
case OpAssign:
// A constant path compiles to setpath, which uses no allocator.
if e.Left.toIndices(nil) != nil {
return nil, nil, 0, false
}
fallthrough
case OpUpdateAdd, OpUpdateSub, OpUpdateMul,
OpUpdateDiv, OpUpdateMod, OpUpdateAlt:
// The right-hand side must neither observe the input value, which it
// could otherwise alias, nor yield multiple values, which would run
// the assignment against an already modified input.
if !e.Right.readsInput() && e.Right.singleValued() {
return e.Left, e.Right, e.Op, true
}
}
return nil, nil, 0, false
}
// readsInput reports whether evaluating the query may observe its input value.
// It answers true unless it can prove otherwise.
func (e *Query) readsInput() bool {
if len(e.FuncDefs) > 0 {
return true
}
if e.Term != nil {
return e.Term.readsInput()
}
switch e.Op {
case OpPipe:
// Binding variables also exposes the input to the right-hand side.
if len(e.Patterns) > 0 {
return e.Left.readsInput() || e.Right.readsInput()
}
return e.Left.readsInput()
case OpComma, OpAdd, OpSub, OpMul, OpDiv, OpMod, OpEq, OpNe,
OpGt, OpLt, OpGe, OpLe, OpAnd, OpOr, OpAlt:
return e.Left.readsInput() || e.Right.readsInput()
default:
return true
}
}
// singleValued reports whether the query yields at most one value.
// It answers false unless it can prove otherwise.
func (e *Query) singleValued() bool {
if len(e.FuncDefs) > 0 {
return false
}
if e.Term != nil {
return e.Term.singleValued()
}
switch e.Op {
case OpPipe:
return len(e.Patterns) == 0 &&
e.Left.singleValued() && e.Right.singleValued()
case OpAdd, OpSub, OpMul, OpDiv, OpMod, OpEq, OpNe,
OpGt, OpLt, OpGe, OpLe, OpAnd, OpOr:
return e.Left.singleValued() && e.Right.singleValued()
default:
return false
}
}
func (e *Term) readsInput() bool {
for _, s := range e.SuffixList {
if s.readsInput() {
return true
}
}
switch e.Type {
case TermTypeNull, TermTypeTrue, TermTypeFalse, TermTypeNumber:
return false
case TermTypeString:
return e.Str.readsInput()
case TermTypeObject:
for _, kv := range e.Object.KeyVals {
if kv.readsInput() {
return true
}
}
return false
case TermTypeArray:
return e.Array.Query != nil && e.Array.Query.readsInput()
case TermTypeUnary:
return e.Unary.Term.readsInput()
case TermTypeFunc:
// Any function may be redefined, so only $x is known not to read it.
return e.Func.Name[0] != '$' || len(e.Func.Args) > 0
case TermTypeQuery:
return e.Query.readsInput()
default:
return true
}
}
func (e *Term) singleValued() bool {
for _, s := range e.SuffixList {
if !s.singleValued() {
return false
}
}
switch e.Type {
case TermTypeNull, TermTypeTrue, TermTypeFalse, TermTypeNumber:
return true
case TermTypeString:
return e.Str.singleValued()
case TermTypeObject:
for _, kv := range e.Object.KeyVals {
if !kv.singleValued() {
return false
}
}
return true
case TermTypeArray:
return true
case TermTypeUnary:
return e.Unary.Term.singleValued()
case TermTypeFunc:
return e.Func.Name[0] == '$' && len(e.Func.Args) == 0
case TermTypeQuery:
return e.Query.singleValued()
default:
return false
}
}
func (e *Suffix) readsInput() bool {
// Only an index is evaluated against the input value.
return e.Index != nil && e.Index.readsInput()
}
func (e *Suffix) singleValued() bool {
return !e.Iter && (e.Index == nil || e.Index.singleValued())
}
func (e *Index) readsInput() bool {
if e.Str != nil {
return e.Str.readsInput()
}
return e.Start != nil && e.Start.readsInput() ||
e.End != nil && e.End.readsInput()
}
func (e *Index) singleValued() bool {
if e.Str != nil {
return e.Str.singleValued()
}
return (e.Start == nil || e.Start.singleValued()) &&
(e.End == nil || e.End.singleValued())
}
func (e *String) readsInput() bool {
for _, q := range e.Queries {
if q.readsInput() {
return true
}
}
return false
}
func (e *String) singleValued() bool {
for _, q := range e.Queries {
if !q.singleValued() {
return false
}
}
return true
}
func (e *ObjectKeyVal) readsInput() bool {
if e.Val == nil {
// {foo} reads the input value, {$foo} does not.
return e.Key == "" || e.Key[0] != '$'
}
if e.KeyString != nil && e.KeyString.readsInput() ||
e.KeyQuery != nil && e.KeyQuery.readsInput() {
return true
}
return e.Val.readsInput()
}
func (e *ObjectKeyVal) singleValued() bool {
if e.KeyString != nil && !e.KeyString.singleValued() ||
e.KeyQuery != nil && !e.KeyQuery.singleValued() {
return false
}
return e.Val == nil || e.Val.singleValued()
}