diff --git a/cli/test.yaml b/cli/test.yaml index 630b3b9..a2ef24f 100644 --- a/cli/test.yaml +++ b/cli/test.yaml @@ -710,27 +710,6 @@ error: | expected a number for indexing an array but got: boolean (true) -- name: indexing number literal error - args: - - '-1[0]' - input: 'null' - error: | - expected an array but got: number (1) - -- name: slicing number literal error - args: - - '1 - -1[1:]' - input: 'null' - error: | - expected an array but got: number (1) - -- name: indexing by indexed number literal error - args: - - '.[1[0]]' - input: '[0, 1]' - error: | - expected an array but got: number (1) - - name: array slicing start error args: - '.[[]:0]' @@ -1346,33 +1325,6 @@ expected: | 9223372036854775808 -- name: abs function against non-numbers - args: - - -c - - 'abs' - input: | - "" - "-42" - [] - [1,-2] - {} - {"a":-1} - expected: | - "" - "-42" - [] - [1,-2] - {} - {"a":-1} - -- name: abs function error - args: - - 'abs' - input: 'null true' - error: | - cannot negate: null - cannot negate: boolean (true) - - name: length function args: - 'length' @@ -2478,17 +2430,6 @@ ["b"] [] -- name: split/1 function error - args: - - -c - - 'try split(",") catch ., try split(1) catch .' - input: '1 "a"' - expected: | - "split(\",\") cannot be applied to: number (1)" - "split(1) cannot be applied to: number (1)" - ["a"] - "split(1) cannot be applied to: string (\"a\")" - - name: join function args: - 'join(",")' @@ -3408,13 +3349,6 @@ expected: | [5,5,false,3,3,[null,false],null,false] -- name: optional operator with alternative operator - args: - - '.a ?// 1' - input: 'null' - expected: | - 1 - - name: condition args: - 'if . then . else [.] end' @@ -3630,29 +3564,6 @@ expected: | 5 -- name: reduce with destructuring alternative operator - args: - - 'reduce .[] as [$i,$j] ?// {$i,$j} (0; . + $i * $j)' - input: '[[1,2], {"i":3,"j":4}, [5,6]]' - expected: | - 44 - -- name: reduce with destructuring alternative operator resetting variables - args: - - -c - - 'reduce .[] as [$a] ?// {$b} ?// $c ([]; . + [[$a, $b, $c]])' - input: '[{"b":1}, [2], 3, [4]]' - expected: | - [[null,1,null],[2,null,null],[null,null,3],[4,null,null]] - -- name: reduce with destructuring alternative operator and error backtrack - args: - - -c - - 'reduce .[] as [$a] ?// $a ([]; . + [if $a == 2 then error else $a end])' - input: '[[1], [2], [3]]' - expected: | - [1,[2],3] - - name: reduce null args: - 'reduce . as $n (.; .)' @@ -3731,68 +3642,6 @@ expected: | [[1,1],[1]] -- name: reduce with assignment in update - args: - - -c - - 'reduce range(3) as $i ({}; .[$i|tostring] = {}), - reduce range(3) as $i ({}; .a[$i|tostring] = $i), - reduce range(3) as $i ([]; .[$i] = $i), - reduce range(3) as $i ({}; .[$i|tostring] += 1)' - input: 'null' - expected: | - {"0":{},"1":{},"2":{}} - {"a":{"0":0,"1":1,"2":2}} - [0,1,2] - {"0":1,"1":1,"2":1} - -- name: reduce with assignment of the accumulated value - args: - - -c - - 'reduce range(3) as $i ({}; .a = .), - reduce range(3) as $i ({}; .a = {v: .}), - reduce range(3) as $i ({}; . as $p | .a = $i | .b = $p)' - input: 'null' - expected: | - {"a":{"a":{"a":{}}}} - {"a":{"v":{"a":{"v":{"a":{"v":{}}}}}}} - {"a":2,"b":{"a":1,"b":{"a":0,"b":{}}}} - -- name: reduce with assignment of multiple values - args: - - -c - - 'reduce range(2) as $i ({"k":"k"}; .[.k] = ("x","y")), - reduce range(2) as $i ({}; (.a = 1), (.b = 2))' - input: 'null' - expected: | - {"k":"y","y":"y"} - {"b":2} - -- name: reduce with assignment of a shared value - args: - - -c - - '{"a":1} as $x | reduce range(2) as $i ({}; .[$i|tostring] = $x) | - [(.["0"].a = 9), $x]' - input: 'null' - expected: | - [{"0":{"a":9},"1":{"a":1}},{"a":1}] - -- name: reduce with assignment result reused - args: - - -c - - 'reduce range(3) as $i ({}; .[$i|tostring] = $i) as $r | - [($r | .["0"] = 9), $r]' - input: 'null' - expected: | - [{"0":9,"1":1,"2":2},{"0":0,"1":1,"2":2}] - -- name: foreach with assignment in update - args: - - -c - - '[foreach range(3) as $i ({}; .[$i|tostring] = $i)]' - input: 'null' - expected: | - [{"0":0},{"0":0,"1":1},{"0":0,"1":1,"2":2}] - - name: foreach iterator args: - 'foreach range(5) as $item (0; $item)' @@ -3822,37 +3671,6 @@ -1 -4 -- name: foreach with destructuring alternative operator - args: - - 'foreach .[] as [$i,$j] ?// {$i,$j} (0; . + $i * $j)' - input: '[[1,2], {"i":3,"j":4}, [5,6]]' - expected: | - 2 - 14 - 44 - -- name: foreach with destructuring alternative operator resetting variables - args: - - -c - - 'foreach .[] as [$a] ?// {$b} ?// $c (null; [$a, $b, $c])' - input: '[{"b":1}, [2], 3, [4]]' - expected: | - [null,1,null] - [2,null,null] - [null,null,3] - [4,null,null] - -- name: foreach with destructuring alternative operator and error backtrack in extract - args: - - -c - - 'foreach .[] as {$a} ?// [$a] ?// $a (null; $a; if 1 < . and . < 4 then error else . end)' - input: '[[1], [2], {"a":3}, 4]' - expected: | - 1 - [2] - {"a":3} - 4 - - name: foreach with iterator in update args: - 'foreach .[] as $i (1; ., . + $i, range(.))' @@ -4677,17 +4495,6 @@ [-3,-3,-2,-1,-1,-0,-0,0,0,0,0,1,1,2,3,3] [3.7,3.3,2.5,1.7,1,0.7,0.3,0,0.3,0.5,0.7,1,1.7,2.5,3.3,3.7] -- name: math functions against large number literals - args: - - -c - - '[floor, ceil, sqrt, exp10]' - input: | - 1e1000 - -1e1000 - expected: | - [1.7976931348623157e+308,1.7976931348623157e+308,1.7976931348623157e+308,1.7976931348623157e+308] - [-1.7976931348623157e+308,-1.7976931348623157e+308,null,0] - - name: round, nearbyint, rint functions args: - -c @@ -6068,7 +5875,7 @@ args: - -c - 'gmtime, localtime | map(. * 1000000 | floor / 1000000)' - input: '1500000000 1567890123.456 1600000000.111 -1 -0.5 0.5' + input: '1500000000 1567890123.456 1600000000.111' expected: | # UTC-7 (TZ=Etc/GMT+7) [2017,6,14,2,40,0,5,194] [2017,6,13,19,40,0,4,193] @@ -6076,12 +5883,6 @@ [2019,8,7,14,2,3.456,6,249] [2020,8,13,12,26,40.111,0,256] [2020,8,13,5,26,40.111,0,256] - [1969,11,31,23,59,59,3,364] - [1969,11,31,16,59,59,3,364] - [1969,11,31,23,59,59.5,3,364] - [1969,11,31,16,59,59.5,3,364] - [1970,0,1,0,0,0.5,4,0] - [1969,11,31,17,0,0.5,3,364] - name: mktime, strftime, strflocaltime, todate functions args: @@ -6093,7 +5894,6 @@ [2020,8,13,12,26,40.111,0,256] [2024,7,20] [2025] - [1969,11,31,23,59,59.5,3,364] expected: | [2017,6,14,2,40,0,5,194] 2017-07-14T02:40:00Z @@ -6130,105 +5930,15 @@ 2024-12-31T00:00:00Z 2024-12-30T17:00:00-0700 2024-12-31T00:00:00Z - [1969,11,31,23,59,59.5,3,364] - 1969-12-31T23:59:59Z - 1969-12-31T23:59:59-0700 - -0.5 - 1969-12-31T23:59:59Z - 1969-12-31T16:59:59-0700 - 1969-12-31T23:59:59Z - -- name: date functions with fractional seconds - args: - - -r - - 'mktime, strftime("%Y-%m-%dT%H:%M:%S.%fZ"), strflocaltime("%Y-%m-%dT%H:%M:%S.%f%z")' - input: | - [1970,0,1,0,0,-0.5] - [1970,0,1,0,0,-1.25] - [1970,0,1,0,0,-60.25] - [1970,0,1,0,0,-1] - [1970,0,1,0,0,0] - [1970,0,1,0,0,0.5] - [1970,0,1,0,0,60.25] - expected: | # UTC-7 (TZ=Etc/GMT+7) - -0.5 - 1969-12-31T23:59:59.500000Z - 1969-12-31T23:59:59.500000-0700 - -1.25 - 1969-12-31T23:59:58.750000Z - 1969-12-31T23:59:58.750000-0700 - -60.25 - 1969-12-31T23:58:59.750000Z - 1969-12-31T23:58:59.750000-0700 - -1 - 1969-12-31T23:59:59.000000Z - 1969-12-31T23:59:59.000000-0700 - 0 - 1970-01-01T00:00:00.000000Z - 1970-01-01T00:00:00.000000-0700 - 0.5 - 1970-01-01T00:00:00.500000Z - 1970-01-01T00:00:00.500000-0700 - 60.25 - 1970-01-01T00:01:00.250000Z - 1970-01-01T00:01:00.250000-0700 - -- name: mktime after subtracting fractional seconds - args: - - 'gmtime | .[5] -= 0.5 | mktime' - input: '0 60 86400 -60' - expected: | - -0.5 - 59.5 - 86399.5 - -60.5 - -- name: date functions with non-finite or out of range epoch timestamps - args: - - -n - - '[infinite, -infinite, nan, 1e19, -1e19, 9223372036854775807, -9223372036854775808] | .[] | try gmtime catch .' - expected: | - "gmtime cannot be applied to 1.7976931348623157e+308: number of seconds is out of range" - "gmtime cannot be applied to -1.7976931348623157e+308: number of seconds is out of range" - "gmtime cannot be applied to null: number of seconds is out of range" - "gmtime cannot be applied to 10000000000000000000: number of seconds is out of range" - "gmtime cannot be applied to -10000000000000000000: number of seconds is out of range" - "gmtime cannot be applied to 9223372036854775807: number of seconds is out of range" - "gmtime cannot be applied to -9223372036854775808: number of seconds is out of range" - -- name: date functions with out of range numbers in the array - args: - - -n - - '[[infinite,0,1], [nan,0,1], [1e19,0,1], [0,1e19,1], [2001,0,1,0,0,infinite], [2001,0,1,0,0,nan], [2001,0,1,0,0,1e19], [9223372036854775807,0,1], ["x",0,1]] | .[] | try mktime catch .' - expected: | - "mktime cannot be applied to [1.7976931348623157e+308,0,1]: number is out of range" - "mktime cannot be applied to [null,0,1]: number is out of range" - "mktime cannot be applied to [10000000000000000000,0,1]: number is out of range" - "mktime cannot be applied to [0,10000000000000000000,1]: number is out of range" - "mktime cannot be applied to [2001,0,1,0,0,1.797693134 ...]: number is out of range" - "mktime cannot be applied to [2001,0,1,0,0,null]: number is out of range" - "mktime cannot be applied to [2001,0,1,0,0,10000000000 ...]: number is out of range" - "mktime cannot be applied to [9223372036854775807,0,1]: number is out of range" - "mktime cannot be applied to [\"x\",0,1]: expected an array of 8 numbers" - -- name: strftime function with out of range datetime - args: - - -n - - 'try (infinite | todate) catch ., try ([1e19,0,1] | strftime("%Y")) catch .' - expected: | - "strftime(\"%Y-%m-%dT%H:%M:%SZ\") cannot be applied to 1.7976931348623157e+308: number of seconds is out of range" - "strftime(\"%Y\") cannot be applied to [10000000000000000000,0,1]: number is out of range" - name: strptime, fromdate functions args: - -c - - 'strptime("%Y-%m-%dT%H:%M:%S%z")? // strptime("%Y-%m-%dT%H:%M:%S.%f%z"), fromdate?' + - 'strptime("%Y-%m-%dT%H:%M:%S%z"), fromdate' input: | "2017-07-14T02:40:00Z" "2019-09-08T06:02:03+0900" "2020-09-13T07:26:40-05:00" - "1969-12-31T23:59:59.5+0000" - "2345-01-02T03:04:05.678901+0000" expected: | [2017,6,14,2,40,0,5,194] 1500000000 @@ -6236,8 +5946,6 @@ 1567890123 [2020,8,13,12,26,40,0,256] 1600000000 - [1969,11,31,23,59,59.5,3,364] - [2345,0,2,3,4,5.678901,2,1] - name: strftime, strptime functions args: @@ -6953,49 +6661,6 @@ "aabcABC☆★☆ABCabc" "abcABC☆★☆ABCabc" -- name: match function with empty matches - args: - - -c - - '[match("a*"; "g").offset], [scan("[a-z]*")], gsub("[0-9]*"; "#"), split("-*"; ""), [splits("\\s*")]' - input: '"a1b-c d"' - expected: | - [0,1,2,3,4,5,6,7] - ["a","","b","","c","","d",""] - "#a##b#-#c# #d#" - ["","a","1","b","","c"," ","d",""] - ["","a","1","b","-","c","","d",""] - -- name: match function with empty matches and assertions - args: - - -c - - 'gsub("(?m)^a*"; "#"), gsub("\\ba*"; "#"), gsub("(^|-)a*"; "#"), gsub("[a-z]*\\b"; "#")' - input: '"a-a b"' - expected: | - "#-a b" - "##-## #b#" - "## b" - "##-## ##" - -- name: match function with empty matches and captures - args: - - -c - - '[scan("(x)?a*")], [match("(a|)"; "g") | [.offset, .captures[0].offset]], [scan("[^a]*"; "i")]' - input: '"1a-A"' - expected: | - [[null],[null],[null],[null],[null]] - [[0,0],[1,1],[2,2],[3,3],[4,4]] - ["1","","-","",""] - -- name: match function with empty matches against multibyte characters - args: - - -c - - '[scan("[☆★]*")], [match("[☆★]*"; "g").offset], gsub("[☆★]*"; "#")' - input: '"☆★1☆★"' - expected: | - ["☆★","","☆★",""] - [0,2,3,5] - "##1##" - - name: INDEX function args: - -c @@ -7293,17 +6958,6 @@ {"a":3} {"a":4} -- name: destructuring alternative operator with break in try - args: - - -c - - 'first((. as {$a} ?// [$a] ?// $a | {$a})?)' - input: '[1] [2] {"a":3} 4' - expected: | - {"a":1} - {"a":2} - {"a":3} - {"a":4} - - name: destructuring alternative operator with halt args: - -c @@ -7323,28 +6977,6 @@ [1] exit_code: 5 -- name: destructuring alternative operator with halt_error in try - args: - - -c - - 'try (. as {$a} ?// [$a] ?// $a | {$a}, halt_error)' - input: '[1]' - expected: | - {"a":1} - error: | - [1] - exit_code: 5 - -- name: destructuring alternative operator resets variables of later alternatives - args: - - -c - - '.[] as [$a] ?// {$b} ?// $c | [$a, $b, $c]' - input: '[{"b":1}, [2], 3, [4]]' - expected: | - [null,1,null] - [2,null,null] - [null,null,3] - [4,null,null] - - name: destructuring alternative operator with multiple variables args: - -c @@ -8008,7 +7640,7 @@ [["c"],3] [["c"]] -- name: stream option with unterminated input after a comma +- name: stream option with unterminated input args: - -c - --stream @@ -8021,21 +7653,6 @@ {"a":1, ^ unexpected EOF -- name: stream option with unterminated input after a value - args: - - -c - - --stream - - '.' - input: '[1,[2,3' - expected: | - [[0],1] - [[1,0],2] - [[1,1],3] - error: | - invalid json: - [1,[2,3 - ^ unexpected EOF - - name: yaml input option args: - --yaml-input @@ -8205,17 +7822,6 @@ 2 | bar: ] ^ did not find expected node content -- name: yaml input option error with multibyte characters - args: - - --yaml-input - - '.' - input: | - {123: 4, 123: 5} - error: | - invalid yaml: :1 - 1 | {123: 4, 123: 5} - ^ mapping key "123" already defined at line 1 - - name: yaml input option multiple errors args: - --yaml-input diff --git a/compiler.go b/compiler.go index 3dc5866..050951b 100644 --- a/compiler.go +++ b/compiler.go @@ -17,7 +17,6 @@ type compiler struct { variables []string customFuncs map[string]function inputIter Iter - inputMu sync.Mutex codes []*code codeinfos []codeinfo builtinScope *scopeinfo @@ -394,7 +393,7 @@ func (c *compiler) compileQuery(e *Query) error { return c.compileAlt(e.Left, e.Right) case OpAssign, OpModify, OpUpdateAdd, OpUpdateSub, OpUpdateMul, OpUpdateDiv, OpUpdateMod, OpUpdateAlt: - return c.compileQueryUpdate(e.Left, e.Right, e.Op, nil) + return c.compileQueryUpdate(e.Left, e.Right, e.Op) case OpOr: return c.compileIf( &If{ @@ -468,10 +467,7 @@ func (c *compiler) compileAlt(l, r *Query) error { return c.compileQuery(r) } -// compileQueryUpdate compiles an assignment operator. alloc is nil unless the -// assignment may update the input value in place, in which case it yields the -// allocator of the enclosing reduce expression. -func (c *compiler) compileQueryUpdate(l, r *Query, op Operator, alloc *Query) error { +func (c *compiler) compileQueryUpdate(l, r *Query, op Operator) error { switch op { case OpAssign: // optimize assignment operator with constant indexing and slicing @@ -491,7 +487,12 @@ func (c *compiler) compileQueryUpdate(l, r *Query, op Operator, alloc *Query) er } fallthrough case OpModify: - return c.compileUpdateFunc(op.getFunc(), []*Query{l, r}, alloc) + return c.compileFunc( + &Func{ + Name: op.getFunc(), + Args: []*Query{l, r}, + }, + ) default: name := "$%0" c.append(&code{op: opdup}) @@ -499,53 +500,27 @@ func (c *compiler) compileQueryUpdate(l, r *Query, op Operator, alloc *Query) er return err } c.append(&code{op: opstore, v: c.pushVariable(name)}) - return c.compileUpdateFunc( - "_modify", - []*Query{ - l, - {Term: &Term{ - Type: TermTypeFunc, - Func: &Func{ - Name: op.getFunc(), - Args: []*Query{ - {Term: &Term{Type: TermTypeIdentity}}, - {Term: &Term{Type: TermTypeFunc, Func: &Func{Name: name}}}, + return c.compileFunc( + &Func{ + Name: "_modify", + Args: []*Query{ + l, + {Term: &Term{ + Type: TermTypeFunc, + Func: &Func{ + Name: op.getFunc(), + Args: []*Query{ + {Term: &Term{Type: TermTypeIdentity}}, + {Term: &Term{Type: TermTypeFunc, Func: &Func{Name: name}}}, + }, }, - }, - }}, + }}, + }, }, - alloc, ) } } -const ( - // Both names are deliberately invalid identifiers, so that a query cannot - // reach the allocator variants directly. - allocatorArgSuffix = "/3" - allocatorName = "$%allocator" -) - -// compileUpdateFunc calls _assign or _modify, or the variant taking an -// allocator as its third argument. -func (c *compiler) compileUpdateFunc(name string, args []*Query, alloc *Query) error { - if alloc == nil { - return c.compileFunc(&Func{Name: name, Args: args}) - } - args = append(args, alloc) - fn := c.lookupBuiltin(name+allocatorArgSuffix, len(args)) - if fn == nil { - switch name { - case "_assign": - c.compileAssign(true) - case "_modify": - c.compileModify(true) - } - fn = c.lookupBuiltin(name+allocatorArgSuffix, len(args)) - } - return c.compileCallPc(fn, args) -} - func (c *compiler) compileBind(l, r *Query, patterns []*Pattern) error { defer c.newScopeDepth()() c.append(&code{op: opdup}) @@ -553,36 +528,23 @@ func (c *compiler) compileBind(l, r *Query, patterns []*Pattern) error { if err := c.compileQuery(l); err != nil { return err } - return c.compilePatterns(patterns, func() error { - if len(patterns) == 1 && c.codes[len(c.codes)-2].op == opexpbegin { - c.codes[len(c.codes)-2].op = opnop - } else { - c.append(&code{op: opexpend}) - } - return c.compileQuery(r) - }) -} - -func (c *compiler) compilePatterns(patterns []*Pattern, body func() error) error { var pc int var vs [][2]int - if len(patterns) > 1 { - for _, p := range patterns { - vs = c.pushPatternVariables(vs, p) - } - } for i, p := range patterns { var pcc int + var err error if i < len(patterns)-1 { defer c.lazy(func() *code { return &code{op: opforkalt, v: pcc} })() } - for _, v := range vs { - c.append(&code{op: oppush, v: nil}) - c.append(&code{op: opstore, v: v}) + if 0 < i { + for _, v := range vs { + c.append(&code{op: oppush, v: nil}) + c.append(&code{op: opstore, v: v}) + } } - if _, err := c.compilePattern(nil, p); err != nil { + if vs, err = c.compilePattern(vs[:0], p); err != nil { return err } if i < len(patterns)-1 { @@ -595,29 +557,12 @@ func (c *compiler) compilePatterns(patterns []*Pattern, body func() error) error if len(patterns) > 1 { pc = len(c.codes) } - return body() -} - -func (c *compiler) pushPatternVariables(vs [][2]int, p *Pattern) [][2]int { - if p.Name != "" { - if v := c.pushVariable(p.Name); !slices.Contains(vs, v) { - vs = append(vs, v) - } - } else if len(p.Array) > 0 { - for _, p := range p.Array { - vs = c.pushPatternVariables(vs, p) - } - } else if len(p.Object) > 0 { - for _, kv := range p.Object { - if kv.Key != "" && kv.Key[0] == '$' { - vs = c.pushPatternVariables(vs, &Pattern{Name: kv.Key}) - } - if kv.Val != nil { - vs = c.pushPatternVariables(vs, kv.Val) - } - } + if len(patterns) == 1 && c.codes[len(c.codes)-2].op == opexpbegin { + c.codes[len(c.codes)-2].op = opnop + } else { + c.append(&code{op: opexpend}) } - return vs + return c.compileQuery(r) } func (c *compiler) compilePattern(vs [][2]int, p *Pattern) ([][2]int, error) { @@ -771,46 +716,23 @@ func (c *compiler) compileReduce(e *Reduce) error { } f() c.append(&code{op: opstore, v: v}) - // An assignment forming the whole update query is the only thing that - // observes the accumulated value, so it may modify it in place; see - // Query#inPlaceUpdate. The allocator lives across the whole reduction. - l, r, op, inPlace := e.Update.inPlaceUpdate() - var alloc *Query - if inPlace { - c.appends( - &code{op: oppush, v: nil}, - &code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}}, - &code{op: opstore, v: c.pushVariable(allocatorName)}, - ) - alloc = &Query{Term: &Term{ - Type: TermTypeFunc, Func: &Func{Name: allocatorName}, - }} - } setfork := c.lazy(func() *code { return &code{op: opfork, v: len(c.codes)} }) if err := c.compileQuery(e.Query); err != nil { return err } - if err := c.compilePatterns(e.Patterns, func() error { - c.append(&code{op: opload, v: v}) - f := c.newScopeDepth() - var err error - if inPlace { - err = c.compileQueryUpdate(l, r, op, alloc) - } else { - err = c.compileQuery(e.Update) - } - if err != nil { - return err - } - f() - c.append(&code{op: opstore, v: v}) - c.append(&code{op: opbacktrack}) - return nil - }); err != nil { + if _, err := c.compilePattern(nil, e.Pattern); err != nil { return err } + c.append(&code{op: opload, v: v}) + f = c.newScopeDepth() + if err := c.compileQuery(e.Update); err != nil { + return err + } + f() + c.append(&code{op: opstore, v: v}) + c.append(&code{op: opbacktrack}) setfork() c.append(&code{op: oppop}) c.append(&code{op: opload, v: v}) @@ -831,21 +753,22 @@ func (c *compiler) compileForeach(e *Foreach) error { if err := c.compileQuery(e.Query); err != nil { return err } - return c.compilePatterns(e.Patterns, func() error { - c.append(&code{op: opload, v: v}) - f := c.newScopeDepth() - if err := c.compileQuery(e.Update); err != nil { - return err - } - f() - c.append(&code{op: opdup}) - c.append(&code{op: opstore, v: v}) - if e.Extract != nil { - defer c.newScopeDepth()() - return c.compileQuery(e.Extract) - } - return nil - }) + if _, err := c.compilePattern(nil, e.Pattern); err != nil { + return err + } + c.append(&code{op: opload, v: v}) + f = c.newScopeDepth() + if err := c.compileQuery(e.Update); err != nil { + return err + } + f() + c.append(&code{op: opdup}) + c.append(&code{op: opstore, v: v}) + if e.Extract != nil { + defer c.newScopeDepth()() + return c.compileQuery(e.Extract) + } + return nil } func (c *compiler) compileLabel(e *Label) error { @@ -1009,9 +932,9 @@ func (c *compiler) compileFunc(e *Func) error { if !compiled { switch e.Name { case "_assign": - c.compileAssign(false) + c.compileAssign() case "_modify": - c.compileModify(false) + c.compileModify() case "_last": c.compileLast() } @@ -1107,54 +1030,29 @@ func (c *compiler) compileFunc(e *Func) error { // // To overcome the difficulty of reducing allocations on `setpath`, we use the // `allocator` type and track the allocated addresses during the reduction. -// -// The shared variant reuses the allocator of the enclosing reduce expression -// instead of creating a new one, modifying the value in place. -func (c *compiler) compileAssign(shared bool) { - name, argcnt := "_assign", 2 - if shared { - name, argcnt = name+allocatorArgSuffix, 3 - } - defer c.appendBuiltin(name, argcnt)() +func (c *compiler) compileAssign() { + defer c.appendBuiltin("_assign", 2)() scope := c.newScope() v, p := [2]int{scope.id, 0}, [2]int{scope.id, 1} x, a := [2]int{scope.id, 2}, [2]int{scope.id, 3} // Cannot reuse v, p due to backtracking in x. w, q := [2]int{scope.id, 4}, [2]int{scope.id, 5} c.appends( - &code{op: opscope, v: [3]int{scope.id, 6, argcnt}}, + &code{op: opscope, v: [3]int{scope.id, 6, 2}}, &code{op: opstore, v: v}, // def _assign(p; $x): &code{op: opstore, v: p}, &code{op: opstore, v: x}, - ) - if shared { - c.append(&code{op: opstore, v: a}) // the allocator argument - } - c.appends( &code{op: opload, v: v}, &code{op: opexpbegin}, &code{op: opload, v: x}, &code{op: opcallpc}, &code{op: opstore, v: x}, &code{op: opexpend}, - ) - if shared { - c.appends( - &code{op: opload, v: v}, - &code{op: opload, v: a}, - &code{op: opcallpc}, - &code{op: opstore, v: a}, - ) - } else { - c.appends( - &code{op: oppush, v: nil}, - &code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}}, - &code{op: opstore, v: a}, - ) - } - c.appends( + &code{op: oppush, v: nil}, + &code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}}, + &code{op: opstore, v: a}, &code{op: opload, v: v}, - &code{op: opfork, v: len(c.codes) + 17}, // reduce [L1] + &code{op: opfork, v: len(c.codes) + 30}, // reduce [L1] &code{op: opdup}, &code{op: opstore, v: w}, &code{op: oppathbegin}, // path(p) @@ -1178,49 +1076,24 @@ func (c *compiler) compileAssign(shared bool) { // Appends the compiled code for the update-assignment operator (`|=`) to // maximize performance. We use the `allocator` type, just like `_assign/2`. -// -// The shared variant reuses the allocator of the enclosing reduce expression; -// see compiler#compileAssign. -func (c *compiler) compileModify(shared bool) { - name, argcnt := "_modify", 2 - if shared { - name, argcnt = name+allocatorArgSuffix, 3 - } - defer c.appendBuiltin(name, argcnt)() +func (c *compiler) compileModify() { + defer c.appendBuiltin("_modify", 2)() scope := c.newScope() v, p := [2]int{scope.id, 0}, [2]int{scope.id, 1} f, d := [2]int{scope.id, 2}, [2]int{scope.id, 3} a, l := [2]int{scope.id, 4}, [2]int{scope.id, 5} c.appends( - &code{op: opscope, v: [3]int{scope.id, 6, argcnt}}, + &code{op: opscope, v: [3]int{scope.id, 6, 2}}, &code{op: opstore, v: v}, // def _modify(p; f): &code{op: opstore, v: p}, &code{op: opstore, v: f}, - ) - if shared { - c.append(&code{op: opstore, v: a}) // the allocator argument - } - c.appends( &code{op: oppush, v: []any{}}, &code{op: opstore, v: d}, - ) - if shared { - c.appends( - &code{op: opload, v: v}, - &code{op: opload, v: a}, - &code{op: opcallpc}, - &code{op: opstore, v: a}, - ) - } else { - c.appends( - &code{op: oppush, v: nil}, - &code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}}, - &code{op: opstore, v: a}, - ) - } - c.appends( + &code{op: oppush, v: nil}, + &code{op: opcall, v: [3]any{funcAllocator, 0, "_allocator"}}, + &code{op: opstore, v: a}, &code{op: opload, v: v}, - &code{op: opfork, v: len(c.codes) + 31}, // reduce [L1] + &code{op: opfork, v: len(c.codes) + 40}, // reduce [L1] &code{op: oppathbegin}, // path(p) &code{op: opload, v: p}, &code{op: opcallpc}, @@ -1229,7 +1102,7 @@ func (c *compiler) compileModify(shared bool) { &code{op: opstore, v: p}, // as $p (.; &code{op: opforklabel, v: l}, // label $l | &code{op: opload, v: v}, // - &code{op: opfork, v: len(c.codes) + 28}, // [L2] + &code{op: opfork, v: len(c.codes) + 37}, // [L2] &code{op: oppop}, // (getpath($p) | &code{op: opload, v: a}, &code{op: opload, v: a}, @@ -1243,8 +1116,8 @@ func (c *compiler) compileModify(shared bool) { &code{op: opcall, v: [3]any{funcSetpathWithAllocator, 3, "_setpath"}}, &code{op: opstore, v: v}, &code{op: opload, v: v}, // ., break $l - &code{op: opfork, v: len(c.codes) + 26}, // [L4] - &code{op: opjump, v: len(c.codes) + 30}, // [L3] + &code{op: opfork, v: len(c.codes) + 35}, // [L4] + &code{op: opjump, v: len(c.codes) + 39}, // [L3] &code{op: opload, v: l}, // [L4] &code{op: opcall, v: [3]any{funcBreak(""), 0, "_break"}}, &code{op: opload, v: p}, // append $p to $d [L2] @@ -1330,8 +1203,6 @@ func (c *compiler) funcBuiltins(any, []any) any { } func (c *compiler) funcInput(any, []any) any { - c.inputMu.Lock() - defer c.inputMu.Unlock() v, ok := c.inputIter.Next() if !ok { return errors.New("break") diff --git a/compiler_test.go b/compiler_test.go index 47601e1..fe67bf5 100644 --- a/compiler_test.go +++ b/compiler_test.go @@ -7,7 +7,6 @@ import ( "log" "os" "reflect" - "runtime" "sync" "testing" "time" @@ -375,95 +374,6 @@ func TestCodeRun_RaceRegexp(t *testing.T) { wg.Wait() } -func TestCodeRun_RaceInputIter(t *testing.T) { - const n = 10 - values := make([]int, n) - for i := range values { - values[i] = i - } - query, err := gojq.Parse("input") - if err != nil { - t.Fatal(err) - } - code, err := gojq.Compile(query, gojq.WithInputIter(gojq.NewIter(values...))) - if err != nil { - t.Fatal(err) - } - var seen sync.Map - var wg sync.WaitGroup - for range n { - wg.Add(1) - go func() { - defer wg.Done() - v, ok := code.Run(nil).Next() - if !ok { - t.Error("expected a value") - return - } - if _, loaded := seen.LoadOrStore(v, struct{}{}); loaded { - t.Errorf("input %v was consumed more than once", v) - } - }() - } - wg.Wait() - for i := range n { - if _, ok := seen.Load(i); !ok { - t.Errorf("expected input %d to be consumed", i) - } - } -} - -// Building an object or an array of n elements must allocate memory -// proportional to n, not to n squared. -func TestCodeRun_ReduceAssignmentAllocation(t *testing.T) { - for _, src := range []string{ - `reduce .[] as $x ({}; .[$x|tostring] = $x)`, - `reduce .[] as $x ({}; .[$x|tostring] |= $x)`, - `reduce .[] as $x ({}; .[$x|tostring] += 1)`, - `reduce .[] as $x ([]; .[$x] = $x)`, - } { - t.Run(src, func(t *testing.T) { - query, err := gojq.Parse(src) - if err != nil { - t.Fatal(err) - } - code, err := gojq.Compile(query) - if err != nil { - t.Fatal(err) - } - small, large := allocatedBytes(t, code, 1000), allocatedBytes(t, code, 4000) - // Quadratic growth would be about sixteen times as much. - if expected := small * 8; large > expected { - t.Errorf("expected at most %v bytes for 4000 elements, got %v (%v bytes for 1000 elements)", - expected, large, small) - } - }) - } -} - -func allocatedBytes(t *testing.T, code *gojq.Code, n int) uint64 { - t.Helper() - input := make([]any, n) - for i := range input { - input[i] = i - } - var before, after runtime.MemStats - runtime.GC() - runtime.ReadMemStats(&before) - iter := code.Run(input) - for { - v, ok := iter.Next() - if !ok { - break - } - if err, ok := v.(error); ok { - t.Fatal(err) - } - } - runtime.ReadMemStats(&after) - return after.TotalAlloc - before.TotalAlloc -} - func BenchmarkCompile(b *testing.B) { cnt, err := os.ReadFile("builtin.jq") if err != nil { diff --git a/inplace.go b/inplace.go deleted file mode 100644 index 442babb..0000000 --- a/inplace.go +++ /dev/null @@ -1,204 +0,0 @@ -package gojq - -// Static analysis deciding whether the accumulated value of a reduce -// expression may be updated in place; see compiler#compileReduce. The checks -// are conservative: failing them only keeps the previous copying behavior. - -// inPlaceUpdate reports whether the query is a single assignment applied to -// the input value, and returns its operands. -func (e *Query) inPlaceUpdate() (l, r *Query, op Operator, ok bool) { - if e.Term != nil || len(e.FuncDefs) > 0 { - return nil, nil, 0, false - } - switch e.Op { - case OpModify: - // _modify releases the allocator before calling the update function. - return e.Left, e.Right, e.Op, true - case OpAssign: - // A constant path compiles to setpath, which uses no allocator. - if e.Left.toIndices(nil) != nil { - return nil, nil, 0, false - } - fallthrough - case OpUpdateAdd, OpUpdateSub, OpUpdateMul, - OpUpdateDiv, OpUpdateMod, OpUpdateAlt: - // The right-hand side must neither observe the input value, which it - // could otherwise alias, nor yield multiple values, which would run - // the assignment against an already modified input. - if !e.Right.readsInput() && e.Right.singleValued() { - return e.Left, e.Right, e.Op, true - } - } - return nil, nil, 0, false -} - -// readsInput reports whether evaluating the query may observe its input value. -// It answers true unless it can prove otherwise. -func (e *Query) readsInput() bool { - if len(e.FuncDefs) > 0 { - return true - } - if e.Term != nil { - return e.Term.readsInput() - } - switch e.Op { - case OpPipe: - // Binding variables also exposes the input to the right-hand side. - if len(e.Patterns) > 0 { - return e.Left.readsInput() || e.Right.readsInput() - } - return e.Left.readsInput() - case OpComma, OpAdd, OpSub, OpMul, OpDiv, OpMod, OpEq, OpNe, - OpGt, OpLt, OpGe, OpLe, OpAnd, OpOr, OpAlt: - return e.Left.readsInput() || e.Right.readsInput() - default: - return true - } -} - -// singleValued reports whether the query yields at most one value. -// It answers false unless it can prove otherwise. -func (e *Query) singleValued() bool { - if len(e.FuncDefs) > 0 { - return false - } - if e.Term != nil { - return e.Term.singleValued() - } - switch e.Op { - case OpPipe: - return len(e.Patterns) == 0 && - e.Left.singleValued() && e.Right.singleValued() - case OpAdd, OpSub, OpMul, OpDiv, OpMod, OpEq, OpNe, - OpGt, OpLt, OpGe, OpLe, OpAnd, OpOr: - return e.Left.singleValued() && e.Right.singleValued() - default: - return false - } -} - -func (e *Term) readsInput() bool { - for _, s := range e.SuffixList { - if s.readsInput() { - return true - } - } - switch e.Type { - case TermTypeNull, TermTypeTrue, TermTypeFalse, TermTypeNumber: - return false - case TermTypeString: - return e.Str.readsInput() - case TermTypeObject: - for _, kv := range e.Object.KeyVals { - if kv.readsInput() { - return true - } - } - return false - case TermTypeArray: - return e.Array.Query != nil && e.Array.Query.readsInput() - case TermTypeUnary: - return e.Unary.Term.readsInput() - case TermTypeFunc: - // Any function may be redefined, so only $x is known not to read it. - return e.Func.Name[0] != '$' || len(e.Func.Args) > 0 - case TermTypeQuery: - return e.Query.readsInput() - default: - return true - } -} - -func (e *Term) singleValued() bool { - for _, s := range e.SuffixList { - if !s.singleValued() { - return false - } - } - switch e.Type { - case TermTypeNull, TermTypeTrue, TermTypeFalse, TermTypeNumber: - return true - case TermTypeString: - return e.Str.singleValued() - case TermTypeObject: - for _, kv := range e.Object.KeyVals { - if !kv.singleValued() { - return false - } - } - return true - case TermTypeArray: - return true - case TermTypeUnary: - return e.Unary.Term.singleValued() - case TermTypeFunc: - return e.Func.Name[0] == '$' && len(e.Func.Args) == 0 - case TermTypeQuery: - return e.Query.singleValued() - default: - return false - } -} - -func (e *Suffix) readsInput() bool { - // Only an index is evaluated against the input value. - return e.Index != nil && e.Index.readsInput() -} - -func (e *Suffix) singleValued() bool { - return !e.Iter && (e.Index == nil || e.Index.singleValued()) -} - -func (e *Index) readsInput() bool { - if e.Str != nil { - return e.Str.readsInput() - } - return e.Start != nil && e.Start.readsInput() || - e.End != nil && e.End.readsInput() -} - -func (e *Index) singleValued() bool { - if e.Str != nil { - return e.Str.singleValued() - } - return (e.Start == nil || e.Start.singleValued()) && - (e.End == nil || e.End.singleValued()) -} - -func (e *String) readsInput() bool { - for _, q := range e.Queries { - if q.readsInput() { - return true - } - } - return false -} - -func (e *String) singleValued() bool { - for _, q := range e.Queries { - if !q.singleValued() { - return false - } - } - return true -} - -func (e *ObjectKeyVal) readsInput() bool { - if e.Val == nil { - // {foo} reads the input value, {$foo} does not. - return e.Key == "" || e.Key[0] != '$' - } - if e.KeyString != nil && e.KeyString.readsInput() || - e.KeyQuery != nil && e.KeyQuery.readsInput() { - return true - } - return e.Val.readsInput() -} - -func (e *ObjectKeyVal) singleValued() bool { - if e.KeyString != nil && !e.KeyString.singleValued() || - e.KeyQuery != nil && !e.KeyQuery.singleValued() { - return false - } - return e.Val == nil || e.Val.singleValued() -}