diff --git a/cli/test.yaml b/cli/test.yaml index 809d68c..357f58f 100644 --- a/cli/test.yaml +++ b/cli/test.yaml @@ -710,6 +710,27 @@ error: | expected a number for indexing an array but got: boolean (true) +- name: indexing number literal error + args: + - '-1[0]' + input: 'null' + error: | + expected an array but got: number (1) + +- name: slicing number literal error + args: + - '1 - -1[1:]' + input: 'null' + error: | + expected an array but got: number (1) + +- name: indexing by indexed number literal error + args: + - '.[1[0]]' + input: '[0, 1]' + error: | + expected an array but got: number (1) + - name: array slicing start error args: - '.[[]:0]' @@ -1325,6 +1346,33 @@ expected: | 9223372036854775808 +- name: abs function against non-numbers + args: + - -c + - 'abs' + input: | + "" + "-42" + [] + [1,-2] + {} + {"a":-1} + expected: | + "" + "-42" + [] + [1,-2] + {} + {"a":-1} + +- name: abs function error + args: + - 'abs' + input: 'null true' + error: | + cannot negate: null + cannot negate: boolean (true) + - name: length function args: - 'length' @@ -2407,6 +2455,40 @@ "/a,b/c/d/e,f/" ",/ /a/,/b/,/ /c/,/ /d/,/ /e/,/f/,/ " +- name: split/1 function against empty string + args: + - -c + - '.[] as $t | .[] as $s | $s | split($t)' + input: '["abc", "a", "b", ""]' + expected: | + ["",""] + ["a"] + ["b"] + [] + ["","bc"] + ["",""] + ["b"] + [] + ["a","c"] + ["a"] + ["",""] + [] + ["a","b","c"] + ["a"] + ["b"] + [] + +- name: split/1 function error + args: + - -c + - 'try split(",") catch ., try split(1) catch .' + input: '1 "a"' + expected: | + "split(\",\") cannot be applied to: number (1)" + "split(1) cannot be applied to: number (1)" + ["a"] + "split(1) cannot be applied to: string (\"a\")" + - name: join function args: - 'join(",")' @@ -3326,6 +3408,13 @@ expected: | [5,5,false,3,3,[null,false],null,false] +- name: optional operator with alternative operator + args: + - '.a ?// 1' + input: 'null' + expected: | + 1 + - name: condition args: - 'if . then . else [.] end' @@ -3541,6 +3630,29 @@ expected: | 5 +- name: reduce with destructuring alternative operator + args: + - 'reduce .[] as [$i,$j] ?// {$i,$j} (0; . + $i * $j)' + input: '[[1,2], {"i":3,"j":4}, [5,6]]' + expected: | + 44 + +- name: reduce with destructuring alternative operator resetting variables + args: + - -c + - 'reduce .[] as [$a] ?// {$b} ?// $c ([]; . + [[$a, $b, $c]])' + input: '[{"b":1}, [2], 3, [4]]' + expected: | + [[null,1,null],[2,null,null],[null,null,3],[4,null,null]] + +- name: reduce with destructuring alternative operator and error backtrack + args: + - -c + - 'reduce .[] as [$a] ?// $a ([]; . + [if $a == 2 then error else $a end])' + input: '[[1], [2], [3]]' + expected: | + [1,[2],3] + - name: reduce null args: - 'reduce . as $n (.; .)' @@ -3585,6 +3697,16 @@ expected: | 8 +- name: reduce with assignment of multiple paths in update + args: + - -c + - 'reduce (1,2) as $i (.; (.[0], .[.[0]]) = $i), + reduce (1,2) as $i (.; (.[0], .[.[0]]) |= $i)' + input: '[0,0,0]' + expected: | + [2,2,0] + [2,2,0] + - name: reduce with query in start args: - 'reduce range(5) as $x (range(3); . + $x)' @@ -3638,6 +3760,37 @@ -1 -4 +- name: foreach with destructuring alternative operator + args: + - 'foreach .[] as [$i,$j] ?// {$i,$j} (0; . + $i * $j)' + input: '[[1,2], {"i":3,"j":4}, [5,6]]' + expected: | + 2 + 14 + 44 + +- name: foreach with destructuring alternative operator resetting variables + args: + - -c + - 'foreach .[] as [$a] ?// {$b} ?// $c (null; [$a, $b, $c])' + input: '[{"b":1}, [2], 3, [4]]' + expected: | + [null,1,null] + [2,null,null] + [null,null,3] + [4,null,null] + +- name: foreach with destructuring alternative operator and error backtrack in extract + args: + - -c + - 'foreach .[] as {$a} ?// [$a] ?// $a (null; $a; if 1 < . and . < 4 then error else . end)' + input: '[[1], [2], {"a":3}, 4]' + expected: | + 1 + [2] + {"a":3} + 4 + - name: foreach with iterator in update args: - 'foreach .[] as $i (1; ., . + $i, range(.))' @@ -4462,6 +4615,17 @@ [-3,-3,-2,-1,-1,-0,-0,0,0,0,0,1,1,2,3,3] [3.7,3.3,2.5,1.7,1,0.7,0.3,0,0.3,0.5,0.7,1,1.7,2.5,3.3,3.7] +- name: math functions against large number literals + args: + - -c + - '[floor, ceil, sqrt, exp10]' + input: | + 1e1000 + -1e1000 + expected: | + [1.7976931348623157e+308,1.7976931348623157e+308,1.7976931348623157e+308,1.7976931348623157e+308] + [-1.7976931348623157e+308,-1.7976931348623157e+308,null,0] + - name: round, nearbyint, rint functions args: - -c @@ -4989,6 +5153,46 @@ expected: | {"a":[{},{"b":[1,2,3]}]} +- name: getpath function against string + args: + - -c + - '[getpath([{"start":1,"end":3}]), getpath([{"start":-2,"end":null}]), getpath([1]), getpath([])]' + input: '"abcde"' + expected: | + ["bc","de","b","abcde"] + +- name: getpath function against string in a container + args: + - -c + - 'getpath(["a",{"start":0,"end":2}]), getpath(["a",1])' + input: '{"a":"abc"}' + expected: | + "ab" + "b" + +- name: getpath function against string round trip with path function + args: + - -c + - '[getpath(path(.[1:3])), getpath(path(.[1]))]' + input: '"abcde"' + expected: | + ["bc","b"] + +- name: getpath function against string round trip in a container + args: + - -c + - '. as $d | [path(.a[0:2]), path(.b[1])] | map(. as $p | $d | getpath($p))' + input: '{"a":"abc","b":"xyz"}' + expected: | + ["ab","y"] + +- name: getpath function against string with object key + args: + - 'getpath(["k"])' + input: '"abc"' + error: | + getpath(["k"]) cannot be applied to "abc": expected an object but got: string ("abc") + - name: getpath function error args: - 'getpath(["a",1,"b","c"])' @@ -5021,9 +5225,9 @@ expected: | {"a":[{"b":5}]} {"a":[{"b":5}],"b":0} - "getpath([\"a\",0,\"b\"]) cannot be applied to: object ({\"a\":0})" + "getpath([\"a\",0,\"b\"]) cannot be applied to {\"a\":0}: expected an array but got: number (0)" {"a":[{"b":5}]} - "getpath([\"a\",0,\"b\"]) cannot be applied to: object ({\"a\":[0,1]})" + "getpath([\"a\",0,\"b\"]) cannot be applied to {\"a\":[0,1]}: expected an object but got: number (0)" "getpath([\"a\",0,\"b\"]) cannot be applied to {\"a\":{\"b\":1}}: expected an array but got: object ({\"b\":1})" {"a":[{"b":5}]} {"a":[{"b":5,"c":3}]} @@ -5437,6 +5641,14 @@ expected: | {"a":{"p":{"b":{"p":1,"q":1},"z":{"p":null,"q":null}},"q":{"b":{"p":1,"q":1}}}} +- name: update-assignment operator against nested object with overlapping paths + args: + - -c + - '(.a.b.c,.a,.a.p.b.z) |= {p: ., q: .}' + input: '{"a":{"b":{"c":1}}}' + expected: | + {"a":{"p":{"b":{"c":{"p":1,"q":1},"z":{"p":null,"q":null}}},"q":{"b":{"c":{"p":1,"q":1}}}}} + - name: update-assignment operator with optional operator args: - -c @@ -5794,7 +6006,7 @@ args: - -c - 'gmtime, localtime | map(. * 1000000 | floor / 1000000)' - input: '1500000000 1567890123.456 1600000000.111' + input: '1500000000 1567890123.456 1600000000.111 -1 -0.5 0.5' expected: | # UTC-7 (TZ=Etc/GMT+7) [2017,6,14,2,40,0,5,194] [2017,6,13,19,40,0,4,193] @@ -5802,6 +6014,12 @@ [2019,8,7,14,2,3.456,6,249] [2020,8,13,12,26,40.111,0,256] [2020,8,13,5,26,40.111,0,256] + [1969,11,31,23,59,59,3,364] + [1969,11,31,16,59,59,3,364] + [1969,11,31,23,59,59.5,3,364] + [1969,11,31,16,59,59.5,3,364] + [1970,0,1,0,0,0.5,4,0] + [1969,11,31,17,0,0.5,3,364] - name: mktime, strftime, strflocaltime, todate functions args: @@ -5813,6 +6031,7 @@ [2020,8,13,12,26,40.111,0,256] [2024,7,20] [2025] + [1969,11,31,23,59,59.5,3,364] expected: | [2017,6,14,2,40,0,5,194] 2017-07-14T02:40:00Z @@ -5849,15 +6068,105 @@ 2024-12-31T00:00:00Z 2024-12-30T17:00:00-0700 2024-12-31T00:00:00Z + [1969,11,31,23,59,59.5,3,364] + 1969-12-31T23:59:59Z + 1969-12-31T23:59:59-0700 + -0.5 + 1969-12-31T23:59:59Z + 1969-12-31T16:59:59-0700 + 1969-12-31T23:59:59Z + +- name: date functions with fractional seconds + args: + - -r + - 'mktime, strftime("%Y-%m-%dT%H:%M:%S.%fZ"), strflocaltime("%Y-%m-%dT%H:%M:%S.%f%z")' + input: | + [1970,0,1,0,0,-0.5] + [1970,0,1,0,0,-1.25] + [1970,0,1,0,0,-60.25] + [1970,0,1,0,0,-1] + [1970,0,1,0,0,0] + [1970,0,1,0,0,0.5] + [1970,0,1,0,0,60.25] + expected: | # UTC-7 (TZ=Etc/GMT+7) + -0.5 + 1969-12-31T23:59:59.500000Z + 1969-12-31T23:59:59.500000-0700 + -1.25 + 1969-12-31T23:59:58.750000Z + 1969-12-31T23:59:58.750000-0700 + -60.25 + 1969-12-31T23:58:59.750000Z + 1969-12-31T23:58:59.750000-0700 + -1 + 1969-12-31T23:59:59.000000Z + 1969-12-31T23:59:59.000000-0700 + 0 + 1970-01-01T00:00:00.000000Z + 1970-01-01T00:00:00.000000-0700 + 0.5 + 1970-01-01T00:00:00.500000Z + 1970-01-01T00:00:00.500000-0700 + 60.25 + 1970-01-01T00:01:00.250000Z + 1970-01-01T00:01:00.250000-0700 + +- name: mktime after subtracting fractional seconds + args: + - 'gmtime | .[5] -= 0.5 | mktime' + input: '0 60 86400 -60' + expected: | + -0.5 + 59.5 + 86399.5 + -60.5 + +- name: date functions with non-finite or out of range epoch timestamps + args: + - -n + - '[infinite, -infinite, nan, 1e19, -1e19, 9223372036854775807, -9223372036854775808] | .[] | try gmtime catch .' + expected: | + "gmtime cannot be applied to 1.7976931348623157e+308: number of seconds is out of range" + "gmtime cannot be applied to -1.7976931348623157e+308: number of seconds is out of range" + "gmtime cannot be applied to null: number of seconds is out of range" + "gmtime cannot be applied to 10000000000000000000: number of seconds is out of range" + "gmtime cannot be applied to -10000000000000000000: number of seconds is out of range" + "gmtime cannot be applied to 9223372036854775807: number of seconds is out of range" + "gmtime cannot be applied to -9223372036854775808: number of seconds is out of range" + +- name: date functions with out of range numbers in the array + args: + - -n + - '[[infinite,0,1], [nan,0,1], [1e19,0,1], [0,1e19,1], [2001,0,1,0,0,infinite], [2001,0,1,0,0,nan], [2001,0,1,0,0,1e19], [9223372036854775807,0,1], ["x",0,1]] | .[] | try mktime catch .' + expected: | + "mktime cannot be applied to [1.7976931348623157e+308,0,1]: number is out of range" + "mktime cannot be applied to [null,0,1]: number is out of range" + "mktime cannot be applied to [10000000000000000000,0,1]: number is out of range" + "mktime cannot be applied to [0,10000000000000000000,1]: number is out of range" + "mktime cannot be applied to [2001,0,1,0,0,1.797693134 ...]: number is out of range" + "mktime cannot be applied to [2001,0,1,0,0,null]: number is out of range" + "mktime cannot be applied to [2001,0,1,0,0,10000000000 ...]: number is out of range" + "mktime cannot be applied to [9223372036854775807,0,1]: number is out of range" + "mktime cannot be applied to [\"x\",0,1]: expected an array of 8 numbers" + +- name: strftime function with out of range datetime + args: + - -n + - 'try (infinite | todate) catch ., try ([1e19,0,1] | strftime("%Y")) catch .' + expected: | + "strftime(\"%Y-%m-%dT%H:%M:%SZ\") cannot be applied to 1.7976931348623157e+308: number of seconds is out of range" + "strftime(\"%Y\") cannot be applied to [10000000000000000000,0,1]: number is out of range" - name: strptime, fromdate functions args: - -c - - 'strptime("%Y-%m-%dT%H:%M:%S%z"), fromdate' + - 'strptime("%Y-%m-%dT%H:%M:%S%z")? // strptime("%Y-%m-%dT%H:%M:%S.%f%z"), fromdate?' input: | "2017-07-14T02:40:00Z" "2019-09-08T06:02:03+0900" "2020-09-13T07:26:40-05:00" + "1969-12-31T23:59:59.5+0000" + "2345-01-02T03:04:05.678901+0000" expected: | [2017,6,14,2,40,0,5,194] 1500000000 @@ -5865,6 +6174,8 @@ 1567890123 [2020,8,13,12,26,40,0,256] 1600000000 + [1969,11,31,23,59,59.5,3,364] + [2345,0,2,3,4,5.678901,2,1] - name: strftime, strptime functions args: @@ -6580,6 +6891,49 @@ "aabcABC☆★☆ABCabc" "abcABC☆★☆ABCabc" +- name: match function with empty matches + args: + - -c + - '[match("a*"; "g").offset], [scan("[a-z]*")], gsub("[0-9]*"; "#"), split("-*"; ""), [splits("\\s*")]' + input: '"a1b-c d"' + expected: | + [0,1,2,3,4,5,6,7] + ["a","","b","","c","","d",""] + "#a##b#-#c# #d#" + ["","a","1","b","","c"," ","d",""] + ["","a","1","b","-","c","","d",""] + +- name: match function with empty matches and assertions + args: + - -c + - 'gsub("(?m)^a*"; "#"), gsub("\\ba*"; "#"), gsub("(^|-)a*"; "#"), gsub("[a-z]*\\b"; "#")' + input: '"a-a b"' + expected: | + "#-a b" + "##-## #b#" + "## b" + "##-## ##" + +- name: match function with empty matches and captures + args: + - -c + - '[scan("(x)?a*")], [match("(a|)"; "g") | [.offset, .captures[0].offset]], [scan("[^a]*"; "i")]' + input: '"1a-A"' + expected: | + [[null],[null],[null],[null],[null]] + [[0,0],[1,1],[2,2],[3,3],[4,4]] + ["1","","-","",""] + +- name: match function with empty matches against multibyte characters + args: + - -c + - '[scan("[☆★]*")], [match("[☆★]*"; "g").offset], gsub("[☆★]*"; "#")' + input: '"☆★1☆★"' + expected: | + ["☆★","","☆★",""] + [0,2,3,5] + "##1##" + - name: INDEX function args: - -c @@ -6877,6 +7231,17 @@ {"a":3} {"a":4} +- name: destructuring alternative operator with break in try + args: + - -c + - 'first((. as {$a} ?// [$a] ?// $a | {$a})?)' + input: '[1] [2] {"a":3} 4' + expected: | + {"a":1} + {"a":2} + {"a":3} + {"a":4} + - name: destructuring alternative operator with halt args: - -c @@ -6896,6 +7261,28 @@ [1] exit_code: 5 +- name: destructuring alternative operator with halt_error in try + args: + - -c + - 'try (. as {$a} ?// [$a] ?// $a | {$a}, halt_error)' + input: '[1]' + expected: | + {"a":1} + error: | + [1] + exit_code: 5 + +- name: destructuring alternative operator resets variables of later alternatives + args: + - -c + - '.[] as [$a] ?// {$b} ?// $c | [$a, $b, $c]' + input: '[{"b":1}, [2], 3, [4]]' + expected: | + [null,1,null] + [2,null,null] + [null,null,3] + [4,null,null] + - name: destructuring alternative operator with multiple variables args: - -c @@ -7559,7 +7946,7 @@ [["c"],3] [["c"]] -- name: stream option with unterminated input +- name: stream option with unterminated input after a comma args: - -c - --stream @@ -7572,6 +7959,21 @@ {"a":1, ^ unexpected EOF +- name: stream option with unterminated input after a value + args: + - -c + - --stream + - '.' + input: '[1,[2,3' + expected: | + [[0],1] + [[1,0],2] + [[1,1],3] + error: | + invalid json: + [1,[2,3 + ^ unexpected EOF + - name: yaml input option args: - --yaml-input @@ -7741,6 +8143,17 @@ 2 | bar: ] ^ did not find expected node content +- name: yaml input option error with multibyte characters + args: + - --yaml-input + - '.' + input: | + {１２３: 4, １２３: 5} + error: | + invalid yaml: :1 + 1 | {１２３: 4, １２３: 5} + ^ mapping key "１２３" already defined at line 1 + - name: yaml input option multiple errors args: - --yaml-input diff --git a/compiler.go b/compiler.go index 477650c..ad79aa5 100644 --- a/compiler.go +++ b/compiler.go @@ -17,6 +17,7 @@ type compiler struct { variables []string customFuncs map[string]function inputIter Iter + inputMu sync.Mutex codes []*code codeinfos []codeinfo builtinScope *scopeinfo @@ -132,18 +133,13 @@ func (c *compiler) compile(q *Query) error { } func (c *compiler) compileImport(i *Import) error { - var path, alias string - var err error - if i.ImportPath != "" { - path, alias = i.ImportPath, i.ImportAlias - } else { - path = i.IncludePath - } + path, alias := cmp.Or(i.ImportPath, i.IncludePath), i.ImportAlias if c.moduleLoader == nil { return fmt.Errorf("cannot load module: %q", path) } if strings.HasPrefix(alias, "$") { var vals any + var err error if moduleLoader, ok := c.moduleLoader.(interface { LoadJSONWithMeta(string, map[string]any) (any, error) }); ok { @@ -166,6 +162,7 @@ func (c *compiler) compileImport(i *Import) error { return nil } var q *Query + var err error if moduleLoader, ok := c.moduleLoader.(interface { LoadModuleWithMeta(string, map[string]any) (*Query, error) }); ok { @@ -532,23 +529,36 @@ func (c *compiler) compileBind(l, r *Query, patterns []*Pattern) error { if err := c.compileQuery(l); err != nil { return err } + return c.compilePatterns(patterns, func() error { + if len(patterns) == 1 && c.codes[len(c.codes)-2].op == opexpbegin { + c.codes[len(c.codes)-2].op = opnop + } else { + c.append(&code{op: opexpend}) + } + return c.compileQuery(r) + }) +} + +func (c *compiler) compilePatterns(patterns []*Pattern, body func() error) error { var pc int var vs [][2]int + if len(patterns) > 1 { + for _, p := range patterns { + vs = c.pushPatternVariables(vs, p) + } + } for i, p := range patterns { var pcc int - var err error if i < len(patterns)-1 { defer c.lazy(func() *code { return &code{op: opforkalt, v: pcc} })() } - if 0 < i { - for _, v := range vs { - c.append(&code{op: oppush, v: nil}) - c.append(&code{op: opstore, v: v}) - } + for _, v := range vs { + c.append(&code{op: oppush, v: nil}) + c.append(&code{op: opstore, v: v}) } - if vs, err = c.compilePattern(vs[:0], p); err != nil { + if _, err := c.compilePattern(nil, p); err != nil { return err } if i < len(patterns)-1 { @@ -561,12 +571,29 @@ func (c *compiler) compileBind(l, r *Query, patterns []*Pattern) error { if len(patterns) > 1 { pc = len(c.codes) } - if len(patterns) == 1 && c.codes[len(c.codes)-2].op == opexpbegin { - c.codes[len(c.codes)-2].op = opnop - } else { - c.append(&code{op: opexpend}) + return body() +} + +func (c *compiler) pushPatternVariables(vs [][2]int, p *Pattern) [][2]int { + if p.Name != "" { + if v := c.pushVariable(p.Name); !slices.Contains(vs, v) { + vs = append(vs, v) + } + } else if len(p.Array) > 0 { + for _, p := range p.Array { + vs = c.pushPatternVariables(vs, p) + } + } else if len(p.Object) > 0 { + for _, kv := range p.Object { + if kv.Key != "" && kv.Key[0] == '$' { + vs = c.pushPatternVariables(vs, &Pattern{Name: kv.Key}) + } + if kv.Val != nil { + vs = c.pushPatternVariables(vs, kv.Val) + } + } } - return c.compileQuery(r) + return vs } func (c *compiler) compilePattern(vs [][2]int, p *Pattern) ([][2]int, error) { @@ -726,17 +753,19 @@ func (c *compiler) compileReduce(e *Reduce) error { if err := c.compileQuery(e.Query); err != nil { return err } - if _, err := c.compilePattern(nil, e.Pattern); err != nil { + if err := c.compilePatterns(e.Patterns, func() error { + c.append(&code{op: opload, v: v}) + f := c.newScopeDepth() + if err := c.compileQuery(e.Update); err != nil { + return err + } + f() + c.append(&code{op: opstore, v: v}) + c.append(&code{op: opbacktrack}) + return nil + }); err != nil { return err } - c.append(&code{op: opload, v: v}) - f = c.newScopeDepth() - if err := c.compileQuery(e.Update); err != nil { - return err - } - f() - c.append(&code{op: opstore, v: v}) - c.append(&code{op: opbacktrack}) setfork() c.append(&code{op: oppop}) c.append(&code{op: opload, v: v}) @@ -757,22 +786,21 @@ func (c *compiler) compileForeach(e *Foreach) error { if err := c.compileQuery(e.Query); err != nil { return err } - if _, err := c.compilePattern(nil, e.Pattern); err != nil { - return err - } - c.append(&code{op: opload, v: v}) - f = c.newScopeDepth() - if err := c.compileQuery(e.Update); err != nil { - return err - } - f() - c.append(&code{op: opdup}) - c.append(&code{op: opstore, v: v}) - if e.Extract != nil { - defer c.newScopeDepth()() - return c.compileQuery(e.Extract) - } - return nil + return c.compilePatterns(e.Patterns, func() error { + c.append(&code{op: opload, v: v}) + f := c.newScopeDepth() + if err := c.compileQuery(e.Update); err != nil { + return err + } + f() + c.append(&code{op: opdup}) + c.append(&code{op: opstore, v: v}) + if e.Extract != nil { + defer c.newScopeDepth()() + return c.compileQuery(e.Extract) + } + return nil + }) } func (c *compiler) compileLabel(e *Label) error { @@ -1207,6 +1235,8 @@ func (c *compiler) funcBuiltins(any, []any) any { } func (c *compiler) funcInput(any, []any) any { + c.inputMu.Lock() + defer c.inputMu.Unlock() v, ok := c.inputIter.Next() if !ok { return errors.New("break") @@ -1274,11 +1304,7 @@ func listModuleDeps(q *Query) []any { if v == nil { v = make(map[string]any) } - relpath := i.ImportPath - if relpath == "" { - relpath = i.IncludePath - } - v["relpath"] = relpath + v["relpath"] = cmp.Or(i.ImportPath, i.IncludePath) if i.ImportAlias != "" { v["as"] = strings.TrimPrefix(i.ImportAlias, "$") } diff --git a/compiler_test.go b/compiler_test.go new file mode 100644 index 0000000..a817734 --- /dev/null +++ b/compiler_test.go @@ -0,0 +1,433 @@ +package gojq_test + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "reflect" + "sync" + "testing" + "time" + "unsafe" + + "github.com/itchyny/gojq" +) + +func ExampleCompile() { + query, err := gojq.Parse(".[] | .foo") + if err != nil { + log.Fatalln(err) + } + code, err := gojq.Compile(query) + if err != nil { + log.Fatalln(err) + } + iter := code.Run([]any{ + nil, + "string", + 42, + []any{"foo"}, + map[string]any{"foo": 42}, + }) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + fmt.Println(err) + continue + } + fmt.Printf("%#v\n", v) + } + + // Output: + // + // expected an object but got: string ("string") + // expected an object but got: number (42) + // expected an object but got: array (["foo"]) + // 42 +} + +func ExampleCode_Run() { + query, err := gojq.Parse(".foo") + if err != nil { + log.Fatalln(err) + } + code, err := gojq.Compile(query) + if err != nil { + log.Fatalln(err) + } + input := map[string]any{"foo": 42} + iter := code.Run(input) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + log.Fatalln(err) + } + fmt.Printf("%#v\n", v) + } + + // Output: + // 42 +} + +func ExampleCode_RunWithContext() { + query, err := gojq.Parse("def f: f; f, f") + if err != nil { + log.Fatalln(err) + } + code, err := gojq.Compile(query) + if err != nil { + log.Fatalln(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + iter := code.RunWithContext(ctx, nil) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + fmt.Println(err) + continue + } + _ = v + } + + // Output: + // context deadline exceeded +} + +func TestCodeCompile_OptimizeConstants(t *testing.T) { + query, err := gojq.Parse(`[1,{foo:2,"bar":+3},[-4]]`) + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query) + if err != nil { + t.Fatal(err) + } + codes := reflect.ValueOf(code).Elem().FieldByName("codes") + if got, expected := codes.Len(), 3; expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } + iter := code.Run(nil) + for { + got, ok := iter.Next() + if !ok { + break + } + if expected := []any{ + 1, map[string]any{"foo": 2, "bar": 3}, []any{-4}, + }; !reflect.DeepEqual(got, expected) { + t.Errorf("expected: %v, got: %v", expected, got) + } + } +} + +func TestCodeCompile_OptimizeIndexSlice(t *testing.T) { + query, err := gojq.Parse(`.foo."bar".["baz"].[-1]."".[0:1]`) + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query) + if err != nil { + t.Fatal(err) + } + codes := reflect.ValueOf(code).Elem().FieldByName("codes") + if got, expected := codes.Len(), 8; expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } + iter := code.Run(nil) + for { + got, ok := iter.Next() + if !ok { + break + } + if got != nil { + t.Errorf("expected: %v, got: %v", nil, got) + } + } +} + +func TestCodeCompile_OptimizeIndexSliceAssign(t *testing.T) { + query, err := gojq.Parse(`.foo."bar".["baz"].[0]."".[0:1] = [0]`) + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query) + if err != nil { + t.Fatal(err) + } + codes := reflect.ValueOf(code).Elem().FieldByName("codes") + if got, expected := codes.Len(), 8; expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } + iter := code.Run(nil) + for { + got, ok := iter.Next() + if !ok { + break + } + if expected := map[string]any{ + "foo": map[string]any{ + "bar": map[string]any{ + "baz": []any{map[string]any{"": []any{0}}}, + }, + }, + }; !reflect.DeepEqual(got, expected) { + t.Errorf("expected: %v, got: %v", expected, got) + } + } +} + +func TestCodeCompile_OptimizeTailRec_While(t *testing.T) { + query, err := gojq.Parse("0 | while(. < 10; . + 1)") + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query) + if err != nil { + t.Fatal(err) + } + codes := reflect.ValueOf(code).Elem().FieldByName("codes") + if got, expected := codes.Len(), 48; expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } + op1 := codes.Index(2).Elem().FieldByName("op") + op2 := codes.Index(21).Elem().FieldByName("op") // test jump of call _while + if got, expected := *(*int)(unsafe.Pointer(op2.UnsafeAddr())), + *(*int)(unsafe.Pointer(op1.UnsafeAddr())); expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } + iter := code.Run(nil) + n := 0 + for { + got, ok := iter.Next() + if !ok { + break + } + if !reflect.DeepEqual(got, n) { + t.Errorf("expected: %v, got: %v", n, got) + } + n++ + } + if expected := 10; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } +} + +func TestCodeCompile_OptimizeTailRec_CallRec(t *testing.T) { + query, err := gojq.Parse(` + def f: . as $x | $x, (if $x < 3 then $x + 1 | f else empty end), $x; f + `) + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query) + if err != nil { + t.Fatal(err) + } + codes := reflect.ValueOf(code).Elem().FieldByName("codes") + if got, expected := codes.Len(), 48; expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } + op1 := codes.Index(39).Elem().FieldByName("op") // callrec f + op2 := codes.Index(38).Elem().FieldByName("op") // call _add/2 + if got, expected := *(*int)(unsafe.Pointer(op2.UnsafeAddr()))+1, + *(*int)(unsafe.Pointer(op1.UnsafeAddr())); expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } +} + +func TestCodeCompile_OptimizeJumps(t *testing.T) { + query, err := gojq.Parse("def f: 1; def g: 2; def h: 3; f") + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query) + if err != nil { + t.Fatal(err) + } + codes := reflect.ValueOf(code).Elem().FieldByName("codes") + if got, expected := codes.Len(), 15; expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } + v := codes.Index(1).Elem().FieldByName("v") + if got, expected := *(*any)(unsafe.Pointer(v.UnsafeAddr())), 13; expected != got { + t.Errorf("expected: %v, got: %v", expected, got) + } + iter := code.Run(nil) + for { + got, ok := iter.Next() + if !ok { + break + } + if expected := 1; !reflect.DeepEqual(got, expected) { + t.Errorf("expected: %v, got: %v", expected, got) + } + } +} + +func TestParseErrorTokenOffset(t *testing.T) { + testCases := []struct { + src string + offset int + }{ + {src: "^", offset: 1}, + {src: " ^", offset: 2}, + {src: " ^ ", offset: 2}, + {src: "👍", offset: 4}, + {src: " 👍", offset: 5}, + {src: " 👍 ", offset: 5}, + {src: "test👍", offset: 8}, + } + for _, tc := range testCases { + t.Run(tc.src, func(t *testing.T) { + _, err := gojq.Parse(tc.src) + if err == nil { + t.Fatal("expected: error") + } + var pe *gojq.ParseError + if !errors.As(err, &pe) { + t.Fatalf("expected: *gojq.ParseError, got %v", err) + } + if pe.Offset != tc.offset { + t.Fatalf("expected: %v, got %v", tc.offset, pe.Offset) + } + }) + } +} + +func TestCodeRun_Race(t *testing.T) { + query, err := gojq.Parse("range(10)") + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for range 10 { + wg.Add(1) + go func() { + defer wg.Done() + iter := code.Run(nil) + n := 0 + for { + got, ok := iter.Next() + if !ok { + break + } + if got != n { + t.Errorf("expected: %v, got: %v", n, got) + } + n++ + } + if expected := 10; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } + }() + } + wg.Wait() +} + +func TestCodeRun_RaceRegexp(t *testing.T) { + query, err := gojq.Parse(` + "abcdefg" | test("cd"), split("e") | . == true or length > 0 + `) + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for range 10 { + wg.Add(1) + go func() { + defer wg.Done() + iter := code.Run(nil) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + t.Errorf("unexpected error: %v", err) + } + if v != true { + t.Errorf("expected: true, got: %v", v) + } + } + }() + } + wg.Wait() +} + +func TestCodeRun_RaceInputIter(t *testing.T) { + const n = 10 + values := make([]int, n) + for i := range values { + values[i] = i + } + query, err := gojq.Parse("input") + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query, gojq.WithInputIter(gojq.NewIter(values...))) + if err != nil { + t.Fatal(err) + } + var seen sync.Map + var wg sync.WaitGroup + for range n { + wg.Add(1) + go func() { + defer wg.Done() + v, ok := code.Run(nil).Next() + if !ok { + t.Error("expected a value") + return + } + if _, loaded := seen.LoadOrStore(v, struct{}{}); loaded { + t.Errorf("input %v was consumed more than once", v) + } + }() + } + wg.Wait() + for i := range n { + if _, ok := seen.Load(i); !ok { + t.Errorf("expected input %d to be consumed", i) + } + } +} + +func BenchmarkCompile(b *testing.B) { + cnt, err := os.ReadFile("builtin.jq") + if err != nil { + b.Fatal(err) + } + query, err := gojq.Parse(string(cnt) + ".") + if err != nil { + b.Fatal(err) + } + for b.Loop() { + _, err := gojq.Compile( + query, + gojq.WithInputIter(gojq.NewIter[any]()), + ) + if err != nil { + b.Fatal(err) + } + } +}