diff --git a/pkg/yqlib/operator_sort.go b/pkg/yqlib/operator_sort.go new file mode 100644 index 0000000..7f364ec --- /dev/null +++ b/pkg/yqlib/operator_sort.go @@ -0,0 +1,194 @@ +package yqlib + +import ( + "container/list" + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +func sortOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} + expressionNode.RHS = selfExpression + return sortByOperator(d, context, expressionNode) +} + +// context represents the current matching nodes in the expression pipeline +// expressionNode is your current expression (sort_by) +func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + + results := list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + var sortableArray sortableNodeArray + + if candidate.CanVisitValues() { + sortableArray = make(sortableNodeArray, 0) + visitor := func(valueNode *CandidateNode) error { + compareContext, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(valueNode), expressionNode.RHS) + if err != nil { + return err + } + sortableNode := sortableNode{Node: valueNode, CompareContext: compareContext, dateTimeLayout: context.GetDateTimeLayout()} + sortableArray = append(sortableArray, sortableNode) + return nil + } + if err := candidate.VisitValues(visitor); err != nil { + return context, err + } + } else { + return context, fmt.Errorf("node at path [%v] is not an array or map (it's a %v)", candidate.GetNicePath(), candidate.Tag) + } + + sort.Stable(sortableArray) + + sortedList := candidate.CopyWithoutContent() + switch candidate.Kind { + case MappingNode: + for _, sortedNode := range sortableArray { + sortedList.AddKeyValueChild(sortedNode.Node.Key, sortedNode.Node) + } + case SequenceNode: + for _, sortedNode := range sortableArray { + sortedList.AddChild(sortedNode.Node) + } + } + + // convert array of value nodes back to map + results.PushBack(sortedList) + } + return context.ChildContext(results), nil +} + +type sortableNode struct { + Node *CandidateNode + CompareContext Context + dateTimeLayout string +} + +type sortableNodeArray []sortableNode + +func (a sortableNodeArray) Len() int { return len(a) } +func (a sortableNodeArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +func (a sortableNodeArray) Less(i, j int) bool { + lhsContext := a[i].CompareContext + rhsContext := a[j].CompareContext + + rhsEl := rhsContext.MatchingNodes.Front() + + for lhsEl := lhsContext.MatchingNodes.Front(); lhsEl != nil && rhsEl != nil; lhsEl = lhsEl.Next() { + lhs := lhsEl.Value.(*CandidateNode) + rhs := rhsEl.Value.(*CandidateNode) + + result := a.compare(lhs, rhs, a[i].dateTimeLayout) + + if result < 0 { + return true + } else if result > 0 { + return false + } + + rhsEl = rhsEl.Next() + } + return lhsContext.MatchingNodes.Len() < rhsContext.MatchingNodes.Len() +} + +func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateTimeLayout string) int { + lhsTag := lhs.Tag + rhsTag := rhs.Tag + + if !strings.HasPrefix(lhsTag, "!!") { + // custom tag - we have to have a guess + lhsTag = lhs.guessTagFromCustomType() + } + + if !strings.HasPrefix(rhsTag, "!!") { + // custom tag - we have to have a guess + rhsTag = rhs.guessTagFromCustomType() + } + + isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp" + layout := dateTimeLayout + // if the lhs is a string, it might be a timestamp in a custom format. + if lhsTag == "!!str" && layout != time.RFC3339 { + _, errLhs := parseDateTime(layout, lhs.Value) + _, errRhs := parseDateTime(layout, rhs.Value) + isDateTime = errLhs == nil && errRhs == nil + } + + if lhsTag == "!!null" && rhsTag != "!!null" { + return -1 + } else if lhsTag != "!!null" && rhsTag == "!!null" { + return 1 + } else if lhsTag == "!!bool" && rhsTag != "!!bool" { + return -1 + } else if lhsTag != "!!bool" && rhsTag == "!!bool" { + return 1 + } else if lhsTag == "!!bool" && rhsTag == "!!bool" { + lhsTruthy := isTruthyNode(lhs) + + rhsTruthy := isTruthyNode(rhs) + if lhsTruthy == rhsTruthy { + return 0 + } else if lhsTruthy { + return 1 + } + return -1 + } else if isDateTime { + lhsTime, err := parseDateTime(layout, lhs.Value) + if err != nil { + log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %v", lhs.Value, layout, err) + return strings.Compare(lhs.Value, rhs.Value) + } + rhsTime, err := parseDateTime(layout, rhs.Value) + if err != nil { + log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %v", rhs.Value, layout, err) + return strings.Compare(lhs.Value, rhs.Value) + } + if lhsTime.Equal(rhsTime) { + return 0 + } else if lhsTime.Before(rhsTime) { + return -1 + } + + return 1 + } else if lhsTag == "!!int" && rhsTag == "!!int" { + _, lhsNum, err := parseInt64(lhs.Value) + if err != nil { + panic(err) + } + _, rhsNum, err := parseInt64(rhs.Value) + if err != nil { + panic(err) + } + if lhsNum < rhsNum { + return -1 + } else if lhsNum > rhsNum { + return 1 + } + return 0 + } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { + lhsNum, err := strconv.ParseFloat(lhs.Value, 64) + if err != nil { + panic(err) + } + rhsNum, err := strconv.ParseFloat(rhs.Value, 64) + if err != nil { + panic(err) + } + if lhsNum == rhsNum { + return 0 + } else if lhsNum < rhsNum { + return -1 + } + + return 1 + } + + return strings.Compare(lhs.Value, rhs.Value) +} diff --git a/pkg/yqlib/operator_sort_test.go b/pkg/yqlib/operator_sort_test.go new file mode 100644 index 0000000..33f1f5d --- /dev/null +++ b/pkg/yqlib/operator_sort_test.go @@ -0,0 +1,190 @@ +package yqlib + +import "testing" + +var sortByOperatorScenarios = []expressionScenario{ + { + description: "Sort by string field", + document: "[{a: banana},{a: cat},{a: apple}]", + expression: `sort_by(.a)`, + expected: []string{ + "D0, P[], (!!seq)::[{a: apple}, {a: banana}, {a: cat}]\n", + }, + }, + { + description: "Sort by string field", + skipDoc: true, + document: "[{a: banana},{a: apple}]", + expression: `sort_by(.a)[]`, + expected: []string{ + "D0, P[0], (!!map)::{a: apple}\n", + "D0, P[1], (!!map)::{a: banana}\n", + }, + }, + { + description: "Sort by with null", + skipDoc: true, + document: "[{a: banana},null,{a: apple}]", + expression: `sort_by(.a)[]`, + expected: []string{ + "D0, P[0], (!!null)::null\n", + "D0, P[1], (!!map)::{a: apple}\n", + "D0, P[2], (!!map)::{a: banana}\n", + }, + }, + { + description: "Sort by multiple fields", + document: "[{a: dog},{a: cat, b: banana},{a: cat, b: apple}]", + expression: `sort_by(.a, .b)`, + expected: []string{ + "D0, P[], (!!seq)::[{a: cat, b: apple}, {a: cat, b: banana}, {a: dog}]\n", + }, + }, + { + description: "Sort by multiple fields", + skipDoc: true, + document: "[{a: dog, b: good},{a: cat, c: things},{a: cat, b: apple}]", + expression: `sort_by(.a, .b)`, + expected: []string{ + "D0, P[], (!!seq)::[{a: cat, c: things}, {a: cat, b: apple}, {a: dog, b: good}]\n", + }, + }, + { + description: "Sort by multiple fields", + skipDoc: true, + document: "[{a: dog, b: 0.1},{a: cat, b: 0.01},{a: cat, b: 0.001}]", + expression: `sort_by(.a, .b)`, + expected: []string{ + "D0, P[], (!!seq)::[{a: cat, b: 0.001}, {a: cat, b: 0.01}, {a: dog, b: 0.1}]\n", + }, + }, + { + description: "Sort descending by string field", + subdescription: "Use sort with reverse to sort in descending order.", + document: "[{a: banana},{a: cat},{a: apple}]", + expression: `sort_by(.a) | reverse`, + expected: []string{ + "D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\n", + }, + }, + { + description: "Sort array in place", + document: "cool: [{a: banana},{a: cat},{a: apple}]", + expression: `.cool |= sort_by(.a)`, + expected: []string{ + "D0, P[], (!!map)::cool: [{a: apple}, {a: banana}, {a: cat}]\n", + }, + }, + { + description: "Sort array of objects by key", + subdescription: "Note that you can give sort_by complex expressions, not just paths", + document: "cool: [{b: banana},{a: banana},{c: banana}]", + expression: `.cool |= sort_by(keys | .[0])`, + expected: []string{ + "D0, P[], (!!map)::cool: [{a: banana}, {b: banana}, {c: banana}]\n", + }, + }, + { + description: "Sort a map", + subdescription: "Sorting a map, by default this will sort by the values", + document: "y: b\nz: a\nx: c\n", + expression: `sort`, + expected: []string{ + "D0, P[], (!!map)::z: a\ny: b\nx: c\n", + }, + }, + { + description: "Sort a map by keys", + subdescription: "Use sort_by to sort a map using a custom function", + document: "Y: b\nz: a\nx: c\n", + expression: `sort_by(key | downcase)`, + expected: []string{ + "D0, P[], (!!map)::x: c\nY: b\nz: a\n", + }, + }, + { + description: "Sort is stable", + subdescription: "Note the order of the elements in unchanged when equal in sorting.", + document: "[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]", + expression: `sort_by(.a)`, + expected: []string{ + "D0, P[], (!!seq)::[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]\n", + }, + }, + { + description: "Sort by numeric field", + document: "[{a: 10},{a: 100},{a: 1}]", + expression: `sort_by(.a)`, + expected: []string{ + "D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\n", + }, + }, + { + description: "Sort by custom date field", + document: `[{a: "12-Jun-2011"},{a: "23-Dec-2010"},{a: "10-Aug-2011"}]`, + expression: `with_dtf("02-Jan-2006"; sort_by(.a))`, + expected: []string{ + "D0, P[], (!!seq)::[{a: \"23-Dec-2010\"}, {a: \"12-Jun-2011\"}, {a: \"10-Aug-2011\"}]\n", + }, + }, + { + skipDoc: true, + document: "[{a: 1.1},{a: 1.001},{a: 1.01}]", + expression: `sort_by(.a)`, + expected: []string{ + "D0, P[], (!!seq)::[{a: 1.001}, {a: 1.01}, {a: 1.1}]\n", + }, + }, + { + description: "Sort, nulls come first", + document: "[8,3,null,6, true, false, cat]", + expression: `sort`, + expected: []string{ + "D0, P[], (!!seq)::[null, false, true, 3, 6, 8, cat]\n", + }, + }, + { + description: "Sort, nulls come first", + skipDoc: true, + document: "[8,null]", + expression: `sort[]`, + expected: []string{ + "D0, P[0], (!!null)::null\n", + "D0, P[1], (!!int)::8\n", + }, + }, + { + skipDoc: true, + description: "false before true", + document: "[{a: false, b: 1}, {a: true, b: 2}, {a: false, b: 3}]", + expression: `sort_by(.a)`, + expected: []string{ + "D0, P[], (!!seq)::[{a: false, b: 1}, {a: false, b: 3}, {a: true, b: 2}]\n", + }, + }, + { + skipDoc: true, + description: "head comment", + document: "# abc\n- def\n# ghi", + expression: `sort`, + expected: []string{ + "D0, P[], (!!seq)::# abc\n- def\n# ghi\n", + }, + }, + { + skipDoc: true, + description: "Sort large integers (no int64 subtraction overflow)", + document: "[5000000000000000000, -5000000000000000000]", + expression: `sort`, + expected: []string{ + "D0, P[], (!!seq)::[-5000000000000000000, 5000000000000000000]\n", + }, + }, +} + +func TestSortByOperatorScenarios(t *testing.T) { + for _, tt := range sortByOperatorScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "sort", sortByOperatorScenarios) +}