suyiiyii
/
yq
mirror of https://github.com/mikefarah/yq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed sorting by date #1412

This commit is contained in:
Mike Farah 2022-11-04 12:21:12 +11:00
parent 7f5dda93c6
commit cf02b90624
11 changed files with 114 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/yqlib/compare_operators.go
View File

@ -54,12 +54,12 @@ func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context,
}
func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
lhsTime, err := time.Parse(layout, lhs.Value)
lhsTime, err := parseDateTime(layout, lhs.Value)
if err != nil {
return false, err
}
rhsTime, err := time.Parse(layout, rhs.Value)
rhsTime, err := parseDateTime(layout, rhs.Value)
if err != nil {
return false, err
}
@ -81,7 +81,7 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs
isDateTime := lhs.Tag == "!!timestamp"
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
if isDateTime {

2
pkg/yqlib/doc/operators/datetime.md
View File

@ -60,7 +60,7 @@ a: 2001-12-15
## Format: get the day of the week
Given a sample.yml file of:
```yaml
a: 2001-12-15T02:59:43.1Z
a: 2001-12-15
```
then
```bash

18
pkg/yqlib/doc/operators/sort.md
View File

@ -135,6 +135,24 @@ will output
- a: 100
```
## Sort by custom date field
Given a sample.yml file of:
```yaml
- a: 12-Jun-2011
- a: 23-Dec-2010
- a: 10-Aug-2011
```
then
```bash
yq 'with_dtf("02-Jan-2006"; sort_by(.a))' sample.yml
```
will output
```yaml
- a: 23-Dec-2010
- a: 12-Jun-2011
- a: 10-Aug-2011
```
## Sort, nulls come first
Given a sample.yml file of:
```yaml

4
pkg/yqlib/operator_add.go
View File

@ -98,7 +98,7 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
@ -152,7 +152,7 @@ func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yam
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
}
currentTime, err := time.Parse(layout, lhs.Value)
currentTime, err := parseDateTime(layout, lhs.Value)
if err != nil {
return err
}

9
pkg/yqlib/operator_add_test.go
View File

@ -252,6 +252,15 @@ var addOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
},
},
{
description: "Date addition -date only",
skipDoc: true,
document: `a: 2021-01-01`,
expression: `.a += "24h"`,
expected: []string{
"D0, P[], (doc)::a: 2021-01-02T00:00:00Z\n",
},
},
{
description: "Date addition - custom format",
subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",

15
pkg/yqlib/operator_datetime.go
View File

@ -50,6 +50,17 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode
}
func parseDateTime(layout string, datestring string) (time.Time, error) {
parsedTime, err := time.Parse(layout, datestring)
if err != nil && layout == time.RFC3339 {
// try parsing the date time with only the date
return time.Parse("2006-01-02", datestring)
}
return parsedTime, err
}
func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
format, err := getStringParamter("format", d, context, expressionNode.RHS)
layout := context.GetDateTimeLayout()
@ -62,7 +73,7 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
parsedTime, err := time.Parse(layout, candidate.Node.Value)
parsedTime, err := parseDateTime(layout, candidate.Node.Value)
if err != nil {
return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
}
@ -101,7 +112,7 @@ func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode)
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
parsedTime, err := time.Parse(layout, candidate.Node.Value)
parsedTime, err := parseDateTime(layout, candidate.Node.Value)
if err != nil {
return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
}

2
pkg/yqlib/operator_datetime_test.go
View File

@ -25,7 +25,7 @@ var dateTimeOperatorScenarios = []expressionScenario{
},
{
description: "Format: get the day of the week",
document: `a: 2001-12-15T02:59:43.1Z`,
document: `a: 2001-12-15`,
expression: `.a | format_datetime("Monday")`,
expected: []string{
"D0, P[a], (!!str)::Saturday\n",

60
pkg/yqlib/operator_sort.go
View File

@ -6,6 +6,7 @@ import (
"sort"
"strconv"
"strings"
"time"
yaml "gopkg.in/yaml.v3"
)
@ -48,7 +49,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
log.Debug("going to compare %v by %v", NodeToString(candidate.CreateReplacement(originalNode)), NodeToString(candidate.CreateReplacement(nodeToCompare)))
sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare}
sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare, dateTimeLayout: context.GetDateTimeLayout()}
if nodeToCompare.Kind != yaml.ScalarNode {
return Context{}, fmt.Errorf("sort only works for scalars, got %v", nodeToCompare.Tag)
@ -70,8 +71,9 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
}
type sortableNode struct {
Node *yaml.Node
NodeToCompare *yaml.Node
Node *yaml.Node
NodeToCompare *yaml.Node
dateTimeLayout string
}
type sortableNodeArray []sortableNode
@ -83,15 +85,37 @@ func (a sortableNodeArray) Less(i, j int) bool {
lhs := a[i].NodeToCompare
rhs := a[j].NodeToCompare
if lhs.Tag == "!!null" && rhs.Tag != "!!null" {
lhsTag := lhs.Tag
rhsTag := rhs.Tag
if !strings.HasPrefix(lhsTag, "!!") {
// custom tag - we have to have a guess
lhsTag = guessTagFromCustomType(lhs)
}
if !strings.HasPrefix(rhsTag, "!!") {
// custom tag - we have to have a guess
rhsTag = guessTagFromCustomType(rhs)
}
isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp"
layout := a[i].dateTimeLayout
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && layout != time.RFC3339 {
_, errLhs := parseDateTime(layout, lhs.Value)
_, errRhs := parseDateTime(layout, rhs.Value)
isDateTime = errLhs == nil && errRhs == nil
}
if lhsTag == "!!null" && rhsTag != "!!null" {
return true
} else if lhs.Tag != "!!null" && rhs.Tag == "!!null" {
} else if lhsTag != "!!null" && rhsTag == "!!null" {
return false
} else if lhs.Tag == "!!bool" && rhs.Tag != "!!bool" {
} else if lhsTag == "!!bool" && rhsTag != "!!bool" {
return true
} else if lhs.Tag != "!!bool" && rhs.Tag == "!!bool" {
} else if lhsTag != "!!bool" && rhsTag == "!!bool" {
return false
} else if lhs.Tag == "!!bool" && rhs.Tag == "!!bool" {
} else if lhsTag == "!!bool" && rhsTag == "!!bool" {
lhsTruthy, err := isTruthyNode(lhs)
if err != nil {
panic(fmt.Errorf("could not parse %v as boolean: %w", lhs.Value, err))
@ -103,9 +127,19 @@ func (a sortableNodeArray) Less(i, j int) bool {
}
return !lhsTruthy && rhsTruthy
} else if lhs.Tag != rhs.Tag || lhs.Tag == "!!str" {
return strings.Compare(lhs.Value, rhs.Value) < 0
} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
} else if isDateTime {
lhsTime, err := parseDateTime(layout, lhs.Value)
if err != nil {
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err)
return strings.Compare(lhs.Value, rhs.Value) < 0
}
rhsTime, err := parseDateTime(layout, rhs.Value)
if err != nil {
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err)
return strings.Compare(lhs.Value, rhs.Value) < 0
}
return lhsTime.Before(rhsTime)
} else if lhsTag == "!!int" && rhsTag == "!!int" {
_, lhsNum, err := parseInt64(lhs.Value)
if err != nil {
panic(err)
@ -115,7 +149,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
panic(err)
}
return lhsNum < rhsNum
} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
if err != nil {
panic(err)
@ -127,5 +161,5 @@ func (a sortableNodeArray) Less(i, j int) bool {
return lhsNum < rhsNum
}
return true
return strings.Compare(lhs.Value, rhs.Value) < 0
}

8
pkg/yqlib/operator_sort_test.go
View File

@ -54,6 +54,14 @@ var sortByOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\n",
},
},
{
description: "Sort by custom date field",
document: `[{a: "12-Jun-2011"},{a: "23-Dec-2010"},{a: "10-Aug-2011"}]`,
expression: `with_dtf("02-Jan-2006"; sort_by(.a))`,
expected: []string{
"D0, P[], (!!seq)::[{a: \"23-Dec-2010\"}, {a: \"12-Jun-2011\"}, {a: \"10-Aug-2011\"}]\n",
},
},
{
skipDoc: true,
document: "[{a: 1.1},{a: 1.001},{a: 1.01}]",

6
pkg/yqlib/operator_subtract.go
View File

@ -92,10 +92,10 @@ func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs
rhsTag = guessTagFromCustomType(rhs)
}
isDateTime := lhs.Tag == "!!timestamp"
isDateTime := lhsTag == "!!timestamp"
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
@ -151,7 +151,7 @@ func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
}
currentTime, err := time.Parse(layout, lhs.Value)
currentTime, err := parseDateTime(layout, lhs.Value)
if err != nil {
return err
}

9
pkg/yqlib/operator_subtract_test.go
View File

@ -102,6 +102,15 @@ var subtractOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n",
},
},
{
description: "Date subtraction - only date",
skipDoc: true,
document: `a: 2021-01-01`,
expression: `.a -= "24h"`,
expected: []string{
"D0, P[], (doc)::a: 2020-12-31T00:00:00Z\n",
},
},
{
description: "Date subtraction - custom format",
subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",