Fixed sorting by date #1412

pkg/yqlib/compare_operators.go

@@ -54,12 +54,12 @@ func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context,
 }
 
 func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
-	lhsTime, err := time.Parse(layout, lhs.Value)
+	lhsTime, err := parseDateTime(layout, lhs.Value)
 	if err != nil {
 		return false, err
 	}
 
-	rhsTime, err := time.Parse(layout, rhs.Value)
+	rhsTime, err := parseDateTime(layout, rhs.Value)
 	if err != nil {
 		return false, err
 	}
@@ -81,7 +81,7 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs
 	isDateTime := lhs.Tag == "!!timestamp"
 	// if the lhs is a string, it might be a timestamp in a custom format.
 	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
-		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
+		_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
 		isDateTime = err == nil
 	}
 	if isDateTime {

pkg/yqlib/doc/operators/datetime.md

@@ -60,7 +60,7 @@ a: 2001-12-15
 ## Format: get the day of the week
 Given a sample.yml file of:
 ```yaml
-a: 2001-12-15T02:59:43.1Z
+a: 2001-12-15
 ```
 then
 ```bash

pkg/yqlib/doc/operators/sort.md

@@ -135,6 +135,24 @@ will output
 - a: 100
 ```
 
+## Sort by custom date field
+Given a sample.yml file of:
+```yaml
+- a: 12-Jun-2011
+- a: 23-Dec-2010
+- a: 10-Aug-2011
+```
+then
+```bash
+yq 'with_dtf("02-Jan-2006"; sort_by(.a))' sample.yml
+```
+will output
+```yaml
+- a: 23-Dec-2010
+- a: 12-Jun-2011
+- a: 10-Aug-2011
+```
+
 ## Sort, nulls come first
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/operator_add.go

@@ -98,7 +98,7 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
 
 	// if the lhs is a string, it might be a timestamp in a custom format.
 	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
-		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
+		_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
 		isDateTime = err == nil
 	}
 
@@ -152,7 +152,7 @@ func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yam
 		return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
 	}
 
-	currentTime, err := time.Parse(layout, lhs.Value)
+	currentTime, err := parseDateTime(layout, lhs.Value)
 	if err != nil {
 		return err
 	}

pkg/yqlib/operator_add_test.go

@@ -252,6 +252,15 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
 		},
 	},
+	{
+		description: "Date addition -date only",
+		skipDoc:     true,
+		document:    `a: 2021-01-01`,
+		expression:  `.a += "24h"`,
+		expected: []string{
+			"D0, P[], (doc)::a: 2021-01-02T00:00:00Z\n",
+		},
+	},
 	{
 		description:    "Date addition - custom format",
 		subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",

pkg/yqlib/operator_datetime.go

@@ -50,6 +50,17 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode
 
 }
 
+func parseDateTime(layout string, datestring string) (time.Time, error) {
+
+	parsedTime, err := time.Parse(layout, datestring)
+	if err != nil && layout == time.RFC3339 {
+		// try parsing the date time with only the date
+		return time.Parse("2006-01-02", datestring)
+	}
+	return parsedTime, err
+
+}
+
 func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	format, err := getStringParamter("format", d, context, expressionNode.RHS)
 	layout := context.GetDateTimeLayout()
@@ -62,7 +73,7 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		parsedTime, err := time.Parse(layout, candidate.Node.Value)
+		parsedTime, err := parseDateTime(layout, candidate.Node.Value)
 		if err != nil {
 			return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
 		}
@@ -101,7 +112,7 @@ func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode)
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		parsedTime, err := time.Parse(layout, candidate.Node.Value)
+		parsedTime, err := parseDateTime(layout, candidate.Node.Value)
 		if err != nil {
 			return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
 		}

pkg/yqlib/operator_datetime_test.go

@@ -25,7 +25,7 @@ var dateTimeOperatorScenarios = []expressionScenario{
 	},
 	{
 		description: "Format: get the day of the week",
-		document:    `a: 2001-12-15T02:59:43.1Z`,
+		document:    `a: 2001-12-15`,
 		expression:  `.a | format_datetime("Monday")`,
 		expected: []string{
 			"D0, P[a], (!!str)::Saturday\n",

pkg/yqlib/operator_sort.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -48,7 +49,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 			log.Debug("going to compare %v by %v", NodeToString(candidate.CreateReplacement(originalNode)), NodeToString(candidate.CreateReplacement(nodeToCompare)))
 
-			sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare}
+			sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare, dateTimeLayout: context.GetDateTimeLayout()}
 
 			if nodeToCompare.Kind != yaml.ScalarNode {
 				return Context{}, fmt.Errorf("sort only works for scalars, got %v", nodeToCompare.Tag)
@@ -70,8 +71,9 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 }
 
 type sortableNode struct {
-	Node          *yaml.Node
+	Node           *yaml.Node
-	NodeToCompare *yaml.Node
+	NodeToCompare  *yaml.Node
+	dateTimeLayout string
 }
 
 type sortableNodeArray []sortableNode
@@ -83,15 +85,37 @@ func (a sortableNodeArray) Less(i, j int) bool {
 	lhs := a[i].NodeToCompare
 	rhs := a[j].NodeToCompare
 
-	if lhs.Tag == "!!null" && rhs.Tag != "!!null" {
+	lhsTag := lhs.Tag
+	rhsTag := rhs.Tag
+
+	if !strings.HasPrefix(lhsTag, "!!") {
+		// custom tag - we have to have a guess
+		lhsTag = guessTagFromCustomType(lhs)
+	}
+
+	if !strings.HasPrefix(rhsTag, "!!") {
+		// custom tag - we have to have a guess
+		rhsTag = guessTagFromCustomType(rhs)
+	}
+
+	isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp"
+	layout := a[i].dateTimeLayout
+	// if the lhs is a string, it might be a timestamp in a custom format.
+	if lhsTag == "!!str" && layout != time.RFC3339 {
+		_, errLhs := parseDateTime(layout, lhs.Value)
+		_, errRhs := parseDateTime(layout, rhs.Value)
+		isDateTime = errLhs == nil && errRhs == nil
+	}
+
+	if lhsTag == "!!null" && rhsTag != "!!null" {
 		return true
-	} else if lhs.Tag != "!!null" && rhs.Tag == "!!null" {
+	} else if lhsTag != "!!null" && rhsTag == "!!null" {
 		return false
-	} else if lhs.Tag == "!!bool" && rhs.Tag != "!!bool" {
+	} else if lhsTag == "!!bool" && rhsTag != "!!bool" {
 		return true
-	} else if lhs.Tag != "!!bool" && rhs.Tag == "!!bool" {
+	} else if lhsTag != "!!bool" && rhsTag == "!!bool" {
 		return false
-	} else if lhs.Tag == "!!bool" && rhs.Tag == "!!bool" {
+	} else if lhsTag == "!!bool" && rhsTag == "!!bool" {
 		lhsTruthy, err := isTruthyNode(lhs)
 		if err != nil {
 			panic(fmt.Errorf("could not parse %v as boolean: %w", lhs.Value, err))
@@ -103,9 +127,19 @@ func (a sortableNodeArray) Less(i, j int) bool {
 		}
 
 		return !lhsTruthy && rhsTruthy
-	} else if lhs.Tag != rhs.Tag || lhs.Tag == "!!str" {
+	} else if isDateTime {
-		return strings.Compare(lhs.Value, rhs.Value) < 0
+		lhsTime, err := parseDateTime(layout, lhs.Value)
-	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
+		if err != nil {
+			log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err)
+			return strings.Compare(lhs.Value, rhs.Value) < 0
+		}
+		rhsTime, err := parseDateTime(layout, rhs.Value)
+		if err != nil {
+			log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err)
+			return strings.Compare(lhs.Value, rhs.Value) < 0
+		}
+		return lhsTime.Before(rhsTime)
+	} else if lhsTag == "!!int" && rhsTag == "!!int" {
 		_, lhsNum, err := parseInt64(lhs.Value)
 		if err != nil {
 			panic(err)
@@ -115,7 +149,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 			panic(err)
 		}
 		return lhsNum < rhsNum
-	} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
+	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {
 			panic(err)
@@ -127,5 +161,5 @@ func (a sortableNodeArray) Less(i, j int) bool {
 		return lhsNum < rhsNum
 	}
 
-	return true
+	return strings.Compare(lhs.Value, rhs.Value) < 0
 }

pkg/yqlib/operator_sort_test.go

@@ -54,6 +54,14 @@ var sortByOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\n",
 		},
 	},
+	{
+		description: "Sort by custom date field",
+		document:    `[{a: "12-Jun-2011"},{a: "23-Dec-2010"},{a: "10-Aug-2011"}]`,
+		expression:  `with_dtf("02-Jan-2006"; sort_by(.a))`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: \"23-Dec-2010\"}, {a: \"12-Jun-2011\"}, {a: \"10-Aug-2011\"}]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "[{a: 1.1},{a: 1.001},{a: 1.01}]",

pkg/yqlib/operator_subtract.go

@@ -92,10 +92,10 @@ func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs
 		rhsTag = guessTagFromCustomType(rhs)
 	}
 
-	isDateTime := lhs.Tag == "!!timestamp"
+	isDateTime := lhsTag == "!!timestamp"
 	// if the lhs is a string, it might be a timestamp in a custom format.
 	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
-		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
+		_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
 		isDateTime = err == nil
 	}
 
@@ -151,7 +151,7 @@ func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs
 		return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
 	}
 
-	currentTime, err := time.Parse(layout, lhs.Value)
+	currentTime, err := parseDateTime(layout, lhs.Value)
 	if err != nil {
 		return err
 	}

pkg/yqlib/operator_subtract_test.go

@@ -102,6 +102,15 @@ var subtractOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n",
 		},
 	},
+	{
+		description: "Date subtraction - only date",
+		skipDoc:     true,
+		document:    `a: 2021-01-01`,
+		expression:  `.a -= "24h"`,
+		expected: []string{
+			"D0, P[], (doc)::a: 2020-12-31T00:00:00Z\n",
+		},
+	},
 	{
 		description:    "Date subtraction - custom format",
 		subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",