suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-12-19 20:19:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added datetime operators

Browse Source
This commit is contained in:
Mike Farah 2022-02-12 12:35:06 +11:00
parent 4b2b47af48
commit 246d1af8d6
11 changed files with 550 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/yqlib/context.go
View File

@ -3,6 +3,7 @@ package yqlib
import (
"container/list"
"fmt"
"time"
"github.com/jinzhu/copier"
logging "gopkg.in/op/go-logging.v1"
@ -12,6 +13,7 @@ type Context struct {
MatchingNodes *list.List
Variables map[string]*list.List
DontAutoCreate bool
datetimeLayout string
}
func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {
@ -28,6 +30,17 @@ func (n *Context) SingleChildContext(candidate *CandidateNode) Context {
return n.ChildContext(list)
}
func (n *Context) SetDateTimeLayout(newDateTimeLayout string) {
n.datetimeLayout = newDateTimeLayout
}
func (n *Context) GetDateTimeLayout() string {
if n.datetimeLayout != "" {
return n.datetimeLayout
}
return time.RFC3339
}
func (n *Context) GetVariable(name string) *list.List {
if n.Variables == nil {
return nil
@ -43,7 +56,7 @@ func (n *Context) SetVariable(name string, value *list.List) {
}
func (n *Context) ChildContext(results *list.List) Context {
clone := Context{DontAutoCreate: n.DontAutoCreate}
clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout}
clone.Variables = make(map[string]*list.List)
if len(n.Variables) > 0 {
err := copier.Copy(&clone.Variables, n.Variables)

32
pkg/yqlib/doc/operators/add.md
View File

@ -206,6 +206,38 @@ a: 4
b: 6
```
## Date addition
You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: 2021-01-01T00:00:00Z
```
then
```bash
yq '.a += "3h10m"' sample.yml
```
will output
```yaml
a: 2021-01-01T03:10:00Z
```
## Date addition - custom format
You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq 'with_dtformat("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 6:00AM GMT
```
## Add to null
Adding to null simply returns the rhs

172
pkg/yqlib/doc/operators/datetime.md Normal file
View File

@ -0,0 +1,172 @@
# Date Time
Various operators for parsing and manipulating dates.
## Date time formattings
This uses the golangs built in time library for parsing and formatting date times.
When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00`.
To use a custom format, use the `with_dtformat` operator to set the formatting context. Expressions in the second parameter then assume that date format.
```bash
yq 'with_dtformat("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
```
See https://pkg.go.dev/time#pkg-constants for more examples.
## Timezones
This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
## Durations
Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
You can durations to time using the `+` operator.
{% hint style="warning" %}
Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
`yq e <exp> <file>`
{% endhint %}
## Format: from standard RFC3339 format
Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.
Given a sample.yml file of:
```yaml
a: 2001-12-15T02:59:43.1Z
```
then
```bash
yq '.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 2:59AM
```
## Format: from custom date time
Use with_dtformat to set a custom datetime format for parsing.
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM
```
then
```bash
yq '.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml
```
will output
```yaml
a: 2001-12-15
```
## Format: get the day of the week
Given a sample.yml file of:
```yaml
a: 2001-12-15T02:59:43.1Z
```
then
```bash
yq '.a | format_datetime("Monday")' sample.yml
```
will output
```yaml
Saturday
```
## Now
Given a sample.yml file of:
```yaml
a: cool
```
then
```bash
yq '.updated = now' sample.yml
```
will output
```yaml
a: cool
updated: 2021-05-19T01:02:03Z
```
## Timezone: from standard RFC3339 format
Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.
Given a sample.yml file of:
```yaml
a: cool
```
then
```bash
yq '.updated = (now | tz("Australia/Sydney"))' sample.yml
```
will output
```yaml
a: cool
updated: 2021-05-19T11:02:03+10:00
```
## Timezone: with custom format
Specify standard IANA Time Zone format or 'utc', 'local'
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq '.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 1:59PM AEDT
```
## Add and tz custom format
Specify standard IANA Time Zone format or 'utc', 'local'
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq '.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 1:59PM AEDT
```
## Date addition
Given a sample.yml file of:
```yaml
a: 2021-01-01T00:00:00Z
```
then
```bash
yq '.a += "3h10m"' sample.yml
```
will output
```yaml
a: 2021-01-01T03:10:00Z
```
## Date addition - custom format
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq 'with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 6:00AM GMT
```

27
pkg/yqlib/doc/operators/headers/datetime.md Normal file
View File

@ -0,0 +1,27 @@
# Date Time
Various operators for parsing and manipulating dates.
## Date time formattings
This uses the golangs built in time library for parsing and formatting date times.
When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00`.
To use a custom format, use the `with_dtformat` operator to set the formatting context. Expressions in the second parameter then assume that date format.
```bash
yq 'with_dtformat("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
```
See https://pkg.go.dev/time#pkg-constants for more examples.
## Timezones
This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
## Durations
Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
You can durations to time using the `+` operator.

5
pkg/yqlib/expression_tokeniser.go
View File

@ -324,6 +324,11 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType))
lexer.Add([]byte(`now`), opToken(nowOpType))
lexer.Add([]byte(`tz`), opToken(tzOpType))
lexer.Add([]byte(`with_dtformat`), opToken(withDtFormatOpType))
lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))

6
pkg/yqlib/lib.go
View File

@ -84,6 +84,12 @@ var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50,
var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}

36
pkg/yqlib/operator_add.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"strconv"
"strings"
"time"
yaml "gopkg.in/yaml.v3"
)
@ -71,7 +72,7 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
}
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhsNode.Style
if err := addScalars(target, lhsNode, rhs.Node); err != nil {
if err := addScalars(context, target, lhsNode, rhs.Node); err != nil {
return nil, err
}
}
@ -97,7 +98,7 @@ func guessTagFromCustomType(node *yaml.Node) string {
return guessedTag
}
func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
lhsTag := lhs.Tag
rhsTag := rhs.Tag
lhsIsCustom := false
@ -112,7 +113,18 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
rhsTag = guessTagFromCustomType(rhs)
}
if lhsTag == "!!str" {
isDateTime := lhs.Tag == "!!timestamp"
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
if isDateTime {
return addDateTimes(context.GetDateTimeLayout(), target, lhs, rhs)
} else if lhsTag == "!!str" {
target.Node.Tag = lhs.Tag
target.Node.Value = lhs.Value + rhs.Value
} else if lhsTag == "!!int" && rhsTag == "!!int" {
@ -149,6 +161,24 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
return nil
}
func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
duration, err := time.ParseDuration(rhs.Value)
if err != nil {
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
}
currentTime, err := time.Parse(layout, lhs.Value)
if err != nil {
return err
}
newTime := currentTime.Add(duration)
target.Node.Value = newTime.Format(layout)
return nil
}
func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
target.Node.Kind = yaml.SequenceNode
if len(lhs.Node.Content) > 0 {

28
pkg/yqlib/operator_add_test.go
View File

@ -208,6 +208,34 @@ var addOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::{a: 4, b: 6}\n",
},
},
{
description: "Date addition",
subdescription: "You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: 2021-01-01T00:00:00Z`,
expression: `.a += "3h10m"`,
expected: []string{
"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
},
},
{
description: "Date addition - custom format",
subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`,
expression: `with_dtformat("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n",
},
},
{
skipDoc: true,
description: "Date addition - custom format",
subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`,
expression: `with_dtformat("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`,
expected: []string{
"D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n",
},
},
{
description: "Add to null",
subdescription: "Adding to null simply returns the rhs",

126
pkg/yqlib/operator_datetime.go Normal file
View File

@ -0,0 +1,126 @@
package yqlib
import (
"container/list"
"errors"
"fmt"
"strings"
"time"
"gopkg.in/yaml.v3"
)
func getStringParamter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) {
result, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode)
if err != nil {
return "", err
} else if result.MatchingNodes.Len() == 0 {
return "", fmt.Errorf("could not find %v for format_time", parameterName)
}
return result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value, nil
}
func withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
if expressionNode.RHS.Operation.OperationType == blockOpType || expressionNode.RHS.Operation.OperationType == unionOpType {
layout, err := getStringParamter("layout", d, context, expressionNode.RHS.LHS)
if err != nil {
return Context{}, fmt.Errorf("could not get date time format: %w", err)
}
context.SetDateTimeLayout(layout)
return d.GetMatchingNodes(context, expressionNode.RHS.RHS)
}
return Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; <exp>)`)
}
// for unit tests
var Now = time.Now
func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
node := &yaml.Node{
Tag: "!!timestamp",
Kind: yaml.ScalarNode,
Value: Now().Format(time.RFC3339),
}
return context.SingleChildContext(&CandidateNode{Node: node}), nil
}
func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
format, err := getStringParamter("format", d, context, expressionNode.RHS)
layout := context.GetDateTimeLayout()
decoder := NewYamlDecoder()
if err != nil {
return Context{}, err
}
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
parsedTime, err := time.Parse(layout, candidate.Node.Value)
if err != nil {
return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
}
formattedTimeStr := parsedTime.Format(format)
decoder.Init(strings.NewReader(formattedTimeStr))
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket)
var node *yaml.Node
if errorReading != nil {
log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr)
node = &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: formattedTimeStr,
}
} else {
node = unwrapDoc(&dataBucket)
}
results.PushBack(candidate.CreateReplacement(node))
}
return context.ChildContext(results), nil
}
func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
timezoneStr, err := getStringParamter("timezone", d, context, expressionNode.RHS)
layout := context.GetDateTimeLayout()
if err != nil {
return Context{}, err
}
var results = list.New()
timezone, err := time.LoadLocation(timezoneStr)
if err != nil {
return Context{}, fmt.Errorf("could not load tz [%v]: %w", timezoneStr, err)
}
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
parsedTime, err := time.Parse(layout, candidate.Node.Value)
if err != nil {
return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
}
tzTime := parsedTime.In(timezone)
node := &yaml.Node{
Kind: yaml.ScalarNode,
Tag: candidate.Node.Tag,
Value: tzTime.Format(layout),
}
results.PushBack(candidate.CreateReplacement(node))
}
return context.ChildContext(results), nil
}

103
pkg/yqlib/operator_datetime_test.go Normal file
View File

@ -0,0 +1,103 @@
package yqlib
import (
"testing"
)
var dateTimeOperatorScenarios = []expressionScenario{
{
description: "Format: from standard RFC3339 format",
subdescription: "Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.",
document: `a: 2001-12-15T02:59:43.1Z`,
expression: `.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM\n",
},
},
{
description: "Format: from custom date time",
subdescription: "Use with_dtformat to set a custom datetime format for parsing.",
document: `a: Saturday, 15-Dec-01 at 2:59AM`,
expression: `.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))`,
expected: []string{
"D0, P[], (doc)::a: 2001-12-15\n",
},
},
{
description: "Format: get the day of the week",
document: `a: 2001-12-15T02:59:43.1Z`,
expression: `.a | format_datetime("Monday")`,
expected: []string{
"D0, P[a], (!!str)::Saturday\n",
},
},
{
description: "Now",
document: "a: cool",
expression: `.updated = now`,
expected: []string{
"D0, P[], (doc)::a: cool\nupdated: 2021-05-19T01:02:03Z\n",
},
},
{
description: "Timezone: from standard RFC3339 format",
subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.",
document: "a: cool",
expression: `.updated = (now | tz("Australia/Sydney"))`,
expected: []string{
"D0, P[], (doc)::a: cool\nupdated: 2021-05-19T11:02:03+10:00\n",
},
},
{
description: "Timezone: with custom format",
subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'",
document: "a: Saturday, 15-Dec-01 at 2:59AM GMT",
expression: `.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
},
},
{
description: "Add and tz custom format",
subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'",
document: "a: Saturday, 15-Dec-01 at 2:59AM GMT",
expression: `.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
},
},
{
description: "Date addition",
document: `a: 2021-01-01T00:00:00Z`,
expression: `.a += "3h10m"`,
expected: []string{
"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
},
},
{
description: "Date addition - custom format",
document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`,
expression: `with_dtformat("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n",
},
},
{
description: "allow comma",
skipDoc: true,
document: "a: Saturday, 15-Dec-01 at 2:59AM GMT",
expression: `.a |= with_dtformat("Monday, 02-Jan-06 at 3:04PM MST", tz("Australia/Sydney"))`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
},
},
}
func TestDatetimeOperatorScenarios(t *testing.T) {
for _, tt := range dateTimeOperatorScenarios {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "datetime", dateTimeOperatorScenarios)
}

4
pkg/yqlib/operators_test.go
View File

@ -10,6 +10,7 @@ import (
"sort"
"strings"
"testing"
"time"
"github.com/mikefarah/yq/v4/test"
"gopkg.in/op/go-logging.v1"
@ -31,6 +32,9 @@ type expressionScenario struct {
func TestMain(m *testing.M) {
logging.SetLevel(logging.ERROR, "")
Now = func() time.Time {
return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
}
code := m.Run()
os.Exit(code)
}