Compare commits

..
5 Commits
25 changed files with 56 additions and 762 deletions
+2 -1
View File
@@ -124,6 +124,7 @@ rm /etc/myfile.tmp
```
### Run with Docker or Podman
#### Oneshot use:
```bash
@@ -193,7 +194,7 @@ Or, in your Dockerfile:
FROM mikefarah/yq
USER root
RUN apk add --no-cache bash
RUN apk add bash
USER yq
```
-9
View File
@@ -1,9 +0,0 @@
#!/bin/bash
testCompletionRuns() {
result=$(./yq __complete "" 2>&1)
assertEquals 0 $?
assertContains "$result" "Completion ended with directive:"
}
source ./scripts/shunit2
+1 -14
View File
@@ -3,7 +3,6 @@ package yqlib
import (
"container/list"
"fmt"
"time"
"github.com/jinzhu/copier"
logging "gopkg.in/op/go-logging.v1"
@@ -13,7 +12,6 @@ type Context struct {
MatchingNodes *list.List
Variables map[string]*list.List
DontAutoCreate bool
datetimeLayout string
}
func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {
@@ -30,17 +28,6 @@ func (n *Context) SingleChildContext(candidate *CandidateNode) Context {
return n.ChildContext(list)
}
func (n *Context) SetDateTimeLayout(newDateTimeLayout string) {
n.datetimeLayout = newDateTimeLayout
}
func (n *Context) GetDateTimeLayout() string {
if n.datetimeLayout != "" {
return n.datetimeLayout
}
return time.RFC3339
}
func (n *Context) GetVariable(name string) *list.List {
if n.Variables == nil {
return nil
@@ -56,7 +43,7 @@ func (n *Context) SetVariable(name string, value *list.List) {
}
func (n *Context) ChildContext(results *list.List) Context {
clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout}
clone := Context{DontAutoCreate: n.DontAutoCreate}
clone.Variables = make(map[string]*list.List)
if len(n.Variables) > 0 {
err := copier.Copy(&clone.Variables, n.Variables)
-32
View File
@@ -206,38 +206,6 @@ a: 4
b: 6
```
## Date addition
You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: 2021-01-01T00:00:00Z
```
then
```bash
yq '.a += "3h10m"' sample.yml
```
will output
```yaml
a: 2021-01-01T03:10:00Z
```
## Date addition - custom format
You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 6:00AM GMT
```
## Add to null
Adding to null simply returns the rhs
+9 -9
View File
@@ -23,7 +23,7 @@ a: cat
```
then
```bash
yq '.a line_comment="single"' sample.yml
yq '.a lineComment="single"' sample.yml
```
will output
```yaml
@@ -38,7 +38,7 @@ b: dog
```
then
```bash
yq '.. line_comment |= .' sample.yml
yq '.. lineComment |= .' sample.yml
```
will output
```yaml
@@ -53,7 +53,7 @@ a: cat
```
then
```bash
yq '. head_comment="single"' sample.yml
yq '. headComment="single"' sample.yml
```
will output
```yaml
@@ -69,7 +69,7 @@ a: cat
```
then
```bash
yq '. foot_comment=.a' sample.yml
yq '. footComment=.a' sample.yml
```
will output
```yaml
@@ -86,7 +86,7 @@ b: dog # leave this
```
then
```bash
yq '.a line_comment=""' sample.yml
yq '.a lineComment=""' sample.yml
```
will output
```yaml
@@ -120,7 +120,7 @@ a: cat # meow
```
then
```bash
yq '.a | line_comment' sample.yml
yq '.a | lineComment' sample.yml
```
will output
```yaml
@@ -138,7 +138,7 @@ a: cat # meow
```
then
```bash
yq '. | head_comment' sample.yml
yq '. | headComment' sample.yml
```
will output
```yaml
@@ -157,7 +157,7 @@ a: cat # meow
```
then
```bash
yq 'head_comment' sample.yml
yq 'headComment' sample.yml
```
will output
```yaml
@@ -177,7 +177,7 @@ a: cat # meow
```
then
```bash
yq '. | foot_comment' sample.yml
yq '. | footComment' sample.yml
```
will output
```yaml
-203
View File
@@ -1,203 +0,0 @@
# Date Time
Various operators for parsing and manipulating dates.
## Date time formattings
This uses the golangs built in time library for parsing and formatting date times.
When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.
To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree.
```bash
yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
```
See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
## Timezones
This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
## Durations
Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
You can durations to time using the `+` operator.
{% hint style="warning" %}
Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
`yq e <exp> <file>`
{% endhint %}
## Format: from standard RFC3339 format
Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.
Given a sample.yml file of:
```yaml
a: 2001-12-15T02:59:43.1Z
```
then
```bash
yq '.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 2:59AM
```
## Format: from custom date time
Use with_dtf to set a custom datetime format for parsing.
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM
```
then
```bash
yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml
```
will output
```yaml
a: 2001-12-15
```
## Format: get the day of the week
Given a sample.yml file of:
```yaml
a: 2001-12-15T02:59:43.1Z
```
then
```bash
yq '.a | format_datetime("Monday")' sample.yml
```
will output
```yaml
Saturday
```
## Now
Given a sample.yml file of:
```yaml
a: cool
```
then
```bash
yq '.updated = now' sample.yml
```
will output
```yaml
a: cool
updated: 2021-05-19T01:02:03Z
```
## Timezone: from standard RFC3339 format
Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.
Given a sample.yml file of:
```yaml
a: cool
```
then
```bash
yq '.updated = (now | tz("Australia/Sydney"))' sample.yml
```
will output
```yaml
a: cool
updated: 2021-05-19T11:02:03+10:00
```
## Timezone: with custom format
Specify standard IANA Time Zone format or 'utc', 'local'
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 1:59PM AEDT
```
## Add and tz custom format
Specify standard IANA Time Zone format or 'utc', 'local'
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 1:59PM AEDT
```
## Date addition
Given a sample.yml file of:
```yaml
a: 2021-01-01T00:00:00Z
```
then
```bash
yq '.a += "3h10m"' sample.yml
```
will output
```yaml
a: 2021-01-01T03:10:00Z
```
## Date subtraction
You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: 2021-01-01T03:10:00Z
```
then
```bash
yq '.a -= "3h10m"' sample.yml
```
will output
```yaml
a: 2021-01-01T00:00:00Z
```
## Date addition - custom format
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 6:00AM GMT
```
## Date script with custom format
You can embed full expressions in with_dtf if needed.
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 2:00PM AWST
```
+3 -3
View File
@@ -17,7 +17,7 @@ a: frog
```
then
```bash
yq '.a | document_index' sample.yml
yq '.a | documentIndex' sample.yml
```
will output
```yaml
@@ -53,7 +53,7 @@ a: frog
```
then
```bash
yq 'select(document_index == 1)' sample.yml
yq 'select(documentIndex == 1)' sample.yml
```
will output
```yaml
@@ -85,7 +85,7 @@ a: frog
```
then
```bash
yq '.a | ({"match": ., "doc": document_index})' sample.yml
yq '.a | ({"match": ., "doc": documentIndex})' sample.yml
```
will output
```yaml
+2 -2
View File
@@ -37,7 +37,7 @@ a: cat
```
then
```bash
yq 'file_index' sample.yml
yq 'fileIndex' sample.yml
```
will output
```yaml
@@ -55,7 +55,7 @@ a: cat
```
then
```bash
yq eval-all 'file_index' sample.yml another.yml
yq eval-all 'fileIndex' sample.yml another.yml
```
will output
```yaml
@@ -1,26 +0,0 @@
# Date Time
Various operators for parsing and manipulating dates.
## Date time formattings
This uses the golangs built in time library for parsing and formatting date times.
When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.
To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree.
```bash
yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
```
See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
## Timezones
This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
## Durations
Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
You can durations to time using the `+` operator.
@@ -11,7 +11,7 @@ Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#
## Split empty
Running
```bash
yq --null-input 'split_doc'
yq --null-input 'splitDoc'
```
will output
```yaml
@@ -26,7 +26,7 @@ Given a sample.yml file of:
```
then
```bash
yq '.[] | split_doc' sample.yml
yq '.[] | splitDoc' sample.yml
```
will output
```yaml
-32
View File
@@ -117,38 +117,6 @@ a: 2
b: 4
```
## Date subtraction
You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: 2021-01-01T03:10:00Z
```
then
```bash
yq '.a -= "3h10m"' sample.yml
```
will output
```yaml
a: 2021-01-01T00:00:00Z
```
## Date subtraction - custom format
Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 6:00AM GMT
```
then
```bash
yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
## Custom types: that are really numbers
When custom tags are encountered, yq will try to decode the underlying type.
+2 -17
View File
@@ -324,11 +324,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType))
lexer.Add([]byte(`now`), opToken(nowOpType))
lexer.Add([]byte(`tz`), opToken(tzOpType))
lexer.Add([]byte(`with_dtf`), opToken(withDtFormatOpType))
lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
@@ -398,12 +393,8 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\/\/`), opToken(alternativeOpType))
lexer.Add([]byte(`documentIndex`), opToken(getDocumentIndexOpType))
lexer.Add([]byte(`document_index`), opToken(getDocumentIndexOpType))
lexer.Add([]byte(`di`), opToken(getDocumentIndexOpType))
lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
lexer.Add([]byte(`split_doc`), opToken(splitDocumentOpType))
lexer.Add([]byte(`join`), opToken(joinStringOpType))
lexer.Add([]byte(`sub`), opToken(subStringOpType))
@@ -432,10 +423,7 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`anchor`), opAssignableToken(getAnchorOpType, assignAnchorOpType))
lexer.Add([]byte(`alias`), opAssignableToken(getAliasOptype, assignAliasOpType))
lexer.Add([]byte(`filename`), opToken(getFilenameOpType))
lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
lexer.Add([]byte(`file_index`), opToken(getFileIndexOpType))
lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
lexer.Add([]byte(`path`), opToken(getPathOpType))
lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
@@ -445,13 +433,10 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`with`), opToken(withOpType))
lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
lexer.Add([]byte(`line_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
lexer.Add([]byte(`head_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
lexer.Add([]byte(`footComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
lexer.Add([]byte(`foot_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
lexer.Add([]byte(`comments\s*=`), assignAllCommentsOp(false))
lexer.Add([]byte(`comments\s*\|=`), assignAllCommentsOp(true))
@@ -534,7 +519,7 @@ func (p *expressionTokeniserImpl) Tokenise(expression string) ([]*token, error)
scanner, err := p.lexer.Scanner([]byte(expression))
if err != nil {
return nil, fmt.Errorf("parsing expression: %w", err)
return nil, fmt.Errorf("Parsing expression: %w", err)
}
var tokens []*token
for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() {
@@ -545,7 +530,7 @@ func (p *expressionTokeniserImpl) Tokenise(expression string) ([]*token, error)
tokens = append(tokens, currentToken)
}
if err != nil {
return nil, fmt.Errorf("parsing expression: %w", err)
return nil, fmt.Errorf("Parsing expression: %w", err)
}
}
var postProcessedTokens = make([]*token, 0)
-6
View File
@@ -84,12 +84,6 @@ var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50,
var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
+3 -33
View File
@@ -4,7 +4,6 @@ import (
"fmt"
"strconv"
"strings"
"time"
yaml "gopkg.in/yaml.v3"
)
@@ -72,7 +71,7 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
}
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhsNode.Style
if err := addScalars(context, target, lhsNode, rhs.Node); err != nil {
if err := addScalars(target, lhsNode, rhs.Node); err != nil {
return nil, err
}
}
@@ -98,7 +97,7 @@ func guessTagFromCustomType(node *yaml.Node) string {
return guessedTag
}
func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
lhsTag := lhs.Tag
rhsTag := rhs.Tag
lhsIsCustom := false
@@ -113,18 +112,7 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
rhsTag = guessTagFromCustomType(rhs)
}
isDateTime := lhs.Tag == "!!timestamp"
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
if isDateTime {
return addDateTimes(context.GetDateTimeLayout(), target, lhs, rhs)
} else if lhsTag == "!!str" {
if lhsTag == "!!str" {
target.Node.Tag = lhs.Tag
target.Node.Value = lhs.Value + rhs.Value
} else if lhsTag == "!!int" && rhsTag == "!!int" {
@@ -161,24 +149,6 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
return nil
}
func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
duration, err := time.ParseDuration(rhs.Value)
if err != nil {
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
}
currentTime, err := time.Parse(layout, lhs.Value)
if err != nil {
return err
}
newTime := currentTime.Add(duration)
target.Node.Value = newTime.Format(layout)
return nil
}
func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
target.Node.Kind = yaml.SequenceNode
if len(lhs.Node.Content) > 0 {
-28
View File
@@ -208,34 +208,6 @@ var addOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::{a: 4, b: 6}\n",
},
},
{
description: "Date addition",
subdescription: "You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: 2021-01-01T00:00:00Z`,
expression: `.a += "3h10m"`,
expected: []string{
"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
},
},
{
description: "Date addition - custom format",
subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`,
expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n",
},
},
{
skipDoc: true,
description: "Date addition - custom format",
subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`,
expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`,
expected: []string{
"D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n",
},
},
{
description: "Add to null",
subdescription: "Adding to null simply returns the rhs",
+13 -13
View File
@@ -8,7 +8,7 @@ var commentOperatorScenarios = []expressionScenario{
{
description: "Set line comment",
document: `a: cat`,
expression: `.a line_comment="single"`,
expression: `.a lineComment="single"`,
expected: []string{
"D0, P[], (doc)::a: cat # single\n",
},
@@ -16,7 +16,7 @@ var commentOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: "a: cat\nb: dog",
expression: `.a line_comment=.b`,
expression: `.a lineComment=.b`,
expected: []string{
"D0, P[], (doc)::a: cat # dog\nb: dog\n",
},
@@ -24,7 +24,7 @@ var commentOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: "a: cat\n---\na: dog",
expression: `.a line_comment |= documentIndex`,
expression: `.a lineComment |= documentIndex`,
expected: []string{
"D0, P[], (doc)::a: cat # 0\n",
"D1, P[], (doc)::a: dog # 1\n",
@@ -33,7 +33,7 @@ var commentOperatorScenarios = []expressionScenario{
{
description: "Use update assign to perform relative updates",
document: "a: cat\nb: dog",
expression: `.. line_comment |= .`,
expression: `.. lineComment |= .`,
expected: []string{
"D0, P[], (!!map)::a: cat # cat\nb: dog # dog\n",
},
@@ -49,7 +49,7 @@ var commentOperatorScenarios = []expressionScenario{
{
description: "Set head comment",
document: `a: cat`,
expression: `. head_comment="single"`,
expression: `. headComment="single"`,
expected: []string{
"D0, P[], (doc)::# single\n\na: cat\n",
},
@@ -57,7 +57,7 @@ var commentOperatorScenarios = []expressionScenario{
{
description: "Set foot comment, using an expression",
document: `a: cat`,
expression: `. foot_comment=.a`,
expression: `. footComment=.a`,
expected: []string{
"D0, P[], (doc)::a: cat\n\n# cat\n",
},
@@ -65,7 +65,7 @@ var commentOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: `a: cat`,
expression: `. foot_comment=.b.d`,
expression: `. footComment=.b.d`,
expected: []string{
"D0, P[], (doc)::a: cat\n",
},
@@ -73,7 +73,7 @@ var commentOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: `a: cat`,
expression: `. foot_comment|=.b.d`,
expression: `. footComment|=.b.d`,
expected: []string{
"D0, P[], (doc)::a: cat\n",
},
@@ -81,7 +81,7 @@ var commentOperatorScenarios = []expressionScenario{
{
description: "Remove comment",
document: "a: cat # comment\nb: dog # leave this",
expression: `.a line_comment=""`,
expression: `.a lineComment=""`,
expected: []string{
"D0, P[], (doc)::a: cat\nb: dog # leave this\n",
},
@@ -98,7 +98,7 @@ var commentOperatorScenarios = []expressionScenario{
{
description: "Get line comment",
document: "# welcome!\n\na: cat # meow\n\n# have a great day",
expression: `.a | line_comment`,
expression: `.a | lineComment`,
expected: []string{
"D0, P[a], (!!str)::meow\n",
},
@@ -107,7 +107,7 @@ var commentOperatorScenarios = []expressionScenario{
description: "Get head comment",
dontFormatInputForDoc: true,
document: "# welcome!\n\na: cat # meow\n\n# have a great day",
expression: `. | head_comment`,
expression: `. | headComment`,
expected: []string{
"D0, P[], (!!str)::welcome!\n",
},
@@ -116,7 +116,7 @@ var commentOperatorScenarios = []expressionScenario{
description: "Head comment with document split",
dontFormatInputForDoc: true,
document: "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day",
expression: `head_comment`,
expression: `headComment`,
expected: []string{
"D0, P[], (!!str)::welcome!\nbob\n",
},
@@ -125,7 +125,7 @@ var commentOperatorScenarios = []expressionScenario{
description: "Get foot comment",
dontFormatInputForDoc: true,
document: "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really",
expression: `. | foot_comment`,
expression: `. | footComment`,
expected: []string{
"D0, P[], (!!str)::have a great day\nno really\n",
},
-126
View File
@@ -1,126 +0,0 @@
package yqlib
import (
"container/list"
"errors"
"fmt"
"strings"
"time"
"gopkg.in/yaml.v3"
)
func getStringParamter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) {
result, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode)
if err != nil {
return "", err
} else if result.MatchingNodes.Len() == 0 {
return "", fmt.Errorf("could not find %v for format_time", parameterName)
}
return result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value, nil
}
func withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
if expressionNode.RHS.Operation.OperationType == blockOpType || expressionNode.RHS.Operation.OperationType == unionOpType {
layout, err := getStringParamter("layout", d, context, expressionNode.RHS.LHS)
if err != nil {
return Context{}, fmt.Errorf("could not get date time format: %w", err)
}
context.SetDateTimeLayout(layout)
return d.GetMatchingNodes(context, expressionNode.RHS.RHS)
}
return Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; <exp>)`)
}
// for unit tests
var Now = time.Now
func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
node := &yaml.Node{
Tag: "!!timestamp",
Kind: yaml.ScalarNode,
Value: Now().Format(time.RFC3339),
}
return context.SingleChildContext(&CandidateNode{Node: node}), nil
}
func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
format, err := getStringParamter("format", d, context, expressionNode.RHS)
layout := context.GetDateTimeLayout()
decoder := NewYamlDecoder()
if err != nil {
return Context{}, err
}
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
parsedTime, err := time.Parse(layout, candidate.Node.Value)
if err != nil {
return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
}
formattedTimeStr := parsedTime.Format(format)
decoder.Init(strings.NewReader(formattedTimeStr))
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket)
var node *yaml.Node
if errorReading != nil {
log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr)
node = &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: formattedTimeStr,
}
} else {
node = unwrapDoc(&dataBucket)
}
results.PushBack(candidate.CreateReplacement(node))
}
return context.ChildContext(results), nil
}
func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
timezoneStr, err := getStringParamter("timezone", d, context, expressionNode.RHS)
layout := context.GetDateTimeLayout()
if err != nil {
return Context{}, err
}
var results = list.New()
timezone, err := time.LoadLocation(timezoneStr)
if err != nil {
return Context{}, fmt.Errorf("could not load tz [%v]: %w", timezoneStr, err)
}
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
parsedTime, err := time.Parse(layout, candidate.Node.Value)
if err != nil {
return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
}
tzTime := parsedTime.In(timezone)
node := &yaml.Node{
Kind: yaml.ScalarNode,
Tag: candidate.Node.Tag,
Value: tzTime.Format(layout),
}
results.PushBack(candidate.CreateReplacement(node))
}
return context.ChildContext(results), nil
}
-120
View File
@@ -1,120 +0,0 @@
package yqlib
import (
"testing"
)
var dateTimeOperatorScenarios = []expressionScenario{
{
description: "Format: from standard RFC3339 format",
subdescription: "Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.",
document: `a: 2001-12-15T02:59:43.1Z`,
expression: `.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM\n",
},
},
{
description: "Format: from custom date time",
subdescription: "Use with_dtf to set a custom datetime format for parsing.",
document: `a: Saturday, 15-Dec-01 at 2:59AM`,
expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))`,
expected: []string{
"D0, P[], (doc)::a: 2001-12-15\n",
},
},
{
description: "Format: get the day of the week",
document: `a: 2001-12-15T02:59:43.1Z`,
expression: `.a | format_datetime("Monday")`,
expected: []string{
"D0, P[a], (!!str)::Saturday\n",
},
},
{
description: "Now",
document: "a: cool",
expression: `.updated = now`,
expected: []string{
"D0, P[], (doc)::a: cool\nupdated: 2021-05-19T01:02:03Z\n",
},
},
{
description: "Timezone: from standard RFC3339 format",
subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.",
document: "a: cool",
expression: `.updated = (now | tz("Australia/Sydney"))`,
expected: []string{
"D0, P[], (doc)::a: cool\nupdated: 2021-05-19T11:02:03+10:00\n",
},
},
{
description: "Timezone: with custom format",
subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'",
document: "a: Saturday, 15-Dec-01 at 2:59AM GMT",
expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
},
},
{
description: "Add and tz custom format",
subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'",
document: "a: Saturday, 15-Dec-01 at 2:59AM GMT",
expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
},
},
{
description: "Date addition",
document: `a: 2021-01-01T00:00:00Z`,
expression: `.a += "3h10m"`,
expected: []string{
"D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n",
},
},
{
description: "Date subtraction",
subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: 2021-01-01T03:10:00Z`,
expression: `.a -= "3h10m"`,
expected: []string{
"D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n",
},
},
{
description: "Date addition - custom format",
document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`,
expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n",
},
},
{
description: "Date script with custom format",
subdescription: "You can embed full expressions in with_dtf if needed.",
document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`,
expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:00PM AWST\n",
},
},
{
description: "allow comma",
skipDoc: true,
document: "a: Saturday, 15-Dec-01 at 2:59AM GMT",
expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST", tz("Australia/Sydney"))`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n",
},
},
}
func TestDatetimeOperatorScenarios(t *testing.T) {
for _, tt := range dateTimeOperatorScenarios {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "datetime", dateTimeOperatorScenarios)
}
+3 -3
View File
@@ -8,7 +8,7 @@ var documentIndexScenarios = []expressionScenario{
{
description: "Retrieve a document index",
document: "a: cat\n---\na: frog\n",
expression: `.a | document_index`,
expression: `.a | documentIndex`,
expected: []string{
"D0, P[a], (!!int)::0\n",
"D1, P[a], (!!int)::1\n",
@@ -26,7 +26,7 @@ var documentIndexScenarios = []expressionScenario{
{
description: "Filter by document index",
document: "a: cat\n---\na: frog\n",
expression: `select(document_index == 1)`,
expression: `select(documentIndex == 1)`,
expected: []string{
"D1, P[], (doc)::a: frog\n",
},
@@ -42,7 +42,7 @@ var documentIndexScenarios = []expressionScenario{
{
description: "Print Document Index with matches",
document: "a: cat\n---\na: frog\n",
expression: `.a | ({"match": ., "doc": document_index})`,
expression: `.a | ({"match": ., "doc": documentIndex})`,
expected: []string{
"D0, P[], (!!map)::match: cat\ndoc: 0\n",
"D0, P[], (!!map)::match: frog\ndoc: 1\n",
+2 -2
View File
@@ -16,7 +16,7 @@ var fileOperatorScenarios = []expressionScenario{
{
description: "Get file index",
document: `{a: cat}`,
expression: `file_index`,
expression: `fileIndex`,
expected: []string{
"D0, P[], (!!int)::0\n",
},
@@ -25,7 +25,7 @@ var fileOperatorScenarios = []expressionScenario{
description: "Get file indices of multiple documents",
document: `{a: cat}`,
document2: `{a: cat}`,
expression: `file_index`,
expression: `fileIndex`,
expected: []string{
"D0, P[], (!!int)::0\n",
"D0, P[], (!!int)::1\n",
+2 -2
View File
@@ -8,7 +8,7 @@ var splitDocOperatorScenarios = []expressionScenario{
{
description: "Split empty",
document: ``,
expression: `split_doc`,
expression: `splitDoc`,
expected: []string{
"D0, P[], (!!null)::\n",
},
@@ -16,7 +16,7 @@ var splitDocOperatorScenarios = []expressionScenario{
{
description: "Split array",
document: `[{a: cat}, {b: dog}]`,
expression: `.[] | split_doc`,
expression: `.[] | splitDoc`,
expected: []string{
"D0, P[0], (!!map)::{a: cat}\n",
"D1, P[1], (!!map)::{b: dog}\n",
+11 -46
View File
@@ -4,7 +4,6 @@ import (
"fmt"
"strconv"
"strings"
"time"
"gopkg.in/yaml.v3"
)
@@ -57,7 +56,7 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca
switch lhsNode.Kind {
case yaml.MappingNode:
return nil, fmt.Errorf("maps not yet supported for subtraction")
return nil, fmt.Errorf("Maps not yet supported for subtraction")
case yaml.SequenceNode:
if rhs.Node.Kind != yaml.SequenceNode {
return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
@@ -69,15 +68,13 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca
}
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhsNode.Style
if err := subtractScalars(context, target, lhsNode, rhs.Node); err != nil {
return nil, err
}
return subtractScalars(target, lhsNode, rhs.Node)
}
return target, nil
}
func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) {
lhsTag := lhs.Tag
rhsTag := rhs.Tag
lhsIsCustom := false
@@ -92,25 +89,16 @@ func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs
rhsTag = guessTagFromCustomType(rhs)
}
isDateTime := lhs.Tag == "!!timestamp"
// if the lhs is a string, it might be a timestamp in a custom format.
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
isDateTime = err == nil
}
if isDateTime {
return subtractDateTime(context.GetDateTimeLayout(), target, lhs, rhs)
} else if lhsTag == "!!str" {
return fmt.Errorf("strings cannot be subtracted")
if lhsTag == "!!str" {
return nil, fmt.Errorf("strings cannot be subtracted")
} else if lhsTag == "!!int" && rhsTag == "!!int" {
format, lhsNum, err := parseInt(lhs.Value)
if err != nil {
return err
return nil, err
}
_, rhsNum, err := parseInt(rhs.Value)
if err != nil {
return err
return nil, err
}
result := lhsNum - rhsNum
target.Node.Tag = lhs.Tag
@@ -118,11 +106,11 @@ func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
if err != nil {
return err
return nil, err
}
rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
if err != nil {
return err
return nil, err
}
result := lhsNum - rhsNum
if lhsIsCustom {
@@ -132,31 +120,8 @@ func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs
}
target.Node.Value = fmt.Sprintf("%v", result)
} else {
return fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag)
return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag)
}
return nil
}
func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
var durationStr string
if strings.HasPrefix(rhs.Value, "-") {
durationStr = rhs.Value[1:]
} else {
durationStr = "-" + rhs.Value
}
duration, err := time.ParseDuration(durationStr)
if err != nil {
return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err)
}
currentTime, err := time.Parse(layout, lhs.Value)
if err != nil {
return err
}
newTime := currentTime.Add(duration)
target.Node.Value = newTime.Format(layout)
return nil
return target, nil
}
-28
View File
@@ -93,34 +93,6 @@ var subtractOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::{a: 2, b: 4}\n",
},
},
{
description: "Date subtraction",
subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: 2021-01-01T03:10:00Z`,
expression: `.a -= "3h10m"`,
expected: []string{
"D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n",
},
},
{
description: "Date subtraction - custom format",
subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: Saturday, 15-Dec-01 at 6:00AM GMT`,
expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`,
expected: []string{
"D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM GMT\n",
},
},
{
skipDoc: true,
description: "Date subtraction - custom format",
subdescription: "You can subtract durations from dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
document: `a: !cat Saturday, 15-Dec-01 at 6:00AM GMT`,
expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`,
expected: []string{
"D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 2:59AM GMT\n",
},
},
{
description: "Custom types: that are really numbers",
subdescription: "When custom tags are encountered, yq will try to decode the underlying type.",
-4
View File
@@ -10,7 +10,6 @@ import (
"sort"
"strings"
"testing"
"time"
"github.com/mikefarah/yq/v4/test"
"gopkg.in/op/go-logging.v1"
@@ -32,9 +31,6 @@ type expressionScenario struct {
func TestMain(m *testing.M) {
logging.SetLevel(logging.ERROR, "")
Now = func() time.Time {
return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
}
code := m.Run()
os.Exit(code)
}
+1 -1
View File
@@ -12,7 +12,7 @@ func main() {
args := os.Args[1:]
_, _, err := cmd.Find(args)
if err != nil && args[0] != "__complete" {
if err != nil {
// default command when nothing matches...
newArgs := []string{"eval"}
cmd.SetArgs(append(newArgs, os.Args[1:]...))