From 4eca302efc3ce90b55f37dd466dd6c283ebe49f2 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 15 Apr 2023 14:10:12 +1000 Subject: [PATCH] yaml includes comments --- pkg/yqlib/all_at_once_evaluator_test.go | 2 +- pkg/yqlib/candidate_node.go | 2 +- pkg/yqlib/candidate_node_yaml.go | 20 +- pkg/yqlib/decoder_yaml.go | 7 +- pkg/yqlib/doc/operators/add.md | 319 +++++++++++++++++++++++- pkg/yqlib/encoder_yaml.go | 8 +- pkg/yqlib/lib.go | 15 +- pkg/yqlib/yaml_test.go | 21 +- 8 files changed, 367 insertions(+), 27 deletions(-) diff --git a/pkg/yqlib/all_at_once_evaluator_test.go b/pkg/yqlib/all_at_once_evaluator_test.go index f83a2156..cbfb1111 100644 --- a/pkg/yqlib/all_at_once_evaluator_test.go +++ b/pkg/yqlib/all_at_once_evaluator_test.go @@ -20,7 +20,7 @@ var evaluateNodesScenario = []expressionScenario{ document: `a: hello`, expression: `.`, expected: []string{ - "D0, P[], (!!map)::a: hello\n", + "D0, P[], (doc)::a: hello\n", }, }, { diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index f079885c..9f3c1d92 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -135,7 +135,7 @@ func (n *CandidateNode) getParsedKey() interface{} { } func (n *CandidateNode) GetPath() []interface{} { - if n.Parent != nil { + if n.Parent != nil && n.getParsedKey() != nil { return append(n.Parent.GetPath(), n.getParsedKey()) } key := n.getParsedKey() diff --git a/pkg/yqlib/candidate_node_yaml.go b/pkg/yqlib/candidate_node_yaml.go index 2d67ed38..c1467d2e 100644 --- a/pkg/yqlib/candidate_node_yaml.go +++ b/pkg/yqlib/candidate_node_yaml.go @@ -51,7 +51,6 @@ func (o *CandidateNode) copyFromYamlNode(node *yaml.Node) { // o.Alias = TODO - find Alias in our own structure // might need to be a post process thing - o.HeadComment = node.HeadComment o.LineComment = node.LineComment o.FootComment = node.FootComment @@ -71,6 +70,7 @@ func (o *CandidateNode) copyToYamlNode(node *yaml.Node) { // might need to be a post process thing node.HeadComment = o.HeadComment + node.LineComment = o.LineComment node.FootComment = o.FootComment @@ -89,7 +89,7 @@ func (o *CandidateNode) decodeIntoChild(childNode *yaml.Node) (*CandidateNode, e return newChild, nil } - err := childNode.Decode(newChild) + err := newChild.UnmarshalYAML(childNode) return newChild, err } @@ -110,6 +110,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error { return err } o.Content = []*CandidateNode{singleChild} + log.Debugf("UnmarshalYAML - finished document node") return nil case yaml.AliasNode: log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag) @@ -145,6 +146,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error { o.Content[i] = keyNode o.Content[i+1] = valueNode } + log.Debugf("UnmarshalYAML - finished mapping node") return nil case yaml.SequenceNode: log.Debugf("UnmarshalYAML - a sequence: %v", len(node.Content)) @@ -168,16 +170,16 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error { } return nil case 0: - log.Debugf("UnmarshalYAML - errr.. %v", node.Tag) // not sure when this happens o.copyFromYamlNode(node) + log.Debugf("UnmarshalYAML - errr.. %v", NodeToString(o)) return nil default: return fmt.Errorf("orderedMap: invalid yaml node") } } -func (o *CandidateNode) MarshalYAML() (interface{}, error) { +func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) { log.Debug("MarshalYAML to yaml: %v", o.Tag) switch o.Kind { case DocumentNode: @@ -185,10 +187,9 @@ func (o *CandidateNode) MarshalYAML() (interface{}, error) { target := &yaml.Node{Kind: yaml.DocumentNode} o.copyToYamlNode(target) - singleChild := &yaml.Node{} - err := singleChild.Encode(o.Content[0]) + singleChild, err := o.Content[0].MarshalYAML() - log.Debug("MarshalYAML its a document - singChild is %v", singleChild.Tag) + log.Debug("MarshalYAML its a document - singChild is %v", singleChild) if err != nil { return nil, err } @@ -213,8 +214,9 @@ func (o *CandidateNode) MarshalYAML() (interface{}, error) { o.copyToYamlNode(target) target.Content = make([]*yaml.Node, len(o.Content)) for i := 0; i < len(o.Content); i += 1 { - child := &yaml.Node{} - err := child.Encode(o.Content[i]) + + child, err := o.Content[i].MarshalYAML() + if err != nil { return nil, err } diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go index a7ba89ab..d38f0632 100644 --- a/pkg/yqlib/decoder_yaml.go +++ b/pkg/yqlib/decoder_yaml.go @@ -97,8 +97,8 @@ func (dec *yamlDecoder) Init(reader io.Reader) error { } func (dec *yamlDecoder) Decode() (*CandidateNode, error) { - var candidateNode CandidateNode - err := dec.decoder.Decode(&candidateNode) + var yamlNode yaml.Node + err := dec.decoder.Decode(&yamlNode) if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything { // force returning an empty node with a comment. @@ -117,6 +117,9 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) { return nil, err } + candidateNode := CandidateNode{} + candidateNode.UnmarshalYAML(&yamlNode) + if dec.leadingContent != "" { candidateNode.LeadingContent = dec.leadingContent dec.leadingContent = "" diff --git a/pkg/yqlib/doc/operators/add.md b/pkg/yqlib/doc/operators/add.md index 384a702f..469e13d8 100644 --- a/pkg/yqlib/doc/operators/add.md +++ b/pkg/yqlib/doc/operators/add.md @@ -9,17 +9,328 @@ Add behaves differently according to the type of the LHS: Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. -## +## Concatenate arrays Given a sample.yml file of: ```yaml -a: hello +a: + - 1 + - 2 +b: + - 3 + - 4 ``` then ```bash -yq sample.yml +yq '.a + .b' sample.yml ``` will output ```yaml -a: hello +- 1 +- 2 +- 3 +- 4 +``` + +## Concatenate to existing array +Note that the styling of `a` is kept. + +Given a sample.yml file of: +```yaml +a: [1,2] +b: + - 3 + - 4 +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: [1, 2, 3, 4] +b: + - 3 + - 4 +``` + +## Concatenate null to array +Given a sample.yml file of: +```yaml +a: + - 1 + - 2 +``` +then +```bash +yq '.a + null' sample.yml +``` +will output +```yaml +- 1 +- 2 +``` + +## Append to existing array +Note that the styling is copied from existing array elements + +Given a sample.yml file of: +```yaml +a: ['dog'] +``` +then +```bash +yq '.a += "cat"' sample.yml +``` +will output +```yaml +a: ['dog', 'cat'] +``` + +## Prepend to existing array +Given a sample.yml file of: +```yaml +a: + - dog +``` +then +```bash +yq '.a = ["cat"] + .a' sample.yml +``` +will output +```yaml +a: + - cat + - dog +``` + +## Add new object to array +Given a sample.yml file of: +```yaml +a: + - dog: woof +``` +then +```bash +yq '.a + {"cat": "meow"}' sample.yml +``` +will output +```yaml +- dog: woof +- cat: meow +``` + +## Relative append +Given a sample.yml file of: +```yaml +a: + a1: + b: + - cat + a2: + b: + - dog + a3: {} +``` +then +```bash +yq '.a[].b += ["mouse"]' sample.yml +``` +will output +```yaml +a: + a1: + b: + - cat + - mouse + a2: + b: + - dog + - mouse + a3: + b: + - mouse +``` + +## String concatenation +Given a sample.yml file of: +```yaml +a: cat +b: meow +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: catmeow +b: meow +``` + +## Number addition - float +If the lhs or rhs are floats then the expression will be calculated with floats. + +Given a sample.yml file of: +```yaml +a: 3 +b: 4.9 +``` +then +```bash +yq '.a = .a + .b' sample.yml +``` +will output +```yaml +a: 7.9 +b: 4.9 +``` + +## Number addition - int +If both the lhs and rhs are ints then the expression will be calculated with ints. + +Given a sample.yml file of: +```yaml +a: 3 +b: 4 +``` +then +```bash +yq '.a = .a + .b' sample.yml +``` +will output +```yaml +a: 7 +b: 4 +``` + +## Increment numbers +Given a sample.yml file of: +```yaml +a: 3 +b: 5 +``` +then +```bash +yq '.[] += 1' sample.yml +``` +will output +```yaml +a: 4 +b: 6 +``` + +## Date addition +You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: 2021-01-01T00:00:00Z +``` +then +```bash +yq '.a += "3h10m"' sample.yml +``` +will output +```yaml +a: 2021-01-01T03:10:00Z +``` + +## Date addition - custom format +You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` +then +```bash +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 6:00AM GMT +``` + +## Add to null +Adding to null simply returns the rhs + +Running +```bash +yq --null-input 'null + "cat"' +``` +will output +```yaml +cat +``` + +## Add maps to shallow merge +Adding objects together shallow merges them. Use `*` to deeply merge. + +Given a sample.yml file of: +```yaml +a: + thing: + name: Astuff + value: x + a1: cool +b: + thing: + name: Bstuff + legs: 3 + b1: neat +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: + thing: + name: Bstuff + legs: 3 + a1: cool + b1: neat +b: + thing: + name: Bstuff + legs: 3 + b1: neat +``` + +## Custom types: that are really strings +When custom tags are encountered, yq will try to decode the underlying type. + +Given a sample.yml file of: +```yaml +a: !horse cat +b: !goat _meow +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: !horse cat_meow +b: !goat _meow +``` + +## Custom types: that are really numbers +When custom tags are encountered, yq will try to decode the underlying type. + +Given a sample.yml file of: +```yaml +a: !horse 1.2 +b: !goat 2.3 +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: !horse 3.5 +b: !goat 2.3 ``` diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go index 418cda8b..bf2e2cd5 100644 --- a/pkg/yqlib/encoder_yaml.go +++ b/pkg/yqlib/encoder_yaml.go @@ -90,7 +90,13 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { encoder.SetIndent(ye.indent) - if err := encoder.Encode(node.unwrapDocument()); err != nil { + target, err := node.MarshalYAML() + + if err != nil { + return err + } + + if err := encoder.Encode(target); err != nil { return err } diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index cbf0d57c..ba9d9e52 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -440,20 +440,19 @@ func NodeToString(node *CandidateNode) string { } else if node.Kind == AliasNode { tag = "alias" } - return fmt.Sprintf(`D%v, P%v, (%v)::%v`, node.Document, node.GetNicePath(), tag, node.Value) + return fmt.Sprintf(`D%v, P%v, %v (%v)::%v`, node.Document, node.GetNicePath(), KindString(node.Kind), tag, node.Value) } - -func KindString(kind yaml.Kind) string { +func KindString(kind Kind) string { switch kind { - case yaml.ScalarNode: + case ScalarNode: return "ScalarNode" - case yaml.SequenceNode: + case SequenceNode: return "SequenceNode" - case yaml.MappingNode: + case MappingNode: return "MappingNode" - case yaml.DocumentNode: + case DocumentNode: return "DocumentNode" - case yaml.AliasNode: + case AliasNode: return "AliasNode" default: return "unknown!" diff --git a/pkg/yqlib/yaml_test.go b/pkg/yqlib/yaml_test.go index 936525fd..0d1a0fb3 100644 --- a/pkg/yqlib/yaml_test.go +++ b/pkg/yqlib/yaml_test.go @@ -49,13 +49,32 @@ var yamlFormatScenarios = []formatScenario{ input: "3.1", expected: "3.1\n", }, + { + description: "basic - float", + skipDoc: true, + input: "[1, 2]", + expected: "[1, 2]\n", + }, } var yamlParseScenarios = []expressionScenario{ { document: `a: hello # things`, expected: []string{ - "D0, P[], (doc)::a: hello #things\n", + "D0, P[], (doc)::a: hello # things\n", + }, + }, + { + document: "a: &a apple\nb: *a", + expression: ".b", + expected: []string{ + "D0, P[b], (!!str)::apple\n", + }, + }, + { + document: `a: [1,2]`, + expected: []string{ + "D0, P[], (doc)::a: [1, 2]\n", }, }, }