suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-03-10 02:58:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mike Farah 2023-04-13 15:40:41 +10:00
parent 2de69a19f5
commit c38841ce20
10 changed files with 192 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/data1.yaml
View File

@ -1 +1,3 @@
["foobar", "foobaz", "blarp"] ---
# cool
a: hello #things

2
pkg/yqlib/candidate_node.go
View File

@ -324,7 +324,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences
} }
func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) { func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
log.Debug("UpdateAttributesFrom: n: %v other: %v", n.Key.Value, other.Key.Value) log.Debug("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
if n.Kind != other.Kind { if n.Kind != other.Kind {
// clear out the contents when switching to a different type // clear out the contents when switching to a different type
// e.g. map to array // e.g. map to array

9
pkg/yqlib/candidate_node_yaml.go
View File

@ -178,26 +178,29 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node) error {
} }
func (o *CandidateNode) MarshalYAML() (interface{}, error) { func (o *CandidateNode) MarshalYAML() (interface{}, error) {
log.Debug("encoding to yaml: %v", o.Tag) log.Debug("MarshalYAML to yaml: %v", o.Tag)
switch o.Kind { switch o.Kind {
case DocumentNode: case DocumentNode:
log.Debug("MarshalYAML its a document")
target := &yaml.Node{Kind: yaml.DocumentNode} target := &yaml.Node{Kind: yaml.DocumentNode}
o.copyToYamlNode(target) o.copyToYamlNode(target)
singleChild := &yaml.Node{} singleChild := &yaml.Node{}
err := singleChild.Encode(o.Content[0]) err := singleChild.Encode(o.Content[0])
log.Debug("MarshalYAML its a document - singChild is %v", singleChild.Tag)
if err != nil { if err != nil {
return nil, err return nil, err
} }
target.Content = []*yaml.Node{singleChild} target.Content = []*yaml.Node{singleChild}
return target, nil return target, nil
case AliasNode: case AliasNode:
log.Debug("encoding alias to yaml: %v", o.Tag) log.Debug("MarshalYAML - alias to yaml: %v", o.Tag)
target := &yaml.Node{Kind: yaml.AliasNode} target := &yaml.Node{Kind: yaml.AliasNode}
o.copyToYamlNode(target) o.copyToYamlNode(target)
return target, nil return target, nil
case ScalarNode: case ScalarNode:
log.Debug("encoding scalar: %v", o.Value) log.Debug("MarshalYAML - scalar: %v", o.Value)
target := &yaml.Node{Kind: yaml.ScalarNode} target := &yaml.Node{Kind: yaml.ScalarNode}
o.copyToYamlNode(target) o.copyToYamlNode(target)
return target, nil return target, nil

113
pkg/yqlib/candidiate_node_json.go
View File

@ -2,10 +2,123 @@ package yqlib
import ( import (
"bytes" "bytes"
"errors"
"fmt"
"io"
"github.com/goccy/go-json" "github.com/goccy/go-json"
) )
func (o *CandidateNode) setScalarFromJson(value interface{}) error {
o.Kind = ScalarNode
switch rawData := value.(type) {
case nil:
o.Tag = "!!null"
o.Value = "null"
case float64, float32:
o.Value = fmt.Sprintf("%v", value)
o.Tag = "!!float"
// json decoder returns ints as float.
if value == float64(int(rawData.(float64))) {
// aha it's an int disguised as a float
o.Tag = "!!int"
}
case int, int64, int32:
o.Value = fmt.Sprintf("%v", value)
o.Tag = "!!int"
case bool:
o.Value = fmt.Sprintf("%v", value)
o.Tag = "!!bool"
case string:
o.Value = rawData
o.Tag = "!!str"
default:
return fmt.Errorf("unrecognised type :( %v", rawData)
}
return nil
}
func (o *CandidateNode) UnmarshalJSON(data []byte) error {
log.Debug("UnmarshalJSON")
switch data[0] {
case '{':
log.Debug("UnmarshalJSON - its a map!")
// its a map
o.Kind = MappingNode
o.Tag = "!!map"
dec := json.NewDecoder(bytes.NewReader(data))
_, err := dec.Token() // open object
if err != nil {
return err
}
// cycle through k/v
var tok json.Token
for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
// we can expect two types: string or Delim. Delim automatically means
// that it is the closing bracket of the object, whereas string means
// that there is another key.
if _, ok := tok.(json.Delim); ok {
break
}
childKey := o.CreateChild()
childKey.IsMapKey = true
childKey.Value = tok.(string)
childKey.Kind = ScalarNode
childKey.Tag = "!!str"
childValue := o.CreateChild()
childValue.Key = childKey
if err := dec.Decode(childValue); err != nil {
return err
}
o.Content = append(o.Content, childKey, childValue)
}
// unexpected error
if err != nil && !errors.Is(err, io.EOF) {
return err
}
return nil
case '[':
log.Debug("UnmarshalJSON - its an array!")
var children []*CandidateNode
if err := json.Unmarshal(data, &children); err != nil {
return err
}
// now we put the children into the content, and set a key value for them
for i, child := range children {
childKey := o.CreateChild()
childKey.Kind = ScalarNode
childKey.Tag = "!!int"
childKey.Value = fmt.Sprintf("%v", i)
child.Parent = o
child.Document = o.Document
child.FileIndex = o.FileIndex
child.Filename = o.Filename
child.Key = childKey
o.Content[i] = child
}
return nil
}
log.Debug("UnmarshalJSON - its a scalar!")
// otherwise, must be a scalar
var scalar interface{}
err := json.Unmarshal(data, &scalar)
if err != nil {
return err
}
log.Debug("UnmarshalJSON - scalar is %v", scalar)
return o.setScalarFromJson(scalar)
}
func (o *CandidateNode) MarshalJSON() ([]byte, error) { func (o *CandidateNode) MarshalJSON() ([]byte, error) {
log.Debugf("MarshalJSON %v", NodeToString(o)) log.Debugf("MarshalJSON %v", NodeToString(o))
buf := new(bytes.Buffer) buf := new(bytes.Buffer)

17
pkg/yqlib/decoder_json.go
View File

@ -24,22 +24,21 @@ func (dec *jsonDecoder) Init(reader io.Reader) error {
func (dec *jsonDecoder) Decode() (*CandidateNode, error) { func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
var dataBucket orderedMap var dataBucket CandidateNode
log.Debug("going to decode json")
err := dec.decoder.Decode(&dataBucket) err := dec.decoder.Decode(&dataBucket)
if err != nil { if err != nil {
return nil, err return nil, err
} }
log.Debug("convert to yaml") // log.Debug("convert to yaml")
node, err := dec.convertToYamlNode(&dataBucket) // node, err := dec.convertToYamlNode(&dataBucket)
log.Debug("done, %w", err) // log.Debug("done, %w", err)
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
return &CandidateNode{ return &CandidateNode{
Kind: DocumentNode, Kind: DocumentNode,
Content: []*CandidateNode{node}, Content: []*CandidateNode{&dataBucket},
}, nil }, nil
} }

2
pkg/yqlib/decoder_test.go
View File

@ -33,7 +33,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
return "", err return "", err
} }
log.Debugf("read the documents") log.Debugf("done reading the documents")
expression := s.expression expression := s.expression
if expression == "" { if expression == "" {

319
pkg/yqlib/doc/operators/add.md
View File

@ -9,328 +9,17 @@ Add behaves differently according to the type of the LHS:
Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`.
## Concatenate arrays ##
Given a sample.yml file of: Given a sample.yml file of:
```yaml ```yaml
a: a: hello
- 1
- 2
b:
- 3
- 4
``` ```
then then
```bash ```bash
yq '.a + .b' sample.yml yq sample.yml
``` ```
will output will output
```yaml ```yaml
- 1 a: hello
- 2
- 3
- 4
```
## Concatenate to existing array
Note that the styling of `a` is kept.
Given a sample.yml file of:
```yaml
a: [1,2]
b:
- 3
- 4
```
then
```bash
yq '.a += .b' sample.yml
```
will output
```yaml
a: [1, 2, 3, 4]
b:
- 3
- 4
```
## Concatenate null to array
Given a sample.yml file of:
```yaml
a:
- 1
- 2
```
then
```bash
yq '.a + null' sample.yml
```
will output
```yaml
- 1
- 2
```
## Append to existing array
Note that the styling is copied from existing array elements
Given a sample.yml file of:
```yaml
a: ['dog']
```
then
```bash
yq '.a += "cat"' sample.yml
```
will output
```yaml
a: ['dog', 'cat']
```
## Prepend to existing array
Given a sample.yml file of:
```yaml
a:
- dog
```
then
```bash
yq '.a = ["cat"] + .a' sample.yml
```
will output
```yaml
a:
- cat
- dog
```
## Add new object to array
Given a sample.yml file of:
```yaml
a:
- dog: woof
```
then
```bash
yq '.a + {"cat": "meow"}' sample.yml
```
will output
```yaml
- dog: woof
- cat: meow
```
## Relative append
Given a sample.yml file of:
```yaml
a:
a1:
b:
- cat
a2:
b:
- dog
a3: {}
```
then
```bash
yq '.a[].b += ["mouse"]' sample.yml
```
will output
```yaml
a:
a1:
b:
- cat
- mouse
a2:
b:
- dog
- mouse
a3:
b:
- mouse
```
## String concatenation
Given a sample.yml file of:
```yaml
a: cat
b: meow
```
then
```bash
yq '.a += .b' sample.yml
```
will output
```yaml
a: catmeow
b: meow
```
## Number addition - float
If the lhs or rhs are floats then the expression will be calculated with floats.
Given a sample.yml file of:
```yaml
a: 3
b: 4.9
```
then
```bash
yq '.a = .a + .b' sample.yml
```
will output
```yaml
a: 7.9
b: 4.9
```
## Number addition - int
If both the lhs and rhs are ints then the expression will be calculated with ints.
Given a sample.yml file of:
```yaml
a: 3
b: 4
```
then
```bash
yq '.a = .a + .b' sample.yml
```
will output
```yaml
a: 7
b: 4
```
## Increment numbers
Given a sample.yml file of:
```yaml
a: 3
b: 5
```
then
```bash
yq '.[] += 1' sample.yml
```
will output
```yaml
a: 4
b: 6
```
## Date addition
You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: 2021-01-01T00:00:00Z
```
then
```bash
yq '.a += "3h10m"' sample.yml
```
will output
```yaml
a: 2021-01-01T03:10:00Z
```
## Date addition - custom format
You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
Given a sample.yml file of:
```yaml
a: Saturday, 15-Dec-01 at 2:59AM GMT
```
then
```bash
yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml
```
will output
```yaml
a: Saturday, 15-Dec-01 at 6:00AM GMT
```
## Add to null
Adding to null simply returns the rhs
Running
```bash
yq --null-input 'null + "cat"'
```
will output
```yaml
cat
```
## Add maps to shallow merge
Adding objects together shallow merges them. Use `*` to deeply merge.
Given a sample.yml file of:
```yaml
a:
thing:
name: Astuff
value: x
a1: cool
b:
thing:
name: Bstuff
legs: 3
b1: neat
```
then
```bash
yq '.a += .b' sample.yml
```
will output
```yaml
a:
thing:
name: Bstuff
legs: 3
a1: cool
b1: neat
b:
thing:
name: Bstuff
legs: 3
b1: neat
```
## Custom types: that are really strings
When custom tags are encountered, yq will try to decode the underlying type.
Given a sample.yml file of:
```yaml
a: !horse cat
b: !goat _meow
```
then
```bash
yq '.a += .b' sample.yml
```
will output
```yaml
a: !horse cat_meow
b: !goat _meow
```
## Custom types: that are really numbers
When custom tags are encountered, yq will try to decode the underlying type.
Given a sample.yml file of:
```yaml
a: !horse 1.2
b: !goat 2.3
```
then
```bash
yq '.a += .b' sample.yml
```
will output
```yaml
a: !horse 3.5
b: !goat 2.3
``` ```

5
pkg/yqlib/encoder_yaml.go
View File

@ -74,7 +74,8 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
} }
func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
log.Debug("encoderYaml - going to print %v", NodeToString(node))
log.Debug("encoderYaml - going to print u %v", NodeToString(node.unwrapDocument()))
if node.Kind == ScalarNode && ye.prefs.UnwrapScalar { if node.Kind == ScalarNode && ye.prefs.UnwrapScalar {
return writeString(writer, node.Value+"\n") return writeString(writer, node.Value+"\n")
} }
@ -89,7 +90,7 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
encoder.SetIndent(ye.indent) encoder.SetIndent(ye.indent)
if err := encoder.Encode(node); err != nil { if err := encoder.Encode(node.unwrapDocument()); err != nil {
return err return err
} }

70
pkg/yqlib/json_test.go
View File

@ -212,13 +212,13 @@ var jsonScenarios = []formatScenario{
// expected: expectedJsonKeysInOrder, // expected: expectedJsonKeysInOrder,
// scenarioType: "decode-ndjson", // scenarioType: "decode-ndjson",
// }, // },
{ // {
description: "numbers", // description: "numbers",
skipDoc: true, // skipDoc: true,
input: "[3, 3.0, 3.1, -1]", // input: "[3, 3.0, 3.1, -1]",
expected: "- 3\n- 3\n- 3.1\n- -1\n", // expected: "- 3\n- 3\n- 3.1\n- -1\n",
scenarioType: "decode-ndjson", // scenarioType: "decode-ndjson",
}, // },
{ {
description: "number single", description: "number single",
skipDoc: true, skipDoc: true,
@ -226,34 +226,34 @@ var jsonScenarios = []formatScenario{
expected: "3\n", expected: "3\n",
scenarioType: "decode-ndjson", scenarioType: "decode-ndjson",
}, },
{ // {
description: "empty string", // description: "empty string",
skipDoc: true, // skipDoc: true,
input: `""`, // input: `""`,
expected: "\"\"\n", // expected: "\"\"\n",
scenarioType: "decode-ndjson", // scenarioType: "decode-ndjson",
}, // },
{ // {
description: "strings", // description: "strings",
skipDoc: true, // skipDoc: true,
input: `["", "cat"]`, // input: `["", "cat"]`,
expected: "- \"\"\n- cat\n", // expected: "- \"\"\n- cat\n",
scenarioType: "decode-ndjson", // scenarioType: "decode-ndjson",
}, // },
{ // {
description: "null", // description: "null",
skipDoc: true, // skipDoc: true,
input: `null`, // input: `null`,
expected: "null\n", // expected: "null\n",
scenarioType: "decode-ndjson", // scenarioType: "decode-ndjson",
}, // },
{ // {
description: "booleans", // description: "booleans",
skipDoc: true, // skipDoc: true,
input: `[true, false]`, // input: `[true, false]`,
expected: "- true\n- false\n", // expected: "- true\n- false\n",
scenarioType: "decode-ndjson", // scenarioType: "decode-ndjson",
}, // },
} }
func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) { func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) {

21
pkg/yqlib/yaml_test.go
View File

@ -6,7 +6,7 @@ import (
"github.com/mikefarah/yq/v4/test" "github.com/mikefarah/yq/v4/test"
) )
var yamlScenarios = []formatScenario{ var yamlFormatScenarios = []formatScenario{
{ {
description: "basic - null", description: "basic - null",
skipDoc: true, skipDoc: true,
@ -51,12 +51,27 @@ var yamlScenarios = []formatScenario{
}, },
} }
var yamlParseScenarios = []expressionScenario{
{
document: `a: hello # things`,
expected: []string{
"D0, P[], (doc)::a: hello #things\n",
},
},
}
func testYamlScenario(t *testing.T, s formatScenario) { func testYamlScenario(t *testing.T, s formatScenario) {
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
} }
func TestYamlScenarios(t *testing.T) { func TestYamlParseScenarios(t *testing.T) {
for _, tt := range yamlScenarios { for _, tt := range yamlParseScenarios {
testScenario(t, &tt)
}
}
func TestYamlFormatScenarios(t *testing.T) {
for _, tt := range yamlFormatScenarios {
testYamlScenario(t, tt) testYamlScenario(t, tt)
} }
} }