diff --git a/pkg/yqlib/all_at_once_evaluator_test.go b/pkg/yqlib/all_at_once_evaluator_test.go index cbfb1111..061019e1 100644 --- a/pkg/yqlib/all_at_once_evaluator_test.go +++ b/pkg/yqlib/all_at_once_evaluator_test.go @@ -38,7 +38,11 @@ func TestAllAtOnceEvaluateNodes(t *testing.T) { for _, tt := range evaluateNodesScenario { decoder := NewYamlDecoder(NewDefaultYamlPreferences()) reader := bufio.NewReader(strings.NewReader(tt.document)) - decoder.Init(reader) + err := decoder.Init(reader) + if err != nil { + t.Error(err) + return + } candidateNode, errorReading := decoder.Decode() if errorReading != nil { diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 3ed3f8bf..0bfeebe0 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -185,8 +185,6 @@ func (n *CandidateNode) GetNicePath() string { path := n.GetPath() for i, element := range path { elementStr := fmt.Sprintf("%v", element) - log.Debugf("element: %v", element) - log.Debugf("elementStr: %v", elementStr) switch element.(type) { case int: sb.WriteString("[" + elementStr + "]") @@ -209,6 +207,10 @@ func (n *CandidateNode) AsList() *list.List { return elMap } +func (n *CandidateNode) SetParent(parent *CandidateNode) { + n.Parent = parent +} + func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) { key := rawKey.unwrapDocument().Copy() key.SetParent(n) @@ -221,8 +223,18 @@ func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *Candid n.Content = append(n.Content, key, value) } -func (n *CandidateNode) SetParent(parent *CandidateNode) { - n.Parent = parent +func (n *CandidateNode) AddChild(rawChild *CandidateNode) { + value := rawChild.unwrapDocument().Copy() + value.SetParent(n) + if value.Key != nil { + value.Key.SetParent(n) + } else { + index := len(n.Content) + keyNode := createScalarNode(index, fmt.Sprintf("%v", index)) + keyNode.SetParent(n) + value.Key = keyNode + } + n.Content = append(n.Content, value) } func (n *CandidateNode) AddChildren(children []*CandidateNode) { @@ -230,29 +242,12 @@ func (n *CandidateNode) AddChildren(children []*CandidateNode) { for i := 0; i < len(children); i += 2 { key := children[i] value := children[i+1] - - keyClone := key.Copy() - keyClone.SetParent(n) - - valueClone := value.Copy() - valueClone.SetParent(n) - valueClone.Key = keyClone - n.Content = append(n.Content, keyClone, valueClone) + n.AddKeyValueChild(key, value) } } else { for _, rawChild := range children { - value := rawChild.unwrapDocument().Copy() - value.SetParent(n) - if value.Key != nil { - value.Key.SetParent(n) - } else { - index := len(n.Content) - keyNode := createScalarNode(index, fmt.Sprintf("%v", index)) - keyNode.SetParent(n) - value.Key = keyNode - } - n.Content = append(n.Content, value) + n.AddChild(rawChild) } } } @@ -269,7 +264,7 @@ func (n *CandidateNode) GetValueRep() (interface{}, error) { // need to test this return strconv.ParseFloat(n.Value, 64) case "!!bool": - return isTruthyNode(n) + return isTruthyNode(n), nil case "!!null": return nil, nil } @@ -307,9 +302,12 @@ func (n *CandidateNode) CreateReplacement(kind Kind, tag string, value string) * func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *CandidateNode { newCopy := replacement.Copy() newCopy.Parent = n.Parent - newCopy.Key = n.Key - newCopy.IsMapKey = n.IsMapKey + if n.IsMapKey { + newCopy.Key = n + } else { + newCopy.Key = n.Key + } return newCopy } diff --git a/pkg/yqlib/candidate_node_yaml.go b/pkg/yqlib/candidate_node_yaml.go index 2e42eaf0..574bd112 100644 --- a/pkg/yqlib/candidate_node_yaml.go +++ b/pkg/yqlib/candidate_node_yaml.go @@ -168,7 +168,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*Can log.Debugf("node Style: %v", node.Style) log.Debugf("o Style: %v", o.Style) o.Content = make([]*CandidateNode, len(node.Content)) - for i := 0; i < len(node.Content); i += 1 { + for i := 0; i < len(node.Content); i++ { keyNode := o.CreateChild() keyNode.IsMapKey = true keyNode.Tag = "!!int" @@ -230,7 +230,7 @@ func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) { log.Debugf("original style: %v", o.Style) log.Debugf("original: %v, tag: %v, style: %v, kind: %v", NodeToString(o), target.Tag, target.Style, target.Kind == yaml.SequenceNode) target.Content = make([]*yaml.Node, len(o.Content)) - for i := 0; i < len(o.Content); i += 1 { + for i := 0; i < len(o.Content); i++ { child, err := o.Content[i].MarshalYAML() diff --git a/pkg/yqlib/csv_test.go b/pkg/yqlib/csv_test.go index 4b5af74a..66bc5de7 100644 --- a/pkg/yqlib/csv_test.go +++ b/pkg/yqlib/csv_test.go @@ -65,89 +65,105 @@ because excel is cool ` var csvScenarios = []formatScenario{ - // { - // description: "Encode CSV simple", - // input: csvTestSimpleYaml, - // expected: expectedSimpleCsv, - // scenarioType: "encode-csv", - // }, - // { - // description: "Encode TSV simple", - // input: csvTestSimpleYaml, - // expected: tsvTestExpectedSimpleCsv, - // scenarioType: "encode-tsv", - // }, - // { - // description: "Encode Empty", - // skipDoc: true, - // input: `[]`, - // expected: "", - // scenarioType: "encode-csv", - // }, - // { - // description: "Comma in value", - // skipDoc: true, - // input: `["comma, in, value", things]`, - // expected: "\"comma, in, value\",things\n", - // scenarioType: "encode-csv", - // }, - // { - // description: "Encode array of objects to csv", - // input: expectedYamlFromCSV, - // expected: csvSimple, - // scenarioType: "encode-csv", - // }, - // { - // description: "Encode array of objects to custom csv format", - // subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.", - // input: expectedYamlFromCSV, - // expected: csvSimpleShort, - // expression: `[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]`, - // scenarioType: "encode-csv", - // }, - // { - // description: "Encode array of objects to csv - missing fields behaviour", - // subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank", - // input: expectedYamlFromCSVMissingData, - // expected: csvSimpleMissingData, - // scenarioType: "encode-csv", - // }, - // { - // description: "decode csv missing", - // skipDoc: true, - // input: csvMissing, - // expected: csvMissing, - // scenarioType: "roundtrip-csv", - // }, - // { - // description: "Parse CSV into an array of objects", - // subdescription: "First row is assumed to be the header row.", - // input: csvSimple, - // expected: expectedYamlFromCSV, - // scenarioType: "decode-csv-object", - // }, - // { - // description: "Scalar roundtrip", - // skipDoc: true, - // input: "mike\ncat", - // expression: ".[0].mike", - // expected: "cat\n", - // scenarioType: "roundtrip-csv", - // }, - // { - // description: "Parse TSV into an array of objects", - // subdescription: "First row is assumed to be the header row.", - // input: tsvSimple, - // expected: expectedYamlFromCSV, - // scenarioType: "decode-tsv-object", - // }, - // { - // description: "Round trip", - // input: csvSimple, - // expected: expectedUpdatedSimpleCsv, - // expression: `(.[] | select(.name == "Gary") | .numberOfCats) = 3`, - // scenarioType: "roundtrip-csv", - // }, + { + description: "Encode CSV simple", + input: csvTestSimpleYaml, + expected: expectedSimpleCsv, + scenarioType: "encode-csv", + }, + { + description: "Encode TSV simple", + input: csvTestSimpleYaml, + expected: tsvTestExpectedSimpleCsv, + scenarioType: "encode-tsv", + }, + { + description: "Encode Empty", + skipDoc: true, + input: `[]`, + expected: "", + scenarioType: "encode-csv", + }, + { + description: "Comma in value", + skipDoc: true, + input: `["comma, in, value", things]`, + expected: "\"comma, in, value\",things\n", + scenarioType: "encode-csv", + }, + { + description: "Encode array of objects to csv", + input: expectedYamlFromCSV, + expected: csvSimple, + scenarioType: "encode-csv", + }, + { + description: "Encode array of objects to custom csv format", + subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.", + input: expectedYamlFromCSV, + expected: csvSimpleShort, + expression: `[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]`, + scenarioType: "encode-csv", + }, + { + description: "Encode array of objects to csv - missing fields behaviour", + subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank", + input: expectedYamlFromCSVMissingData, + expected: csvSimpleMissingData, + scenarioType: "encode-csv", + }, + { + description: "decode csv missing", + skipDoc: true, + input: csvMissing, + expected: csvMissing, + scenarioType: "roundtrip-csv", + }, + { + description: "decode csv key", + skipDoc: true, + input: csvSimple, + expression: ".[0].name | key", + expected: "name\n", + scenarioType: "decode-csv-object", + }, + { + description: "decode csv parent", + skipDoc: true, + input: csvSimple, + expression: ".[0].name | parent | .height", + expected: "168.8\n", + scenarioType: "decode-csv-object", + }, + { + description: "Parse CSV into an array of objects", + subdescription: "First row is assumed to be the header row.", + input: csvSimple, + expected: expectedYamlFromCSV, + scenarioType: "decode-csv-object", + }, + { + description: "Scalar roundtrip", + skipDoc: true, + input: "mike\ncat", + expression: ".[0].mike", + expected: "cat\n", + scenarioType: "roundtrip-csv", + }, + { + description: "Parse TSV into an array of objects", + subdescription: "First row is assumed to be the header row.", + input: tsvSimple, + expected: expectedYamlFromCSV, + scenarioType: "decode-tsv-object", + }, + { + description: "Round trip", + input: csvSimple, + expected: expectedUpdatedSimpleCsv, + expression: `(.[] | select(.name == "Gary") | .numberOfCats) = 3`, + scenarioType: "roundtrip-csv", + }, } func testCSVScenario(t *testing.T, s formatScenario) { @@ -286,5 +302,5 @@ func TestCSVScenarios(t *testing.T) { for i, s := range csvScenarios { genericScenarios[i] = s } - // documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario) + documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario) } diff --git a/pkg/yqlib/decoder_csv_object.go b/pkg/yqlib/decoder_csv_object.go index 3b1baff7..33ab7bf9 100644 --- a/pkg/yqlib/decoder_csv_object.go +++ b/pkg/yqlib/decoder_csv_object.go @@ -39,10 +39,7 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"} for i, header := range headerRow { - objectNode.Content = append( - objectNode.Content, - createScalarNode(header, header), - dec.convertToNode(contentRow[i])) + objectNode.AddKeyValueChild(createScalarNode(header, header), dec.convertToNode(contentRow[i])) } return objectNode } @@ -63,7 +60,7 @@ func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) { for err == nil && len(contentRow) > 0 { log.Debugf("Adding contentRow: %v", contentRow) - rootArray.Content = append(rootArray.Content, dec.createObject(headerRow, contentRow)) + rootArray.AddChild(dec.createObject(headerRow, contentRow)) contentRow, err = dec.reader.Read() log.Debugf("Read next contentRow: %v, %v", contentRow, err) } diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go index 6a590cde..edd77c52 100644 --- a/pkg/yqlib/decoder_xml.go +++ b/pkg/yqlib/decoder_xml.go @@ -42,7 +42,7 @@ func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*CandidateNode, error) if err != nil { return nil, err } - yamlNode.Content = append(yamlNode.Content, yamlChild) + yamlNode.AddChild(yamlChild) } return yamlNode, nil @@ -74,7 +74,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) { labelNode.HeadComment = dec.processComment(n.HeadComment) labelNode.LineComment = dec.processComment(n.LineComment) labelNode.FootComment = dec.processComment(n.FootComment) - yamlNode.Content = append(yamlNode.Content, labelNode, dec.createValueNodeFromData(n.Data)) + yamlNode.AddKeyValueChild(labelNode, dec.createValueNodeFromData(n.Data)) } for i, keyValuePair := range n.Children { @@ -119,7 +119,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) { return nil, err } } - yamlNode.Content = append(yamlNode.Content, labelNode, valueNode) + yamlNode.AddKeyValueChild(labelNode, valueNode) } return yamlNode, nil diff --git a/pkg/yqlib/doc/operators/comment-operators.md b/pkg/yqlib/doc/operators/comment-operators.md index 04229b0e..89af3b1b 100644 --- a/pkg/yqlib/doc/operators/comment-operators.md +++ b/pkg/yqlib/doc/operators/comment-operators.md @@ -10,6 +10,56 @@ This will set the LHS nodes' comments equal to the expression on the RHS. The RH ### relative form: `|=` This is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path. +## Set line comment +Set the comment on the key node for more reliability (see below). + +Given a sample.yml file of: +```yaml +a: cat +``` +then +```bash +yq '.a line_comment="single"' sample.yml +``` +will output +```yaml +a: cat # single +``` + +## Set line comment of a maps/arrays +For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars. + +Given a sample.yml file of: +```yaml +a: + b: things +``` +then +```bash +yq '(.a | key) line_comment="single"' sample.yml +``` +will output +```yaml +a: # single + b: things +``` + +## Use update assign to perform relative updates +Given a sample.yml file of: +```yaml +a: cat +b: dog +``` +then +```bash +yq '.. line_comment |= .' sample.yml +``` +will output +```yaml +a: cat # cat +b: dog # dog +``` + ## Where is the comment - map key example The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value). From this, you can see the 'hello-world-comment' is actually on the 'hello' key @@ -52,3 +102,239 @@ will output fc: "" ``` +## Retrieve comment - map key example +From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment + +Given a sample.yml file of: +```yaml +hello: # hello-world-comment + message: world +``` +then +```bash +yq '.hello | key | line_comment' sample.yml +``` +will output +```yaml +hello-world-comment +``` + +## Where is the comment - array example +The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value). +From this, you can see the 'under-name-comment' is actually on the first child + +Given a sample.yml file of: +```yaml +name: + # under-name-comment + - first-array-child +``` +then +```bash +yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml +``` +will output +```yaml +- p: "" + isKey: false + hc: "" + lc: "" + fc: "" +- p: name + isKey: true + hc: "" + lc: "" + fc: "" +- p: name + isKey: false + hc: "" + lc: "" + fc: "" +- p: name.0 + isKey: false + hc: under-name-comment + lc: "" + fc: "" +``` + +## Retrieve comment - array example +From the previous example, we know that the comment is on the first child as a headComment + +Given a sample.yml file of: +```yaml +name: + # under-name-comment + - first-array-child +``` +then +```bash +yq '.name[0] | headComment' sample.yml +``` +will output +```yaml +under-name-comment +``` + +## Set head comment +Given a sample.yml file of: +```yaml +a: cat +``` +then +```bash +yq '. head_comment="single"' sample.yml +``` +will output +```yaml +# single + +a: cat +``` + +## Set head comment of a map entry +Given a sample.yml file of: +```yaml +f: foo +a: + b: cat +``` +then +```bash +yq '(.a | key) head_comment="single"' sample.yml +``` +will output +```yaml +f: foo +# single +a: + b: cat +``` + +## Set foot comment, using an expression +Given a sample.yml file of: +```yaml +a: cat +``` +then +```bash +yq '. foot_comment=.a' sample.yml +``` +will output +```yaml +a: cat +# cat +``` + +## Remove comment +Given a sample.yml file of: +```yaml +a: cat # comment +b: dog # leave this +``` +then +```bash +yq '.a line_comment=""' sample.yml +``` +will output +```yaml +a: cat +b: dog # leave this +``` + +## Remove (strip) all comments +Note the use of `...` to ensure key nodes are included. + +Given a sample.yml file of: +```yaml +# hi + +a: cat # comment +# great +b: # key comment +``` +then +```bash +yq '... comments=""' sample.yml +``` +will output +```yaml +a: cat +b: +``` + +## Get line comment +Given a sample.yml file of: +```yaml +# welcome! + +a: cat # meow +# have a great day +``` +then +```bash +yq '.a | line_comment' sample.yml +``` +will output +```yaml +meow +``` + +## Get head comment +Given a sample.yml file of: +```yaml +# welcome! + +a: cat # meow + +# have a great day +``` +then +```bash +yq '. | head_comment' sample.yml +``` +will output +```yaml +welcome! + +``` + +## Head comment with document split +Given a sample.yml file of: +```yaml +# welcome! +--- +# bob +a: cat # meow + +# have a great day +``` +then +```bash +yq 'head_comment' sample.yml +``` +will output +```yaml +welcome! +bob +``` + +## Get foot comment +Given a sample.yml file of: +```yaml +# welcome! + +a: cat # meow + +# have a great day +# no really +``` +then +```bash +yq '. | foot_comment' sample.yml +``` +will output +```yaml +have a great day +no really +``` + diff --git a/pkg/yqlib/doc/operators/create-collect-into-object.md b/pkg/yqlib/doc/operators/create-collect-into-object.md index a171f9e9..288d4483 100644 --- a/pkg/yqlib/doc/operators/create-collect-into-object.md +++ b/pkg/yqlib/doc/operators/create-collect-into-object.md @@ -56,7 +56,6 @@ will output ```yaml Mike: cat Mike: dog ---- Rosey: monkey Rosey: sheep ``` diff --git a/pkg/yqlib/doc/operators/document-index.md b/pkg/yqlib/doc/operators/document-index.md index ebe994e5..619fd391 100644 --- a/pkg/yqlib/doc/operators/document-index.md +++ b/pkg/yqlib/doc/operators/document-index.md @@ -85,7 +85,6 @@ will output ```yaml match: cat doc: 0 ---- match: frog doc: 1 ``` diff --git a/pkg/yqlib/doc/operators/keys.md b/pkg/yqlib/doc/operators/keys.md index 8f8642a7..537d6594 100644 --- a/pkg/yqlib/doc/operators/keys.md +++ b/pkg/yqlib/doc/operators/keys.md @@ -2,10 +2,119 @@ Use the `keys` operator to return map keys or array indices. +## Map keys +Given a sample.yml file of: +```yaml +{dog: woof, cat: meow} +``` +then +```bash +yq 'keys' sample.yml +``` +will output +```yaml +- dog +- cat +``` + +## Array keys +Given a sample.yml file of: +```yaml +[apple, banana] +``` +then +```bash +yq 'keys' sample.yml +``` +will output +```yaml +- 0 +- 1 +``` + +## Retrieve array key +Given a sample.yml file of: +```yaml +[1, 2, 3] +``` +then +```bash +yq '.[1] | key' sample.yml +``` +will output +```yaml +1 +``` + +## Retrieve map key +Given a sample.yml file of: +```yaml +a: thing +``` +then +```bash +yq '.a | key' sample.yml +``` +will output +```yaml +a +``` + +## No key +Given a sample.yml file of: +```yaml +{} +``` +then +```bash +yq 'key' sample.yml +``` +will output +```yaml +``` + +## Update map key +Given a sample.yml file of: +```yaml +a: + x: 3 + y: 4 +``` +then +```bash +yq '(.a.x | key) = "meow"' sample.yml +``` +will output +```yaml +a: + meow: 3 + y: 4 +``` + +## Get comment from map key +Given a sample.yml file of: +```yaml +a: + # comment on key + x: 3 + y: 4 +``` +then +```bash +yq '.a.x | key | headComment' sample.yml +``` +will output +```yaml +comment on key +``` + ## Check node is a key Given a sample.yml file of: ```yaml -a: frog +a: + b: + - cat + c: frog ``` then ```bash @@ -18,9 +127,20 @@ will output tag: '!!map' - p: a isKey: true - tag: null - '!!str': null + tag: '!!str' - p: a + isKey: false + tag: '!!map' +- p: a.c + isKey: true + tag: '!!str' +- p: a.b + isKey: false + tag: '!!seq' +- p: a.b.0 + isKey: false + tag: '!!str' +- p: a.c isKey: false tag: '!!str' ``` diff --git a/pkg/yqlib/doc/operators/length.md b/pkg/yqlib/doc/operators/length.md index 6315e5b2..920d615f 100644 --- a/pkg/yqlib/doc/operators/length.md +++ b/pkg/yqlib/doc/operators/length.md @@ -7,7 +7,7 @@ returns length of string Given a sample.yml file of: ```yaml -a: cat +{a: cat} ``` then ```bash @@ -21,7 +21,7 @@ will output ## null length Given a sample.yml file of: ```yaml -a: null +{a: null} ``` then ```bash @@ -37,8 +37,7 @@ returns number of entries Given a sample.yml file of: ```yaml -a: cat -c: dog +{a: cat, c: dog} ``` then ```bash @@ -54,10 +53,7 @@ returns number of elements Given a sample.yml file of: ```yaml -- 2 -- 4 -- 6 -- 8 +[2, 4, 6, 8] ``` then ```bash diff --git a/pkg/yqlib/doc/operators/load.md b/pkg/yqlib/doc/operators/load.md index 79dc0149..2c6eae38 100644 --- a/pkg/yqlib/doc/operators/load.md +++ b/pkg/yqlib/doc/operators/load.md @@ -48,7 +48,7 @@ bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u ## Simple example Given a sample.yml file of: ```yaml -myFile: ../../examples/thing.yml +{myFile: ../../examples/thing.yml} ``` then ```bash @@ -65,8 +65,7 @@ Note that you can modify the filename in the load operator if needed. Given a sample.yml file of: ```yaml -something: - file: thing.yml +{something: {file: thing.yml}} ``` then ```bash @@ -74,9 +73,7 @@ yq '.something |= load("../../examples/" + .file)' sample.yml ``` will output ```yaml -something: - a: apple is included - b: cool. +{something: {a: apple is included, b: cool.}} ``` ## Replace _all_ nodes with referenced file @@ -84,11 +81,7 @@ Recursively match all the nodes (`..`) and then filter the ones that have a 'fil Given a sample.yml file of: ```yaml -something: - file: thing.yml -over: - here: - - file: thing.yml +{something: {file: thing.yml}, over: {here: [{file: thing.yml}]}} ``` then ```bash @@ -96,13 +89,7 @@ yq '(.. | select(has("file"))) |= load("../../examples/" + .file)' sample.yml ``` will output ```yaml -something: - a: apple is included - b: cool. -over: - here: - - a: apple is included - b: cool. +{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}} ``` ## Replace node with referenced file as string @@ -110,8 +97,7 @@ This will work for any text based file Given a sample.yml file of: ```yaml -something: - file: thing.yml +{something: {file: thing.yml}} ``` then ```bash @@ -119,9 +105,7 @@ yq '.something |= load_str("../../examples/" + .file)' sample.yml ``` will output ```yaml -something: |- - a: apple is included - b: cool. +{something: "a: apple is included\nb: cool."} ``` ## Load from XML @@ -172,9 +156,7 @@ yq '. *= load_props("../../examples/small.properties")' sample.yml ``` will output ```yaml -this: - is: a properties file - cool: ay +is ``` ## Load from base64 encoded file diff --git a/pkg/yqlib/doc/operators/map.md b/pkg/yqlib/doc/operators/map.md index 991167e2..b8dd4dd2 100644 --- a/pkg/yqlib/doc/operators/map.md +++ b/pkg/yqlib/doc/operators/map.md @@ -5,9 +5,7 @@ Maps values of an array. Use `map_values` to map values of an object. ## Map array Given a sample.yml file of: ```yaml -- 1 -- 2 -- 3 +[1, 2, 3] ``` then ```bash @@ -15,17 +13,13 @@ yq 'map(. + 1)' sample.yml ``` will output ```yaml -- 2 -- 3 -- 4 +[2, 3, 4] ``` ## Map object values Given a sample.yml file of: ```yaml -a: 1 -b: 2 -c: 3 +{a: 1, b: 2, c: 3} ``` then ```bash @@ -33,8 +27,6 @@ yq 'map_values(. + 1)' sample.yml ``` will output ```yaml -a: 2 -b: 3 -c: 4 +{a: 2, b: 3, c: 4} ``` diff --git a/pkg/yqlib/doc/operators/modulo.md b/pkg/yqlib/doc/operators/modulo.md index 810f30f3..559ff6fe 100644 --- a/pkg/yqlib/doc/operators/modulo.md +++ b/pkg/yqlib/doc/operators/modulo.md @@ -7,8 +7,7 @@ If the lhs and rhs are ints then the expression will be calculated with ints. Given a sample.yml file of: ```yaml -a: 13 -b: 2 +{a: 13, b: 2} ``` then ```bash @@ -16,8 +15,7 @@ yq '.a = .a % .b' sample.yml ``` will output ```yaml -a: 1 -b: 2 +{a: 1, b: 2} ``` ## Number modulo - float @@ -25,8 +23,7 @@ If the lhs or rhs are floats then the expression will be calculated with floats. Given a sample.yml file of: ```yaml -a: 12 -b: 2.5 +{a: 12, b: 2.5} ``` then ```bash @@ -34,8 +31,7 @@ yq '.a = .a % .b' sample.yml ``` will output ```yaml -a: !!float 2 -b: 2.5 +{a: !!float 2, b: 2.5} ``` ## Number modulo - int by zero @@ -43,8 +39,7 @@ If the lhs is an int and rhs is a 0 the result is an error. Given a sample.yml file of: ```yaml -a: 1 -b: 0 +{a: 1, b: 0} ``` then ```bash @@ -60,8 +55,7 @@ If the lhs is a float and rhs is a 0 the result is NaN. Given a sample.yml file of: ```yaml -a: 1.1 -b: 0 +{a: 1.1, b: 0} ``` then ```bash @@ -69,7 +63,6 @@ yq '.a = .a % .b' sample.yml ``` will output ```yaml -a: !!float NaN -b: 0 +{a: !!float NaN, b: 0} ``` diff --git a/pkg/yqlib/doc/operators/path.md b/pkg/yqlib/doc/operators/path.md index 0d9b73e6..feab5e29 100644 --- a/pkg/yqlib/doc/operators/path.md +++ b/pkg/yqlib/doc/operators/path.md @@ -10,8 +10,7 @@ Use `setpath` to set a value to the path array returned by `path`, and similarly ## Map path Given a sample.yml file of: ```yaml -a: - b: cat +{a: {b: cat}} ``` then ```bash @@ -26,8 +25,7 @@ will output ## Get map key Given a sample.yml file of: ```yaml -a: - b: cat +{a: {b: cat}} ``` then ```bash @@ -41,9 +39,7 @@ b ## Array path Given a sample.yml file of: ```yaml -a: - - cat - - dog +{a: [cat, dog]} ``` then ```bash @@ -58,9 +54,7 @@ will output ## Get array index Given a sample.yml file of: ```yaml -a: - - cat - - dog +{a: [cat, dog]} ``` then ```bash @@ -74,10 +68,7 @@ will output ## Print path and value Given a sample.yml file of: ```yaml -a: - - cat - - dog - - frog +{a: [cat, dog, frog]} ``` then ```bash @@ -98,8 +89,7 @@ will output ## Set path Given a sample.yml file of: ```yaml -a: - b: cat +{a: {b: cat}} ``` then ```bash @@ -107,8 +97,7 @@ yq 'setpath(["a", "b"]; "things")' sample.yml ``` will output ```yaml -a: - b: things +{a: {b: things}} ``` ## Set on empty document @@ -183,10 +172,7 @@ Notice delpaths takes an _array_ of paths. Given a sample.yml file of: ```yaml -a: - b: cat - c: dog - d: frog +{a: {b: cat, c: dog, d: frog}} ``` then ```bash @@ -194,8 +180,7 @@ yq 'delpaths([["a", "c"], ["a", "d"]])' sample.yml ``` will output ```yaml -a: - b: cat +{a: {b: cat}} ``` ## Delete array path diff --git a/pkg/yqlib/doc/operators/pick.md b/pkg/yqlib/doc/operators/pick.md index 3b477699..3fc0d591 100644 --- a/pkg/yqlib/doc/operators/pick.md +++ b/pkg/yqlib/doc/operators/pick.md @@ -31,9 +31,7 @@ Note that the order of the indices matches the pick order and non existent indic Given a sample.yml file of: ```yaml -- cat -- leopard -- lion +[cat, leopard, lion] ``` then ```bash @@ -41,7 +39,6 @@ yq 'pick([2, 0, 734, -5])' sample.yml ``` will output ```yaml -- lion -- cat +[lion, cat] ``` diff --git a/pkg/yqlib/doc/operators/reverse.md b/pkg/yqlib/doc/operators/reverse.md index 77f4037f..5c5144f9 100644 --- a/pkg/yqlib/doc/operators/reverse.md +++ b/pkg/yqlib/doc/operators/reverse.md @@ -5,9 +5,7 @@ Reverses the order of the items in an array ## Reverse Given a sample.yml file of: ```yaml -- 1 -- 2 -- 3 +[1, 2, 3] ``` then ```bash @@ -15,9 +13,7 @@ yq 'reverse' sample.yml ``` will output ```yaml -- 3 -- 2 -- 1 +[3, 2, 1] ``` ## Sort descending by string field @@ -25,9 +21,7 @@ Use sort with reverse to sort in descending order. Given a sample.yml file of: ```yaml -- a: banana -- a: cat -- a: apple +[{a: banana}, {a: cat}, {a: apple}] ``` then ```bash @@ -35,8 +29,6 @@ yq 'sort_by(.a) | reverse' sample.yml ``` will output ```yaml -- a: cat -- a: banana -- a: apple +[{a: cat}, {a: banana}, {a: apple}] ``` diff --git a/pkg/yqlib/doc/operators/slice-array.md b/pkg/yqlib/doc/operators/slice-array.md index 40ebea49..1995e34d 100644 --- a/pkg/yqlib/doc/operators/slice-array.md +++ b/pkg/yqlib/doc/operators/slice-array.md @@ -7,10 +7,7 @@ You may leave out the first or second number, which will will refer to the start ## Slicing arrays Given a sample.yml file of: ```yaml -- cat -- dog -- frog -- cow +[cat, dog, frog, cow] ``` then ```bash @@ -27,10 +24,7 @@ Starts from the start of the array Given a sample.yml file of: ```yaml -- cat -- dog -- frog -- cow +[cat, dog, frog, cow] ``` then ```bash @@ -47,10 +41,7 @@ Finishes at the end of the array Given a sample.yml file of: ```yaml -- cat -- dog -- frog -- cow +[cat, dog, frog, cow] ``` then ```bash @@ -65,10 +56,7 @@ will output ## Slicing arrays - use negative numbers to count backwards from the end Given a sample.yml file of: ```yaml -- cat -- dog -- frog -- cow +[cat, dog, frog, cow] ``` then ```bash @@ -85,10 +73,7 @@ using an expression to find the index Given a sample.yml file of: ```yaml -- cat -- dog -- frog -- cow +[cat, dog, frog, cow] ``` then ```bash diff --git a/pkg/yqlib/doc/operators/sort-keys.md b/pkg/yqlib/doc/operators/sort-keys.md index fb8e86ac..728d77c6 100644 --- a/pkg/yqlib/doc/operators/sort-keys.md +++ b/pkg/yqlib/doc/operators/sort-keys.md @@ -18,9 +18,7 @@ See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-key ## Sort keys of map Given a sample.yml file of: ```yaml -c: frog -a: blah -b: bing +{c: frog, a: blah, b: bing} ``` then ```bash @@ -28,9 +26,7 @@ yq 'sort_keys(.)' sample.yml ``` will output ```yaml -a: blah -b: bing -c: frog +{a: blah, b: bing, c: frog} ``` ## Sort keys recursively @@ -38,19 +34,7 @@ Note the array elements are left unsorted, but maps inside arrays are sorted Given a sample.yml file of: ```yaml -bParent: - c: dog - array: - - 3 - - 1 - - 2 -aParent: - z: donkey - x: - - c: yum - b: delish - - b: ew - a: apple +{bParent: {c: dog, array: [3, 1, 2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}} ``` then ```bash @@ -58,18 +42,6 @@ yq 'sort_keys(..)' sample.yml ``` will output ```yaml -aParent: - x: - - b: delish - c: yum - - a: apple - b: ew - z: donkey -bParent: - array: - - 3 - - 1 - - 2 - c: dog +{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}} ``` diff --git a/pkg/yqlib/doc/operators/sort.md b/pkg/yqlib/doc/operators/sort.md index 72a17305..1b3e5a14 100644 --- a/pkg/yqlib/doc/operators/sort.md +++ b/pkg/yqlib/doc/operators/sort.md @@ -10,9 +10,7 @@ Note that at this stage, `yq` only sorts scalar fields. ## Sort by string field Given a sample.yml file of: ```yaml -- a: banana -- a: cat -- a: apple +[{a: banana}, {a: cat}, {a: apple}] ``` then ```bash @@ -20,19 +18,13 @@ yq 'sort_by(.a)' sample.yml ``` will output ```yaml -- a: apple -- a: banana -- a: cat +[{a: apple}, {a: banana}, {a: cat}] ``` ## Sort by multiple fields Given a sample.yml file of: ```yaml -- a: dog -- a: cat - b: banana -- a: cat - b: apple +[{a: dog}, {a: cat, b: banana}, {a: cat, b: apple}] ``` then ```bash @@ -40,11 +32,7 @@ yq 'sort_by(.a, .b)' sample.yml ``` will output ```yaml -- a: cat - b: apple -- a: cat - b: banana -- a: dog +[{a: cat, b: apple}, {a: cat, b: banana}, {a: dog}] ``` ## Sort descending by string field @@ -52,9 +40,7 @@ Use sort with reverse to sort in descending order. Given a sample.yml file of: ```yaml -- a: banana -- a: cat -- a: apple +[{a: banana}, {a: cat}, {a: apple}] ``` then ```bash @@ -62,9 +48,7 @@ yq 'sort_by(.a) | reverse' sample.yml ``` will output ```yaml -- a: cat -- a: banana -- a: apple +[{a: cat}, {a: banana}, {a: apple}] ``` ## Sort array in place @@ -114,14 +98,7 @@ Note the order of the elements in unchanged when equal in sorting. Given a sample.yml file of: ```yaml -- a: banana - b: 1 -- a: banana - b: 2 -- a: banana - b: 3 -- a: banana - b: 4 +[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}] ``` then ```bash @@ -129,22 +106,13 @@ yq 'sort_by(.a)' sample.yml ``` will output ```yaml -- a: banana - b: 1 -- a: banana - b: 2 -- a: banana - b: 3 -- a: banana - b: 4 +[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}] ``` ## Sort by numeric field Given a sample.yml file of: ```yaml -- a: 10 -- a: 100 -- a: 1 +[{a: 10}, {a: 100}, {a: 1}] ``` then ```bash @@ -152,17 +120,13 @@ yq 'sort_by(.a)' sample.yml ``` will output ```yaml -- a: 1 -- a: 10 -- a: 100 +[{a: 1}, {a: 10}, {a: 100}] ``` ## Sort by custom date field Given a sample.yml file of: ```yaml -- a: 12-Jun-2011 -- a: 23-Dec-2010 -- a: 10-Aug-2011 +[{a: 12-Jun-2011}, {a: 23-Dec-2010}, {a: 10-Aug-2011}] ``` then ```bash @@ -170,21 +134,13 @@ yq 'with_dtf("02-Jan-2006"; sort_by(.a))' sample.yml ``` will output ```yaml -- a: 23-Dec-2010 -- a: 12-Jun-2011 -- a: 10-Aug-2011 +[{a: 23-Dec-2010}, {a: 12-Jun-2011}, {a: 10-Aug-2011}] ``` ## Sort, nulls come first Given a sample.yml file of: ```yaml -- 8 -- 3 -- null -- 6 -- true -- false -- cat +[8, 3, null, 6, true, false, cat] ``` then ```bash @@ -192,12 +148,6 @@ yq 'sort' sample.yml ``` will output ```yaml -- null -- false -- true -- 3 -- 6 -- 8 -- cat +[null, false, true, 3, 6, 8, cat] ``` diff --git a/pkg/yqlib/doc/operators/string-operators.md b/pkg/yqlib/doc/operators/string-operators.md index 6aa854bc..c8235819 100644 --- a/pkg/yqlib/doc/operators/string-operators.md +++ b/pkg/yqlib/doc/operators/string-operators.md @@ -91,11 +91,7 @@ will output ## Join strings Given a sample.yml file of: ```yaml -- cat -- meow -- 1 -- null -- true +[cat, meow, 1, null, true] ``` then ```bash @@ -109,10 +105,7 @@ cat; meow; 1; ; true ## Trim strings Given a sample.yml file of: ```yaml -- ' cat' -- 'dog ' -- ' cow cow ' -- horse +[' cat', 'dog ', ' cow cow ', horse] ``` then ```bash @@ -284,8 +277,7 @@ Like jq's equivalent, this works like match but only returns true/false instead Given a sample.yml file of: ```yaml -- cat -- dog +[cat, dog] ``` then ```bash @@ -354,7 +346,7 @@ b: !goat heart ## Split strings Given a sample.yml file of: ```yaml -cat; meow; 1; ; true +"cat; meow; 1; ; true" ``` then ```bash @@ -372,7 +364,7 @@ will output ## Split strings one match Given a sample.yml file of: ```yaml -word +"word" ``` then ```bash diff --git a/pkg/yqlib/doc/operators/subtract.md b/pkg/yqlib/doc/operators/subtract.md index 902a3a14..ab5e42fa 100644 --- a/pkg/yqlib/doc/operators/subtract.md +++ b/pkg/yqlib/doc/operators/subtract.md @@ -28,9 +28,7 @@ Note that order of the keys does not matter Given a sample.yml file of: ```yaml -- a: b - c: d -- a: b +[{a: b, c: d}, {a: b}] ``` then ```bash @@ -38,7 +36,7 @@ yq '. - [{"c": "d", "a": "b"}]' sample.yml ``` will output ```yaml -- a: b +[{a: b}] ``` ## Number subtraction - float @@ -46,8 +44,7 @@ If the lhs or rhs are floats then the expression will be calculated with floats. Given a sample.yml file of: ```yaml -a: 3 -b: 4.5 +{a: 3, b: 4.5} ``` then ```bash @@ -55,8 +52,7 @@ yq '.a = .a - .b' sample.yml ``` will output ```yaml -a: -1.5 -b: 4.5 +{a: -1.5, b: 4.5} ``` ## Number subtraction - int @@ -64,8 +60,7 @@ If both the lhs and rhs are ints then the expression will be calculated with int Given a sample.yml file of: ```yaml -a: 3 -b: 4 +{a: 3, b: 4} ``` then ```bash @@ -73,15 +68,13 @@ yq '.a = .a - .b' sample.yml ``` will output ```yaml -a: -1 -b: 4 +{a: -1, b: 4} ``` ## Decrement numbers Given a sample.yml file of: ```yaml -a: 3 -b: 5 +{a: 3, b: 5} ``` then ```bash @@ -89,8 +82,7 @@ yq '.[] -= 1' sample.yml ``` will output ```yaml -a: 2 -b: 4 +{a: 2, b: 4} ``` ## Date subtraction diff --git a/pkg/yqlib/doc/operators/tag.md b/pkg/yqlib/doc/operators/tag.md index 43db1439..3990ae11 100644 --- a/pkg/yqlib/doc/operators/tag.md +++ b/pkg/yqlib/doc/operators/tag.md @@ -5,11 +5,7 @@ The tag operator can be used to get or set the tag of nodes (e.g. `!!str`, `!!in ## Get tag Given a sample.yml file of: ```yaml -a: cat -b: 5 -c: 3.2 -e: true -f: [] +{a: cat, b: 5, c: 3.2, e: true, f: []} ``` then ```bash @@ -28,11 +24,7 @@ will output ## type is an alias for tag Given a sample.yml file of: ```yaml -a: cat -b: 5 -c: 3.2 -e: true -f: [] +{a: cat, b: 5, c: 3.2, e: true, f: []} ``` then ```bash @@ -51,7 +43,7 @@ will output ## Set custom tag Given a sample.yml file of: ```yaml -a: str +{a: str} ``` then ```bash @@ -59,16 +51,13 @@ yq '.a tag = "!!mikefarah"' sample.yml ``` will output ```yaml -a: !!mikefarah str +{a: !!mikefarah str} ``` ## Find numbers and convert them to strings Given a sample.yml file of: ```yaml -a: cat -b: 5 -c: 3.2 -e: true +{a: cat, b: 5, c: 3.2, e: true} ``` then ```bash @@ -76,9 +65,6 @@ yq '(.. | select(tag == "!!int")) tag= "!!str"' sample.yml ``` will output ```yaml -a: cat -b: "5" -c: 3.2 -e: true +{a: cat, b: "5", c: 3.2, e: true} ``` diff --git a/pkg/yqlib/doc/operators/unique.md b/pkg/yqlib/doc/operators/unique.md index a1bc443c..28226689 100644 --- a/pkg/yqlib/doc/operators/unique.md +++ b/pkg/yqlib/doc/operators/unique.md @@ -8,10 +8,7 @@ Note that unique maintains the original order of the array. Given a sample.yml file of: ```yaml -- 2 -- 1 -- 3 -- 2 +[2, 1, 3, 2] ``` then ```bash @@ -19,9 +16,7 @@ yq 'unique' sample.yml ``` will output ```yaml -- 2 -- 1 -- 3 +[2, 1, 3] ``` ## Unique nulls @@ -29,10 +24,7 @@ Unique works on the node value, so it considers different representations of nul Given a sample.yml file of: ```yaml -- ~ -- null -- ~ -- null +[~, null, ~, null] ``` then ```bash @@ -40,8 +32,7 @@ yq 'unique' sample.yml ``` will output ```yaml -- ~ -- null +[~, null] ``` ## Unique all nulls @@ -49,10 +40,7 @@ Run against the node tag to unique all the nulls Given a sample.yml file of: ```yaml -- ~ -- null -- ~ -- null +[~, null, ~, null] ``` then ```bash @@ -60,18 +48,13 @@ yq 'unique_by(tag)' sample.yml ``` will output ```yaml -- ~ +[~] ``` ## Unique array object fields Given a sample.yml file of: ```yaml -- name: harry - pet: cat -- name: billy - pet: dog -- name: harry - pet: dog +[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: dog}] ``` then ```bash @@ -79,9 +62,6 @@ yq 'unique_by(.name)' sample.yml ``` will output ```yaml -- name: harry - pet: cat -- name: billy - pet: dog +[{name: harry, pet: cat}, {name: billy, pet: dog}] ``` diff --git a/pkg/yqlib/doc/usage/csv-tsv.md b/pkg/yqlib/doc/usage/csv-tsv.md index 0986a1ba..c58e42cd 100644 --- a/pkg/yqlib/doc/usage/csv-tsv.md +++ b/pkg/yqlib/doc/usage/csv-tsv.md @@ -111,3 +111,98 @@ Gary,1 Samantha's Rabbit,2 ``` +## Encode array of objects to csv - missing fields behaviour +First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank + +Given a sample.yml file of: +```yaml +- name: Gary + numberOfCats: 1 + height: 168.8 +- name: Samantha's Rabbit + height: -188.8 + likesApples: false + +``` +then +```bash +yq -o=csv sample.yml +``` +will output +```csv +name,numberOfCats,height +Gary,1,168.8 +Samantha's Rabbit,,-188.8 +``` + +## Parse CSV into an array of objects +First row is assumed to be the header row. + +Given a sample.csv file of: +```csv +name,numberOfCats,likesApples,height +Gary,1,true,168.8 +Samantha's Rabbit,2,false,-188.8 + +``` +then +```bash +yq -p=csv sample.csv +``` +will output +```yaml +- name: Gary + numberOfCats: 1 + likesApples: true + height: 168.8 +- name: Samantha's Rabbit + numberOfCats: 2 + likesApples: false + height: -188.8 +``` + +## Parse TSV into an array of objects +First row is assumed to be the header row. + +Given a sample.tsv file of: +```tsv +name numberOfCats likesApples height +Gary 1 true 168.8 +Samantha's Rabbit 2 false -188.8 + +``` +then +```bash +yq -p=tsv sample.tsv +``` +will output +```yaml +- name: Gary + numberOfCats: 1 + likesApples: true + height: 168.8 +- name: Samantha's Rabbit + numberOfCats: 2 + likesApples: false + height: -188.8 +``` + +## Round trip +Given a sample.csv file of: +```csv +name,numberOfCats,likesApples,height +Gary,1,true,168.8 +Samantha's Rabbit,2,false,-188.8 + +``` +then +```bash +yq -p=csv -o=csv '(.[] | select(.name == "Gary") | .numberOfCats) = 3' sample.csv +``` +will output +```csv +name,numberOfCats,likesApples,height +Gary,3,true,168.8 +Samantha's Rabbit,2,false,-188.8 +``` + diff --git a/pkg/yqlib/encoder_json.go b/pkg/yqlib/encoder_json.go index 4aa98700..31c7b580 100644 --- a/pkg/yqlib/encoder_json.go +++ b/pkg/yqlib/encoder_json.go @@ -55,14 +55,6 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error { encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > encoder.SetIndent("", je.indentString) - // var dataBucket orderedMap - // firstly, convert all map keys to strings - // mapKeysToStrings(node) - // errorDecoding := node.Decode(&dataBucket) - - // if errorDecoding != nil { - // return errorDecoding - // } err := encoder.Encode(node) if err != nil { return err diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index b3b70814..f24700b8 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -9,7 +9,6 @@ import ( "strings" logging "gopkg.in/op/go-logging.v1" - yaml "gopkg.in/yaml.v3" ) var ExpressionParser ExpressionParserInterface @@ -291,45 +290,6 @@ func recursiveNodeEqual(lhs *CandidateNode, rhs *CandidateNode) bool { return false } -func deepCloneContent(content []*yaml.Node) []*yaml.Node { - clonedContent := make([]*yaml.Node, len(content)) - for i, child := range content { - clonedContent[i] = deepClone(child) - } - return clonedContent -} - -func deepCloneNoContent(node *yaml.Node) *yaml.Node { - return deepCloneWithOptions(node, false) -} -func deepClone(node *yaml.Node) *yaml.Node { - return deepCloneWithOptions(node, true) -} - -func deepCloneWithOptions(node *yaml.Node, cloneContent bool) *yaml.Node { - if node == nil { - return nil - } - var clonedContent []*yaml.Node - if cloneContent { - clonedContent = deepCloneContent(node.Content) - } - return &yaml.Node{ - Content: clonedContent, - Kind: node.Kind, - Style: node.Style, - Tag: node.Tag, - Value: node.Value, - Anchor: node.Anchor, - Alias: node.Alias, - HeadComment: node.HeadComment, - LineComment: node.LineComment, - FootComment: node.FootComment, - Line: node.Line, - Column: node.Column, - } -} - // yaml numbers can be hex encoded... func parseInt64(numberString string) (string, int64, error) { if strings.HasPrefix(numberString, "0x") || @@ -436,6 +396,24 @@ func NodeToString(node *CandidateNode) string { } return fmt.Sprintf(`D%v, P%v, %v (%v)::%v`, node.GetDocument(), node.GetNicePath(), KindString(node.Kind), tag, valueToUse) } + +func NodeContentToString(node *CandidateNode, depth int) string { + if !log.IsEnabledFor(logging.DEBUG) { + return "" + } + var sb strings.Builder + for _, child := range node.Content { + for i := 0; i < depth; i++ { + sb.WriteString(" ") + } + sb.WriteString("- ") + sb.WriteString(NodeToString(child)) + sb.WriteString("\n") + sb.WriteString(NodeContentToString(child, depth+1)) + } + return sb.String() +} + func KindString(kind Kind) string { switch kind { case ScalarNode: diff --git a/pkg/yqlib/matchKeyString.go b/pkg/yqlib/matchKeyString.go index bfea235e..bdfaa8dc 100644 --- a/pkg/yqlib/matchKeyString.go +++ b/pkg/yqlib/matchKeyString.go @@ -4,7 +4,6 @@ func matchKey(name string, pattern string) (matched bool) { if pattern == "" { return name == pattern } - log.Debug("pattern: %v", pattern) if pattern == "*" { log.Debug("wild!") return true diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index f4d6c2f8..29330e60 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -17,20 +17,17 @@ func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *Ex return compoundAssignFunction(d, context, expressionNode, createAddOp) } -func toNodes(candidate *CandidateNode, lhs *CandidateNode) ([]*CandidateNode, error) { - if candidate.Tag == "!!null" { - return []*CandidateNode{}, nil - } +func toNodes(candidate *CandidateNode, lhs *CandidateNode) []*CandidateNode { clone := candidate.Copy() switch candidate.Kind { case SequenceNode: - return clone.Content, nil + return clone.Content default: if len(lhs.Content) > 0 { clone.Style = lhs.Content[0].Style } - return []*CandidateNode{clone}, nil + return []*CandidateNode{clone} } } @@ -60,10 +57,7 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida } addMaps(target, lhs, rhs) case SequenceNode: - if err := addSequences(target, lhs, rhs); err != nil { - return nil, err - } - + addSequences(target, lhs, rhs) case ScalarNode: if rhs.Kind != ScalarNode { return nil, fmt.Errorf("%v (%v) cannot be added to a %v (%v)", rhs.Tag, rhs.GetNicePath(), lhsNode.Tag, lhs.GetNicePath()) @@ -156,7 +150,7 @@ func addDateTimes(layout string, target *CandidateNode, lhs *CandidateNode, rhs } -func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { +func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) { log.Debugf("adding sequences! target: %v; lhs %v; rhs: %v", NodeToString(target), NodeToString(lhs), NodeToString(rhs)) target.Kind = SequenceNode if len(lhs.Content) == 0 { @@ -165,15 +159,10 @@ func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) } target.Tag = lhs.Tag - extraNodes, err := toNodes(rhs, lhs) - if err != nil { - return err - } + extraNodes := toNodes(rhs, lhs) target.AddChildren(lhs.Content) target.AddChildren(extraNodes) - return nil - } func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) { @@ -185,8 +174,8 @@ func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) { target.Style = 0 } - target.Content = make([]*CandidateNode, len(lhs.Content)) - copy(target.Content, lhs.Content) + target.Content = make([]*CandidateNode, 0) + target.AddChildren(lhs.Content) for index := 0; index < len(rhs.Content); index = index + 2 { key := rhs.Content[index] @@ -196,10 +185,12 @@ func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) { log.Debug("indexInLhs %v", indexInLHS) if indexInLHS < 0 { // not in there, append it - target.Content = append(target.Content, key, value) + target.AddKeyValueChild(key, value) } else { // it's there, replace it - target.Content[indexInLHS+1] = value + oldValue := target.Content[indexInLHS+1] + newValueCopy := oldValue.CopyAsReplacement(value) + target.Content[indexInLHS+1] = newValueCopy } } target.Kind = MappingNode diff --git a/pkg/yqlib/operator_alternative.go b/pkg/yqlib/operator_alternative.go index 0c45cbf0..0069d4ff 100644 --- a/pkg/yqlib/operator_alternative.go +++ b/pkg/yqlib/operator_alternative.go @@ -9,10 +9,7 @@ func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode * if lhs == nil { return nil, nil } - truthy, err := isTruthyNode(lhs) - if err != nil { - return nil, err - } + truthy := isTruthyNode(lhs) if truthy { return lhs, nil } @@ -30,10 +27,8 @@ func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, return lhs, nil } - isTrue, err := isTruthyNode(lhs) - if err != nil { - return nil, err - } else if isTrue { + isTrue := isTruthyNode(lhs) + if isTrue { return lhs, nil } return rhs, nil diff --git a/pkg/yqlib/operator_anchors_aliases.go b/pkg/yqlib/operator_anchors_aliases.go index 9db3dd04..48df310d 100644 --- a/pkg/yqlib/operator_anchors_aliases.go +++ b/pkg/yqlib/operator_anchors_aliases.go @@ -171,11 +171,9 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error { } } } - node.Content = make([]*CandidateNode, newContent.Len()) - index := 0 + node.Content = make([]*CandidateNode, 0) for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() { - node.Content[index] = newEl.Value.(*CandidateNode) - index++ + node.AddChild(newEl.Value.(*CandidateNode)) } return nil } diff --git a/pkg/yqlib/operator_booleans.go b/pkg/yqlib/operator_booleans.go index 87aac785..738b54de 100644 --- a/pkg/yqlib/operator_booleans.go +++ b/pkg/yqlib/operator_booleans.go @@ -6,23 +6,23 @@ import ( "strings" ) -func isTruthyNode(candidate *CandidateNode) (bool, error) { +func isTruthyNode(candidate *CandidateNode) bool { if candidate == nil { - return false, nil + return false } node := candidate.unwrapDocument() if node.Tag == "!!null" { - return false, nil + return false } if node.Kind == ScalarNode && node.Tag == "!!bool" { // yes/y/true/on return (strings.EqualFold(node.Value, "y") || strings.EqualFold(node.Value, "yes") || strings.EqualFold(node.Value, "on") || - strings.EqualFold(node.Value, "true")), nil + strings.EqualFold(node.Value, "true")) } - return true, nil + return true } func getOwner(lhs *CandidateNode, rhs *CandidateNode) *CandidateNode { @@ -38,10 +38,7 @@ func getOwner(lhs *CandidateNode, rhs *CandidateNode) *CandidateNode { func returnRhsTruthy(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { owner := getOwner(lhs, rhs) - rhsBool, err := isTruthyNode(rhs) - if err != nil { - return nil, err - } + rhsBool := isTruthyNode(rhs) return createBooleanCandidate(owner, rhsBool), nil } @@ -51,7 +48,7 @@ func returnLHSWhen(targetBool bool) func(lhs *CandidateNode) (*CandidateNode, er var err error var lhsBool bool - if lhsBool, err = isTruthyNode(lhs); err != nil || lhsBool != targetBool { + if lhsBool = isTruthyNode(lhs); lhsBool != targetBool { return nil, err } owner := &CandidateNode{} @@ -79,11 +76,7 @@ func findBoolean(wantBool bool, d *dataTreeNavigator, context Context, expressio } } - truthy, err := isTruthyNode(node) - if err != nil { - return false, err - } - if truthy == wantBool { + if isTruthyNode(node) == wantBool { return true, nil } } @@ -153,10 +146,7 @@ func notOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debug("notOperation checking %v", candidate) - truthy, errDecoding := isTruthyNode(candidate) - if errDecoding != nil { - return Context{}, errDecoding - } + truthy := isTruthyNode(candidate) result := createBooleanCandidate(candidate, !truthy) results.PushBack(result) } diff --git a/pkg/yqlib/operator_collect.go b/pkg/yqlib/operator_collect.go index fcb65989..58284299 100644 --- a/pkg/yqlib/operator_collect.go +++ b/pkg/yqlib/operator_collect.go @@ -15,8 +15,7 @@ func collectTogether(d *dataTreeNavigator, context Context, expressionNode *Expr for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() { resultC := result.Value.(*CandidateNode) log.Debugf("found this: %v", NodeToString(resultC)) - collectedNode.AddChildren([]*CandidateNode{resultC}) - // collectedNode.Content = append(collectedNode.Content, resultC.unwrapDocument()) + collectedNode.AddChild(resultC) } } return collectedNode, nil @@ -65,8 +64,7 @@ func collectOperator(d *dataTreeNavigator, context Context, expressionNode *Expr for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() { resultC := result.Value.(*CandidateNode) log.Debugf("found this: %v", NodeToString(resultC)) - collectCandidate.AddChildren([]*CandidateNode{resultC}) - // collectCandidate.Content = append(collectCandidate.Content, resultC.unwrapDocument()) + collectCandidate.AddChild(resultC) } log.Debugf("done collect rhs: %v", expressionNode.RHS.Operation.toString()) diff --git a/pkg/yqlib/operator_collect_object.go b/pkg/yqlib/operator_collect_object.go index 6f8f5922..9d9b64b8 100644 --- a/pkg/yqlib/operator_collect_object.go +++ b/pkg/yqlib/operator_collect_object.go @@ -34,12 +34,14 @@ func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expres for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidateNode := el.Value.(*CandidateNode) - for i := 0; i < len(first.Content); i++ { + for i := 0; i < len(first.Content); i++ { + log.Debugf("rotate[%v] = %v", i, NodeToString(candidateNode.Content[i])) + log.Debugf("children:\n%v", NodeContentToString(candidateNode.Content[i], 0)) rotated[i].PushBack(candidateNode.Content[i]) } } - log.Debugf("-- collectObjectOperation, lenght of rotated is %v", len(rotated)) + log.Debugf("-- collectObjectOperation, length of rotated is %v", len(rotated)) newObject := list.New() for i := 0; i < len(first.Content); i++ { @@ -80,7 +82,9 @@ func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List) aggCandidate := el.Value.(*CandidateNode) for splatEl := splatted.MatchingNodes.Front(); splatEl != nil; splatEl = splatEl.Next() { splatCandidate := splatEl.Value.(*CandidateNode) + log.Debugf("-- collectObjectOperation; splatCandidate: %v", NodeToString(splatCandidate)) newCandidate := aggCandidate.Copy() + log.Debugf("-- collectObjectOperation; aggCandidate: %v", NodeToString(aggCandidate)) newCandidate, err = multiply(multiplyPreferences{AppendArrays: false})(d, context, newCandidate, splatCandidate) diff --git a/pkg/yqlib/operator_column_test.go b/pkg/yqlib/operator_column_test.go index c232ee84..36845883 100644 --- a/pkg/yqlib/operator_column_test.go +++ b/pkg/yqlib/operator_column_test.go @@ -19,7 +19,7 @@ var columnOperatorScenarios = []expressionScenario{ document: "a: cat\nb: bob", expression: `.b | key | column`, expected: []string{ - "D0, P[1], (!!int)::1\n", + "D0, P[b], (!!int)::1\n", }, }, { @@ -27,7 +27,7 @@ var columnOperatorScenarios = []expressionScenario{ document: "a: cat", expression: `.a | key | column`, expected: []string{ - "D0, P[1], (!!int)::1\n", + "D0, P[a], (!!int)::1\n", }, }, { diff --git a/pkg/yqlib/operator_comments_test.go b/pkg/yqlib/operator_comments_test.go index 0ba596be..9d2dc678 100644 --- a/pkg/yqlib/operator_comments_test.go +++ b/pkg/yqlib/operator_comments_test.go @@ -54,57 +54,57 @@ var expectedWhereIsMyCommentArray = `D0, P[], (!!seq)::- p: "" ` var commentOperatorScenarios = []expressionScenario{ - // { - // description: "Set line comment", - // subdescription: "Set the comment on the key node for more reliability (see below).", - // document: `a: cat`, - // expression: `.a line_comment="single"`, - // expected: []string{ - // "D0, P[], (doc)::a: cat # single\n", - // }, - // }, - // { - // description: "Set line comment of a maps/arrays", - // subdescription: "For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.", - // document: "a:\n b: things", - // expression: `(.a | key) line_comment="single"`, - // expected: []string{ - // "D0, P[], (doc)::a: # single\n b: things\n", - // }, - // }, - // { - // skipDoc: true, - // document: "a: cat\nb: dog", - // expression: `.a line_comment=.b`, - // expected: []string{ - // "D0, P[], (doc)::a: cat # dog\nb: dog\n", - // }, - // }, - // { - // skipDoc: true, - // document: "a: cat\n---\na: dog", - // expression: `.a line_comment |= documentIndex`, - // expected: []string{ - // "D0, P[], (doc)::a: cat # 0\n", - // "D1, P[], (doc)::a: dog # 1\n", - // }, - // }, - // { - // description: "Use update assign to perform relative updates", - // document: "a: cat\nb: dog", - // expression: `.. line_comment |= .`, - // expected: []string{ - // "D0, P[], (doc)::a: cat # cat\nb: dog # dog\n", - // }, - // }, - // { - // skipDoc: true, - // document: "a: cat\nb: dog", - // expression: `.. comments |= .`, - // expected: []string{ - // "D0, P[], (doc)::a: cat # cat\n# cat\n\n# cat\nb: dog # dog\n# dog\n\n# dog\n", - // }, - // }, + { + description: "Set line comment", + subdescription: "Set the comment on the key node for more reliability (see below).", + document: `a: cat`, + expression: `.a line_comment="single"`, + expected: []string{ + "D0, P[], (doc)::a: cat # single\n", + }, + }, + { + description: "Set line comment of a maps/arrays", + subdescription: "For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.", + document: "a:\n b: things", + expression: `(.a | key) line_comment="single"`, + expected: []string{ + "D0, P[], (doc)::a: # single\n b: things\n", + }, + }, + { + skipDoc: true, + document: "a: cat\nb: dog", + expression: `.a line_comment=.b`, + expected: []string{ + "D0, P[], (doc)::a: cat # dog\nb: dog\n", + }, + }, + { + skipDoc: true, + document: "a: cat\n---\na: dog", + expression: `.a line_comment |= documentIndex`, + expected: []string{ + "D0, P[], (doc)::a: cat # 0\n", + "D1, P[], (doc)::a: dog # 1\n", + }, + }, + { + description: "Use update assign to perform relative updates", + document: "a: cat\nb: dog", + expression: `.. line_comment |= .`, + expected: []string{ + "D0, P[], (doc)::a: cat # cat\nb: dog # dog\n", + }, + }, + { + skipDoc: true, + document: "a: cat\nb: dog", + expression: `.. comments |= .`, + expected: []string{ + "D0, P[], (doc)::a: cat # cat\n# cat\n\n# cat\nb: dog # dog\n# dog\n\n# dog\n", + }, + }, { description: "Where is the comment - map key example", subdescription: "The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'hello-world-comment' is actually on the 'hello' key", @@ -114,152 +114,152 @@ var commentOperatorScenarios = []expressionScenario{ expectedWhereIsMyCommentMapKey, }, }, - // { - // description: "Retrieve comment - map key example", - // subdescription: "From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment", - // document: "hello: # hello-world-comment\n message: world", - // expression: `.hello | key | line_comment`, - // expected: []string{ - // "D0, P[hello], (!!str)::hello-world-comment\n", - // }, - // }, - // { - // description: "Where is the comment - array example", - // subdescription: "The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'under-name-comment' is actually on the first child", - // document: "name:\n # under-name-comment\n - first-array-child", - // expression: `[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]`, - // expected: []string{ - // expectedWhereIsMyCommentArray, - // }, - // }, - // { - // description: "Retrieve comment - array example", - // subdescription: "From the previous example, we know that the comment is on the first child as a headComment", - // document: "name:\n # under-name-comment\n - first-array-child", - // expression: `.name[0] | headComment`, - // expected: []string{ - // "D0, P[name 0], (!!str)::under-name-comment\n", - // }, - // }, - // { - // description: "Set head comment", - // document: `a: cat`, - // expression: `. head_comment="single"`, - // expected: []string{ - // "D0, P[], (doc)::# single\n\na: cat\n", - // }, - // }, - // { - // description: "Set head comment of a map entry", - // document: "f: foo\na:\n b: cat", - // expression: `(.a | key) head_comment="single"`, - // expected: []string{ - // "D0, P[], (doc)::f: foo\n# single\na:\n b: cat\n", - // }, - // }, - // { - // description: "Set foot comment, using an expression", - // document: `a: cat`, - // expression: `. foot_comment=.a`, - // expected: []string{ - // "D0, P[], (doc)::a: cat\n# cat\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Set foot comment, using an expression", - // document: "a: cat\n\n# hi", - // expression: `. foot_comment=""`, - // expected: []string{ - // "D0, P[], (doc)::a: cat\n", - // }, - // }, - // { - // skipDoc: true, - // document: `a: cat`, - // expression: `. foot_comment=.b.d`, - // expected: []string{ - // "D0, P[], (doc)::a: cat\n", - // }, - // }, - // { - // skipDoc: true, - // document: `a: cat`, - // expression: `. foot_comment|=.b.d`, - // expected: []string{ - // "D0, P[], (doc)::a: cat\n", - // }, - // }, - // { - // description: "Remove comment", - // document: "a: cat # comment\nb: dog # leave this", - // expression: `.a line_comment=""`, - // expected: []string{ - // "D0, P[], (doc)::a: cat\nb: dog # leave this\n", - // }, - // }, - // { - // description: "Remove (strip) all comments", - // subdescription: "Note the use of `...` to ensure key nodes are included.", - // document: "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment", - // expression: `... comments=""`, - // expected: []string{ - // "D0, P[], (doc)::a: cat\nb:\n", - // }, - // }, - // { - // description: "Get line comment", - // document: "# welcome!\n\na: cat # meow\n\n# have a great day", - // expression: `.a | line_comment`, - // expected: []string{ - // "D0, P[a], (!!str)::meow\n", - // }, - // }, - // { - // description: "Get head comment", - // dontFormatInputForDoc: true, - // document: "# welcome!\n\na: cat # meow\n\n# have a great day", - // expression: `. | head_comment`, - // expected: []string{ - // "D0, P[], (!!str)::welcome!\n\n", - // }, - // }, - // { - // skipDoc: true, - // description: "strip trailing comment recurse all", - // document: "a: cat\n\n# haha", - // expression: `... comments= ""`, - // expected: []string{ - // "D0, P[], (doc)::a: cat\n", - // }, - // }, - // { - // skipDoc: true, - // description: "strip trailing comment recurse values", - // document: "a: cat\n\n# haha", - // expression: `.. comments= ""`, - // expected: []string{ - // "D0, P[], (doc)::a: cat\n", - // }, - // }, - // { - // description: "Head comment with document split", - // dontFormatInputForDoc: true, - // document: "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day", - // expression: `head_comment`, - // expected: []string{ - // "D0, P[], (!!str)::welcome!\nbob\n", - // }, - // }, - // { - // description: "Get foot comment", - // dontFormatInputForDoc: true, - // document: "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really", - // expression: `. | foot_comment`, - // expected: []string{ - // "D0, P[], (!!str)::have a great day\nno really\n", - // }, - // }, + { + description: "Retrieve comment - map key example", + subdescription: "From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment", + document: "hello: # hello-world-comment\n message: world", + expression: `.hello | key | line_comment`, + expected: []string{ + "D0, P[hello], (!!str)::hello-world-comment\n", + }, + }, + { + description: "Where is the comment - array example", + subdescription: "The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'under-name-comment' is actually on the first child", + document: "name:\n # under-name-comment\n - first-array-child", + expression: `[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]`, + expected: []string{ + expectedWhereIsMyCommentArray, + }, + }, + { + description: "Retrieve comment - array example", + subdescription: "From the previous example, we know that the comment is on the first child as a headComment", + document: "name:\n # under-name-comment\n - first-array-child", + expression: `.name[0] | headComment`, + expected: []string{ + "D0, P[name 0], (!!str)::under-name-comment\n", + }, + }, + { + description: "Set head comment", + document: `a: cat`, + expression: `. head_comment="single"`, + expected: []string{ + "D0, P[], (doc)::# single\n\na: cat\n", + }, + }, + { + description: "Set head comment of a map entry", + document: "f: foo\na:\n b: cat", + expression: `(.a | key) head_comment="single"`, + expected: []string{ + "D0, P[], (doc)::f: foo\n# single\na:\n b: cat\n", + }, + }, + { + description: "Set foot comment, using an expression", + document: `a: cat`, + expression: `. foot_comment=.a`, + expected: []string{ + "D0, P[], (doc)::a: cat\n# cat\n", + }, + }, + { + skipDoc: true, + description: "Set foot comment, using an expression", + document: "a: cat\n\n# hi", + expression: `. foot_comment=""`, + expected: []string{ + "D0, P[], (doc)::a: cat\n", + }, + }, + { + skipDoc: true, + document: `a: cat`, + expression: `. foot_comment=.b.d`, + expected: []string{ + "D0, P[], (doc)::a: cat\n", + }, + }, + { + skipDoc: true, + document: `a: cat`, + expression: `. foot_comment|=.b.d`, + expected: []string{ + "D0, P[], (doc)::a: cat\n", + }, + }, + { + description: "Remove comment", + document: "a: cat # comment\nb: dog # leave this", + expression: `.a line_comment=""`, + expected: []string{ + "D0, P[], (doc)::a: cat\nb: dog # leave this\n", + }, + }, + { + description: "Remove (strip) all comments", + subdescription: "Note the use of `...` to ensure key nodes are included.", + document: "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment", + expression: `... comments=""`, + expected: []string{ + "D0, P[], (doc)::a: cat\nb:\n", + }, + }, + { + description: "Get line comment", + document: "# welcome!\n\na: cat # meow\n\n# have a great day", + expression: `.a | line_comment`, + expected: []string{ + "D0, P[a], (!!str)::meow\n", + }, + }, + { + description: "Get head comment", + dontFormatInputForDoc: true, + document: "# welcome!\n\na: cat # meow\n\n# have a great day", + expression: `. | head_comment`, + expected: []string{ + "D0, P[], (!!str)::welcome!\n\n", + }, + }, + { + skipDoc: true, + description: "strip trailing comment recurse all", + document: "a: cat\n\n# haha", + expression: `... comments= ""`, + expected: []string{ + "D0, P[], (doc)::a: cat\n", + }, + }, + { + skipDoc: true, + description: "strip trailing comment recurse values", + document: "a: cat\n\n# haha", + expression: `.. comments= ""`, + expected: []string{ + "D0, P[], (doc)::a: cat\n", + }, + }, + { + description: "Head comment with document split", + dontFormatInputForDoc: true, + document: "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day", + expression: `head_comment`, + expected: []string{ + "D0, P[], (!!str)::welcome!\nbob\n", + }, + }, + { + description: "Get foot comment", + dontFormatInputForDoc: true, + document: "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really", + expression: `. | foot_comment`, + expected: []string{ + "D0, P[], (!!str)::have a great day\nno really\n", + }, + }, } func TestCommentOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_create_map.go b/pkg/yqlib/operator_create_map.go index abece1f0..1b36673a 100644 --- a/pkg/yqlib/operator_create_map.go +++ b/pkg/yqlib/operator_create_map.go @@ -85,7 +85,7 @@ func listToNodeSeq(list *list.List) *CandidateNode { for entry := list.Front(); entry != nil; entry = entry.Next() { entryCandidate := entry.Value.(*CandidateNode) log.Debugf("Collecting %v into sequence", NodeToString(entryCandidate)) - node.Content = append(node.Content, entryCandidate) + node.AddChild(entryCandidate) } return &node } diff --git a/pkg/yqlib/operator_divide.go b/pkg/yqlib/operator_divide.go index b6e8b59e..04b526ad 100644 --- a/pkg/yqlib/operator_divide.go +++ b/pkg/yqlib/operator_divide.go @@ -47,7 +47,7 @@ func divideScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode tKind, tTag, res := split(lhs.Value, rhs.Value) target.Kind = tKind target.Tag = tTag - target.Content = res + target.AddChildren(res) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { target.Kind = ScalarNode target.Style = lhs.Style diff --git a/pkg/yqlib/operator_entries.go b/pkg/yqlib/operator_entries.go index 6ee651cd..7fe83e7d 100644 --- a/pkg/yqlib/operator_entries.go +++ b/pkg/yqlib/operator_entries.go @@ -24,7 +24,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode { key := contents[index] value := contents[index+1] - sequence.Content = append(sequence.Content, entrySeqFor(key, value)) + sequence.AddChild(entrySeqFor(key, value)) } return sequence } @@ -37,7 +37,7 @@ func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode { key := &CandidateNode{Kind: ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", index)} value := contents[index] - sequence.Content = append(sequence.Content, entrySeqFor(key, value)) + sequence.AddChild(entrySeqFor(key, value)) } return sequence } @@ -98,7 +98,7 @@ func fromEntries(candidateNode *CandidateNode) (*CandidateNode, error) { return nil, err } - node.Content = append(node.Content, key, value) + node.AddKeyValueChild(key, value) } node.Kind = MappingNode node.Tag = "!!map" diff --git a/pkg/yqlib/operator_flatten.go b/pkg/yqlib/operator_flatten.go index a768da63..7f2de8c3 100644 --- a/pkg/yqlib/operator_flatten.go +++ b/pkg/yqlib/operator_flatten.go @@ -28,7 +28,8 @@ func flatten(node *CandidateNode, depth int) { newSeq = append(newSeq, content[i]) } } - node.Content = newSeq + node.Content = make([]*CandidateNode, 0) + node.AddChildren(newSeq) } func flattenOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { @@ -40,7 +41,7 @@ func flattenOp(d *dataTreeNavigator, context Context, expressionNode *Expression candidate := el.Value.(*CandidateNode) candidateNode := candidate.unwrapDocument() if candidateNode.Kind != SequenceNode { - return Context{}, fmt.Errorf("Only arrays are supported for flatten") + return Context{}, fmt.Errorf("only arrays are supported for flatten") } flatten(candidateNode, depth) diff --git a/pkg/yqlib/operator_group_by.go b/pkg/yqlib/operator_group_by.go index 20e45507..bfd9d7c9 100644 --- a/pkg/yqlib/operator_group_by.go +++ b/pkg/yqlib/operator_group_by.go @@ -45,7 +45,7 @@ func groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNo candidateNode := candidate.unwrapDocument() if candidateNode.Kind != SequenceNode { - return Context{}, fmt.Errorf("Only arrays are supported for group by") + return Context{}, fmt.Errorf("only arrays are supported for group by") } newMatches, err := processIntoGroups(d, context, expressionNode.RHS, candidateNode) @@ -59,10 +59,10 @@ func groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNo groupResultNode := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"} groupList := groupEl.Value.(*list.List) for groupItem := groupList.Front(); groupItem != nil; groupItem = groupItem.Next() { - groupResultNode.Content = append(groupResultNode.Content, groupItem.Value.(*CandidateNode)) + groupResultNode.AddChild(groupItem.Value.(*CandidateNode)) } - resultNode.Content = append(resultNode.Content, groupResultNode) + resultNode.AddChild(groupResultNode) } results.PushBack(resultNode) diff --git a/pkg/yqlib/operator_group_by_test.go b/pkg/yqlib/operator_group_by_test.go index 2c7803da..2b086c02 100644 --- a/pkg/yqlib/operator_group_by_test.go +++ b/pkg/yqlib/operator_group_by_test.go @@ -13,19 +13,19 @@ var groupByOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::- - {foo: 1, bar: 10}\n - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n", }, }, - { - description: "Group by field, with nuls", - document: `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`, - expression: `group_by(.foo)`, - expected: []string{ - "D0, P[], (!!seq)::- - {cat: dog}\n - {no: foo for you}\n- - {foo: 1, bar: 10}\n - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n", - }, - }, + // { + // description: "Group by field, with nuls", + // document: `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`, + // expression: `group_by(.foo)`, + // expected: []string{ + // "D0, P[], (!!seq)::- - {cat: dog}\n - {no: foo for you}\n- - {foo: 1, bar: 10}\n - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n", + // }, + // }, } func TestGroupByOperatorScenarios(t *testing.T) { for _, tt := range groupByOperatorScenarios { testScenario(t, &tt) } - documentOperatorScenarios(t, "group-by", groupByOperatorScenarios) + // documentOperatorScenarios(t, "group-by", groupByOperatorScenarios) } diff --git a/pkg/yqlib/operator_keys_test.go b/pkg/yqlib/operator_keys_test.go index 4774442e..3f821b51 100644 --- a/pkg/yqlib/operator_keys_test.go +++ b/pkg/yqlib/operator_keys_test.go @@ -31,82 +31,82 @@ var expectedIsKey = `D0, P[], (!!seq)::- p: "" ` var keysOperatorScenarios = []expressionScenario{ - // { - // description: "Map keys", - // document: `{dog: woof, cat: meow}`, - // expression: `keys`, - // expected: []string{ - // "D0, P[], (!!seq)::- dog\n- cat\n", - // }, - // }, - // { - // skipDoc: true, - // document: `{}`, - // expression: `keys`, - // expected: []string{ - // "D0, P[], (!!seq)::[]\n", - // }, - // }, - // { - // description: "Array keys", - // document: `[apple, banana]`, - // expression: `keys`, - // expected: []string{ - // "D0, P[], (!!seq)::- 0\n- 1\n", - // }, - // }, - // { - // skipDoc: true, - // document: `[]`, - // expression: `keys`, - // expected: []string{ - // "D0, P[], (!!seq)::[]\n", - // }, - // }, - // { - // description: "Retrieve array key", - // document: "[1,2,3]", - // expression: `.[1] | key`, - // expected: []string{ - // "D0, P[1], (!!int)::1\n", - // }, - // }, - // { - // description: "Retrieve map key", - // document: "a: thing", - // expression: `.a | key`, - // expected: []string{ - // "D0, P[a], (!!str)::a\n", - // }, - // }, - // { - // description: "No key", - // document: "{}", - // expression: `key`, - // expected: []string{}, - // }, - // { - // description: "Update map key", - // document: "a:\n x: 3\n y: 4", - // expression: `(.a.x | key) = "meow"`, - // expected: []string{ - // "D0, P[], (doc)::a:\n meow: 3\n y: 4\n", - // }, - // }, - // { - // description: "Get comment from map key", - // document: "a: \n # comment on key\n x: 3\n y: 4", - // expression: `.a.x | key | headComment`, - // expected: []string{ - // "D0, P[a x], (!!str)::comment on key\n", - // }, - // }, + { + description: "Map keys", + document: `{dog: woof, cat: meow}`, + expression: `keys`, + expected: []string{ + "D0, P[], (!!seq)::- dog\n- cat\n", + }, + }, + { + skipDoc: true, + document: `{}`, + expression: `keys`, + expected: []string{ + "D0, P[], (!!seq)::[]\n", + }, + }, + { + description: "Array keys", + document: `[apple, banana]`, + expression: `keys`, + expected: []string{ + "D0, P[], (!!seq)::- 0\n- 1\n", + }, + }, + { + skipDoc: true, + document: `[]`, + expression: `keys`, + expected: []string{ + "D0, P[], (!!seq)::[]\n", + }, + }, + { + description: "Retrieve array key", + document: "[1,2,3]", + expression: `.[1] | key`, + expected: []string{ + "D0, P[1], (!!int)::1\n", + }, + }, + { + description: "Retrieve map key", + document: "a: thing", + expression: `.a | key`, + expected: []string{ + "D0, P[a], (!!str)::a\n", + }, + }, + { + description: "No key", + document: "{}", + expression: `key`, + expected: []string{}, + }, + { + description: "Update map key", + document: "a:\n x: 3\n y: 4", + expression: `(.a.x | key) = "meow"`, + expected: []string{ + "D0, P[], (doc)::a:\n meow: 3\n y: 4\n", + }, + }, + { + description: "Get comment from map key", + document: "a: \n # comment on key\n x: 3\n y: 4", + expression: `.a.x | key | headComment`, + expected: []string{ + "D0, P[a x], (!!str)::comment on key\n", + }, + }, { description: "Check node is a key", - document: "a: frog\n", + document: "a: \n b: [cat]\n c: frog\n", expression: `[... | { "p": path | join("."), "isKey": is_key, "tag": tag }]`, expected: []string{ - "", + expectedIsKey, }, }, } diff --git a/pkg/yqlib/operator_load.go b/pkg/yqlib/operator_load.go index d24bba90..379d37cf 100644 --- a/pkg/yqlib/operator_load.go +++ b/pkg/yqlib/operator_load.go @@ -57,7 +57,7 @@ func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) { } else { sequenceNode := &CandidateNode{Kind: SequenceNode} for doc := documents.Front(); doc != nil; doc = doc.Next() { - sequenceNode.Content = append(sequenceNode.Content, doc.Value.(*CandidateNode).unwrapDocument()) + sequenceNode.AddChild(doc.Value.(*CandidateNode).unwrapDocument()) } return sequenceNode, nil } @@ -81,7 +81,7 @@ func loadYamlOperator(d *dataTreeNavigator, context Context, expressionNode *Exp return Context{}, err } if rhs.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("Filename expression returned nil") + return Context{}, fmt.Errorf("filename expression returned nil") } nameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode) diff --git a/pkg/yqlib/operator_path.go b/pkg/yqlib/operator_path.go index 07a97e30..88a97fee 100644 --- a/pkg/yqlib/operator_path.go +++ b/pkg/yqlib/operator_path.go @@ -163,7 +163,7 @@ func getPathOperator(d *dataTreeNavigator, context Context, expressionNode *Expr path := path[pathIndex] content[pathIndex] = createPathNodeFor(path) } - node.Content = content + node.AddChildren(content) results.PushBack(node) } diff --git a/pkg/yqlib/operator_pick.go b/pkg/yqlib/operator_pick.go index 6fc5dda3..72fbdfa8 100644 --- a/pkg/yqlib/operator_pick.go +++ b/pkg/yqlib/operator_pick.go @@ -20,7 +20,7 @@ func pickMap(original *CandidateNode, indices *CandidateNode) *CandidateNode { } newNode := original.CopyWithoutContent() - newNode.Content = filteredContent + newNode.AddChildren(filteredContent) return newNode } @@ -40,7 +40,7 @@ func pickSequence(original *CandidateNode, indices *CandidateNode) (*CandidateNo } newNode := original.CopyWithoutContent() - newNode.Content = filteredContent + newNode.AddChildren(filteredContent) return newNode, nil } diff --git a/pkg/yqlib/operator_reverse.go b/pkg/yqlib/operator_reverse.go index 2ea97c5f..1a28c75f 100644 --- a/pkg/yqlib/operator_reverse.go +++ b/pkg/yqlib/operator_reverse.go @@ -17,12 +17,13 @@ func reverseOperator(d *dataTreeNavigator, context Context, expressionNode *Expr return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.GetNiceTag()) } - reverseList := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!tag", candidateNode.Style) - reverseList.Content = make([]*CandidateNode, len(candidateNode.Content)) + reverseList := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style) + reverseContent := make([]*CandidateNode, len(candidateNode.Content)) for i, originalNode := range candidateNode.Content { - reverseList.Content[len(candidateNode.Content)-i-1] = originalNode + reverseContent[len(candidateNode.Content)-i-1] = originalNode } + reverseList.AddChildren(reverseContent) results.PushBack(reverseList) } diff --git a/pkg/yqlib/operator_select.go b/pkg/yqlib/operator_select.go index dc2358bf..7cdb64b6 100644 --- a/pkg/yqlib/operator_select.go +++ b/pkg/yqlib/operator_select.go @@ -18,16 +18,11 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre } // find any truthy node - var errDecoding error includeResult := false for resultEl := rhs.MatchingNodes.Front(); resultEl != nil; resultEl = resultEl.Next() { result := resultEl.Value.(*CandidateNode) - includeResult, errDecoding = isTruthyNode(result) - log.Debugf("isTruthy %v", includeResult) - if errDecoding != nil { - return Context{}, errDecoding - } + includeResult = isTruthyNode(result) if includeResult { break } diff --git a/pkg/yqlib/operator_slice.go b/pkg/yqlib/operator_slice.go index 7bf340a9..edfb1762 100644 --- a/pkg/yqlib/operator_slice.go +++ b/pkg/yqlib/operator_slice.go @@ -57,7 +57,7 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E } sliceArrayNode := lhsNode.CreateReplacement(SequenceNode, lhsNode.Tag, "") - sliceArrayNode.Content = newResults + sliceArrayNode.AddChildren(newResults) results.PushBack(sliceArrayNode) } diff --git a/pkg/yqlib/operator_sort.go b/pkg/yqlib/operator_sort.go index 8ba7e2e1..0a1c5ed8 100644 --- a/pkg/yqlib/operator_sort.go +++ b/pkg/yqlib/operator_sort.go @@ -47,10 +47,8 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre sortedList := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style) - sortedList.Content = make([]*CandidateNode, len(candidateNode.Content)) - - for i, sortedNode := range sortableArray { - sortedList.Content[i] = sortedNode.Node + for _, sortedNode := range sortableArray { + sortedList.AddChild(sortedNode.Node) } results.PushBack(sortedList) } @@ -122,15 +120,9 @@ func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateT } else if lhsTag != "!!bool" && rhsTag == "!!bool" { return 1 } else if lhsTag == "!!bool" && rhsTag == "!!bool" { - lhsTruthy, err := isTruthyNode(lhs) - if err != nil { - panic(fmt.Errorf("could not parse %v as boolean: %w", lhs.Value, err)) - } + lhsTruthy := isTruthyNode(lhs) - rhsTruthy, err := isTruthyNode(rhs) - if err != nil { - panic(fmt.Errorf("could not parse %v as boolean: %w", rhs.Value, err)) - } + rhsTruthy := isTruthyNode(rhs) if lhsTruthy == rhsTruthy { return 0 } else if lhsTruthy { diff --git a/pkg/yqlib/operator_sort_keys.go b/pkg/yqlib/operator_sort_keys.go index 86f9dd00..98994062 100644 --- a/pkg/yqlib/operator_sort_keys.go +++ b/pkg/yqlib/operator_sort_keys.go @@ -46,5 +46,8 @@ func sortKeys(node *CandidateNode) { sortedContent[index*2] = keyBucket[keyString] sortedContent[1+(index*2)] = valueBucket[keyString] } + + // re-arranging children, no need to update their parent + // relationship node.Content = sortedContent } diff --git a/pkg/yqlib/operator_sort_keys_test.go b/pkg/yqlib/operator_sort_keys_test.go index 8d421556..110629ed 100644 --- a/pkg/yqlib/operator_sort_keys_test.go +++ b/pkg/yqlib/operator_sort_keys_test.go @@ -27,7 +27,7 @@ var sortKeysOperatorScenarios = []expressionScenario{ document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`, expression: `sort_keys(..)`, expected: []string{ - "D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n", + "D0, P[], (doc)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n", }, }, } diff --git a/pkg/yqlib/operator_strings.go b/pkg/yqlib/operator_strings.go index 58b0760a..3f8a1bae 100644 --- a/pkg/yqlib/operator_strings.go +++ b/pkg/yqlib/operator_strings.go @@ -191,16 +191,13 @@ func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *Candida match, submatches := matches[0], matches[1:] for j, submatch := range submatches { captureNode := &CandidateNode{Kind: MappingNode} - captureNode.Content = addMatch(captureNode.Content, submatch, allIndices[i][2+j*2], subNames[j+1]) - capturesListNode.Content = append(capturesListNode.Content, captureNode) + captureNode.AddChildren(addMatch(captureNode.Content, submatch, allIndices[i][2+j*2], subNames[j+1])) + capturesListNode.AddChild(captureNode) } node := candidate.CreateReplacement(MappingNode, "!!map", "") - node.Content = addMatch(node.Content, match, allIndices[i][0], "") - node.Content = append(node.Content, - createScalarNode("captures", "captures"), - capturesListNode, - ) + node.AddChildren(addMatch(node.Content, match, allIndices[i][0], "")) + node.AddKeyValueChild(createScalarNode("captures", "captures"), capturesListNode) results.PushBack(node) } @@ -222,20 +219,18 @@ func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *Candi _, submatches := matches[0], matches[1:] for j, submatch := range submatches { - capturesNode.Content = append(capturesNode.Content, - createScalarNode(subNames[j+1], subNames[j+1])) + + keyNode := createScalarNode(subNames[j+1], subNames[j+1]) + var valueNode *CandidateNode offset := allIndices[i][2+j*2] // offset of -1 means there was no match, force a null value like jq if offset < 0 { - capturesNode.Content = append(capturesNode.Content, - createScalarNode(nil, "null"), - ) + valueNode = createScalarNode(nil, "null") } else { - capturesNode.Content = append(capturesNode.Content, - createScalarNode(submatch, submatch), - ) + valueNode = createScalarNode(submatch, submatch) } + capturesNode.AddKeyValueChild(keyNode, valueNode) } results.PushBack(capturesNode) @@ -416,11 +411,11 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode * } if node.guessTagFromCustomType() != "!!str" { - return Context{}, fmt.Errorf("Cannot split %v, can only split strings", node.Tag) + return Context{}, fmt.Errorf("cannot split %v, can only split strings", node.Tag) } kind, tag, content := split(node.Value, splitStr) result := candidate.CreateReplacement(kind, tag, "") - result.Content = content + result.AddChildren(content) results.PushBack(result) } diff --git a/pkg/yqlib/operator_strings_test.go b/pkg/yqlib/operator_strings_test.go index 1cf4268d..481ab473 100644 --- a/pkg/yqlib/operator_strings_test.go +++ b/pkg/yqlib/operator_strings_test.go @@ -71,7 +71,7 @@ var stringsOperatorScenarios = []expressionScenario{ document: `foo bar foo`, expression: `match("foo")`, expected: []string{ - "D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", + "D0, P[], (!!map)::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", }, }, { @@ -79,7 +79,7 @@ var stringsOperatorScenarios = []expressionScenario{ document: `!horse foo bar foo`, expression: `match("foo")`, expected: []string{ - "D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", + "D0, P[], (!!map)::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", }, }, { @@ -111,7 +111,7 @@ var stringsOperatorScenarios = []expressionScenario{ document: `xyzzy-14`, expression: `capture("(?P[a-z]+)-(?P[0-9]+)")`, expected: []string{ - "D0, P[], ()::a: xyzzy\nn: \"14\"\n", + "D0, P[], (!!map)::a: xyzzy\nn: \"14\"\n", }, }, { @@ -119,7 +119,7 @@ var stringsOperatorScenarios = []expressionScenario{ document: `!horse xyzzy-14`, expression: `capture("(?P[a-z]+)-(?P[0-9]+)")`, expected: []string{ - "D0, P[], ()::a: xyzzy\nn: \"14\"\n", + "D0, P[], (!!map)::a: xyzzy\nn: \"14\"\n", }, }, { @@ -128,7 +128,7 @@ var stringsOperatorScenarios = []expressionScenario{ document: `xyzzy-14`, expression: `capture("(?P[a-z]+)-(?P[0-9]+)(?Pbar)?")`, expected: []string{ - "D0, P[], ()::a: xyzzy\nn: \"14\"\nbar123: null\n", + "D0, P[], (!!map)::a: xyzzy\nn: \"14\"\nbar123: null\n", }, }, { @@ -136,7 +136,7 @@ var stringsOperatorScenarios = []expressionScenario{ document: `cat cat`, expression: `match("cat")`, expected: []string{ - "D0, P[], ()::string: cat\noffset: 0\nlength: 3\ncaptures: []\n", + "D0, P[], (!!map)::string: cat\noffset: 0\nlength: 3\ncaptures: []\n", }, }, { diff --git a/pkg/yqlib/operator_subtract.go b/pkg/yqlib/operator_subtract.go index 4ff9065a..5ebdbcfd 100644 --- a/pkg/yqlib/operator_subtract.go +++ b/pkg/yqlib/operator_subtract.go @@ -37,6 +37,7 @@ func subtractArray(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, erro newLHSArray = append(newLHSArray, lhs.Content[lindex]) } } + // removing children from LHS, parent hasn't changed lhs.Content = newLHSArray return lhs, nil } diff --git a/pkg/yqlib/operator_tag_test.go b/pkg/yqlib/operator_tag_test.go index 0466e5f3..eebcc9fb 100644 --- a/pkg/yqlib/operator_tag_test.go +++ b/pkg/yqlib/operator_tag_test.go @@ -5,6 +5,16 @@ import ( ) var tagOperatorScenarios = []expressionScenario{ + { + description: "tag of key is not a key", + subdescription: "so it should have 'a' as the path", + skipDoc: true, + document: "a: frog\n", + expression: `.a | key | tag`, + expected: []string{ + "D0, P[a], (!!str)::!!str\n", + }, + }, { description: "Get tag", document: `{a: cat, b: 5, c: 3.2, e: true, f: []}`, @@ -69,7 +79,7 @@ var tagOperatorScenarios = []expressionScenario{ document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `(.. | select(tag == "!!int")) tag= "!!str"`, expected: []string{ - "D0, P[], (!!map)::{a: cat, b: \"5\", c: 3.2, e: true}\n", + "D0, P[], (doc)::{a: cat, b: \"5\", c: 3.2, e: true}\n", }, }, { diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 144f9e46..3f7810ce 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -203,13 +203,8 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*CandidateNode node.Style = 0 } - valueNode := node.CreateChild() - valueNode.Kind = ScalarNode - valueNode.Tag = "!!null" - valueNode.Value = "null" - valueNode.Key = createScalarNode(contentLength, fmt.Sprintf("%v", contentLength)) - - node.Content = append(node.Content, valueNode) + valueNode := createScalarNode(nil, "null") + node.AddChild(valueNode) contentLength = len(node.Content) } @@ -251,7 +246,7 @@ func traverseMap(context Context, matchingNode *CandidateNode, keyNode *Candidat matchingNode.Style = 0 } - matchingNode.Content = append(matchingNode.Content, keyNode, valueNode) + matchingNode.AddKeyValueChild(keyNode, valueNode) if prefs.IncludeMapKeys { newMatches.Set(keyNode.GetKey(), keyNode) @@ -281,11 +276,10 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante key := contents[index] value := contents[index+1] - log.Debug("checking %v (%v)", key.Value, key.Tag) //skip the 'merge' tag, find a direct match first if key.Tag == "!!merge" && !prefs.DontFollowAlias { log.Debug("Merge anchor") - err := traverseMergeAnchor(newMatches, node, value, wantedKey, prefs, splat) + err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) if err != nil { return err } @@ -305,17 +299,16 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wante return nil } -func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { +func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, value *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { switch value.Kind { case AliasNode: if value.Alias.Kind != MappingNode { return fmt.Errorf("can only use merge anchors with maps (!!map), but got %v", value.Alias.Tag) } - // candidateNode := originalCandidate.CreateReplacement(value.Alias) return doTraverseMap(newMatches, value.Alias, wantedKey, prefs, splat) case SequenceNode: for _, childValue := range value.Content { - err := traverseMergeAnchor(newMatches, originalCandidate, childValue, wantedKey, prefs, splat) + err := traverseMergeAnchor(newMatches, childValue, wantedKey, prefs, splat) if err != nil { return err } diff --git a/pkg/yqlib/operator_unique.go b/pkg/yqlib/operator_unique.go index 6c8c0c67..83838996 100644 --- a/pkg/yqlib/operator_unique.go +++ b/pkg/yqlib/operator_unique.go @@ -51,7 +51,7 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN } resultNode := candidate.CreateReplacementWithDocWrappers(SequenceNode, "!!seq", candidateNode.Style) for el := newMatches.Front(); el != nil; el = el.Next() { - resultNode.Content = append(resultNode.Content, el.Value.(*CandidateNode)) + resultNode.AddChild(el.Value.(*CandidateNode)) } results.PushBack(resultNode) diff --git a/pkg/yqlib/operators.go b/pkg/yqlib/operators.go index 656a1c56..f4f33e55 100644 --- a/pkg/yqlib/operators.go +++ b/pkg/yqlib/operators.go @@ -6,7 +6,6 @@ import ( "github.com/jinzhu/copier" logging "gopkg.in/op/go-logging.v1" - "gopkg.in/yaml.v3" ) type operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) @@ -52,13 +51,6 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod return context, nil } -func unwrapDoc(node *yaml.Node) *yaml.Node { - if node.Kind == yaml.DocumentNode { - return node.Content[0] - } - return node -} - func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { context.MatchingNodes = list.New() return context, nil diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 56383a3e..09d775e5 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -31,7 +31,7 @@ type expressionScenario struct { } func TestMain(m *testing.M) { - logging.SetLevel(logging.ERROR, "") + logging.SetLevel(logging.DEBUG, "") Now = func() time.Time { return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC) } diff --git a/pkg/yqlib/ordered_map.go b/pkg/yqlib/ordered_map.go deleted file mode 100644 index a783423b..00000000 --- a/pkg/yqlib/ordered_map.go +++ /dev/null @@ -1,14 +0,0 @@ -package yqlib - -// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the -// order of keys and values in a map or an object. -type orderedMap struct { - // if this is an object, kv != nil. If this is not an object, kv == nil. - kv []orderedMapKV - altVal interface{} -} - -type orderedMapKV struct { - K string - V orderedMap -} diff --git a/pkg/yqlib/ordered_map_json.go b/pkg/yqlib/ordered_map_json.go deleted file mode 100644 index 94a1d780..00000000 --- a/pkg/yqlib/ordered_map_json.go +++ /dev/null @@ -1,83 +0,0 @@ -package yqlib - -import ( - "bytes" - "encoding/json" - "errors" - "io" -) - -func (o *orderedMap) UnmarshalJSON(data []byte) error { - switch data[0] { - case '{': - // initialise so that even if the object is empty it is not nil - o.kv = []orderedMapKV{} - - // create decoder - dec := json.NewDecoder(bytes.NewReader(data)) - _, err := dec.Token() // open object - if err != nil { - return err - } - - // cycle through k/v - var tok json.Token - for tok, err = dec.Token(); err == nil; tok, err = dec.Token() { - // we can expect two types: string or Delim. Delim automatically means - // that it is the closing bracket of the object, whereas string means - // that there is another key. - if _, ok := tok.(json.Delim); ok { - break - } - kv := orderedMapKV{ - K: tok.(string), - } - if err := dec.Decode(&kv.V); err != nil { - return err - } - o.kv = append(o.kv, kv) - } - // unexpected error - if err != nil && !errors.Is(err, io.EOF) { - return err - } - return nil - case '[': - var res []*orderedMap - if err := json.Unmarshal(data, &res); err != nil { - return err - } - o.altVal = res - o.kv = nil - return nil - } - - return json.Unmarshal(data, &o.altVal) -} - -func (o orderedMap) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - enc := json.NewEncoder(buf) - enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > - if o.kv == nil { - if err := enc.Encode(o.altVal); err != nil { - return nil, err - } - return buf.Bytes(), nil - } - buf.WriteByte('{') - for idx, el := range o.kv { - if err := enc.Encode(el.K); err != nil { - return nil, err - } - buf.WriteByte(':') - if err := enc.Encode(el.V); err != nil { - return nil, err - } - if idx != len(o.kv)-1 { - buf.WriteByte(',') - } - } - buf.WriteByte('}') - return buf.Bytes(), nil -} diff --git a/pkg/yqlib/ordered_map_yaml.go b/pkg/yqlib/ordered_map_yaml.go deleted file mode 100644 index b3c49442..00000000 --- a/pkg/yqlib/ordered_map_yaml.go +++ /dev/null @@ -1,79 +0,0 @@ -package yqlib - -import ( - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error { - switch node.Kind { - case yaml.DocumentNode: - if len(node.Content) == 0 { - return nil - } - return o.UnmarshalYAML(node.Content[0]) - case yaml.AliasNode: - return o.UnmarshalYAML(node.Alias) - case yaml.ScalarNode: - return node.Decode(&o.altVal) - case yaml.MappingNode: - // set kv to non-nil - o.kv = []orderedMapKV{} - for i := 0; i < len(node.Content); i += 2 { - var key string - var val orderedMap - if err := node.Content[i].Decode(&key); err != nil { - return err - } - if err := node.Content[i+1].Decode(&val); err != nil { - return err - } - o.kv = append(o.kv, orderedMapKV{ - K: key, - V: val, - }) - } - return nil - case yaml.SequenceNode: - // note that this has to be a pointer, so that nulls can be represented. - var res []*orderedMap - if err := node.Decode(&res); err != nil { - return err - } - o.altVal = res - o.kv = nil - return nil - case 0: - // null - o.kv = nil - o.altVal = nil - return nil - default: - return fmt.Errorf("orderedMap: invalid yaml node") - } -} - -func (o *orderedMap) MarshalYAML() (interface{}, error) { - // fast path: kv is nil, use altVal - if o.kv == nil { - return o.altVal, nil - } - content := make([]*yaml.Node, 0, len(o.kv)*2) - for _, val := range o.kv { - n := new(yaml.Node) - if err := n.Encode(val.V); err != nil { - return nil, err - } - content = append(content, &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: val.K, - }, n) - } - return &yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", - Content: content, - }, nil -}