Merge 9883ebc313 into 8bf425b4d1

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23.5 AS builder
+FROM golang:1.23.4 AS builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.23.5
+FROM golang:1.23.4
 
 RUN apt-get update && \
     apt-get install -y npm && \

examples/data1.yaml

@@ -1,3 +1,11 @@
-Foo: 3
+# block comments come through
-apple: 1
+person: # neither do comments on maps
-bar: 2
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    - dog # comments on array values appear
+    - things:
+        - frog
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []

go.mod

@@ -5,7 +5,7 @@ require (
 	github.com/alecthomas/participle/v2 v2.1.1
 	github.com/alecthomas/repr v0.4.0
 	github.com/dimchansky/utfbom v1.1.1
-	github.com/elliotchance/orderedmap v1.8.0
+	github.com/elliotchance/orderedmap v1.7.1
 	github.com/fatih/color v1.18.0
 	github.com/goccy/go-json v0.10.4
 	github.com/goccy/go-yaml v1.13.3
@@ -16,7 +16,7 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/net v0.34.0
+	golang.org/x/net v0.33.0
 	golang.org/x/text v0.21.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
@@ -26,7 +26,9 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
 )
 
-go 1.23.0
+go 1.21.0
+
+toolchain go1.22.5

go.sum

@@ -12,8 +12,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
-github.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCGOhPElnMw=
+github.com/elliotchance/orderedmap v1.7.1 h1:8SR2DB391dw0HVI9572ElrY+KU0Q89OCXYwWZx7aAZc=
-github.com/elliotchance/orderedmap v1.8.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
+github.com/elliotchance/orderedmap v1.7.1/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
@@ -62,14 +62,14 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
-golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

pkg/yqlib/candidate_node.go

@@ -198,29 +198,6 @@ func (n *CandidateNode) SetParent(parent *CandidateNode) {
 	n.Parent = parent
 }
 
-type ValueVisitor func(*CandidateNode) error
-
-func (n *CandidateNode) VisitValues(visitor ValueVisitor) error {
-	if n.Kind == MappingNode {
-		for i := 1; i < len(n.Content); i = i + 2 {
-			if err := visitor(n.Content[i]); err != nil {
-				return err
-			}
-		}
-	} else if n.Kind == SequenceNode {
-		for i := 0; i < len(n.Content); i = i + 1 {
-			if err := visitor(n.Content[i]); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (n *CandidateNode) CanVisitValues() bool {
-	return n.Kind == MappingNode || n.Kind == SequenceNode
-}
-
 func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) (*CandidateNode, *CandidateNode) {
 	key := rawKey.Copy()
 	key.SetParent(n)

pkg/yqlib/doc/operators/headers/sort-keys.md

@@ -12,5 +12,5 @@ diff file1.yml file2.yml
 
 Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if you are using merge anchors.
 
-For more advanced sorting, you can use the [sort_by](https://mikefarah.gitbook.io/yq/operators/sort) function on a map, and give it a custom function like `sort_by(key | downcase)`.
+For more advanced sorting, using `to_entries` to convert the map to an array, then sort/process the array as you like (e.g. using `sort_by`) and convert back to a map using `from_entries`.
-
+See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-keys) for an example. 

pkg/yqlib/doc/operators/sort-keys.md

@@ -12,8 +12,8 @@ diff file1.yml file2.yml
 
 Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if you are using merge anchors.
 
-For more advanced sorting, you can use the [sort_by](https://mikefarah.gitbook.io/yq/operators/sort) function on a map, and give it a custom function like `sort_by(key | downcase)`.
+For more advanced sorting, using `to_entries` to convert the map to an array, then sort/process the array as you like (e.g. using `sort_by`) and convert back to a map using `from_entries`.
-
+See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-keys) for an example. 
 
 ## Sort keys of map
 Given a sample.yml file of:

pkg/yqlib/doc/operators/sort.md

@@ -109,46 +109,6 @@ cool:
   - c: banana
 ```
 
-## Sort a map
-Sorting a map, by default, will sort by the values
-
-Given a sample.yml file of:
-```yaml
-y: b
-z: a
-x: c
-```
-then
-```bash
-yq 'sort' sample.yml
-```
-will output
-```yaml
-z: a
-y: b
-x: c
-```
-
-## Sort a map by keys
-Use sort_by to sort a map using a custom function
-
-Given a sample.yml file of:
-```yaml
-Y: b
-z: a
-x: c
-```
-then
-```bash
-yq 'sort_by(key | downcase)' sample.yml
-```
-will output
-```yaml
-x: c
-Y: b
-z: a
-```
-
 ## Sort is stable
 Note the order of the elements in unchanged when equal in sorting.
 

pkg/yqlib/doc/usage/properties.md

@@ -71,7 +71,7 @@ person.food[0] = pizza
 ```
 
 ## Encode properties - custom separator
-Use the --properties-separator flag to specify your own key/value separator.
+Use the --properties-customer-separator flag to specify your own key/value separator.
 
 Given a sample.yml file of:
 ```yaml
@@ -89,7 +89,7 @@ emptyMap: []
 ```
 then
 ```bash
-yq -o=props --properties-separator=" :@ " sample.yml
+yq -o=props --properties-customer-separator=" :@ " sample.yml
 ```
 will output
 ```properties

pkg/yqlib/operator_sort.go

@@ -24,40 +24,30 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		var sortableArray sortableNodeArray
+		if candidate.Kind != SequenceNode {
+			return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.Tag)
+		}
 
-		if candidate.CanVisitValues() {
+		sortableArray := make(sortableNodeArray, len(candidate.Content))
-			sortableArray = make(sortableNodeArray, 0)
+
-			visitor := func(valueNode *CandidateNode) error {
+		for i, originalNode := range candidate.Content {
-				compareContext, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(valueNode), expressionNode.RHS)
+
-				if err != nil {
+			compareContext, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(originalNode), expressionNode.RHS)
-					return err
+			if err != nil {
-				}
+				return Context{}, err
-				sortableNode := sortableNode{Node: valueNode, CompareContext: compareContext, dateTimeLayout: context.GetDateTimeLayout()}
-				sortableArray = append(sortableArray, sortableNode)
-				return nil
 			}
-			if err := candidate.VisitValues(visitor); err != nil {
+
-				return context, err
+			sortableArray[i] = sortableNode{Node: originalNode, CompareContext: compareContext, dateTimeLayout: context.GetDateTimeLayout()}
-			}
+
-		} else {
-			return context, fmt.Errorf("node at path [%v] is not an array or map (it's a %v)", candidate.GetNicePath(), candidate.Tag)
 		}
 
 		sort.Stable(sortableArray)
 
-		sortedList := candidate.CopyWithoutContent()
+		sortedList := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidate.Style)
-		if candidate.Kind == MappingNode {
-			for _, sortedNode := range sortableArray {
-				sortedList.AddKeyValueChild(sortedNode.Node.Key, sortedNode.Node)
-			}
-		} else if candidate.Kind == SequenceNode {
-			for _, sortedNode := range sortableArray {
-				sortedList.AddChild(sortedNode.Node)
-			}
-		}
 
-		// convert array of value nodes back to map
+		for _, sortedNode := range sortableArray {
+			sortedList.AddChild(sortedNode.Node)
+		}
 		results.PushBack(sortedList)
 	}
 	return context.ChildContext(results), nil

pkg/yqlib/operator_sort_test.go

@@ -84,24 +84,6 @@ var sortByOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::cool: [{a: banana}, {b: banana}, {c: banana}]\n",
 		},
 	},
-	{
-		description:    "Sort a map",
-		subdescription: "Sorting a map, by default this will sort by the values",
-		document:       "y: b\nz: a\nx: c\n",
-		expression:     `sort`,
-		expected: []string{
-			"D0, P[], (!!map)::z: a\ny: b\nx: c\n",
-		},
-	},
-	{
-		description:    "Sort a map by keys",
-		subdescription: "Use sort_by to sort a map using a custom function",
-		document:       "Y: b\nz: a\nx: c\n",
-		expression:     `sort_by(key | downcase)`,
-		expected: []string{
-			"D0, P[], (!!map)::x: c\nY: b\nz: a\n",
-		},
-	},
 	{
 		description:    "Sort is stable",
 		subdescription: "Note the order of the elements in unchanged when equal in sorting.",

pkg/yqlib/properties_test.go

@@ -171,7 +171,7 @@ var propertyScenarios = []formatScenario{
 	},
 	{
 		description:    "Encode properties - custom separator",
-		subdescription: "Use the --properties-separator flag to specify your own key/value separator.",
+		subdescription: "Use the --properties-customer-separator flag to specify your own key/value separator.",
 		input:          samplePropertiesYaml,
 		expected:       expectedPropertiesUnwrappedCustomSeparator,
 		scenarioType:   "encode-custom-separator",
@@ -324,7 +324,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario)
 		prefs.UseArrayBrackets = true
 	} else if s.scenarioType == "encode-custom-separator" {
 		prefs.KeyValueSeparator = " :@ "
-		useCustomSeparatorFlag = ` --properties-separator=" :@ "`
+		useCustomSeparatorFlag = ` --properties-customer-separator=" :@ "`
 	}
 
 	if expression != "" {