Merge branch 'master' into add-comment-style

.golangci.yml

@@ -10,7 +10,9 @@ linters:
     - gofmt
     - goimports
     - gosec
-    - megacheck
+    - gosimple
+    - staticcheck
+    - unused
     - misspell
     - nakedret
     - nolintlint
@@ -31,13 +33,6 @@ linters-settings:
             desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil"
 issues:
   exclude-rules:
-  - linters:
-    - gosec
-    text:  "Implicit memory aliasing in for loop."
-    path: _test\.go
-  - linters:
-    - revive
-    text: "unexported-return"
   - linters:
     - revive
     text: "var-naming"

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23.2 as builder
+FROM golang:1.23.3 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.23.2
+FROM golang:1.23.3
 
 RUN apt-get update && \
     apt-get install -y npm && \

cmd/constant.go

@@ -12,9 +12,6 @@ var outputFormat = ""
 var inputFormat = ""
 
 var exitStatus = false
-var forceColor = false
-var forceNoColor = false
-var colorsEnabled = false
 var indent = 2
 var noDocSeparators = false
 var nullInput = false
@@ -23,6 +20,10 @@ var verbose = false
 var version = false
 var prettyPrint = false
 
+var forceColor = false
+var forceNoColor = false
+var colorsEnabled = false
+
 // can be either "" (off), "extract" or "process"
 var frontMatter = ""
 

cmd/evaluate_all_command.go

@@ -12,7 +12,7 @@ func createEvaluateAllCommand() *cobra.Command {
 		Use:     "eval-all [expression] [yaml_file1]...",
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {
 			if len(args) == 0 {
 				return nil, cobra.ShellCompDirectiveNoFileComp
 			}

cmd/evaluate_sequence_command.go

@@ -13,7 +13,7 @@ func createEvaluateSequenceCommand() *cobra.Command {
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
 		Short:   "(default) Apply the expression to each document in each yaml file in sequence",
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {
 			if len(args) == 0 {
 				return nil, cobra.ShellCompDirectiveNoFileComp
 			}

cmd/root.go

@@ -66,7 +66,7 @@ yq -P -oy sample.json
 			return evaluateSequence(cmd, args)
 
 		},
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SetOut(cmd.OutOrStdout())
 			level := logging.WARNING
 			stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`
@@ -90,6 +90,10 @@ yq -P -oy sample.json
 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
 
+			// when NO_COLOR environment variable presents and not an empty string the coloured output should be disabled;
+			// refer to no-color.org
+			forceNoColor = os.Getenv("NO_COLOR") != ""
+
 			return nil
 		},
 	}
@@ -182,7 +186,7 @@ yq -P -oy sample.json
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
-	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
+	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", forceNoColor, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
 	if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {
 		panic(err)

go.mod

@@ -16,8 +16,8 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/net v0.30.0
+	golang.org/x/net v0.31.0
-	golang.org/x/text v0.19.0
+	golang.org/x/text v0.20.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -26,7 +26,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 )
 
 go 1.21.0

go.sum

@@ -62,16 +62,18 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=

pkg/yqlib/decoder_toml.go

@@ -286,7 +286,7 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
 		}
 
 		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-		if err != nil && !errors.Is(io.EOF, err) {
+		if err != nil && !errors.Is(err, io.EOF) {
 			return false, err
 		}
 	}
@@ -343,7 +343,7 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
 	tableValue := dec.parser.Expression()
 	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
 	log.Debugf("table node err: %w", err)
-	if err != nil && !errors.Is(io.EOF, err) {
+	if err != nil && !errors.Is(err, io.EOF) {
 		return false, err
 	}
 	c := Context{}

pkg/yqlib/lexer_participle.go

@@ -311,7 +311,7 @@ func opTokenWithPrefs(opType *operationType, assignOpType *operationType, prefer
 }
 
 func expressionOpToken(expression string) yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
+	return func(_ lexer.Token) (*token, error) {
 		prefs := expressionOpPreferences{expression: expression}
 		expressionOp := &Operation{OperationType: expressionOpType, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: expressionOp}, nil
@@ -522,7 +522,7 @@ func parentWithLevel() yqAction {
 }
 
 func parentWithDefaultLevel() yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
+	return func(_ lexer.Token) (*token, error) {
 		prefs := parentOpPreferences{Level: 1}
 		op := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: getParentOpType.Type, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil

pkg/yqlib/operator_assign.go

@@ -7,7 +7,7 @@ type assignPreferences struct {
 }
 
 func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
-	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		if !prefs.OnlyWriteNull || lhs.Tag == "!!null" {
 			lhs.UpdateFrom(rhs, prefs)
 		}

pkg/yqlib/operator_compare.go

@@ -18,7 +18,7 @@ func compareOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 }
 
 func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		log.Debugf("compare cross function")
 		if lhs == nil && rhs == nil {
 			owner := &CandidateNode{}

pkg/yqlib/operator_create_map.go

@@ -53,7 +53,7 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
 	log.Debugf("**********sequenceFor %v", NodeToString(matchingNode))
 
 	mapPairs, err := crossFunction(d, context.ChildContext(matches), expressionNode,
-		func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 			node := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
 
 			log.Debugf("**********adding key %v and value %v", NodeToString(lhs), NodeToString(rhs))

pkg/yqlib/operator_equals.go

@@ -6,7 +6,7 @@ func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 }
 
 func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		value := false
 		log.Debugf("isEquals cross function")
 		if lhs == nil && rhs == nil {

pkg/yqlib/operator_error.go

@@ -1,7 +1,7 @@
 package yqlib
 
 import (
-	"fmt"
+	"errors"
 )
 
 func errorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -16,5 +16,5 @@ func errorOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 	if rhs.MatchingNodes.Len() > 0 {
 		errorMessage = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value
 	}
-	return Context{}, fmt.Errorf(errorMessage)
+	return Context{}, errors.New(errorMessage)
 }

pkg/yqlib/operator_sort.go

@@ -69,6 +69,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 	rhsContext := a[j].CompareContext
 
 	rhsEl := rhsContext.MatchingNodes.Front()
+
 	for lhsEl := lhsContext.MatchingNodes.Front(); lhsEl != nil && rhsEl != nil; lhsEl = lhsEl.Next() {
 		lhs := lhsEl.Value.(*CandidateNode)
 		rhs := rhsEl.Value.(*CandidateNode)
@@ -83,7 +84,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 
 		rhsEl = rhsEl.Next()
 	}
-	return false
+	return lhsContext.MatchingNodes.Len() < rhsContext.MatchingNodes.Len()
 }
 
 func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateTimeLayout string) int {

pkg/yqlib/operator_sort_test.go

@@ -21,6 +21,17 @@ var sortByOperatorScenarios = []expressionScenario{
 			"D0, P[0], (!!map)::{a: banana}\n",
 		},
 	},
+	{
+		description: "Sort by with null",
+		skipDoc:     true,
+		document:    "[{a: banana},null,{a: apple}]",
+		expression:  `sort_by(.a)[]`,
+		expected: []string{
+			"D0, P[1], (!!null)::null\n",
+			"D0, P[2], (!!map)::{a: apple}\n",
+			"D0, P[0], (!!map)::{a: banana}\n",
+		},
+	},
 	{
 		description: "Sort by multiple fields",
 		document:    "[{a: dog},{a: cat, b: banana},{a: cat, b: apple}]",

scripts/devtools.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -ex
 go mod download golang.org/x/tools@latest
-curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.2
+curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.62.0
 wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s