Add string slicing support to yq

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/a8525fbb-77a7-4bb0-a3a7-b24f99ae8710

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

pkg/yqlib/doc/operators/headers/slice-array.md

@@ -1,5 +1,5 @@
-# Slice/Splice Array
+# Slice/Splice Array or String
 
-The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
+The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
 
-You may leave out the first or second number, which will refer to the start or end of the array respectively.
+You may leave out the first or second number, which will refer to the start or end of the array or string respectively.

pkg/yqlib/doc/operators/slice-array.md

@@ -1,8 +1,8 @@
-# Slice/Splice Array
+# Slice/Splice Array or String
 
-The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
+The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
 
-You may leave out the first or second number, which will refer to the start or end of the array respectively.
+You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
 
 ## Slicing arrays
 Given a sample.yml file of:
@@ -103,3 +103,65 @@ will output
 - cow
 ```
 
+## Slicing strings
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country[4:]' sample.yml
+```
+will output
+```yaml
+ralia
+```
+
+## Slicing strings - without the second number
+Finishes at the end of the string
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country[0:5]' sample.yml
+```
+will output
+```yaml
+Austr
+```
+
+## Slicing strings - without the first number
+Starts from the start of the string
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country[:5]' sample.yml
+```
+will output
+```yaml
+Austr
+```
+
+## Slicing strings - use negative numbers to count backwards from the end
+Negative indices count from the end of the string
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country[-5:]' sample.yml
+```
+will output
+```yaml
+ralia
+```
+

pkg/yqlib/lexer.go

@@ -127,7 +127,7 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
 	if tokenIsOpType(currentToken, createMapOpType) {
 		log.Debugf("tokenIsOpType: createMapOpType")
 		// check the previous token is '[', means we are slice, but dont have a first number
-		if index > 0 && tokens[index-1].TokenType == traverseArrayCollect {
+		if index > 0 && (tokens[index-1].TokenType == traverseArrayCollect || tokens[index-1].TokenType == openCollect) {
 			log.Debugf("previous token is : traverseArrayOpType")
 			// need to put the number 0 before this token, as that is implied
 			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})

pkg/yqlib/operator_slice.go

@@ -16,6 +16,38 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode,
 	return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value)
 }
 
+func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) (*CandidateNode, error) {
+	runes := []rune(lhsNode.Value)
+	length := len(runes)
+
+	relativeFirstNumber := firstNumber
+	if relativeFirstNumber < 0 {
+		relativeFirstNumber = length + firstNumber
+	}
+	if relativeFirstNumber < 0 {
+		relativeFirstNumber = 0
+	}
+
+	relativeSecondNumber := secondNumber
+	if relativeSecondNumber < 0 {
+		relativeSecondNumber = length + secondNumber
+	} else if relativeSecondNumber > length {
+		relativeSecondNumber = length
+	}
+
+	log.Debugf("sliceStringNode: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
+
+	if relativeFirstNumber > length {
+		relativeFirstNumber = length
+	}
+	if relativeSecondNumber < relativeFirstNumber {
+		relativeSecondNumber = relativeFirstNumber
+	}
+
+	slicedString := string(runes[relativeFirstNumber:relativeSecondNumber])
+	return lhsNode.CreateReplacement(ScalarNode, "!!str", slicedString), nil
+}
+
 func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 
 	log.Debug("slice array operator!")
@@ -28,20 +60,29 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E
 		lhsNode := el.Value.(*CandidateNode)
 
 		firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
-
 		if err != nil {
 			return Context{}, err
 		}
-		relativeFirstNumber := firstNumber
-		if relativeFirstNumber < 0 {
-			relativeFirstNumber = len(lhsNode.Content) + firstNumber
-		}
 
 		secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
 		if err != nil {
 			return Context{}, err
 		}
 
+		if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" {
+			slicedNode, err := sliceStringNode(lhsNode, firstNumber, secondNumber)
+			if err != nil {
+				return Context{}, err
+			}
+			results.PushBack(slicedNode)
+			continue
+		}
+
+		relativeFirstNumber := firstNumber
+		if relativeFirstNumber < 0 {
+			relativeFirstNumber = len(lhsNode.Content) + firstNumber
+		}
+
 		relativeSecondNumber := secondNumber
 		if relativeSecondNumber < 0 {
 			relativeSecondNumber = len(lhsNode.Content) + secondNumber

pkg/yqlib/operator_slice_test.go

@@ -98,6 +98,84 @@ var sliceArrayScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- cat1\n",
 		},
 	},
+	{
+		description: "Slicing strings",
+		document:    `country: Australia`,
+		expression:  `.country[4:]`,
+		expected: []string{
+			"D0, P[country], (!!str)::ralia\n",
+		},
+	},
+	{
+		description:    "Slicing strings - without the second number",
+		subdescription: "Finishes at the end of the string",
+		document:       `country: Australia`,
+		expression:     `.country[0:5]`,
+		expected: []string{
+			"D0, P[country], (!!str)::Austr\n",
+		},
+	},
+	{
+		description:    "Slicing strings - without the first number",
+		subdescription: "Starts from the start of the string",
+		document:       `country: Australia`,
+		expression:     `.country[:5]`,
+		expected: []string{
+			"D0, P[country], (!!str)::Austr\n",
+		},
+	},
+	{
+		description:    "Slicing strings - use negative numbers to count backwards from the end",
+		subdescription: "Negative indices count from the end of the string",
+		document:       `country: Australia`,
+		expression:     `.country[-5:]`,
+		expected: []string{
+			"D0, P[country], (!!str)::ralia\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `country: Australia`,
+		expression: `.country[1:-1]`,
+		expected: []string{
+			"D0, P[country], (!!str)::ustrali\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `country: Australia`,
+		expression: `.country[:]`,
+		expected: []string{
+			"D0, P[country], (!!str)::Australia\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "second index beyond string length clamps",
+		document:    `country: Australia`,
+		expression:  `.country[:100]`,
+		expected: []string{
+			"D0, P[country], (!!str)::Australia\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "first index beyond string length returns empty string",
+		document:    `country: Australia`,
+		expression:  `.country[100:]`,
+		expected: []string{
+			"D0, P[country], (!!str)::\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Unicode string slicing",
+		document:    `greeting: héllo`,
+		expression:  `.greeting[1:3]`,
+		expected: []string{
+			"D0, P[greeting], (!!str)::él\n",
+		},
+	},
 }
 
 func TestSliceOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_traverse_path.go

@@ -79,7 +79,11 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 	log.Debugf("--traverseArrayOperator")
 
 	if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
-		return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
+		lhsContext, err := d.GetMatchingNodes(context, expressionNode.LHS)
+		if err != nil {
+			return Context{}, err
+		}
+		return sliceArrayOperator(d, lhsContext, expressionNode.RHS.RHS)
 	}
 
 	lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)

project-words.txt

@@ -298,4 +298,8 @@ subsubarray
 Ffile
 Fquery
 coverpkg
-gsub
+gsub
+ralia
+Austr
+ustrali
+héllo