From 2692987998d08fa6482e6b1be9ab0465ad701ea0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:01:02 +0000 Subject: [PATCH 1/5] Initial plan From 6d345ac79535600fb664d094f55128ff07d5439e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:12:55 +0000 Subject: [PATCH 2/5] Add string slicing support to yq Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/a8525fbb-77a7-4bb0-a3a7-b24f99ae8710 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- .../doc/operators/headers/slice-array.md | 6 +- pkg/yqlib/doc/operators/slice-array.md | 68 +++++++++++++++- pkg/yqlib/lexer.go | 2 +- pkg/yqlib/operator_slice.go | 51 ++++++++++-- pkg/yqlib/operator_slice_test.go | 78 +++++++++++++++++++ pkg/yqlib/operator_traverse_path.go | 6 +- project-words.txt | 6 +- 7 files changed, 203 insertions(+), 14 deletions(-) diff --git a/pkg/yqlib/doc/operators/headers/slice-array.md b/pkg/yqlib/doc/operators/headers/slice-array.md index 87307bd1..0113a585 100644 --- a/pkg/yqlib/doc/operators/headers/slice-array.md +++ b/pkg/yqlib/doc/operators/headers/slice-array.md @@ -1,5 +1,5 @@ -# Slice/Splice Array +# Slice/Splice Array or String -The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array. +The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string. -You may leave out the first or second number, which will refer to the start or end of the array respectively. +You may leave out the first or second number, which will refer to the start or end of the array or string respectively. diff --git a/pkg/yqlib/doc/operators/slice-array.md b/pkg/yqlib/doc/operators/slice-array.md index 9b89210b..800108d2 100644 --- a/pkg/yqlib/doc/operators/slice-array.md +++ b/pkg/yqlib/doc/operators/slice-array.md @@ -1,8 +1,8 @@ -# Slice/Splice Array +# Slice/Splice Array or String -The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array. +The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string. -You may leave out the first or second number, which will refer to the start or end of the array respectively. +You may leave out the first or second number, which will refer to the start or end of the array or string respectively. ## Slicing arrays Given a sample.yml file of: @@ -103,3 +103,65 @@ will output - cow ``` +## Slicing strings +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country[4:]' sample.yml +``` +will output +```yaml +ralia +``` + +## Slicing strings - without the second number +Finishes at the end of the string + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country[0:5]' sample.yml +``` +will output +```yaml +Austr +``` + +## Slicing strings - without the first number +Starts from the start of the string + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country[:5]' sample.yml +``` +will output +```yaml +Austr +``` + +## Slicing strings - use negative numbers to count backwards from the end +Negative indices count from the end of the string + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country[-5:]' sample.yml +``` +will output +```yaml +ralia +``` + diff --git a/pkg/yqlib/lexer.go b/pkg/yqlib/lexer.go index 04212fce..25ef12ac 100644 --- a/pkg/yqlib/lexer.go +++ b/pkg/yqlib/lexer.go @@ -127,7 +127,7 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke if tokenIsOpType(currentToken, createMapOpType) { log.Debugf("tokenIsOpType: createMapOpType") // check the previous token is '[', means we are slice, but dont have a first number - if index > 0 && tokens[index-1].TokenType == traverseArrayCollect { + if index > 0 && (tokens[index-1].TokenType == traverseArrayCollect || tokens[index-1].TokenType == openCollect) { log.Debugf("previous token is : traverseArrayOpType") // need to put the number 0 before this token, as that is implied postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")}) diff --git a/pkg/yqlib/operator_slice.go b/pkg/yqlib/operator_slice.go index 42606134..c17ed259 100644 --- a/pkg/yqlib/operator_slice.go +++ b/pkg/yqlib/operator_slice.go @@ -16,6 +16,38 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode, return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value) } +func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) (*CandidateNode, error) { + runes := []rune(lhsNode.Value) + length := len(runes) + + relativeFirstNumber := firstNumber + if relativeFirstNumber < 0 { + relativeFirstNumber = length + firstNumber + } + if relativeFirstNumber < 0 { + relativeFirstNumber = 0 + } + + relativeSecondNumber := secondNumber + if relativeSecondNumber < 0 { + relativeSecondNumber = length + secondNumber + } else if relativeSecondNumber > length { + relativeSecondNumber = length + } + + log.Debugf("sliceStringNode: slice from %v to %v", relativeFirstNumber, relativeSecondNumber) + + if relativeFirstNumber > length { + relativeFirstNumber = length + } + if relativeSecondNumber < relativeFirstNumber { + relativeSecondNumber = relativeFirstNumber + } + + slicedString := string(runes[relativeFirstNumber:relativeSecondNumber]) + return lhsNode.CreateReplacement(ScalarNode, "!!str", slicedString), nil +} + func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debug("slice array operator!") @@ -28,20 +60,29 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E lhsNode := el.Value.(*CandidateNode) firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS) - if err != nil { return Context{}, err } - relativeFirstNumber := firstNumber - if relativeFirstNumber < 0 { - relativeFirstNumber = len(lhsNode.Content) + firstNumber - } secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS) if err != nil { return Context{}, err } + if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" { + slicedNode, err := sliceStringNode(lhsNode, firstNumber, secondNumber) + if err != nil { + return Context{}, err + } + results.PushBack(slicedNode) + continue + } + + relativeFirstNumber := firstNumber + if relativeFirstNumber < 0 { + relativeFirstNumber = len(lhsNode.Content) + firstNumber + } + relativeSecondNumber := secondNumber if relativeSecondNumber < 0 { relativeSecondNumber = len(lhsNode.Content) + secondNumber diff --git a/pkg/yqlib/operator_slice_test.go b/pkg/yqlib/operator_slice_test.go index 29a821eb..16e796eb 100644 --- a/pkg/yqlib/operator_slice_test.go +++ b/pkg/yqlib/operator_slice_test.go @@ -98,6 +98,84 @@ var sliceArrayScenarios = []expressionScenario{ "D0, P[], (!!seq)::- cat1\n", }, }, + { + description: "Slicing strings", + document: `country: Australia`, + expression: `.country[4:]`, + expected: []string{ + "D0, P[country], (!!str)::ralia\n", + }, + }, + { + description: "Slicing strings - without the second number", + subdescription: "Finishes at the end of the string", + document: `country: Australia`, + expression: `.country[0:5]`, + expected: []string{ + "D0, P[country], (!!str)::Austr\n", + }, + }, + { + description: "Slicing strings - without the first number", + subdescription: "Starts from the start of the string", + document: `country: Australia`, + expression: `.country[:5]`, + expected: []string{ + "D0, P[country], (!!str)::Austr\n", + }, + }, + { + description: "Slicing strings - use negative numbers to count backwards from the end", + subdescription: "Negative indices count from the end of the string", + document: `country: Australia`, + expression: `.country[-5:]`, + expected: []string{ + "D0, P[country], (!!str)::ralia\n", + }, + }, + { + skipDoc: true, + document: `country: Australia`, + expression: `.country[1:-1]`, + expected: []string{ + "D0, P[country], (!!str)::ustrali\n", + }, + }, + { + skipDoc: true, + document: `country: Australia`, + expression: `.country[:]`, + expected: []string{ + "D0, P[country], (!!str)::Australia\n", + }, + }, + { + skipDoc: true, + description: "second index beyond string length clamps", + document: `country: Australia`, + expression: `.country[:100]`, + expected: []string{ + "D0, P[country], (!!str)::Australia\n", + }, + }, + { + skipDoc: true, + description: "first index beyond string length returns empty string", + document: `country: Australia`, + expression: `.country[100:]`, + expected: []string{ + "D0, P[country], (!!str)::\n", + }, + }, + { + skipDoc: true, + description: "Unicode string slicing", + document: `greeting: héllo`, + expression: `.greeting[1:3]`, + expected: []string{ + "D0, P[greeting], (!!str)::él\n", + }, + }, } func TestSliceOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index da981252..fd795b9b 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -79,7 +79,11 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode log.Debugf("--traverseArrayOperator") if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType { - return sliceArrayOperator(d, context, expressionNode.RHS.RHS) + lhsContext, err := d.GetMatchingNodes(context, expressionNode.LHS) + if err != nil { + return Context{}, err + } + return sliceArrayOperator(d, lhsContext, expressionNode.RHS.RHS) } lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) diff --git a/project-words.txt b/project-words.txt index 86c85e01..ed57a37c 100644 --- a/project-words.txt +++ b/project-words.txt @@ -298,4 +298,8 @@ subsubarray Ffile Fquery coverpkg -gsub \ No newline at end of file +gsub +ralia +Austr +ustrali +héllo From 9a9399ad0005b87032ca9e43224058cfc04b5bbd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 02:01:01 +0000 Subject: [PATCH 3/5] Fix sliceStringNode signature and fix test descriptions/expressions Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/58726b13-68ae-4f93-971f-eb70459edcf4 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- pkg/yqlib/doc/operators/slice-array.md | 8 ++++---- pkg/yqlib/operator_slice.go | 10 +++------- pkg/yqlib/operator_slice_test.go | 8 ++++---- project-words.txt | 1 + 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/pkg/yqlib/doc/operators/slice-array.md b/pkg/yqlib/doc/operators/slice-array.md index 800108d2..aee24811 100644 --- a/pkg/yqlib/doc/operators/slice-array.md +++ b/pkg/yqlib/doc/operators/slice-array.md @@ -110,11 +110,11 @@ country: Australia ``` then ```bash -yq '.country[4:]' sample.yml +yq '.country[0:5]' sample.yml ``` will output ```yaml -ralia +Austr ``` ## Slicing strings - without the second number @@ -126,11 +126,11 @@ country: Australia ``` then ```bash -yq '.country[0:5]' sample.yml +yq '.country[5:]' sample.yml ``` will output ```yaml -Austr +alia ``` ## Slicing strings - without the first number diff --git a/pkg/yqlib/operator_slice.go b/pkg/yqlib/operator_slice.go index c17ed259..d5191b54 100644 --- a/pkg/yqlib/operator_slice.go +++ b/pkg/yqlib/operator_slice.go @@ -16,7 +16,7 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode, return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value) } -func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) (*CandidateNode, error) { +func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) *CandidateNode { runes := []rune(lhsNode.Value) length := len(runes) @@ -45,7 +45,7 @@ func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) } slicedString := string(runes[relativeFirstNumber:relativeSecondNumber]) - return lhsNode.CreateReplacement(ScalarNode, "!!str", slicedString), nil + return lhsNode.CreateReplacement(ScalarNode, "!!str", slicedString) } func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { @@ -70,11 +70,7 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E } if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" { - slicedNode, err := sliceStringNode(lhsNode, firstNumber, secondNumber) - if err != nil { - return Context{}, err - } - results.PushBack(slicedNode) + results.PushBack(sliceStringNode(lhsNode, firstNumber, secondNumber)) continue } diff --git a/pkg/yqlib/operator_slice_test.go b/pkg/yqlib/operator_slice_test.go index 16e796eb..c4a6b783 100644 --- a/pkg/yqlib/operator_slice_test.go +++ b/pkg/yqlib/operator_slice_test.go @@ -101,18 +101,18 @@ var sliceArrayScenarios = []expressionScenario{ { description: "Slicing strings", document: `country: Australia`, - expression: `.country[4:]`, + expression: `.country[0:5]`, expected: []string{ - "D0, P[country], (!!str)::ralia\n", + "D0, P[country], (!!str)::Austr\n", }, }, { description: "Slicing strings - without the second number", subdescription: "Finishes at the end of the string", document: `country: Australia`, - expression: `.country[0:5]`, + expression: `.country[5:]`, expected: []string{ - "D0, P[country], (!!str)::Austr\n", + "D0, P[country], (!!str)::alia\n", }, }, { diff --git a/project-words.txt b/project-words.txt index ed57a37c..8f2fcf9e 100644 --- a/project-words.txt +++ b/project-words.txt @@ -303,3 +303,4 @@ ralia Austr ustrali héllo +alia From 778088d70cb1145b9b1653bab7be88c488bff284 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 28 Mar 2026 14:18:51 +1100 Subject: [PATCH 4/5] Update pkg/yqlib/operator_slice.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/yqlib/operator_slice.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/yqlib/operator_slice.go b/pkg/yqlib/operator_slice.go index d5191b54..24d1629d 100644 --- a/pkg/yqlib/operator_slice.go +++ b/pkg/yqlib/operator_slice.go @@ -45,7 +45,9 @@ func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) } slicedString := string(runes[relativeFirstNumber:relativeSecondNumber]) - return lhsNode.CreateReplacement(ScalarNode, "!!str", slicedString) + replacement := lhsNode.CreateReplacement(ScalarNode, lhsNode.Tag, slicedString) + replacement.Style = lhsNode.Style + return replacement } func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { From 341e2524b97e02755382b165b1bdf2af16d6815b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 03:20:37 +0000 Subject: [PATCH 5/5] Fix array slice out-of-bounds panic with very negative indices Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/7c146762-d251-45fd-8555-2488f59fc57b Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- pkg/yqlib/operator_slice.go | 11 +++++++++++ pkg/yqlib/operator_slice_test.go | 16 ++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/pkg/yqlib/operator_slice.go b/pkg/yqlib/operator_slice.go index 24d1629d..e54a7918 100644 --- a/pkg/yqlib/operator_slice.go +++ b/pkg/yqlib/operator_slice.go @@ -80,13 +80,24 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E if relativeFirstNumber < 0 { relativeFirstNumber = len(lhsNode.Content) + firstNumber } + if relativeFirstNumber < 0 { + relativeFirstNumber = 0 + } else if relativeFirstNumber > len(lhsNode.Content) { + relativeFirstNumber = len(lhsNode.Content) + } relativeSecondNumber := secondNumber if relativeSecondNumber < 0 { relativeSecondNumber = len(lhsNode.Content) + secondNumber + } + if relativeSecondNumber < 0 { + relativeSecondNumber = 0 } else if relativeSecondNumber > len(lhsNode.Content) { relativeSecondNumber = len(lhsNode.Content) } + if relativeSecondNumber < relativeFirstNumber { + relativeSecondNumber = relativeFirstNumber + } log.Debugf("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber) diff --git a/pkg/yqlib/operator_slice_test.go b/pkg/yqlib/operator_slice_test.go index c4a6b783..33ce462f 100644 --- a/pkg/yqlib/operator_slice_test.go +++ b/pkg/yqlib/operator_slice_test.go @@ -98,6 +98,22 @@ var sliceArrayScenarios = []expressionScenario{ "D0, P[], (!!seq)::- cat1\n", }, }, + { + skipDoc: true, + document: `[cat, dog, frog]`, + expression: `.[-100:]`, + expected: []string{ + "D0, P[], (!!seq)::- cat\n- dog\n- frog\n", + }, + }, + { + skipDoc: true, + document: `[cat, dog, frog]`, + expression: `.[:-100]`, + expected: []string{ + "D0, P[], (!!seq)::[]\n", + }, + }, { description: "Slicing strings", document: `country: Australia`,