suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-12-19 20:19:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Parameterised merge array by key example

Browse Source
This commit is contained in:
Mike Farah 2022-02-01 15:11:39 +11:00
parent 0afb59c65e
commit b80080a26d
3 changed files with 30 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/yqlib/doc/operators/eval.md
View File

@ -36,7 +36,7 @@ a:
``` ```
then then
```bash ```bash
valueEnv="moo" pathEnv=".a.b[0].name" yq 'eval(strenv(pathEnv)) = strenv(valueEnv)' sample.yml pathEnv=".a.b[0].name" valueEnv="moo" yq 'eval(strenv(pathEnv)) = strenv(valueEnv)' sample.yml
``` ```
will output will output
```yaml ```yaml

19
pkg/yqlib/doc/operators/multiply-merge.md
View File

@ -262,18 +262,23 @@ will output
``` ```
## Merge arrays of objects together, matching on a key ## Merge arrays of objects together, matching on a key
This is a fairly complex expression - you can use it as is by providing the environment variables as seen in the example below.
It merges in the array provided in the second file into the first - matching on equal keys.
Explanation:
The approach, at a high level, is to reduce into a merged map (keyed by the unique key) The approach, at a high level, is to reduce into a merged map (keyed by the unique key)
and then convert that back into an array. and then convert that back into an array.
First the expression will create a map from the arrays keyed by '.a', the unique field we want to merge by. First the expression will create a map from the arrays keyed by the idPath, the unique field we want to merge by.
The reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together. The reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together.
Next, we convert the map back to an array, using reduce again, concatenating all the map values together. Next, we convert the map back to an array, using reduce again, concatenating all the map values together.
Finally, we set the result of the merged array back into the first doc. Finally, we set the result of the merged array back into the first doc.
To use this, you will need to update '.myArray' in the expression to your array (e.g. .my.array), and '.a' to be the key field of your array (e.g. '.name')
Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223) Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223)
@ -290,7 +295,7 @@ something: else
``` ```
And another sample another.yml file of: And another sample another.yml file of:
```yaml ```yaml
myArray: newArray:
- a: banana - a: banana
c: bananaC c: bananaC
- a: apple - a: apple
@ -300,12 +305,12 @@ myArray:
``` ```
then then
```bash ```bash
yq eval-all ' originalPath=".myArray" otherPath=".newArray" idPath=".a" yq eval-all '
( (
((.myArray[] | {.a: .}) as $item ireduce ({}; . * $item )) as $uniqueMap (( (eval(strenv(originalPath)) + eval(strenv(otherPath))) | .[] | {(eval(strenv(idPath))): .}) as $item ireduce ({}; . * $item )) as $uniqueMap
| ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value)
) as $mergedArray ) as $mergedArray
| select(fi == 0) | .myArray = $mergedArray | select(fi == 0) | (eval(strenv(originalPath))) = $mergedArray
' sample.yml another.yml ' sample.yml another.yml
``` ```
will output will output

28
pkg/yqlib/operator_multiply_test.go
View File

@ -29,27 +29,32 @@ var mergeArrayWithAnchors = `sample:
- <<: *a - <<: *a
` `
var mergeArraysObjectKeysText = `The approach, at a high level, is to reduce into a merged map (keyed by the unique key) var mergeArraysObjectKeysText = `
This is a fairly complex expression - you can use it as is by providing the environment variables as seen in the example below.
It merges in the array provided in the second file into the first - matching on equal keys.
Explanation:
The approach, at a high level, is to reduce into a merged map (keyed by the unique key)
and then convert that back into an array. and then convert that back into an array.
First the expression will create a map from the arrays keyed by '.a', the unique field we want to merge by. First the expression will create a map from the arrays keyed by the idPath, the unique field we want to merge by.
The reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together. The reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together.
Next, we convert the map back to an array, using reduce again, concatenating all the map values together. Next, we convert the map back to an array, using reduce again, concatenating all the map values together.
Finally, we set the result of the merged array back into the first doc. Finally, we set the result of the merged array back into the first doc.
To use this, you will need to update '.myArray' in the expression to your array (e.g. .my.array), and '.a' to be the key field of your array (e.g. '.name')
Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223) Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223)
` `
var mergeExpression = ` var mergeExpression = `
( (
((.myArray[] | {.a: .}) as $item ireduce ({}; . * $item )) as $uniqueMap (( (eval(strenv(originalPath)) + eval(strenv(otherPath))) | .[] | {(eval(strenv(idPath))): .}) as $item ireduce ({}; . * $item )) as $uniqueMap
| ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value)
) as $mergedArray ) as $mergedArray
| select(fi == 0) | .myArray = $mergedArray | select(fi == 0) | (eval(strenv(originalPath))) = $mergedArray
` `
var docWithHeader = `# here var docWithHeader = `# here
@ -413,11 +418,12 @@ var multiplyOperatorScenarios = []expressionScenario{
}, },
}, },
{ {
description: "Merge arrays of objects together, matching on a key", description: "Merge arrays of objects together, matching on a key",
subdescription: mergeArraysObjectKeysText, subdescription: mergeArraysObjectKeysText,
document: `{myArray: [{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}], something: else}`, document: `{myArray: [{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}], something: else}`,
document2: `myArray: [{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`, document2: `newArray: [{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
expression: mergeExpression, environmentVariables: map[string]string{"originalPath": ".myArray", "otherPath": ".newArray", "idPath": ".a"},
expression: mergeExpression,
expected: []string{ expected: []string{
"D0, P[], (doc)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n", "D0, P[], (doc)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n",
}, },