diff --git a/pkg/yqlib/doc/Sort Keys.md b/pkg/yqlib/doc/Sort Keys.md new file mode 100644 index 00000000..a1a99306 --- /dev/null +++ b/pkg/yqlib/doc/Sort Keys.md @@ -0,0 +1,68 @@ +The Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps). + +Sort is particularly useful for diffing two different yaml documents: + +```bash +yq eval -i 'sortKeys(..)' file1.yml +yq eval -i 'sortKeys(..)' file2.yml +diff file1.yml file2.yml +``` + +## Sort keys of map +Given a sample.yml file of: +```yaml +c: frog +a: blah +b: bing +``` +then +```bash +yq eval 'sortKeys(.)' sample.yml +``` +will output +```yaml +a: blah +b: bing +c: frog +``` + +## Sort keys recursively +Note the array elements are left unsorted, but maps inside arrays are sorted + +Given a sample.yml file of: +```yaml +bParent: + c: dog + array: + - 3 + - 1 + - 2 +aParent: + z: donkey + x: + - c: yum + b: delish + - b: ew + a: apple +``` +then +```bash +yq eval 'sortKeys(..)' sample.yml +``` +will output +```yaml +aParent: + x: + - b: delish + c: yum + - a: apple + b: ew + z: donkey +bParent: + array: + - 3 + - 1 + - 2 + c: dog +``` + diff --git a/pkg/yqlib/doc/headers/Sort Keys.md b/pkg/yqlib/doc/headers/Sort Keys.md new file mode 100644 index 00000000..00eb71a9 --- /dev/null +++ b/pkg/yqlib/doc/headers/Sort Keys.md @@ -0,0 +1,9 @@ +The Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps). + +Sort is particularly useful for diffing two different yaml documents: + +```bash +yq eval -i 'sortKeys(..)' file1.yml +yq eval -i 'sortKeys(..)' file2.yml +diff file1.yml file2.yml +``` diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index e3693f06..73be42b2 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -20,8 +20,6 @@ type OperationType struct { // operators TODO: // - cookbook doc for common things -// - existStatus -// - write in place // - mergeEmpty (sets only if the document is empty, do I do that now?) // - compare ?? // - validate ?? @@ -57,6 +55,7 @@ var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence var GetPath = &OperationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: GetPathOperator} var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator} +var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: SortKeysOperator} var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator} var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} diff --git a/pkg/yqlib/operator_sort_keys.go b/pkg/yqlib/operator_sort_keys.go new file mode 100644 index 00000000..e8f39381 --- /dev/null +++ b/pkg/yqlib/operator_sort_keys.go @@ -0,0 +1,53 @@ +package yqlib + +import ( + "container/list" + "sort" + + yaml "gopkg.in/yaml.v3" +) + +func SortKeysOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { + + for el := matchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs) + if err != nil { + return nil, err + } + + for childEl := rhs.Front(); childEl != nil; childEl = childEl.Next() { + node := UnwrapDoc(childEl.Value.(*CandidateNode).Node) + if node.Kind == yaml.MappingNode { + sortKeys(node) + } + if err != nil { + return nil, err + } + } + + } + return matchingNodes, nil +} + +func sortKeys(node *yaml.Node) { + keys := make([]string, len(node.Content)/2) + keyBucket := map[string]*yaml.Node{} + valueBucket := map[string]*yaml.Node{} + var contents = node.Content + for index := 0; index < len(contents); index = index + 2 { + key := contents[index] + value := contents[index+1] + keys[index/2] = key.Value + keyBucket[key.Value] = key + valueBucket[key.Value] = value + } + sort.Strings(keys) + sortedContent := make([]*yaml.Node, len(node.Content)) + for index := 0; index < len(keys); index = index + 1 { + keyString := keys[index] + sortedContent[index*2] = keyBucket[keyString] + sortedContent[1+(index*2)] = valueBucket[keyString] + } + node.Content = sortedContent +} diff --git a/pkg/yqlib/operator_sort_keys_test.go b/pkg/yqlib/operator_sort_keys_test.go new file mode 100644 index 00000000..303b5adc --- /dev/null +++ b/pkg/yqlib/operator_sort_keys_test.go @@ -0,0 +1,32 @@ +package yqlib + +import ( + "testing" +) + +var sortKeysOperatorScenarios = []expressionScenario{ + { + description: "Sort keys of map", + document: `{c: frog, a: blah, b: bing}`, + expression: `sortKeys(.)`, + expected: []string{ + "D0, P[], (doc)::{a: blah, b: bing, c: frog}\n", + }, + }, + { + description: "Sort keys recursively", + subdescription: "Note the array elements are left unsorted, but maps inside arrays are sorted", + document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`, + expression: `sortKeys(..)`, + expected: []string{ + "D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n", + }, + }, +} + +func TestSortKeysOperatorScenarios(t *testing.T) { + for _, tt := range sortKeysOperatorScenarios { + testScenario(t, &tt) + } + documentScenarios(t, "Sort Keys", sortKeysOperatorScenarios) +} diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index d712e191..e6cc4b17 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -190,6 +190,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`,`), opToken(Union)) lexer.Add([]byte(`:\s*`), opToken(CreateMap)) lexer.Add([]byte(`length`), opToken(Length)) + lexer.Add([]byte(`sortKeys`), opToken(SortKeys)) lexer.Add([]byte(`select`), opToken(Select)) lexer.Add([]byte(`has`), opToken(Has)) lexer.Add([]byte(`explode`), opToken(Explode))