From 7288d3477803541830fedc5056f1a7d8554440ab Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 22 Oct 2021 12:37:47 +1100 Subject: [PATCH] Added decoder op --- pkg/yqlib/doc/Encoder and Decoder.md | 110 ++++++++++++++++++ pkg/yqlib/doc/Encoder.md | 77 ------------ pkg/yqlib/doc/headers/Encoder and Decoder.md | 1 + pkg/yqlib/doc/headers/Encoder.md | 1 - pkg/yqlib/expression_tokeniser.go | 6 + pkg/yqlib/lib.go | 3 +- ...encoder.go => operator_encoder_decoder.go} | 22 ++++ ...st.go => operator_encoder_decoder_test.go} | 25 +++- 8 files changed, 162 insertions(+), 83 deletions(-) create mode 100644 pkg/yqlib/doc/Encoder and Decoder.md create mode 100644 pkg/yqlib/doc/headers/Encoder and Decoder.md delete mode 100644 pkg/yqlib/doc/headers/Encoder.md rename pkg/yqlib/{operator_encoder.go => operator_encoder_decoder.go} (60%) rename pkg/yqlib/{operator_encoder_test.go => operator_encoder_decoder_test.go} (59%) diff --git a/pkg/yqlib/doc/Encoder and Decoder.md b/pkg/yqlib/doc/Encoder and Decoder.md new file mode 100644 index 00000000..dbc695f1 --- /dev/null +++ b/pkg/yqlib/doc/Encoder and Decoder.md @@ -0,0 +1,110 @@ +Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure. + +## Encode value as yaml string +Given a sample.yml file of: +```yaml +a: + cool: thing +``` +then +```bash +yq eval '.b = (.a | to_yaml)' sample.yml +``` +will output +```yaml +a: + cool: thing +b: | + cool: thing +``` + +## Encode value as yaml string, using toyaml +Does the same thing as to_yaml, matching jq naming convention. + +Given a sample.yml file of: +```yaml +a: + cool: thing +``` +then +```bash +yq eval '.b = (.a | to_yaml)' sample.yml +``` +will output +```yaml +a: + cool: thing +b: | + cool: thing +``` + +## Encode value as json string +Given a sample.yml file of: +```yaml +a: + cool: thing +``` +then +```bash +yq eval '.b = (.a | to_json)' sample.yml +``` +will output +```yaml +a: + cool: thing +b: | + { + "cool": "thing" + } +``` + +## Encode value as props string +Given a sample.yml file of: +```yaml +a: + cool: thing +``` +then +```bash +yq eval '.b = (.a | to_props)' sample.yml +``` +will output +```yaml +a: + cool: thing +b: | + cool = thing +``` + +## Decode a yaml encoded string +Given a sample.yml file of: +```yaml +a: 'foo: bar' +``` +then +```bash +yq eval '.b = (.a | from_yaml)' sample.yml +``` +will output +```yaml +a: 'foo: bar' +b: + foo: bar +``` + +## Update an encoded yaml string +Given a sample.yml file of: +```yaml +a: | + foo: bar +``` +then +```bash +yq eval '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml +``` +will output +```yaml +a: | + foo: cat +``` + diff --git a/pkg/yqlib/doc/Encoder.md b/pkg/yqlib/doc/Encoder.md index c903de96..e69de29b 100644 --- a/pkg/yqlib/doc/Encoder.md +++ b/pkg/yqlib/doc/Encoder.md @@ -1,77 +0,0 @@ -Encode operators will take the piped in object structure and encode it as a string in the desired format. -## Encode value as yaml string -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq eval '.b = (.a | to_yaml)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: | - cool: thing -``` - -## Encode value as yaml string, using toyaml -Does the same thing as to_yaml, matching jq naming convention. - -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq eval '.b = (.a | to_yaml)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: | - cool: thing -``` - -## Encode value as json string -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq eval '.b = (.a | to_json)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: | - { - "cool": "thing" - } -``` - -## Encode value as props string -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq eval '.b = (.a | to_props)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: | - cool = thing -``` - diff --git a/pkg/yqlib/doc/headers/Encoder and Decoder.md b/pkg/yqlib/doc/headers/Encoder and Decoder.md new file mode 100644 index 00000000..6ad18118 --- /dev/null +++ b/pkg/yqlib/doc/headers/Encoder and Decoder.md @@ -0,0 +1 @@ +Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure. diff --git a/pkg/yqlib/doc/headers/Encoder.md b/pkg/yqlib/doc/headers/Encoder.md deleted file mode 100644 index 93f54639..00000000 --- a/pkg/yqlib/doc/headers/Encoder.md +++ /dev/null @@ -1 +0,0 @@ -Encode operators will take the piped in object structure and encode it as a string in the desired format. \ No newline at end of file diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 932dd8c5..19393ab4 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -281,6 +281,12 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat})) lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat})) + lexer.Add([]byte(`fromyaml`), opToken(decodeOpType)) + lexer.Add([]byte(`fromjson`), opToken(decodeOpType)) + + lexer.Add([]byte(`from_yaml`), opToken(decodeOpType)) + lexer.Add([]byte(`from_json`), opToken(decodeOpType)) + lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType)) lexer.Add([]byte(`select`), opToken(selectOpType)) lexer.Add([]byte(`has`), opToken(hasOpType)) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index fdb598a8..7c2d82c3 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -59,7 +59,8 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator} var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator} -var encodeOpType = &operationType{Type: "TO_YAML", NumArgs: 0, Precedence: 50, Handler: encodeOperator} +var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator} +var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator} var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator} var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator} diff --git a/pkg/yqlib/operator_encoder.go b/pkg/yqlib/operator_encoder_decoder.go similarity index 60% rename from pkg/yqlib/operator_encoder.go rename to pkg/yqlib/operator_encoder_decoder.go index 896dfc17..2b6c53e4 100644 --- a/pkg/yqlib/operator_encoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "container/list" + "strings" "gopkg.in/yaml.v3" ) @@ -38,3 +39,24 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre } return context.ChildContext(results), nil } + +/* takes a string and decodes it back into an object */ +func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + + var results = list.New() + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + var dataBucket yaml.Node + log.Debugf("got: [%v]", candidate.Node.Value) + decoder := yaml.NewDecoder(strings.NewReader(unwrapDoc(candidate.Node).Value)) + errorReading := decoder.Decode(&dataBucket) + if errorReading != nil { + return Context{}, errorReading + } + //first node is a doc + node := unwrapDoc(&dataBucket) + + results.PushBack(candidate.CreateChild(nil, node)) + } + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_encoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go similarity index 59% rename from pkg/yqlib/operator_encoder_test.go rename to pkg/yqlib/operator_encoder_decoder_test.go index e180a6c2..1252d12e 100644 --- a/pkg/yqlib/operator_encoder_test.go +++ b/pkg/yqlib/operator_encoder_decoder_test.go @@ -4,7 +4,7 @@ import ( "testing" ) -var encoderOperatorScenarios = []expressionScenario{ +var encoderDecoderOperatorScenarios = []expressionScenario{ { description: "Encode value as yaml string", document: `{a: {cool: "thing"}}`, @@ -42,11 +42,28 @@ var encoderOperatorScenarios = []expressionScenario{ `, }, }, + { + description: "Decode a yaml encoded string", + document: `a: "foo: bar"`, + expression: `.b = (.a | from_yaml)`, + expected: []string{ + "D0, P[], (doc)::a: \"foo: bar\"\nb:\n foo: bar\n", + }, + }, + { + description: "Update an encoded yaml string", + dontFormatInputForDoc: true, + document: "a: |\n foo: bar", + expression: `.a |= (from_yaml | .foo = "cat" | to_yaml)`, + expected: []string{ + "D0, P[], (doc)::a: |\n foo: cat\n", + }, + }, } -func TestEncoderOperatorScenarios(t *testing.T) { - for _, tt := range encoderOperatorScenarios { +func TestEncoderDecoderOperatorScenarios(t *testing.T) { + for _, tt := range encoderDecoderOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "Encoder", encoderOperatorScenarios) + documentScenarios(t, "Encoder and Decoder", encoderDecoderOperatorScenarios) }