From d9bca65626e7f4baf28d4dc31dd921cbd9a22345 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 23 Feb 2022 09:26:35 +1100 Subject: [PATCH] Added base64 support --- pkg/yqlib/decoder.go | 1 + pkg/yqlib/decoder_base64.go | 44 +++++++++++++ pkg/yqlib/doc/operators/encode-decode.md | 66 +++++++++++++++++++ .../doc/operators/headers/encode-decode.md | 3 + pkg/yqlib/doc/usage/base64.md | 23 +++++++ pkg/yqlib/encoder_base64.go | 38 +++++++++++ pkg/yqlib/expression_tokeniser.go | 3 + pkg/yqlib/operator_encoder_decoder.go | 4 ++ pkg/yqlib/operator_encoder_decoder_test.go | 35 ++++++++++ pkg/yqlib/printer.go | 1 + 10 files changed, 218 insertions(+) create mode 100644 pkg/yqlib/decoder_base64.go create mode 100644 pkg/yqlib/doc/usage/base64.md create mode 100644 pkg/yqlib/encoder_base64.go diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go index f8797607..c4d46708 100644 --- a/pkg/yqlib/decoder.go +++ b/pkg/yqlib/decoder.go @@ -13,6 +13,7 @@ const ( YamlInputFormat = 1 << iota XMLInputFormat PropertiesInputFormat + Base64InputFormat ) type Decoder interface { diff --git a/pkg/yqlib/decoder_base64.go b/pkg/yqlib/decoder_base64.go new file mode 100644 index 00000000..550f8ef3 --- /dev/null +++ b/pkg/yqlib/decoder_base64.go @@ -0,0 +1,44 @@ +package yqlib + +import ( + "bytes" + "encoding/base64" + "io" + + yaml "gopkg.in/yaml.v3" +) + +type base64Decoder struct { + reader io.Reader + finished bool + encoding base64.Encoding +} + +func NewBase64Decoder() Decoder { + return &base64Decoder{finished: false, encoding: *base64.StdEncoding} +} + +func (dec *base64Decoder) Init(reader io.Reader) { + dec.reader = reader + dec.finished = false +} + +func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error { + if dec.finished { + return io.EOF + } + base64Reader := base64.NewDecoder(&dec.encoding, dec.reader) + buf := new(bytes.Buffer) + + if _, err := buf.ReadFrom(base64Reader); err != nil { + return err + } + if buf.Len() == 0 { + dec.finished = true + return io.EOF + } + rootYamlNode.Kind = yaml.ScalarNode + rootYamlNode.Tag = "!!str" + rootYamlNode.Value = buf.String() + return nil +} diff --git a/pkg/yqlib/doc/operators/encode-decode.md b/pkg/yqlib/doc/operators/encode-decode.md index 191a1c56..cf73bcd3 100644 --- a/pkg/yqlib/doc/operators/encode-decode.md +++ b/pkg/yqlib/doc/operators/encode-decode.md @@ -15,6 +15,7 @@ These operators are useful to process yaml documents that have stringified embed | CSV | | to_csv/@csv | | TSV | | to_tsv/@tsv | | XML | from_xml | to_xml(i)/@xml | +| Base64 | @base64d | @base64 | CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). @@ -22,6 +23,8 @@ CSV and TSV format both accept either a single array or scalars (representing a XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields. +Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string. + {% hint style="warning" %} Note that versions prior to 4.18 require the 'eval/e' command to be specified. @@ -354,3 +357,66 @@ b: foo: bar ``` +## Encode a string to base64 +Given a sample.yml file of: +```yaml +coolData: a special string +``` +then +```bash +yq '.coolData | @base64' sample.yml +``` +will output +```yaml +YSBzcGVjaWFsIHN0cmluZw== +``` + +## Encode a yaml document to base64 +Pipe through @yaml first to convert to a string, then use @base64 to encode it. + +Given a sample.yml file of: +```yaml +a: apple +``` +then +```bash +yq '@yaml | @base64' sample.yml +``` +will output +```yaml +YTogYXBwbGUK +``` + +## Decode a base64 encoded string +Decoded data is assumed to be a string. + +Given a sample.yml file of: +```yaml +coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig== +``` +then +```bash +yq '.coolData | @base64d' sample.yml +``` +will output +```yaml +Works with UTF-16 😊 +``` + +## Decode a base64 encoded yaml document +Pipe through `from_yaml` to parse the decoded base64 string as a yaml document. + +Given a sample.yml file of: +```yaml +coolData: YTogYXBwbGUK +``` +then +```bash +yq '.coolData |= (@base64d | from_yaml)' sample.yml +``` +will output +```yaml +coolData: + a: apple +``` + diff --git a/pkg/yqlib/doc/operators/headers/encode-decode.md b/pkg/yqlib/doc/operators/headers/encode-decode.md index ede62504..fc77bd68 100644 --- a/pkg/yqlib/doc/operators/headers/encode-decode.md +++ b/pkg/yqlib/doc/operators/headers/encode-decode.md @@ -15,9 +15,12 @@ These operators are useful to process yaml documents that have stringified embed | CSV | | to_csv/@csv | | TSV | | to_tsv/@tsv | | XML | from_xml | to_xml(i)/@xml | +| Base64 | @base64d | @base64 | CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields. + +Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string. diff --git a/pkg/yqlib/doc/usage/base64.md b/pkg/yqlib/doc/usage/base64.md new file mode 100644 index 00000000..7da21101 --- /dev/null +++ b/pkg/yqlib/doc/usage/base64.md @@ -0,0 +1,23 @@ + +{% hint style="warning" %} +Note that versions prior to 4.18 require the 'eval/e' command to be specified. + +`yq e ` +{% endhint %} + +## Decode Base64 +Decoded data is assumed to be a string. + +Given a sample.yml file of: +```yml +V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig== +``` +then +```bash +yq -p=props sample.properties +``` +will output +```yaml +V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig: = +``` + diff --git a/pkg/yqlib/encoder_base64.go b/pkg/yqlib/encoder_base64.go new file mode 100644 index 00000000..635d27f7 --- /dev/null +++ b/pkg/yqlib/encoder_base64.go @@ -0,0 +1,38 @@ +package yqlib + +import ( + "encoding/base64" + "fmt" + "io" + + yaml "gopkg.in/yaml.v3" +) + +type base64Encoder struct { + encoding base64.Encoding +} + +func NewBase64Encoder() Encoder { + return &base64Encoder{encoding: *base64.StdEncoding} +} + +func (e *base64Encoder) CanHandleAliases() bool { + return false +} + +func (e *base64Encoder) PrintDocumentSeparator(writer io.Writer) error { + return nil +} + +func (e *base64Encoder) PrintLeadingContent(writer io.Writer, content string) error { + return nil +} + +func (e *base64Encoder) Encode(writer io.Writer, originalNode *yaml.Node) error { + node := unwrapDoc(originalNode) + if guessTagFromCustomType(node) != "!!str" { + return fmt.Errorf("cannot encode %v as base64, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag) + } + _, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(originalNode.Value)))) + return err +} diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 9f024536..61e7697d 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -363,6 +363,9 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`to_xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 2})) lexer.Add([]byte(`@xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 0})) + lexer.Add([]byte(`@base64`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: Base64OutputFormat})) + lexer.Add([]byte(`@base64d`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: Base64InputFormat})) + lexer.Add([]byte(`fromyaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat})) lexer.Add([]byte(`fromjson`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat})) lexer.Add([]byte(`fromxml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat})) diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index 0f27a31c..2d0798b5 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -24,6 +24,8 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder { return NewYamlEncoder(indent, false, true, true) case XMLOutputFormat: return NewXMLEncoder(indent, XMLPreferences.AttributePrefix, XMLPreferences.ContentName) + case Base64OutputFormat: + return NewBase64Encoder() } panic("invalid encoder") } @@ -103,6 +105,8 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre decoder = NewYamlDecoder() case XMLInputFormat: decoder = NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName) + case Base64InputFormat: + decoder = NewBase64Decoder() } var results = list.New() diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go index 79a5a8ff..4a5990e5 100644 --- a/pkg/yqlib/operator_encoder_decoder_test.go +++ b/pkg/yqlib/operator_encoder_decoder_test.go @@ -200,6 +200,41 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::a: \"bar\"\nb:\n foo: bar\n", }, }, + { + description: "Encode a string to base64", + document: "coolData: a special string", + expression: ".coolData | @base64", + expected: []string{ + "D0, P[coolData], (!!str)::YSBzcGVjaWFsIHN0cmluZw==\n", + }, + }, + { + description: "Encode a yaml document to base64", + subdescription: "Pipe through @yaml first to convert to a string, then use @base64 to encode it.", + document: "a: apple", + expression: "@yaml | @base64", + expected: []string{ + "D0, P[], (!!str)::YTogYXBwbGUK\n", + }, + }, + { + description: "Decode a base64 encoded string", + subdescription: "Decoded data is assumed to be a string.", + document: "coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==", + expression: ".coolData | @base64d", + expected: []string{ + "D0, P[coolData], (!!str)::Works with UTF-16 😊\n", + }, + }, + { + description: "Decode a base64 encoded yaml document", + subdescription: "Pipe through `from_yaml` to parse the decoded base64 string as a yaml document.", + document: "coolData: YTogYXBwbGUK", + expression: ".coolData |= (@base64d | from_yaml)", + expected: []string{ + "D0, P[], (doc)::coolData:\n a: apple\n", + }, + }, } func TestEncoderDecoderOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 4429ac72..b844f8b7 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -26,6 +26,7 @@ const ( CSVOutputFormat TSVOutputFormat XMLOutputFormat + Base64OutputFormat ) func OutputFormatFromString(format string) (PrinterOutputFormat, error) {