Allow build without json and xml support (#1556)

* Refactor ordered_map into separate files

Separate json and xml, from the regular yaml.

Makes it possible to compile, without those...

* Refactor encoder and decoder creation

Use more consistent parameters vs globals

Return errors instead of calling panic()

* Allow build without json and xml support
This commit is contained in:
Anders Björklund 2023-03-01 03:19:06 +01:00 committed by GitHub
parent 62d167c141
commit cf8cfbd865
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 338 additions and 213 deletions

View File

@ -84,7 +84,10 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
if err != nil {
return err
}
encoder := configureEncoder(format)
encoder, err := configureEncoder()
if err != nil {
return err
}
printer := yqlib.NewPrinter(encoder, printerWriter)

View File

@ -93,7 +93,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
if err != nil {
return err
}
encoder := configureEncoder(format)
encoder, err := configureEncoder()
if err != nil {
return err
}
printer := yqlib.NewPrinter(encoder, printerWriter)

View File

@ -61,7 +61,15 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
if err != nil {
return nil, err
}
switch yqlibInputFormat {
yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
if yqlibDecoder == nil {
return nil, fmt.Errorf("no support for %s input format", inputFormat)
}
return yqlibDecoder, err
}
func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
switch format {
case yqlib.XMLInputFormat:
return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
case yqlib.PropertiesInputFormat:
@ -72,10 +80,12 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
return yqlib.NewCSVObjectDecoder(','), nil
case yqlib.TSVObjectInputFormat:
return yqlib.NewCSVObjectDecoder('\t'), nil
case yqlib.YamlInputFormat:
prefs := yqlib.ConfiguredYamlPreferences
prefs.EvaluateTogether = evaluateTogether
return yqlib.NewYamlDecoder(prefs), nil
}
prefs := yqlib.ConfiguredYamlPreferences
prefs.EvaluateTogether = evaluateTogether
return yqlib.NewYamlDecoder(prefs), nil
return nil, fmt.Errorf("invalid decoder: %v", format)
}
func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@ -95,22 +105,34 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
return printerWriter, nil
}
func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
func configureEncoder() (yqlib.Encoder, error) {
yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
if err != nil {
return nil, err
}
yqlibEncoder, err := createEncoder(yqlibOutputFormat)
if yqlibEncoder == nil {
return nil, fmt.Errorf("no support for %s output format", outputFormat)
}
return yqlibEncoder, err
}
func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
switch format {
case yqlib.JSONOutputFormat:
return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar)
return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
case yqlib.PropsOutputFormat:
return yqlib.NewPropertiesEncoder(unwrapScalar)
return yqlib.NewPropertiesEncoder(unwrapScalar), nil
case yqlib.CSVOutputFormat:
return yqlib.NewCsvEncoder(',')
return yqlib.NewCsvEncoder(','), nil
case yqlib.TSVOutputFormat:
return yqlib.NewCsvEncoder('\t')
return yqlib.NewCsvEncoder('\t'), nil
case yqlib.YamlOutputFormat:
return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
case yqlib.XMLOutputFormat:
return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
}
panic("invalid encoder")
return nil, fmt.Errorf("invalid encoder: %v", format)
}
// this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)

View File

@ -1,3 +1,5 @@
//go:build !yq_nojson
package yqlib
import (

View File

@ -1,3 +1,5 @@
//go:build !yq_noxml
package yqlib
import (

View File

@ -1,10 +1,6 @@
package yqlib
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
yaml "gopkg.in/yaml.v3"
@ -17,162 +13,17 @@ type Encoder interface {
CanHandleAliases() bool
}
// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
// order of keys and values in a map or an object.
type orderedMap struct {
// if this is an object, kv != nil. If this is not an object, kv == nil.
kv []orderedMapKV
altVal interface{}
}
func mapKeysToStrings(node *yaml.Node) {
type orderedMapKV struct {
K string
V orderedMap
}
func (o *orderedMap) UnmarshalJSON(data []byte) error {
switch data[0] {
case '{':
// initialise so that even if the object is empty it is not nil
o.kv = []orderedMapKV{}
// create decoder
dec := json.NewDecoder(bytes.NewReader(data))
_, err := dec.Token() // open object
if err != nil {
return err
}
// cycle through k/v
var tok json.Token
for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
// we can expect two types: string or Delim. Delim automatically means
// that it is the closing bracket of the object, whereas string means
// that there is another key.
if _, ok := tok.(json.Delim); ok {
break
if node.Kind == yaml.MappingNode {
for index, child := range node.Content {
if index%2 == 0 { // its a map key
child.Tag = "!!str"
}
kv := orderedMapKV{
K: tok.(string),
}
if err := dec.Decode(&kv.V); err != nil {
return err
}
o.kv = append(o.kv, kv)
}
// unexpected error
if err != nil && !errors.Is(err, io.EOF) {
return err
}
return nil
case '[':
var res []*orderedMap
if err := json.Unmarshal(data, &res); err != nil {
return err
}
o.altVal = res
o.kv = nil
return nil
}
return json.Unmarshal(data, &o.altVal)
}
func (o orderedMap) MarshalJSON() ([]byte, error) {
buf := new(bytes.Buffer)
enc := json.NewEncoder(buf)
enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
if o.kv == nil {
if err := enc.Encode(o.altVal); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
buf.WriteByte('{')
for idx, el := range o.kv {
if err := enc.Encode(el.K); err != nil {
return nil, err
}
buf.WriteByte(':')
if err := enc.Encode(el.V); err != nil {
return nil, err
}
if idx != len(o.kv)-1 {
buf.WriteByte(',')
}
}
buf.WriteByte('}')
return buf.Bytes(), nil
}
func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.DocumentNode:
if len(node.Content) == 0 {
return nil
}
return o.UnmarshalYAML(node.Content[0])
case yaml.AliasNode:
return o.UnmarshalYAML(node.Alias)
case yaml.ScalarNode:
return node.Decode(&o.altVal)
case yaml.MappingNode:
// set kv to non-nil
o.kv = []orderedMapKV{}
for i := 0; i < len(node.Content); i += 2 {
var key string
var val orderedMap
if err := node.Content[i].Decode(&key); err != nil {
return err
}
if err := node.Content[i+1].Decode(&val); err != nil {
return err
}
o.kv = append(o.kv, orderedMapKV{
K: key,
V: val,
})
}
return nil
case yaml.SequenceNode:
// note that this has to be a pointer, so that nulls can be represented.
var res []*orderedMap
if err := node.Decode(&res); err != nil {
return err
}
o.altVal = res
o.kv = nil
return nil
case 0:
// null
o.kv = nil
o.altVal = nil
return nil
default:
return fmt.Errorf("orderedMap: invalid yaml node")
for _, child := range node.Content {
mapKeysToStrings(child)
}
}
func (o *orderedMap) MarshalYAML() (interface{}, error) {
// fast path: kv is nil, use altVal
if o.kv == nil {
return o.altVal, nil
}
content := make([]*yaml.Node, 0, len(o.kv)*2)
for _, val := range o.kv {
n := new(yaml.Node)
if err := n.Encode(val.V); err != nil {
return nil, err
}
content = append(content, &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: val.K,
}, n)
}
return &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
Content: content,
}, nil
}

View File

@ -1,3 +1,5 @@
//go:build !yq_nojson
package yqlib
import (
@ -14,21 +16,6 @@ type jsonEncoder struct {
UnwrapScalar bool
}
func mapKeysToStrings(node *yaml.Node) {
if node.Kind == yaml.MappingNode {
for index, child := range node.Content {
if index%2 == 0 { // its a map key
child.Tag = "!!str"
}
}
}
for _, child := range node.Content {
mapKeysToStrings(child)
}
}
func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
var indentString = ""

View File

@ -1,3 +1,5 @@
//go:build !yq_nojson
package yqlib
import (

View File

@ -1,3 +1,5 @@
//go:build !yq_noxml
package yqlib
import (

View File

@ -1,3 +1,5 @@
//go:build !yq_nojson
package yqlib
import (

11
pkg/yqlib/no_json.go Normal file
View File

@ -0,0 +1,11 @@
//go:build yq_nojson
package yqlib
func NewJSONDecoder() Decoder {
return nil
}
func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
return nil
}

11
pkg/yqlib/no_xml.go Normal file
View File

@ -0,0 +1,11 @@
//go:build yq_noxml
package yqlib
func NewXMLDecoder(prefs XmlPreferences) Decoder {
return nil
}
func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
return nil
}

View File

@ -4,6 +4,7 @@ import (
"bufio"
"bytes"
"container/list"
"errors"
"regexp"
"strings"
@ -39,6 +40,9 @@ func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string,
log.Debug("printing with indent: %v", prefs.indent)
encoder := configureEncoder(prefs.format, prefs.indent)
if encoder == nil {
return "", errors.New("no support for output format")
}
printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
err := printer.PrintResults(candidate.AsList())
@ -98,13 +102,11 @@ type decoderPreferences struct {
format InputFormat
}
/* takes a string and decodes it back into an object */
func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
preferences := expressionNode.Operation.Preferences.(decoderPreferences)
func createDecoder(format InputFormat) Decoder {
var decoder Decoder
switch preferences.format {
switch format {
case JsonInputFormat:
decoder = NewJSONDecoder()
case YamlInputFormat:
decoder = NewYamlDecoder(ConfiguredYamlPreferences)
case XMLInputFormat:
@ -120,6 +122,18 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
case UriInputFormat:
decoder = NewUriDecoder()
}
return decoder
}
/* takes a string and decodes it back into an object */
func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
preferences := expressionNode.Operation.Preferences.(decoderPreferences)
decoder := createDecoder(preferences.format)
if decoder == nil {
return Context{}, errors.New("no support for input format")
}
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {

View File

@ -8,15 +8,17 @@ var prefix = "D0, P[], (doc)::a:\n cool:\n bob: dylan\n"
var encoderDecoderOperatorScenarios = []expressionScenario{
{
description: "Encode value as json string",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | to_json)`,
requiresFormat: "json",
description: "Encode value as json string",
document: `{a: {cool: "thing"}}`,
expression: `.b = (.a | to_json)`,
expected: []string{
`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n \"cool\": \"thing\"\n}\n"}
`,
},
},
{
requiresFormat: "json",
description: "Encode value as json string, on one line",
subdescription: "Pass in a 0 indent to print json on a single line.",
document: `{a: {cool: "thing"}}`,
@ -27,6 +29,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
},
},
{
requiresFormat: "json",
description: "Encode value as json string, on one line shorthand",
subdescription: "Pass in a 0 indent to print json on a single line.",
document: `{a: {cool: "thing"}}`,
@ -37,6 +40,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
},
},
{
requiresFormat: "json",
description: "Decode a json encoded string",
subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.",
document: `a: '{"cool":"thing"}'`,
@ -193,33 +197,37 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
},
},
{
description: "Encode value as xml string",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
expression: `.a | to_xml`,
requiresFormat: "xml",
description: "Encode value as xml string",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
expression: `.a | to_xml`,
expected: []string{
"D0, P[a], (!!str)::<cool id=\"hi\">\n <foo>bar</foo>\n</cool>\n\n",
},
},
{
description: "Encode value as xml string on a single line",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
expression: `.a | @xml`,
requiresFormat: "xml",
description: "Encode value as xml string on a single line",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
expression: `.a | @xml`,
expected: []string{
"D0, P[a], (!!str)::<cool id=\"hi\"><foo>bar</foo></cool>\n\n",
},
},
{
description: "Encode value as xml string with custom indentation",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
expression: `{"cat": .a | to_xml(1)}`,
requiresFormat: "xml",
description: "Encode value as xml string with custom indentation",
document: `{a: {cool: {foo: "bar", +@id: hi}}}`,
expression: `{"cat": .a | to_xml(1)}`,
expected: []string{
"D0, P[], (!!map)::cat: |\n <cool id=\"hi\">\n <foo>bar</foo>\n </cool>\n",
},
},
{
description: "Decode a xml encoded string",
document: `a: "<foo>bar</foo>"`,
expression: `.b = (.a | from_xml)`,
requiresFormat: "xml",
description: "Decode a xml encoded string",
document: `a: "<foo>bar</foo>"`,
expression: `.b = (.a | from_xml)`,
expected: []string{
"D0, P[], (doc)::a: \"<foo>bar</foo>\"\nb:\n foo: bar\n",
},
@ -303,9 +311,10 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
},
},
{
description: "empty xml decode",
skipDoc: true,
expression: `"" | @xmld`,
requiresFormat: "xml",
description: "empty xml decode",
skipDoc: true,
expression: `"" | @xmld`,
expected: []string{
"D0, P[], (!!null)::\n",
},

View File

@ -34,6 +34,9 @@ func loadString(filename string) (*CandidateNode, error) {
}
func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
if decoder == nil {
return nil, fmt.Errorf("could not load %s", filename)
}
file, err := os.Open(filename) // #nosec
if err != nil {

View File

@ -74,9 +74,10 @@ var loadScenarios = []expressionScenario{
},
},
{
description: "Load from XML",
document: "cool: things",
expression: `.more_stuff = load_xml("../../examples/small.xml")`,
requiresFormat: "xml",
description: "Load from XML",
document: "cool: things",
expression: `.more_stuff = load_xml("../../examples/small.xml")`,
expected: []string{
"D0, P[], (doc)::cool: things\nmore_stuff:\n this: is some xml\n",
},

View File

@ -28,6 +28,7 @@ type expressionScenario struct {
skipDoc bool
expectedError string
dontFormatInputForDoc bool // dont format input doc for documentation generation
requiresFormat string
}
func TestMain(m *testing.M) {
@ -103,6 +104,23 @@ func testScenario(t *testing.T, s *expressionScenario) {
return
}
if s.requiresFormat != "" {
format := s.requiresFormat
inputFormat, err := InputFormatFromString(format)
if err != nil {
t.Error(err)
}
if decoder := createDecoder(inputFormat); decoder == nil {
t.Skipf("no support for %s input format", format)
}
outputFormat, err := OutputFormatFromString(format)
if err != nil {
t.Error(err)
}
if encoder := configureEncoder(outputFormat, 4); encoder == nil {
t.Skipf("no support for %s output format", format)
}
}
if err != nil {
t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression))
return

14
pkg/yqlib/ordered_map.go Normal file
View File

@ -0,0 +1,14 @@
package yqlib
// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
// order of keys and values in a map or an object.
type orderedMap struct {
// if this is an object, kv != nil. If this is not an object, kv == nil.
kv []orderedMapKV
altVal interface{}
}
type orderedMapKV struct {
K string
V orderedMap
}

View File

@ -0,0 +1,83 @@
package yqlib
import (
"bytes"
"encoding/json"
"errors"
"io"
)
func (o *orderedMap) UnmarshalJSON(data []byte) error {
switch data[0] {
case '{':
// initialise so that even if the object is empty it is not nil
o.kv = []orderedMapKV{}
// create decoder
dec := json.NewDecoder(bytes.NewReader(data))
_, err := dec.Token() // open object
if err != nil {
return err
}
// cycle through k/v
var tok json.Token
for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
// we can expect two types: string or Delim. Delim automatically means
// that it is the closing bracket of the object, whereas string means
// that there is another key.
if _, ok := tok.(json.Delim); ok {
break
}
kv := orderedMapKV{
K: tok.(string),
}
if err := dec.Decode(&kv.V); err != nil {
return err
}
o.kv = append(o.kv, kv)
}
// unexpected error
if err != nil && !errors.Is(err, io.EOF) {
return err
}
return nil
case '[':
var res []*orderedMap
if err := json.Unmarshal(data, &res); err != nil {
return err
}
o.altVal = res
o.kv = nil
return nil
}
return json.Unmarshal(data, &o.altVal)
}
func (o orderedMap) MarshalJSON() ([]byte, error) {
buf := new(bytes.Buffer)
enc := json.NewEncoder(buf)
enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
if o.kv == nil {
if err := enc.Encode(o.altVal); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
buf.WriteByte('{')
for idx, el := range o.kv {
if err := enc.Encode(el.K); err != nil {
return nil, err
}
buf.WriteByte(':')
if err := enc.Encode(el.V); err != nil {
return nil, err
}
if idx != len(o.kv)-1 {
buf.WriteByte(',')
}
}
buf.WriteByte('}')
return buf.Bytes(), nil
}

View File

@ -0,0 +1,79 @@
package yqlib
import (
"fmt"
yaml "gopkg.in/yaml.v3"
)
func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.DocumentNode:
if len(node.Content) == 0 {
return nil
}
return o.UnmarshalYAML(node.Content[0])
case yaml.AliasNode:
return o.UnmarshalYAML(node.Alias)
case yaml.ScalarNode:
return node.Decode(&o.altVal)
case yaml.MappingNode:
// set kv to non-nil
o.kv = []orderedMapKV{}
for i := 0; i < len(node.Content); i += 2 {
var key string
var val orderedMap
if err := node.Content[i].Decode(&key); err != nil {
return err
}
if err := node.Content[i+1].Decode(&val); err != nil {
return err
}
o.kv = append(o.kv, orderedMapKV{
K: key,
V: val,
})
}
return nil
case yaml.SequenceNode:
// note that this has to be a pointer, so that nulls can be represented.
var res []*orderedMap
if err := node.Decode(&res); err != nil {
return err
}
o.altVal = res
o.kv = nil
return nil
case 0:
// null
o.kv = nil
o.altVal = nil
return nil
default:
return fmt.Errorf("orderedMap: invalid yaml node")
}
}
func (o *orderedMap) MarshalYAML() (interface{}, error) {
// fast path: kv is nil, use altVal
if o.kv == nil {
return o.altVal, nil
}
content := make([]*yaml.Node, 0, len(o.kv)*2)
for _, val := range o.kv {
n := new(yaml.Node)
if err := n.Encode(val.V); err != nil {
return nil, err
}
content = append(content, &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: val.K,
}, n)
}
return &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
Content: content,
}, nil
}

View File

@ -314,7 +314,11 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
var writer = bufio.NewWriter(&output)
// note printDocSeparators is true, it should still not print document separators
// when outputing JSON.
printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer))
encoder := NewJSONEncoder(0, false, false)
if encoder == nil {
t.Skipf("no support for %s output format", "json")
}
printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {

View File

@ -1,3 +1,5 @@
//go:build !yq_noxml
package yqlib
import (