From 5f9e6dae76f42aea78c1bbc26e5bf462f99d05c7 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 16 Dec 2021 17:50:09 +1100 Subject: [PATCH] wipt --- cmd/constant.go | 1 + examples/mike.xml | 6 +++ go.mod | 3 ++ go.sum | 4 ++ pkg/yqlib/decoder_xml.go | 90 +++++++++++++++++++++++++++++++++++ pkg/yqlib/decoder_yaml.go | 23 +++++++++ pkg/yqlib/stream_evaluator.go | 3 +- 7 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 examples/mike.xml create mode 100644 pkg/yqlib/decoder_xml.go create mode 100644 pkg/yqlib/decoder_yaml.go diff --git a/cmd/constant.go b/cmd/constant.go index 0b921410..c7fc820a 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -6,6 +6,7 @@ var unwrapScalar = true var writeInplace = false var outputToJSON = false var outputFormat = "yaml" +var inputFormat = "yaml" var exitStatus = false var forceColor = false diff --git a/examples/mike.xml b/examples/mike.xml new file mode 100644 index 00000000..b67be84e --- /dev/null +++ b/examples/mike.xml @@ -0,0 +1,6 @@ + + + + bar + bar23 + \ No newline at end of file diff --git a/go.mod b/go.mod index ac72186a..c86e61f1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/mikefarah/yq/v4 require ( + github.com/basgys/goxml2json v1.1.0 github.com/elliotchance/orderedmap v1.4.0 github.com/fatih/color v1.13.0 github.com/goccy/go-yaml v1.9.4 @@ -18,7 +19,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/timtadh/data-structures v0.5.3 // indirect + golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) diff --git a/go.sum b/go.sum index 1cdc7256..cb30932a 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw= +github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -449,6 +451,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -553,6 +556,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go new file mode 100644 index 00000000..563e3ba6 --- /dev/null +++ b/pkg/yqlib/decoder_xml.go @@ -0,0 +1,90 @@ +package yqlib + +import ( + "io" + + xj "github.com/basgys/goxml2json" + yaml "gopkg.in/yaml.v3" +) + +type xmlDecoder struct { + reader io.Reader + attributePrefix string + contentPrefix string + finished bool +} + +func NewXmlDecoder(reader io.Reader, attributePrefix string, contentPrefix string) Decoder { + return &xmlDecoder{reader: reader, attributePrefix: attributePrefix, contentPrefix: contentPrefix, finished: false} +} + +func (dec *xmlDecoder) createSequence(nodes xj.Nodes) (*yaml.Node, error) { + yamlNode := &yaml.Node{Kind: yaml.SequenceNode} + for _, child := range nodes { + yamlChild, err := dec.convertToYamlNode(child) + if err != nil { + return nil, err + } + yamlNode.Content = append(yamlNode.Content, yamlChild) + } + + return yamlNode, nil +} + +func (dec *xmlDecoder) createMap(n *xj.Node) (*yaml.Node, error) { + yamlNode := &yaml.Node{Kind: yaml.MappingNode} + + if len(n.Data) > 0 { + label := dec.contentPrefix + "content" + yamlNode.Content = append(yamlNode.Content, createScalarNode(label, label), createScalarNode(n.Data, n.Data)) + } + + for label, children := range n.Children { + labelNode := createScalarNode(label, label) + var valueNode *yaml.Node + var err error + if len(children) > 1 { + valueNode, err = dec.createSequence(children) + if err != nil { + return nil, err + } + } else { + valueNode, err = dec.convertToYamlNode(children[0]) + if err != nil { + return nil, err + } + } + yamlNode.Content = append(yamlNode.Content, labelNode, valueNode) + } + + return yamlNode, nil +} + +func (dec *xmlDecoder) convertToYamlNode(n *xj.Node) (*yaml.Node, error) { + if n.IsComplex() { + return dec.createMap(n) + } + return createScalarNode(n.Data, n.Data), nil +} + +func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error { + if dec.finished { + return io.EOF + } + root := &xj.Node{} + // cant use xj - it doesn't keep map order. + err := xj.NewDecoder(dec.reader).Decode(root) + + if err != nil { + return err + } + firstNode, err := dec.convertToYamlNode(root) + + if err != nil { + return err + } + rootYamlNode.Kind = yaml.DocumentNode + rootYamlNode.Content = []*yaml.Node{firstNode} + dec.finished = true + return nil +} diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go new file mode 100644 index 00000000..c1908284 --- /dev/null +++ b/pkg/yqlib/decoder_yaml.go @@ -0,0 +1,23 @@ +package yqlib + +import ( + "io" + + yaml "gopkg.in/yaml.v3" +) + +type Decoder interface { + Decode(node *yaml.Node) error +} + +type yamlDecoder struct { + decoder *yaml.Decoder +} + +func NewYamlDecoder(reader io.Reader) Decoder { + return &yamlDecoder{decoder: yaml.NewDecoder(reader)} +} + +func (dec *yamlDecoder) Decode(rootYamlNode *yaml.Node) error { + return dec.decoder.Decode(rootYamlNode) +} diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index ddb777fb..3c9504b7 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -23,6 +23,7 @@ type streamEvaluator struct { treeNavigator DataTreeNavigator treeCreator ExpressionParser fileIndex int + decoder Decoder } func NewStreamEvaluator() StreamEvaluator { @@ -92,7 +93,7 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error) { var currentIndex uint - decoder := yaml.NewDecoder(reader) + decoder := NewXmlDecoder(reader, "+", "c") for { var dataBucket yaml.Node errorReading := decoder.Decode(&dataBucket)