diff --git a/LICENSE b/LICENSE index e2f1fc03..dfbcc195 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,5 @@ +The MIT License (MIT) + Copyright (c) 2017 Mike Farah Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/cmd/constant.go b/cmd/constant.go index 0b921410..5cca3c5c 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -6,6 +6,10 @@ var unwrapScalar = true var writeInplace = false var outputToJSON = false var outputFormat = "yaml" +var inputFormat = "yaml" + +var xmlAttributePrefix = "+" +var xmlContentName = "+content" var exitStatus = false var forceColor = false diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index af3c6ec7..c52e2526 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -75,6 +75,11 @@ func evaluateAll(cmd *cobra.Command, args []string) error { return err } + decoder, err := configureDecoder() + if err != nil { + return err + } + printerWriter := configurePrinterWriter(format, out) printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) @@ -99,7 +104,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error { switch len(args) { case 0: if pipingStdIn { - err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing) + err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing, decoder) } else { cmd.Println(cmd.UsageString()) return nil @@ -108,10 +113,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error { if nullInput { err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "") } else { - err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing) + err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing, decoder) } default: - err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing) + err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing, decoder) } completedSuccessfully = err == nil diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index e875c4bd..2a371ed4 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -92,6 +92,11 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) + decoder, err := configureDecoder() + if err != nil { + return err + } + streamEvaluator := yqlib.NewStreamEvaluator() if frontMatter != "" { @@ -113,7 +118,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { switch len(args) { case 0: if pipingStdIn { - err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing) + err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing, decoder) } else { cmd.Println(cmd.UsageString()) return nil @@ -122,10 +127,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { if nullInput { err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "") } else { - err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing) + err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing, decoder) } default: - err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing) + err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing, decoder) } completedSuccessfully = err == nil diff --git a/cmd/root.go b/cmd/root.go index 11471d27..28e22648 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -49,6 +49,11 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`, } rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p] output format type.") + rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|xml|x] input format type.") + + rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes") + rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).") + rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.") rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)") diff --git a/cmd/utils.go b/cmd/utils.go index 483ea426..7ab518a4 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -45,6 +45,18 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err return firstFileIndex, nil } +func configureDecoder() (yqlib.Decoder, error) { + yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat) + if err != nil { + return nil, err + } + switch yqlibInputFormat { + case yqlib.XmlInputFormat: + return yqlib.NewXmlDecoder(xmlAttributePrefix, xmlContentName), nil + } + return yqlib.NewYamlDecoder(), nil +} + func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) yqlib.PrinterWriter { var printerWriter yqlib.PrinterWriter diff --git a/examples/mike.xml b/examples/mike.xml new file mode 100644 index 00000000..e64b598b --- /dev/null +++ b/examples/mike.xml @@ -0,0 +1,4 @@ + + 3f + meow:as + true \ No newline at end of file diff --git a/examples/mike2.xml b/examples/mike2.xml new file mode 100644 index 00000000..0ad80955 --- /dev/null +++ b/examples/mike2.xml @@ -0,0 +1,11 @@ + + + + + + + cool + + ba2234r + bar2234233 + \ No newline at end of file diff --git a/go.mod b/go.mod index ac72186a..be0a0302 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/magiconair/properties v1.8.5 github.com/spf13/cobra v1.3.0 github.com/timtadh/lexmachine v0.2.2 + golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) @@ -19,6 +20,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/timtadh/data-structures v0.5.3 // indirect golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) diff --git a/go.sum b/go.sum index 1cdc7256..aad032da 100644 --- a/go.sum +++ b/go.sum @@ -449,6 +449,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -553,6 +554,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index d23c56c6..371bc48f 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -8,7 +8,7 @@ import ( // A yaml expression evaluator that runs the expression once against all files/nodes in memory. type Evaluator interface { - EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error + EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error // EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) @@ -46,7 +46,7 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand return context.MatchingNodes, nil } -func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error { +func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error { fileIndex := 0 firstFileLeadingContent := "" @@ -61,7 +61,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string firstFileLeadingContent = leadingContent } - fileDocuments, err := readDocuments(reader, filename, fileIndex) + fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder) if err != nil { return err } diff --git a/pkg/yqlib/decode_xml_test.go b/pkg/yqlib/decode_xml_test.go new file mode 100644 index 00000000..e0d72d21 --- /dev/null +++ b/pkg/yqlib/decode_xml_test.go @@ -0,0 +1,110 @@ +package yqlib + +import ( + "bufio" + "bytes" + "fmt" + "strings" + "testing" + + "github.com/mikefarah/yq/v4/test" + yaml "gopkg.in/yaml.v3" +) + +func decodeXml(t *testing.T, xml string) *CandidateNode { + decoder := NewXmlDecoder("+", "+content") + + decoder.Init(strings.NewReader(xml)) + + node := &yaml.Node{} + err := decoder.Decode(node) + if err != nil { + t.Error(err, "fail to decode", xml) + } + return &CandidateNode{Node: node} +} + +type xmlScenario struct { + inputXml string + expected string + description string + subdescription string + skipDoc bool +} + +var xmlScenarios = []xmlScenario{ + { + description: "Parse xml: simple", + inputXml: "\nmeow", + expected: "D0, P[], (doc)::cat: meow\n", + }, + { + description: "Parse xml: array", + subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.", + inputXml: "\n1\n2", + expected: "D0, P[], (doc)::animal:\n - \"1\"\n - \"2\"\n", + }, + { + description: "Parse xml: attributes", + subdescription: "Attributes are converted to fields, with the attribute prefix.", + inputXml: "\n\n 7\n", + expected: "D0, P[], (doc)::cat:\n +legs: \"4\"\n legs: \"7\"\n", + }, + { + description: "Parse xml: attributes with content", + subdescription: "Content is added as a field, using the content name", + inputXml: "\nmeow", + expected: "D0, P[], (doc)::cat:\n +content: meow\n +legs: \"4\"\n", + }, +} + +func testXmlScenario(t *testing.T, s *xmlScenario) { + var actual = resultToString(t, decodeXml(t, s.inputXml)) + test.AssertResult(t, s.expected, actual) +} + +func documentXmlScenario(t *testing.T, w *bufio.Writer, i interface{}) { + s := i.(xmlScenario) + + if s.skipDoc { + return + } + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.xml file of:\n") + writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.inputXml)) + + writeOrPanic(w, "then\n") + writeOrPanic(w, "```bash\nyq e sample.xml\n```\n") + writeOrPanic(w, "will output\n") + + var output bytes.Buffer + printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) + + node := decodeXml(t, s.inputXml) + + err := printer.PrintResults(node.AsList()) + if err != nil { + t.Error(err) + return + } + + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String())) + +} + +func TestXmlScenarios(t *testing.T) { + for _, tt := range xmlScenarios { + testXmlScenario(t, &tt) + } + genericScenarios := make([]interface{}, len(xmlScenarios)) + for i, s := range xmlScenarios { + genericScenarios[i] = s + } + documentScenarios(t, "usage", "xml", genericScenarios, documentXmlScenario) +} diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go new file mode 100644 index 00000000..ada214ca --- /dev/null +++ b/pkg/yqlib/decoder_xml.go @@ -0,0 +1,249 @@ +package yqlib + +import ( + "encoding/xml" + "fmt" + "io" + "unicode" + + "golang.org/x/net/html/charset" + yaml "gopkg.in/yaml.v3" +) + +type InputFormat uint + +const ( + YamlInputFormat = 1 << iota + XmlInputFormat +) + +func InputFormatFromString(format string) (InputFormat, error) { + switch format { + case "yaml", "y": + return YamlInputFormat, nil + case "xml", "x": + return XmlInputFormat, nil + default: + return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml]", format) + } +} + +type xmlDecoder struct { + reader io.Reader + attributePrefix string + contentPrefix string + finished bool +} + +func NewXmlDecoder(attributePrefix string, contentPrefix string) Decoder { + if contentPrefix == "" { + contentPrefix = "content" + } + return &xmlDecoder{attributePrefix: attributePrefix, contentPrefix: contentPrefix, finished: false} +} + +func (dec *xmlDecoder) Init(reader io.Reader) { + dec.reader = reader + dec.finished = false +} + +func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) { + yamlNode := &yaml.Node{Kind: yaml.SequenceNode} + for _, child := range nodes { + yamlChild, err := dec.convertToYamlNode(child) + if err != nil { + return nil, err + } + yamlNode.Content = append(yamlNode.Content, yamlChild) + } + + return yamlNode, nil +} + +func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) { + yamlNode := &yaml.Node{Kind: yaml.MappingNode, HeadComment: n.Comment} + + if len(n.Data) > 0 { + label := dec.contentPrefix + yamlNode.Content = append(yamlNode.Content, createScalarNode(label, label), createScalarNode(n.Data, n.Data)) + } + + for _, keyValuePair := range n.Children { + label := keyValuePair.K + children := keyValuePair.V + labelNode := createScalarNode(label, label) + var valueNode *yaml.Node + var err error + log.Debug("len of children in %v is %v", label, len(children)) + if len(children) > 1 { + valueNode, err = dec.createSequence(children) + if err != nil { + return nil, err + } + } else { + valueNode, err = dec.convertToYamlNode(children[0]) + if err != nil { + return nil, err + } + } + yamlNode.Content = append(yamlNode.Content, labelNode, valueNode) + } + + return yamlNode, nil +} + +func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) { + if len(n.Children) > 0 { + return dec.createMap(n) + } + scalar := createScalarNode(n.Data, n.Data) + scalar.HeadComment = n.Comment + return scalar, nil +} + +func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error { + if dec.finished { + return io.EOF + } + root := &xmlNode{} + // cant use xj - it doesn't keep map order. + err := dec.decodeXml(root) + + if err != nil { + return err + } + firstNode, err := dec.convertToYamlNode(root) + + if err != nil { + return err + } + rootYamlNode.Kind = yaml.DocumentNode + rootYamlNode.Content = []*yaml.Node{firstNode} + dec.finished = true + return nil +} + +type xmlNode struct { + Children []*xmlChildrenKv + Comment string + Data string +} + +type xmlChildrenKv struct { + K string + V []*xmlNode +} + +// AddChild appends a node to the list of children +func (n *xmlNode) AddChild(s string, c *xmlNode) { + + if n.Children == nil { + n.Children = make([]*xmlChildrenKv, 0) + } + log.Debug("looking for %s", s) + // see if we can find an existing entry to add to + for _, childEntry := range n.Children { + if childEntry.K == s { + log.Debug("found it, appending an entry%s", s) + childEntry.V = append(childEntry.V, c) + log.Debug("yay len of children in %v is %v", s, len(childEntry.V)) + return + } + } + log.Debug("not there, making a new one %s", s) + n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}}) +} + +type element struct { + parent *element + n *xmlNode + label string +} + +// this code is heavily based on https://github.com/basgys/goxml2json +// main changes are to decode into a structure that preserves the original order +// of the map keys. +func (dec *xmlDecoder) decodeXml(root *xmlNode) error { + xmlDec := xml.NewDecoder(dec.reader) + + // That will convert the charset if the provided XML is non-UTF-8 + xmlDec.CharsetReader = charset.NewReaderLabel + + // Create first element from the root node + elem := &element{ + parent: nil, + n: root, + } + + for { + t, _ := xmlDec.Token() + if t == nil { + break + } + + switch se := t.(type) { + case xml.StartElement: + // Build new a new current element and link it to its parent + elem = &element{ + parent: elem, + n: &xmlNode{}, + label: se.Name.Local, + } + + // Extract attributes as children + for _, a := range se.Attr { + elem.n.AddChild(dec.attributePrefix+a.Name.Local, &xmlNode{Data: a.Value}) + } + case xml.CharData: + // Extract XML data (if any) + elem.n.Data = trimNonGraphic(string(se)) + case xml.EndElement: + // And add it to its parent list + if elem.parent != nil { + elem.parent.n.AddChild(elem.label, elem.n) + } + + // Then change the current element to its parent + elem = elem.parent + case xml.Comment: + elem.n.Comment = trimNonGraphic(string(xml.CharData(se))) + } + } + + return nil +} + +// trimNonGraphic returns a slice of the string s, with all leading and trailing +// non graphic characters and spaces removed. +// +// Graphic characters include letters, marks, numbers, punctuation, symbols, +// and spaces, from categories L, M, N, P, S, Zs. +// Spacing characters are set by category Z and property Pattern_White_Space. +func trimNonGraphic(s string) string { + if s == "" { + return s + } + + var first *int + var last int + for i, r := range []rune(s) { + if !unicode.IsGraphic(r) || unicode.IsSpace(r) { + continue + } + + if first == nil { + f := i // copy i + first = &f + last = i + } else { + last = i + } + } + + // If first is nil, it means there are no graphic characters + if first == nil { + return "" + } + + return string([]rune(s)[*first : last+1]) +} diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go new file mode 100644 index 00000000..58f1c520 --- /dev/null +++ b/pkg/yqlib/decoder_yaml.go @@ -0,0 +1,28 @@ +package yqlib + +import ( + "io" + + yaml "gopkg.in/yaml.v3" +) + +type Decoder interface { + Init(reader io.Reader) + Decode(node *yaml.Node) error +} + +type yamlDecoder struct { + decoder yaml.Decoder +} + +func NewYamlDecoder() Decoder { + return &yamlDecoder{} +} + +func (dec *yamlDecoder) Init(reader io.Reader) { + dec.decoder = *yaml.NewDecoder(reader) +} + +func (dec *yamlDecoder) Decode(rootYamlNode *yaml.Node) error { + return dec.decoder.Decode(rootYamlNode) +} diff --git a/pkg/yqlib/doc/add.md b/pkg/yqlib/doc/operators/add.md similarity index 100% rename from pkg/yqlib/doc/add.md rename to pkg/yqlib/doc/operators/add.md diff --git a/pkg/yqlib/doc/alternative-default-value.md b/pkg/yqlib/doc/operators/alternative-default-value.md similarity index 100% rename from pkg/yqlib/doc/alternative-default-value.md rename to pkg/yqlib/doc/operators/alternative-default-value.md diff --git a/pkg/yqlib/doc/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md similarity index 100% rename from pkg/yqlib/doc/anchor-and-alias-operators.md rename to pkg/yqlib/doc/operators/anchor-and-alias-operators.md diff --git a/pkg/yqlib/doc/assign-update.md b/pkg/yqlib/doc/operators/assign-update.md similarity index 100% rename from pkg/yqlib/doc/assign-update.md rename to pkg/yqlib/doc/operators/assign-update.md diff --git a/pkg/yqlib/doc/boolean-operators.md b/pkg/yqlib/doc/operators/boolean-operators.md similarity index 100% rename from pkg/yqlib/doc/boolean-operators.md rename to pkg/yqlib/doc/operators/boolean-operators.md diff --git a/pkg/yqlib/doc/collect-into-array.md b/pkg/yqlib/doc/operators/collect-into-array.md similarity index 100% rename from pkg/yqlib/doc/collect-into-array.md rename to pkg/yqlib/doc/operators/collect-into-array.md diff --git a/pkg/yqlib/doc/comment-operators.md b/pkg/yqlib/doc/operators/comment-operators.md similarity index 100% rename from pkg/yqlib/doc/comment-operators.md rename to pkg/yqlib/doc/operators/comment-operators.md diff --git a/pkg/yqlib/doc/contains.md b/pkg/yqlib/doc/operators/contains.md similarity index 100% rename from pkg/yqlib/doc/contains.md rename to pkg/yqlib/doc/operators/contains.md diff --git a/pkg/yqlib/doc/create-collect-into-object.md b/pkg/yqlib/doc/operators/create-collect-into-object.md similarity index 100% rename from pkg/yqlib/doc/create-collect-into-object.md rename to pkg/yqlib/doc/operators/create-collect-into-object.md diff --git a/pkg/yqlib/doc/delete.md b/pkg/yqlib/doc/operators/delete.md similarity index 100% rename from pkg/yqlib/doc/delete.md rename to pkg/yqlib/doc/operators/delete.md diff --git a/pkg/yqlib/doc/document-index.md b/pkg/yqlib/doc/operators/document-index.md similarity index 100% rename from pkg/yqlib/doc/document-index.md rename to pkg/yqlib/doc/operators/document-index.md diff --git a/pkg/yqlib/doc/encode-decode.md b/pkg/yqlib/doc/operators/encode-decode.md similarity index 95% rename from pkg/yqlib/doc/encode-decode.md rename to pkg/yqlib/doc/operators/encode-decode.md index d4416608..d77aa917 100644 --- a/pkg/yqlib/doc/encode-decode.md +++ b/pkg/yqlib/doc/operators/encode-decode.md @@ -271,3 +271,19 @@ cat thing1,thing2 true 3.40 dog thing3 false 12 ``` +## Decode a xml encoded string +Given a sample.yml file of: +```yaml +a: bar +``` +then +```bash +yq eval '.b = (.a | from_xml)' sample.yml +``` +will output +```yaml +a: bar +b: + foo: bar +``` + diff --git a/pkg/yqlib/doc/entries.md b/pkg/yqlib/doc/operators/entries.md similarity index 100% rename from pkg/yqlib/doc/entries.md rename to pkg/yqlib/doc/operators/entries.md diff --git a/pkg/yqlib/doc/env-variable-operators.md b/pkg/yqlib/doc/operators/env-variable-operators.md similarity index 100% rename from pkg/yqlib/doc/env-variable-operators.md rename to pkg/yqlib/doc/operators/env-variable-operators.md diff --git a/pkg/yqlib/doc/equals.md b/pkg/yqlib/doc/operators/equals.md similarity index 100% rename from pkg/yqlib/doc/equals.md rename to pkg/yqlib/doc/operators/equals.md diff --git a/pkg/yqlib/doc/file-operators.md b/pkg/yqlib/doc/operators/file-operators.md similarity index 100% rename from pkg/yqlib/doc/file-operators.md rename to pkg/yqlib/doc/operators/file-operators.md diff --git a/pkg/yqlib/doc/flatten.md b/pkg/yqlib/doc/operators/flatten.md similarity index 100% rename from pkg/yqlib/doc/flatten.md rename to pkg/yqlib/doc/operators/flatten.md diff --git a/pkg/yqlib/doc/group-by.md b/pkg/yqlib/doc/operators/group-by.md similarity index 100% rename from pkg/yqlib/doc/group-by.md rename to pkg/yqlib/doc/operators/group-by.md diff --git a/pkg/yqlib/doc/has.md b/pkg/yqlib/doc/operators/has.md similarity index 100% rename from pkg/yqlib/doc/has.md rename to pkg/yqlib/doc/operators/has.md diff --git a/pkg/yqlib/doc/headers/Main.md b/pkg/yqlib/doc/operators/headers/Main.md similarity index 100% rename from pkg/yqlib/doc/headers/Main.md rename to pkg/yqlib/doc/operators/headers/Main.md diff --git a/pkg/yqlib/doc/headers/add.md b/pkg/yqlib/doc/operators/headers/add.md similarity index 100% rename from pkg/yqlib/doc/headers/add.md rename to pkg/yqlib/doc/operators/headers/add.md diff --git a/pkg/yqlib/doc/headers/alternative-default-value.md b/pkg/yqlib/doc/operators/headers/alternative-default-value.md similarity index 100% rename from pkg/yqlib/doc/headers/alternative-default-value.md rename to pkg/yqlib/doc/operators/headers/alternative-default-value.md diff --git a/pkg/yqlib/doc/headers/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md similarity index 100% rename from pkg/yqlib/doc/headers/anchor-and-alias-operators.md rename to pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md diff --git a/pkg/yqlib/doc/headers/assign-update.md b/pkg/yqlib/doc/operators/headers/assign-update.md similarity index 100% rename from pkg/yqlib/doc/headers/assign-update.md rename to pkg/yqlib/doc/operators/headers/assign-update.md diff --git a/pkg/yqlib/doc/headers/boolean-operators.md b/pkg/yqlib/doc/operators/headers/boolean-operators.md similarity index 100% rename from pkg/yqlib/doc/headers/boolean-operators.md rename to pkg/yqlib/doc/operators/headers/boolean-operators.md diff --git a/pkg/yqlib/doc/headers/collect-into-array.md b/pkg/yqlib/doc/operators/headers/collect-into-array.md similarity index 100% rename from pkg/yqlib/doc/headers/collect-into-array.md rename to pkg/yqlib/doc/operators/headers/collect-into-array.md diff --git a/pkg/yqlib/doc/headers/comment-operators.md b/pkg/yqlib/doc/operators/headers/comment-operators.md similarity index 100% rename from pkg/yqlib/doc/headers/comment-operators.md rename to pkg/yqlib/doc/operators/headers/comment-operators.md diff --git a/pkg/yqlib/doc/headers/contains.md b/pkg/yqlib/doc/operators/headers/contains.md similarity index 100% rename from pkg/yqlib/doc/headers/contains.md rename to pkg/yqlib/doc/operators/headers/contains.md diff --git a/pkg/yqlib/doc/headers/create-collect-into-object.md b/pkg/yqlib/doc/operators/headers/create-collect-into-object.md similarity index 100% rename from pkg/yqlib/doc/headers/create-collect-into-object.md rename to pkg/yqlib/doc/operators/headers/create-collect-into-object.md diff --git a/pkg/yqlib/doc/headers/delete.md b/pkg/yqlib/doc/operators/headers/delete.md similarity index 100% rename from pkg/yqlib/doc/headers/delete.md rename to pkg/yqlib/doc/operators/headers/delete.md diff --git a/pkg/yqlib/doc/headers/document-index.md b/pkg/yqlib/doc/operators/headers/document-index.md similarity index 100% rename from pkg/yqlib/doc/headers/document-index.md rename to pkg/yqlib/doc/operators/headers/document-index.md diff --git a/pkg/yqlib/doc/headers/encode-decode.md b/pkg/yqlib/doc/operators/headers/encode-decode.md similarity index 97% rename from pkg/yqlib/doc/headers/encode-decode.md rename to pkg/yqlib/doc/operators/headers/encode-decode.md index 92ff9f5a..847240b3 100644 --- a/pkg/yqlib/doc/headers/encode-decode.md +++ b/pkg/yqlib/doc/operators/headers/encode-decode.md @@ -14,6 +14,7 @@ These operators are useful to process yaml documents that have stringified embed | Properties | | to_props/@props | | CSV | | to_csv/@csv | | TSV | | to_tsv/@tsv | +| XML | from_xml | | CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). diff --git a/pkg/yqlib/doc/headers/entries.md b/pkg/yqlib/doc/operators/headers/entries.md similarity index 100% rename from pkg/yqlib/doc/headers/entries.md rename to pkg/yqlib/doc/operators/headers/entries.md diff --git a/pkg/yqlib/doc/headers/env-variable-operators.md b/pkg/yqlib/doc/operators/headers/env-variable-operators.md similarity index 100% rename from pkg/yqlib/doc/headers/env-variable-operators.md rename to pkg/yqlib/doc/operators/headers/env-variable-operators.md diff --git a/pkg/yqlib/doc/headers/equals.md b/pkg/yqlib/doc/operators/headers/equals.md similarity index 100% rename from pkg/yqlib/doc/headers/equals.md rename to pkg/yqlib/doc/operators/headers/equals.md diff --git a/pkg/yqlib/doc/headers/file-operators.md b/pkg/yqlib/doc/operators/headers/file-operators.md similarity index 100% rename from pkg/yqlib/doc/headers/file-operators.md rename to pkg/yqlib/doc/operators/headers/file-operators.md diff --git a/pkg/yqlib/doc/headers/flatten.md b/pkg/yqlib/doc/operators/headers/flatten.md similarity index 100% rename from pkg/yqlib/doc/headers/flatten.md rename to pkg/yqlib/doc/operators/headers/flatten.md diff --git a/pkg/yqlib/doc/headers/group-by.md b/pkg/yqlib/doc/operators/headers/group-by.md similarity index 100% rename from pkg/yqlib/doc/headers/group-by.md rename to pkg/yqlib/doc/operators/headers/group-by.md diff --git a/pkg/yqlib/doc/headers/has.md b/pkg/yqlib/doc/operators/headers/has.md similarity index 100% rename from pkg/yqlib/doc/headers/has.md rename to pkg/yqlib/doc/operators/headers/has.md diff --git a/pkg/yqlib/doc/headers/keys.md b/pkg/yqlib/doc/operators/headers/keys.md similarity index 100% rename from pkg/yqlib/doc/headers/keys.md rename to pkg/yqlib/doc/operators/headers/keys.md diff --git a/pkg/yqlib/doc/headers/length.md b/pkg/yqlib/doc/operators/headers/length.md similarity index 100% rename from pkg/yqlib/doc/headers/length.md rename to pkg/yqlib/doc/operators/headers/length.md diff --git a/pkg/yqlib/doc/headers/load.md b/pkg/yqlib/doc/operators/headers/load.md similarity index 100% rename from pkg/yqlib/doc/headers/load.md rename to pkg/yqlib/doc/operators/headers/load.md diff --git a/pkg/yqlib/doc/headers/map.md b/pkg/yqlib/doc/operators/headers/map.md similarity index 100% rename from pkg/yqlib/doc/headers/map.md rename to pkg/yqlib/doc/operators/headers/map.md diff --git a/pkg/yqlib/doc/headers/multiply-merge.md b/pkg/yqlib/doc/operators/headers/multiply-merge.md similarity index 100% rename from pkg/yqlib/doc/headers/multiply-merge.md rename to pkg/yqlib/doc/operators/headers/multiply-merge.md diff --git a/pkg/yqlib/doc/headers/parent.md b/pkg/yqlib/doc/operators/headers/parent.md similarity index 100% rename from pkg/yqlib/doc/headers/parent.md rename to pkg/yqlib/doc/operators/headers/parent.md diff --git a/pkg/yqlib/doc/headers/path.md b/pkg/yqlib/doc/operators/headers/path.md similarity index 100% rename from pkg/yqlib/doc/headers/path.md rename to pkg/yqlib/doc/operators/headers/path.md diff --git a/pkg/yqlib/doc/headers/pipe.md b/pkg/yqlib/doc/operators/headers/pipe.md similarity index 100% rename from pkg/yqlib/doc/headers/pipe.md rename to pkg/yqlib/doc/operators/headers/pipe.md diff --git a/pkg/yqlib/doc/headers/recursive-descent-glob.md b/pkg/yqlib/doc/operators/headers/recursive-descent-glob.md similarity index 100% rename from pkg/yqlib/doc/headers/recursive-descent-glob.md rename to pkg/yqlib/doc/operators/headers/recursive-descent-glob.md diff --git a/pkg/yqlib/doc/headers/reduce.md b/pkg/yqlib/doc/operators/headers/reduce.md similarity index 100% rename from pkg/yqlib/doc/headers/reduce.md rename to pkg/yqlib/doc/operators/headers/reduce.md diff --git a/pkg/yqlib/doc/headers/select.md b/pkg/yqlib/doc/operators/headers/select.md similarity index 100% rename from pkg/yqlib/doc/headers/select.md rename to pkg/yqlib/doc/operators/headers/select.md diff --git a/pkg/yqlib/doc/headers/sort-keys.md b/pkg/yqlib/doc/operators/headers/sort-keys.md similarity index 100% rename from pkg/yqlib/doc/headers/sort-keys.md rename to pkg/yqlib/doc/operators/headers/sort-keys.md diff --git a/pkg/yqlib/doc/headers/sort.md b/pkg/yqlib/doc/operators/headers/sort.md similarity index 100% rename from pkg/yqlib/doc/headers/sort.md rename to pkg/yqlib/doc/operators/headers/sort.md diff --git a/pkg/yqlib/doc/headers/split-into-documents.md b/pkg/yqlib/doc/operators/headers/split-into-documents.md similarity index 100% rename from pkg/yqlib/doc/headers/split-into-documents.md rename to pkg/yqlib/doc/operators/headers/split-into-documents.md diff --git a/pkg/yqlib/doc/headers/string-operators.md b/pkg/yqlib/doc/operators/headers/string-operators.md similarity index 100% rename from pkg/yqlib/doc/headers/string-operators.md rename to pkg/yqlib/doc/operators/headers/string-operators.md diff --git a/pkg/yqlib/doc/headers/style.md b/pkg/yqlib/doc/operators/headers/style.md similarity index 100% rename from pkg/yqlib/doc/headers/style.md rename to pkg/yqlib/doc/operators/headers/style.md diff --git a/pkg/yqlib/doc/headers/subtract.md b/pkg/yqlib/doc/operators/headers/subtract.md similarity index 100% rename from pkg/yqlib/doc/headers/subtract.md rename to pkg/yqlib/doc/operators/headers/subtract.md diff --git a/pkg/yqlib/doc/headers/tag.md b/pkg/yqlib/doc/operators/headers/tag.md similarity index 100% rename from pkg/yqlib/doc/headers/tag.md rename to pkg/yqlib/doc/operators/headers/tag.md diff --git a/pkg/yqlib/doc/headers/traverse-read.md b/pkg/yqlib/doc/operators/headers/traverse-read.md similarity index 100% rename from pkg/yqlib/doc/headers/traverse-read.md rename to pkg/yqlib/doc/operators/headers/traverse-read.md diff --git a/pkg/yqlib/doc/headers/union.md b/pkg/yqlib/doc/operators/headers/union.md similarity index 100% rename from pkg/yqlib/doc/headers/union.md rename to pkg/yqlib/doc/operators/headers/union.md diff --git a/pkg/yqlib/doc/headers/unique.md b/pkg/yqlib/doc/operators/headers/unique.md similarity index 100% rename from pkg/yqlib/doc/headers/unique.md rename to pkg/yqlib/doc/operators/headers/unique.md diff --git a/pkg/yqlib/doc/headers/variable-operators.md b/pkg/yqlib/doc/operators/headers/variable-operators.md similarity index 100% rename from pkg/yqlib/doc/headers/variable-operators.md rename to pkg/yqlib/doc/operators/headers/variable-operators.md diff --git a/pkg/yqlib/doc/headers/with.md b/pkg/yqlib/doc/operators/headers/with.md similarity index 100% rename from pkg/yqlib/doc/headers/with.md rename to pkg/yqlib/doc/operators/headers/with.md diff --git a/pkg/yqlib/doc/keys.md b/pkg/yqlib/doc/operators/keys.md similarity index 100% rename from pkg/yqlib/doc/keys.md rename to pkg/yqlib/doc/operators/keys.md diff --git a/pkg/yqlib/doc/length.md b/pkg/yqlib/doc/operators/length.md similarity index 100% rename from pkg/yqlib/doc/length.md rename to pkg/yqlib/doc/operators/length.md diff --git a/pkg/yqlib/doc/load.md b/pkg/yqlib/doc/operators/load.md similarity index 100% rename from pkg/yqlib/doc/load.md rename to pkg/yqlib/doc/operators/load.md diff --git a/pkg/yqlib/doc/map.md b/pkg/yqlib/doc/operators/map.md similarity index 100% rename from pkg/yqlib/doc/map.md rename to pkg/yqlib/doc/operators/map.md diff --git a/pkg/yqlib/doc/multiply-merge.md b/pkg/yqlib/doc/operators/multiply-merge.md similarity index 100% rename from pkg/yqlib/doc/multiply-merge.md rename to pkg/yqlib/doc/operators/multiply-merge.md diff --git a/pkg/yqlib/doc/parent.md b/pkg/yqlib/doc/operators/parent.md similarity index 100% rename from pkg/yqlib/doc/parent.md rename to pkg/yqlib/doc/operators/parent.md diff --git a/pkg/yqlib/doc/path.md b/pkg/yqlib/doc/operators/path.md similarity index 100% rename from pkg/yqlib/doc/path.md rename to pkg/yqlib/doc/operators/path.md diff --git a/pkg/yqlib/doc/pipe.md b/pkg/yqlib/doc/operators/pipe.md similarity index 100% rename from pkg/yqlib/doc/pipe.md rename to pkg/yqlib/doc/operators/pipe.md diff --git a/pkg/yqlib/doc/recursive-descent-glob.md b/pkg/yqlib/doc/operators/recursive-descent-glob.md similarity index 100% rename from pkg/yqlib/doc/recursive-descent-glob.md rename to pkg/yqlib/doc/operators/recursive-descent-glob.md diff --git a/pkg/yqlib/doc/reduce.md b/pkg/yqlib/doc/operators/reduce.md similarity index 100% rename from pkg/yqlib/doc/reduce.md rename to pkg/yqlib/doc/operators/reduce.md diff --git a/pkg/yqlib/doc/select.md b/pkg/yqlib/doc/operators/select.md similarity index 100% rename from pkg/yqlib/doc/select.md rename to pkg/yqlib/doc/operators/select.md diff --git a/pkg/yqlib/doc/sort-keys.md b/pkg/yqlib/doc/operators/sort-keys.md similarity index 100% rename from pkg/yqlib/doc/sort-keys.md rename to pkg/yqlib/doc/operators/sort-keys.md diff --git a/pkg/yqlib/doc/sort.md b/pkg/yqlib/doc/operators/sort.md similarity index 100% rename from pkg/yqlib/doc/sort.md rename to pkg/yqlib/doc/operators/sort.md diff --git a/pkg/yqlib/doc/split-into-documents.md b/pkg/yqlib/doc/operators/split-into-documents.md similarity index 100% rename from pkg/yqlib/doc/split-into-documents.md rename to pkg/yqlib/doc/operators/split-into-documents.md diff --git a/pkg/yqlib/doc/string-operators.md b/pkg/yqlib/doc/operators/string-operators.md similarity index 100% rename from pkg/yqlib/doc/string-operators.md rename to pkg/yqlib/doc/operators/string-operators.md diff --git a/pkg/yqlib/doc/style.md b/pkg/yqlib/doc/operators/style.md similarity index 100% rename from pkg/yqlib/doc/style.md rename to pkg/yqlib/doc/operators/style.md diff --git a/pkg/yqlib/doc/subtract.md b/pkg/yqlib/doc/operators/subtract.md similarity index 100% rename from pkg/yqlib/doc/subtract.md rename to pkg/yqlib/doc/operators/subtract.md diff --git a/pkg/yqlib/doc/tag.md b/pkg/yqlib/doc/operators/tag.md similarity index 100% rename from pkg/yqlib/doc/tag.md rename to pkg/yqlib/doc/operators/tag.md diff --git a/pkg/yqlib/doc/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md similarity index 100% rename from pkg/yqlib/doc/traverse-read.md rename to pkg/yqlib/doc/operators/traverse-read.md diff --git a/pkg/yqlib/doc/union.md b/pkg/yqlib/doc/operators/union.md similarity index 100% rename from pkg/yqlib/doc/union.md rename to pkg/yqlib/doc/operators/union.md diff --git a/pkg/yqlib/doc/unique.md b/pkg/yqlib/doc/operators/unique.md similarity index 100% rename from pkg/yqlib/doc/unique.md rename to pkg/yqlib/doc/operators/unique.md diff --git a/pkg/yqlib/doc/variable-operators.md b/pkg/yqlib/doc/operators/variable-operators.md similarity index 100% rename from pkg/yqlib/doc/variable-operators.md rename to pkg/yqlib/doc/operators/variable-operators.md diff --git a/pkg/yqlib/doc/with.md b/pkg/yqlib/doc/operators/with.md similarity index 100% rename from pkg/yqlib/doc/with.md rename to pkg/yqlib/doc/operators/with.md diff --git a/pkg/yqlib/doc/usage/headers/xml.md b/pkg/yqlib/doc/usage/headers/xml.md new file mode 100644 index 00000000..9258afd0 --- /dev/null +++ b/pkg/yqlib/doc/usage/headers/xml.md @@ -0,0 +1,12 @@ +# XML + +At the moment, `yq` only supports decoding `xml` (into one of the other supported output formats). + +As yaml does not have the concept of attributes, these are converted to regular fields with a prefix to prevent clobbering. Consecutive xml nodes with the same name are assumed to be arrays. + +All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types: + + +``` +yq e -p=xml '.myNumberField |= from_yaml' my.xml +``` diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md new file mode 100644 index 00000000..e084149e --- /dev/null +++ b/pkg/yqlib/doc/usage/xml.md @@ -0,0 +1,88 @@ +# XML + +At the moment, `yq` only supports decoding `xml` (into one of the other supported output formats). + +As yaml does not have the concept of attributes, these are converted to regular fields with a prefix to prevent clobbering. Consecutive xml nodes with the same name are assumed to be arrays. + +All values in XML are assumed to be strings - but you can use `from_yaml` to parse them into their correct types: + + +``` +yq e -p=xml '.myNumberField |= from_yaml' my.xml +``` + +## Parse xml: simple +Given a sample.xml file of: +```xml + +meow +``` +then +```bash +yq e sample.xml +``` +will output +```yaml +cat: meow +``` + +## Parse xml: array +Consecutive nodes with identical xml names are assumed to be arrays. + +Given a sample.xml file of: +```xml + +1 +2 +``` +then +```bash +yq e sample.xml +``` +will output +```yaml +animal: + - "1" + - "2" +``` + +## Parse xml: attributes +Attributes are converted to fields, with the attribute prefix. + +Given a sample.xml file of: +```xml + + + 7 + +``` +then +```bash +yq e sample.xml +``` +will output +```yaml +cat: + +legs: "4" + legs: "7" +``` + +## Parse xml: attributes with content +Content is added as a field, using the content name + +Given a sample.xml file of: +```xml + +meow +``` +then +```bash +yq e sample.xml +``` +will output +```yaml +cat: + +content: meow + +legs: "4" +``` + diff --git a/pkg/yqlib/encoder_csv_test.go b/pkg/yqlib/encoder_csv_test.go index 1db2859e..816cbfe4 100644 --- a/pkg/yqlib/encoder_csv_test.go +++ b/pkg/yqlib/encoder_csv_test.go @@ -14,7 +14,7 @@ func yamlToCsv(sampleYaml string, separator rune) string { writer := bufio.NewWriter(&output) var jsonEncoder = NewCsvEncoder(writer, separator) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_properties_test.go b/pkg/yqlib/encoder_properties_test.go index df973734..f82af12a 100644 --- a/pkg/yqlib/encoder_properties_test.go +++ b/pkg/yqlib/encoder_properties_test.go @@ -14,7 +14,7 @@ func yamlToProps(sampleYaml string) string { writer := bufio.NewWriter(&output) var propsEncoder = NewPropertiesEncoder(writer) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_test.go b/pkg/yqlib/encoder_test.go index 283e89ee..576cfb95 100644 --- a/pkg/yqlib/encoder_test.go +++ b/pkg/yqlib/encoder_test.go @@ -14,7 +14,7 @@ func yamlToJson(sampleYaml string, indent int) string { writer := bufio.NewWriter(&output) var jsonEncoder = NewJsonEncoder(writer, indent) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 0fb05642..d812039c 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -326,36 +326,47 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat)) lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2})) + lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2})) // 0 indent doesn't work with yaml. lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2})) lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2})) - lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2})) - lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2})) - - lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2})) lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2})) lexer.Add([]byte(`@json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 0})) + lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2})) + lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2})) + lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2})) + + lexer.Add([]byte(`tocsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CsvOutputFormat})) lexer.Add([]byte(`to_csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CsvOutputFormat})) lexer.Add([]byte(`@csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CsvOutputFormat})) + lexer.Add([]byte(`totsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TsvOutputFormat})) lexer.Add([]byte(`to_tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TsvOutputFormat})) lexer.Add([]byte(`@tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TsvOutputFormat})) - lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2})) + lexer.Add([]byte(`fromyaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat})) + lexer.Add([]byte(`fromjson`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat})) + lexer.Add([]byte(`fromxml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XmlInputFormat})) - lexer.Add([]byte(`fromyaml`), opToken(decodeOpType)) - lexer.Add([]byte(`fromjson`), opToken(decodeOpType)) - - lexer.Add([]byte(`from_yaml`), opToken(decodeOpType)) - lexer.Add([]byte(`from_json`), opToken(decodeOpType)) + lexer.Add([]byte(`from_yaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat})) + lexer.Add([]byte(`from_json`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat})) + lexer.Add([]byte(`from_xml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XmlInputFormat})) lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType)) lexer.Add([]byte(`sort_keys`), opToken(sortKeysOpType)) - lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false})) + lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewYamlDecoder()})) + + lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder("+", "+content")})) + lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder("+", "+content")})) + lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXmlDecoder("+", "+content")})) + lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true})) + lexer.Add([]byte(`load_str`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true})) + lexer.Add([]byte(`loadstr`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true})) + lexer.Add([]byte(`select`), opToken(selectOpType)) lexer.Add([]byte(`has`), opToken(hasOpType)) lexer.Add([]byte(`unique`), opToken(uniqueOpType)) diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index 445977e8..b303866a 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -150,5 +150,5 @@ func TestAddOperatorScenarios(t *testing.T) { for _, tt := range addOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "add", addOperatorScenarios) + documentOperatorScenarios(t, "add", addOperatorScenarios) } diff --git a/pkg/yqlib/operator_alternative_test.go b/pkg/yqlib/operator_alternative_test.go index 0331213d..e23c8dbb 100644 --- a/pkg/yqlib/operator_alternative_test.go +++ b/pkg/yqlib/operator_alternative_test.go @@ -91,5 +91,5 @@ func TestAlternativeOperatorScenarios(t *testing.T) { for _, tt := range alternativeOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "alternative-default-value", alternativeOperatorScenarios) + documentOperatorScenarios(t, "alternative-default-value", alternativeOperatorScenarios) } diff --git a/pkg/yqlib/operator_anchors_aliases_test.go b/pkg/yqlib/operator_anchors_aliases_test.go index 30642dcd..9222c177 100644 --- a/pkg/yqlib/operator_anchors_aliases_test.go +++ b/pkg/yqlib/operator_anchors_aliases_test.go @@ -233,5 +233,5 @@ func TestAnchorAliasOperatorScenarios(t *testing.T) { for _, tt := range anchorOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "anchor-and-alias-operators", anchorOperatorScenarios) + documentOperatorScenarios(t, "anchor-and-alias-operators", anchorOperatorScenarios) } diff --git a/pkg/yqlib/operator_assign_test.go b/pkg/yqlib/operator_assign_test.go index 9ecca5d4..b5f6813c 100644 --- a/pkg/yqlib/operator_assign_test.go +++ b/pkg/yqlib/operator_assign_test.go @@ -178,5 +178,5 @@ func TestAssignOperatorScenarios(t *testing.T) { for _, tt := range assignOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "assign-update", assignOperatorScenarios) + documentOperatorScenarios(t, "assign-update", assignOperatorScenarios) } diff --git a/pkg/yqlib/operator_booleans_test.go b/pkg/yqlib/operator_booleans_test.go index ac6de3eb..e9b72f99 100644 --- a/pkg/yqlib/operator_booleans_test.go +++ b/pkg/yqlib/operator_booleans_test.go @@ -229,5 +229,5 @@ func TestBooleanOperatorScenarios(t *testing.T) { for _, tt := range booleanOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "boolean-operators", booleanOperatorScenarios) + documentOperatorScenarios(t, "boolean-operators", booleanOperatorScenarios) } diff --git a/pkg/yqlib/operator_collect_object_test.go b/pkg/yqlib/operator_collect_object_test.go index edbe63d2..4db923f7 100644 --- a/pkg/yqlib/operator_collect_object_test.go +++ b/pkg/yqlib/operator_collect_object_test.go @@ -160,5 +160,5 @@ func TestCollectObjectOperatorScenarios(t *testing.T) { for _, tt := range collectObjectOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "create-collect-into-object", collectObjectOperatorScenarios) + documentOperatorScenarios(t, "create-collect-into-object", collectObjectOperatorScenarios) } diff --git a/pkg/yqlib/operator_collect_test.go b/pkg/yqlib/operator_collect_test.go index 0e23e340..fce899fb 100644 --- a/pkg/yqlib/operator_collect_test.go +++ b/pkg/yqlib/operator_collect_test.go @@ -108,5 +108,5 @@ func TestCollectOperatorScenarios(t *testing.T) { for _, tt := range collectOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "collect-into-array", collectOperatorScenarios) + documentOperatorScenarios(t, "collect-into-array", collectOperatorScenarios) } diff --git a/pkg/yqlib/operator_comments_test.go b/pkg/yqlib/operator_comments_test.go index cc3ce4fe..223eeb6c 100644 --- a/pkg/yqlib/operator_comments_test.go +++ b/pkg/yqlib/operator_comments_test.go @@ -136,5 +136,5 @@ func TestCommentOperatorScenarios(t *testing.T) { for _, tt := range commentOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "comment-operators", commentOperatorScenarios) + documentOperatorScenarios(t, "comment-operators", commentOperatorScenarios) } diff --git a/pkg/yqlib/operator_contains_test.go b/pkg/yqlib/operator_contains_test.go index e1d3dd01..5cef9d7c 100644 --- a/pkg/yqlib/operator_contains_test.go +++ b/pkg/yqlib/operator_contains_test.go @@ -78,5 +78,5 @@ func TestContainsOperatorScenarios(t *testing.T) { for _, tt := range containsOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "contains", containsOperatorScenarios) + documentOperatorScenarios(t, "contains", containsOperatorScenarios) } diff --git a/pkg/yqlib/operator_delete_test.go b/pkg/yqlib/operator_delete_test.go index 02efcb75..76d95224 100644 --- a/pkg/yqlib/operator_delete_test.go +++ b/pkg/yqlib/operator_delete_test.go @@ -171,5 +171,5 @@ func TestDeleteOperatorScenarios(t *testing.T) { for _, tt := range deleteOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "delete", deleteOperatorScenarios) + documentOperatorScenarios(t, "delete", deleteOperatorScenarios) } diff --git a/pkg/yqlib/operator_document_index_test.go b/pkg/yqlib/operator_document_index_test.go index 1e35cd19..23195505 100644 --- a/pkg/yqlib/operator_document_index_test.go +++ b/pkg/yqlib/operator_document_index_test.go @@ -54,5 +54,5 @@ func TestDocumentIndexScenarios(t *testing.T) { for _, tt := range documentIndexScenarios { testScenario(t, &tt) } - documentScenarios(t, "document-index", documentIndexScenarios) + documentOperatorScenarios(t, "document-index", documentIndexScenarios) } diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index d4d61b3f..a702d0f1 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -68,9 +68,23 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre return context.ChildContext(results), nil } +type decoderPreferences struct { + format InputFormat +} + /* takes a string and decodes it back into an object */ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + preferences := expressionNode.Operation.Preferences.(decoderPreferences) + + var decoder Decoder + switch preferences.format { + case YamlInputFormat: + decoder = NewYamlDecoder() + case XmlInputFormat: + decoder = NewXmlDecoder("+a", "+content") + } + var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) @@ -79,7 +93,9 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre var dataBucket yaml.Node log.Debugf("got: [%v]", candidate.Node.Value) - decoder := yaml.NewDecoder(strings.NewReader(unwrapDoc(candidate.Node).Value)) + + decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value)) + errorReading := decoder.Decode(&dataBucket) if errorReading != nil { return Context{}, errorReading diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go index c7597533..1c394356 100644 --- a/pkg/yqlib/operator_encoder_decoder_test.go +++ b/pkg/yqlib/operator_encoder_decoder_test.go @@ -168,11 +168,19 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::a: \"foo:\\n a: frog\"\n", }, }, + { + description: "Decode a xml encoded string", + document: `a: "bar"`, + expression: `.b = (.a | from_xml)`, + expected: []string{ + "D0, P[], (doc)::a: \"bar\"\nb:\n foo: bar\n", + }, + }, } func TestEncoderDecoderOperatorScenarios(t *testing.T) { for _, tt := range encoderDecoderOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "encode-decode", encoderDecoderOperatorScenarios) + documentOperatorScenarios(t, "encode-decode", encoderDecoderOperatorScenarios) } diff --git a/pkg/yqlib/operator_entries_test.go b/pkg/yqlib/operator_entries_test.go index 4bee227d..1dd120c3 100644 --- a/pkg/yqlib/operator_entries_test.go +++ b/pkg/yqlib/operator_entries_test.go @@ -76,5 +76,5 @@ func TestEntriesOperatorScenarios(t *testing.T) { for _, tt := range entriesOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "entries", entriesOperatorScenarios) + documentOperatorScenarios(t, "entries", entriesOperatorScenarios) } diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go index 1cc1ba67..eac19734 100644 --- a/pkg/yqlib/operator_env_test.go +++ b/pkg/yqlib/operator_env_test.go @@ -68,5 +68,5 @@ func TestEnvOperatorScenarios(t *testing.T) { for _, tt := range envOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "env-variable-operators", envOperatorScenarios) + documentOperatorScenarios(t, "env-variable-operators", envOperatorScenarios) } diff --git a/pkg/yqlib/operator_equals_test.go b/pkg/yqlib/operator_equals_test.go index 69cd6e09..517dae36 100644 --- a/pkg/yqlib/operator_equals_test.go +++ b/pkg/yqlib/operator_equals_test.go @@ -186,5 +186,5 @@ func TestEqualOperatorScenarios(t *testing.T) { for _, tt := range equalsOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "equals", equalsOperatorScenarios) + documentOperatorScenarios(t, "equals", equalsOperatorScenarios) } diff --git a/pkg/yqlib/operator_file_test.go b/pkg/yqlib/operator_file_test.go index 88d1422f..875d4187 100644 --- a/pkg/yqlib/operator_file_test.go +++ b/pkg/yqlib/operator_file_test.go @@ -53,5 +53,5 @@ func TestFileOperatorsScenarios(t *testing.T) { for _, tt := range fileOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "file-operators", fileOperatorScenarios) + documentOperatorScenarios(t, "file-operators", fileOperatorScenarios) } diff --git a/pkg/yqlib/operator_flatten_test.go b/pkg/yqlib/operator_flatten_test.go index 4e826471..add71e1f 100644 --- a/pkg/yqlib/operator_flatten_test.go +++ b/pkg/yqlib/operator_flatten_test.go @@ -44,5 +44,5 @@ func TestFlattenOperatorScenarios(t *testing.T) { for _, tt := range flattenOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "flatten", flattenOperatorScenarios) + documentOperatorScenarios(t, "flatten", flattenOperatorScenarios) } diff --git a/pkg/yqlib/operator_group_by_test.go b/pkg/yqlib/operator_group_by_test.go index 60462206..2c7803da 100644 --- a/pkg/yqlib/operator_group_by_test.go +++ b/pkg/yqlib/operator_group_by_test.go @@ -27,5 +27,5 @@ func TestGroupByOperatorScenarios(t *testing.T) { for _, tt := range groupByOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "group-by", groupByOperatorScenarios) + documentOperatorScenarios(t, "group-by", groupByOperatorScenarios) } diff --git a/pkg/yqlib/operator_has_test.go b/pkg/yqlib/operator_has_test.go index fbd31d5b..3a631330 100644 --- a/pkg/yqlib/operator_has_test.go +++ b/pkg/yqlib/operator_has_test.go @@ -77,5 +77,5 @@ func TestHasOperatorScenarios(t *testing.T) { for _, tt := range hasOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "has", hasOperatorScenarios) + documentOperatorScenarios(t, "has", hasOperatorScenarios) } diff --git a/pkg/yqlib/operator_keys_test.go b/pkg/yqlib/operator_keys_test.go index bd5087fb..b42306f7 100644 --- a/pkg/yqlib/operator_keys_test.go +++ b/pkg/yqlib/operator_keys_test.go @@ -81,5 +81,5 @@ func TestKeysOperatorScenarios(t *testing.T) { for _, tt := range keysOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "keys", keysOperatorScenarios) + documentOperatorScenarios(t, "keys", keysOperatorScenarios) } diff --git a/pkg/yqlib/operator_length_test.go b/pkg/yqlib/operator_length_test.go index d08c1326..a67e627d 100644 --- a/pkg/yqlib/operator_length_test.go +++ b/pkg/yqlib/operator_length_test.go @@ -62,5 +62,5 @@ func TestLengthOperatorScenarios(t *testing.T) { for _, tt := range lengthOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "length", lengthOperatorScenarios) + documentOperatorScenarios(t, "length", lengthOperatorScenarios) } diff --git a/pkg/yqlib/operator_load.go b/pkg/yqlib/operator_load.go index 4741b09a..426b8f25 100644 --- a/pkg/yqlib/operator_load.go +++ b/pkg/yqlib/operator_load.go @@ -12,6 +12,7 @@ import ( type loadPrefs struct { loadAsString bool + decoder Decoder } func loadString(filename string) (*CandidateNode, error) { @@ -26,7 +27,7 @@ func loadString(filename string) (*CandidateNode, error) { return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: string(filebytes)}}, nil } -func loadYaml(filename string) (*CandidateNode, error) { +func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) { file, err := os.Open(filename) // #nosec if err != nil { @@ -34,7 +35,7 @@ func loadYaml(filename string) (*CandidateNode, error) { } reader := bufio.NewReader(file) - documents, err := readDocuments(reader, filename, 0) + documents, err := readDocuments(reader, filename, 0, decoder) if err != nil { return nil, err } @@ -83,7 +84,7 @@ func loadYamlOperator(d *dataTreeNavigator, context Context, expressionNode *Exp if loadPrefs.loadAsString { contentsCandidate, err = loadString(filename) } else { - contentsCandidate, err = loadYaml(filename) + contentsCandidate, err = loadYaml(filename, loadPrefs.decoder) } if err != nil { return Context{}, fmt.Errorf("Failed to load %v: %w", filename, err) diff --git a/pkg/yqlib/operator_load_test.go b/pkg/yqlib/operator_load_test.go index 4fe690f1..5cfae4ad 100644 --- a/pkg/yqlib/operator_load_test.go +++ b/pkg/yqlib/operator_load_test.go @@ -46,5 +46,5 @@ func TestLoadScenarios(t *testing.T) { for _, tt := range loadScenarios { testScenario(t, &tt) } - documentScenarios(t, "load", loadScenarios) + documentOperatorScenarios(t, "load", loadScenarios) } diff --git a/pkg/yqlib/operator_map_test.go b/pkg/yqlib/operator_map_test.go index 1dd5db57..715df336 100644 --- a/pkg/yqlib/operator_map_test.go +++ b/pkg/yqlib/operator_map_test.go @@ -47,5 +47,5 @@ func TestMapOperatorScenarios(t *testing.T) { for _, tt := range mapOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "map", mapOperatorScenarios) + documentOperatorScenarios(t, "map", mapOperatorScenarios) } diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index 59a08849..ab136f13 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -441,5 +441,5 @@ func TestMultiplyOperatorScenarios(t *testing.T) { for _, tt := range multiplyOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "multiply-merge", multiplyOperatorScenarios) + documentOperatorScenarios(t, "multiply-merge", multiplyOperatorScenarios) } diff --git a/pkg/yqlib/operator_parent_test.go b/pkg/yqlib/operator_parent_test.go index f4cbf236..35afd67e 100644 --- a/pkg/yqlib/operator_parent_test.go +++ b/pkg/yqlib/operator_parent_test.go @@ -33,5 +33,5 @@ func TestParentOperatorScenarios(t *testing.T) { for _, tt := range parentOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "parent", parentOperatorScenarios) + documentOperatorScenarios(t, "parent", parentOperatorScenarios) } diff --git a/pkg/yqlib/operator_path_test.go b/pkg/yqlib/operator_path_test.go index a93830a2..faf06ce9 100644 --- a/pkg/yqlib/operator_path_test.go +++ b/pkg/yqlib/operator_path_test.go @@ -69,5 +69,5 @@ func TestPathOperatorsScenarios(t *testing.T) { for _, tt := range pathOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "path", pathOperatorScenarios) + documentOperatorScenarios(t, "path", pathOperatorScenarios) } diff --git a/pkg/yqlib/operator_pipe_test.go b/pkg/yqlib/operator_pipe_test.go index fc36cde9..9e979603 100644 --- a/pkg/yqlib/operator_pipe_test.go +++ b/pkg/yqlib/operator_pipe_test.go @@ -27,5 +27,5 @@ func TestPipeOperatorScenarios(t *testing.T) { for _, tt := range pipeOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "pipe", pipeOperatorScenarios) + documentOperatorScenarios(t, "pipe", pipeOperatorScenarios) } diff --git a/pkg/yqlib/operator_recursive_descent_test.go b/pkg/yqlib/operator_recursive_descent_test.go index 8d4ddc7e..45e63cfd 100644 --- a/pkg/yqlib/operator_recursive_descent_test.go +++ b/pkg/yqlib/operator_recursive_descent_test.go @@ -233,5 +233,5 @@ func TestRecursiveDescentOperatorScenarios(t *testing.T) { for _, tt := range recursiveDescentOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "recursive-descent-glob", recursiveDescentOperatorScenarios) + documentOperatorScenarios(t, "recursive-descent-glob", recursiveDescentOperatorScenarios) } diff --git a/pkg/yqlib/operator_reduce_test.go b/pkg/yqlib/operator_reduce_test.go index dde2c33f..80b45a56 100644 --- a/pkg/yqlib/operator_reduce_test.go +++ b/pkg/yqlib/operator_reduce_test.go @@ -36,5 +36,5 @@ func TestReduceOperatorScenarios(t *testing.T) { for _, tt := range reduceOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "reduce", reduceOperatorScenarios) + documentOperatorScenarios(t, "reduce", reduceOperatorScenarios) } diff --git a/pkg/yqlib/operator_select_test.go b/pkg/yqlib/operator_select_test.go index 4a1e8723..43c16a9b 100644 --- a/pkg/yqlib/operator_select_test.go +++ b/pkg/yqlib/operator_select_test.go @@ -99,5 +99,5 @@ func TestSelectOperatorScenarios(t *testing.T) { for _, tt := range selectOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "select", selectOperatorScenarios) + documentOperatorScenarios(t, "select", selectOperatorScenarios) } diff --git a/pkg/yqlib/operator_sort_keys_test.go b/pkg/yqlib/operator_sort_keys_test.go index 462771e6..8d421556 100644 --- a/pkg/yqlib/operator_sort_keys_test.go +++ b/pkg/yqlib/operator_sort_keys_test.go @@ -36,5 +36,5 @@ func TestSortKeysOperatorScenarios(t *testing.T) { for _, tt := range sortKeysOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "sort-keys", sortKeysOperatorScenarios) + documentOperatorScenarios(t, "sort-keys", sortKeysOperatorScenarios) } diff --git a/pkg/yqlib/operator_sort_test.go b/pkg/yqlib/operator_sort_test.go index 6828d73e..7c4128d6 100644 --- a/pkg/yqlib/operator_sort_test.go +++ b/pkg/yqlib/operator_sort_test.go @@ -76,5 +76,5 @@ func TestSortByOperatorScenarios(t *testing.T) { for _, tt := range sortByOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "sort", sortByOperatorScenarios) + documentOperatorScenarios(t, "sort", sortByOperatorScenarios) } diff --git a/pkg/yqlib/operator_split_document_test.go b/pkg/yqlib/operator_split_document_test.go index 813f3173..ca490512 100644 --- a/pkg/yqlib/operator_split_document_test.go +++ b/pkg/yqlib/operator_split_document_test.go @@ -28,5 +28,5 @@ func TestSplitDocOperatorScenarios(t *testing.T) { for _, tt := range splitDocOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "split-into-documents", splitDocOperatorScenarios) + documentOperatorScenarios(t, "split-into-documents", splitDocOperatorScenarios) } diff --git a/pkg/yqlib/operator_strings_test.go b/pkg/yqlib/operator_strings_test.go index c94a2fc8..c4e26214 100644 --- a/pkg/yqlib/operator_strings_test.go +++ b/pkg/yqlib/operator_strings_test.go @@ -170,5 +170,5 @@ func TestStringsOperatorScenarios(t *testing.T) { for _, tt := range stringsOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "string-operators", stringsOperatorScenarios) + documentOperatorScenarios(t, "string-operators", stringsOperatorScenarios) } diff --git a/pkg/yqlib/operator_style_test.go b/pkg/yqlib/operator_style_test.go index 5f2490fd..ce263a7b 100644 --- a/pkg/yqlib/operator_style_test.go +++ b/pkg/yqlib/operator_style_test.go @@ -152,5 +152,5 @@ func TestStyleOperatorScenarios(t *testing.T) { for _, tt := range styleOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "style", styleOperatorScenarios) + documentOperatorScenarios(t, "style", styleOperatorScenarios) } diff --git a/pkg/yqlib/operator_subtract_test.go b/pkg/yqlib/operator_subtract_test.go index 7e209d5e..e43e0bfe 100644 --- a/pkg/yqlib/operator_subtract_test.go +++ b/pkg/yqlib/operator_subtract_test.go @@ -99,5 +99,5 @@ func TestSubtractOperatorScenarios(t *testing.T) { for _, tt := range subtractOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "subtract", subtractOperatorScenarios) + documentOperatorScenarios(t, "subtract", subtractOperatorScenarios) } diff --git a/pkg/yqlib/operator_tag_test.go b/pkg/yqlib/operator_tag_test.go index 08b22741..380f6028 100644 --- a/pkg/yqlib/operator_tag_test.go +++ b/pkg/yqlib/operator_tag_test.go @@ -64,5 +64,5 @@ func TestTagOperatorScenarios(t *testing.T) { for _, tt := range tagOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "tag", tagOperatorScenarios) + documentOperatorScenarios(t, "tag", tagOperatorScenarios) } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index b556ba28..94400ba2 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -519,5 +519,5 @@ func TestTraversePathOperatorScenarios(t *testing.T) { for _, tt := range traversePathOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "traverse-read", traversePathOperatorScenarios) + documentOperatorScenarios(t, "traverse-read", traversePathOperatorScenarios) } diff --git a/pkg/yqlib/operator_union_test.go b/pkg/yqlib/operator_union_test.go index a5f12de7..1409fd7f 100644 --- a/pkg/yqlib/operator_union_test.go +++ b/pkg/yqlib/operator_union_test.go @@ -44,5 +44,5 @@ func TestUnionOperatorScenarios(t *testing.T) { for _, tt := range unionOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "union", unionOperatorScenarios) + documentOperatorScenarios(t, "union", unionOperatorScenarios) } diff --git a/pkg/yqlib/operator_unique_test.go b/pkg/yqlib/operator_unique_test.go index 347f4d71..4a3ce79b 100644 --- a/pkg/yqlib/operator_unique_test.go +++ b/pkg/yqlib/operator_unique_test.go @@ -61,5 +61,5 @@ func TestUniqueOperatorScenarios(t *testing.T) { for _, tt := range uniqueOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "unique", uniqueOperatorScenarios) + documentOperatorScenarios(t, "unique", uniqueOperatorScenarios) } diff --git a/pkg/yqlib/operator_variables_test.go b/pkg/yqlib/operator_variables_test.go index a6e8d854..e40f8dd8 100644 --- a/pkg/yqlib/operator_variables_test.go +++ b/pkg/yqlib/operator_variables_test.go @@ -66,5 +66,5 @@ func TestVariableOperatorScenarios(t *testing.T) { for _, tt := range variableOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "variable-operators", variableOperatorScenarios) + documentOperatorScenarios(t, "variable-operators", variableOperatorScenarios) } diff --git a/pkg/yqlib/operator_with_test.go b/pkg/yqlib/operator_with_test.go index 11ea08c1..aad4d15c 100644 --- a/pkg/yqlib/operator_with_test.go +++ b/pkg/yqlib/operator_with_test.go @@ -34,5 +34,5 @@ func TestWithOperatorScenarios(t *testing.T) { for _, tt := range withOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "with", withOperatorScenarios) + documentOperatorScenarios(t, "with", withOperatorScenarios) } diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index b99d3865..c99c63a7 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -32,7 +32,7 @@ func readDocumentWithLeadingContent(content string, fakefilename string, fakeFil return nil, err } - inputs, err := readDocuments(reader, fakefilename, fakeFileIndex) + inputs, err := readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder()) if err != nil { return nil, err } @@ -90,27 +90,32 @@ func testScenario(t *testing.T, s *expressionScenario) { test.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document)) } +func resultToString(t *testing.T, n *CandidateNode) string { + var valueBuffer bytes.Buffer + printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true) + + err := printer.PrintResults(n.AsList()) + if err != nil { + t.Error(err) + return "" + } + + tag := n.Node.Tag + if n.Node.Kind == yaml.DocumentNode { + tag = "doc" + } else if n.Node.Kind == yaml.AliasNode { + tag = "alias" + } + return fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.Document, n.Path, tag, valueBuffer.String()) +} + func resultsToString(t *testing.T, results *list.List) []string { var pretty = make([]string, 0) for el := results.Front(); el != nil; el = el.Next() { n := el.Value.(*CandidateNode) - var valueBuffer bytes.Buffer - printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true) - err := printer.PrintResults(n.AsList()) - if err != nil { - t.Error(err) - return nil - } - - tag := n.Node.Tag - if n.Node.Kind == yaml.DocumentNode { - tag = "doc" - } else if n.Node.Kind == yaml.AliasNode { - tag = "alias" - } - output := fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.Document, n.Path, tag, valueBuffer.String()) + output := resultToString(t, n) pretty = append(pretty, output) } return pretty @@ -123,8 +128,8 @@ func writeOrPanic(w *bufio.Writer, text string) { } } -func copyFromHeader(title string, out *os.File) error { - source := fmt.Sprintf("doc/headers/%v.md", title) +func copyFromHeader(folder string, title string, out *os.File) error { + source := fmt.Sprintf("doc/%v/headers/%v.md", folder, title) _, err := os.Stat(source) if os.IsNotExist(err) { return nil @@ -147,15 +152,17 @@ func formatYaml(yaml string, filename string) string { panic(err) } streamEvaluator := NewStreamEvaluator() - _, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "") + _, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "", NewYamlDecoder()) if err != nil { panic(err) } return output.String() } -func documentScenarios(t *testing.T, title string, scenarios []expressionScenario) { - f, err := os.Create(fmt.Sprintf("doc/%v.md", title)) +type documentScenarioFunc func(t *testing.T, writer *bufio.Writer, scenario interface{}) + +func documentScenarios(t *testing.T, folder string, title string, scenarios []interface{}, documentScenario documentScenarioFunc) { + f, err := os.Create(fmt.Sprintf("doc/%v/%v.md", folder, title)) if err != nil { t.Error(err) @@ -163,7 +170,7 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari } defer f.Close() - err = copyFromHeader(title, f) + err = copyFromHeader(folder, title, f) if err != nil { t.Error(err) return @@ -173,14 +180,26 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari writeOrPanic(w, "\n") for _, s := range scenarios { - if !s.skipDoc { - documentScenario(t, w, s) - } + documentScenario(t, w, s) } w.Flush() } -func documentScenario(t *testing.T, w *bufio.Writer, s expressionScenario) { +func documentOperatorScenarios(t *testing.T, title string, scenarios []expressionScenario) { + genericScenarios := make([]interface{}, len(scenarios)) + for i, s := range scenarios { + genericScenarios[i] = s + } + + documentScenarios(t, "operators", title, genericScenarios, documentOperatorScenario) +} + +func documentOperatorScenario(t *testing.T, w *bufio.Writer, i interface{}) { + s := i.(expressionScenario) + + if s.skipDoc { + return + } writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index 8c74af87..d3172906 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -38,7 +38,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } @@ -76,7 +76,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } @@ -118,7 +118,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } @@ -165,7 +165,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } @@ -215,7 +215,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } @@ -234,7 +234,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } @@ -263,7 +263,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } @@ -294,7 +294,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) { panic(err) } streamEvaluator := NewStreamEvaluator() - _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n") + _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n", NewYamlDecoder()) if err != nil { panic(err) } @@ -316,7 +316,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) { // when outputing JSON. printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) if err != nil { panic(err) } diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index ddb777fb..96826f30 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -14,8 +14,8 @@ import ( // Uses less memory than loading all documents and running the expression once, but this cannot process // cross document expressions. type StreamEvaluator interface { - Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error) - EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error + Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error) + EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error EvaluateNew(expression string, printer Printer, leadingContent string) error } @@ -51,7 +51,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin return printer.PrintResults(result.MatchingNodes) } -func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error { +func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error { var totalProcessDocs uint node, err := s.treeCreator.ParseExpression(expression) if err != nil { @@ -70,7 +70,7 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p if err != nil { return err } - processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent) + processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent, decoder) if err != nil { return err } @@ -89,10 +89,10 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p return nil } -func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error) { +func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error) { var currentIndex uint - decoder := yaml.NewDecoder(reader) + decoder.Init(reader) for { var dataBucket yaml.Node errorReading := decoder.Decode(&dataBucket) diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index f8291a40..ecfa6a99 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -107,8 +107,8 @@ func processReadStream(reader *bufio.Reader) (io.Reader, string, error) { } } -func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) { - decoder := yaml.NewDecoder(reader) +func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) { + decoder.Init(reader) inputList := list.New() var currentIndex uint diff --git a/scripts/copy-docs.sh b/scripts/copy-docs.sh index e3f39341..5c27efaa 100755 --- a/scripts/copy-docs.sh +++ b/scripts/copy-docs.sh @@ -1,4 +1,5 @@ #!/bin/bash cp how-it-works.md ../yq-gitbook/. -cp pkg/yqlib/doc/*.md ../yq-gitbook/operators/. \ No newline at end of file +cp pkg/yqlib/doc/operators/*.md ../yq-gitbook/operators/. +cp pkg/yqlib/doc/usage/*.md ../yq-gitbook/usage/. \ No newline at end of file