From 1c9f001171999e7f5efb7b1b822aed5e3af0623f Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 12 Mar 2024 16:43:53 +1100 Subject: [PATCH] wip --- cmd/root.go | 24 +++---- cmd/utils.go | 8 ++- pkg/properties/decoder_properties.go | 17 +---- pkg/properties/encoder_properties.go | 14 ++--- pkg/properties/encoder_properties_test.go | 2 +- pkg/properties/properties.go | 9 +-- pkg/xml/decoder_xml.go | 70 ++++++++++----------- pkg/xml/encoder_xml.go | 58 ++++++++--------- pkg/xml/xml.go | 17 +++-- pkg/yqlib/all_at_once_evaluator.go | 2 +- pkg/yqlib/candidate_node.go | 37 ++++++++++- pkg/yqlib/candidate_node_test.go | 16 ++--- pkg/yqlib/candidiate_node_json.go | 2 +- pkg/yqlib/data_tree_navigator.go | 11 +++- pkg/yqlib/decoder_base64.go | 2 +- pkg/yqlib/decoder_csv_object.go | 4 +- pkg/yqlib/decoder_toml.go | 10 +-- pkg/yqlib/decoder_uri.go | 2 +- pkg/yqlib/decoder_yaml.go | 2 +- pkg/yqlib/encoder.go | 15 ----- pkg/yqlib/encoder_csv.go | 4 +- pkg/yqlib/encoder_json.go | 2 +- pkg/yqlib/encoder_lua.go | 58 ++++++++--------- pkg/yqlib/encoder_sh.go | 2 +- pkg/yqlib/encoder_shellvariables.go | 2 +- pkg/yqlib/encoder_toml.go | 2 +- pkg/yqlib/encoder_yaml.go | 8 +-- pkg/yqlib/expression_postfix.go | 2 +- pkg/yqlib/format.go | 77 ++++++++++++++++------- pkg/yqlib/lexer_participle.go | 46 +++++++------- pkg/yqlib/lib.go | 18 +----- pkg/yqlib/operation.go | 2 +- pkg/yqlib/operator_encoder_decoder.go | 24 +------ pkg/yqlib/operator_entries.go | 4 +- pkg/yqlib/operator_strings.go | 32 +++++----- pkg/yqlib/operator_traverse_path.go | 6 +- pkg/yqlib/printer_writer.go | 9 +-- pkg/yqlib/stream_evaluator.go | 2 +- pkg/yqlib/utils.go | 2 +- yq.go | 6 +- 40 files changed, 321 insertions(+), 309 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 071c679e..8acbc9bf 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,6 +5,8 @@ import ( "os" "strings" + "github.com/mikefarah/yq/v4/pkg/properties" + "github.com/mikefarah/yq/v4/pkg/xml" "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" logging "gopkg.in/op/go-logging.v1" @@ -117,27 +119,27 @@ yq -P -oy sample.json panic(err) } - rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes") + rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", xml.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes") if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil { panic(err) } - rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).") + rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ContentName, "xml-content-name", xml.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).") if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil { panic(err) } - rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.") - rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes") - rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.") - rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. )") + rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", xml.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.") + rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", xml.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes") + rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", xml.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.") + rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", xml.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. )") if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil { panic(err) } - rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. )") + rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", xml.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. )") if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil { panic(err) } - rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. )") - rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. )") + rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", xml.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. )") + rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", xml.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. )") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values") rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character") @@ -155,8 +157,8 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables") - rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values") - rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)") + rootCmd.PersistentFlags().StringVar(&properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values") + rootCmd.PersistentFlags().BoolVar(&properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)") rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)") diff --git a/cmd/utils.go b/cmd/utils.go index 7bd96a28..72eb922b 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -6,6 +6,8 @@ import ( "os" "strings" + "github.com/mikefarah/yq/v4/pkg/properties" + "github.com/mikefarah/yq/v4/pkg/xml" "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" "gopkg.in/op/go-logging.v1" @@ -102,7 +104,7 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) { yqlib.GetLogger().Debug("Using output format %v", outputFormat) if outputFormatType == yqlib.YamlFormat || - outputFormatType == yqlib.PropertiesFormat { + outputFormatType == properties.PropertiesFormat { unwrapScalar = true } if unwrapScalarFlag.IsExplicitlySet() { @@ -148,12 +150,12 @@ func configureEncoder() (yqlib.Encoder, error) { if err != nil { return nil, err } - yqlib.ConfiguredXMLPreferences.Indent = indent + xml.ConfiguredXMLPreferences.Indent = indent yqlib.ConfiguredYamlPreferences.Indent = indent yqlib.ConfiguredJSONPreferences.Indent = indent yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar - yqlib.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar + properties.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled diff --git a/pkg/properties/decoder_properties.go b/pkg/properties/decoder_properties.go index abcead29..0b263c00 100644 --- a/pkg/properties/decoder_properties.go +++ b/pkg/properties/decoder_properties.go @@ -49,27 +49,14 @@ func (dec *propertiesDecoder) processComment(c string) string { } func (dec *propertiesDecoder) applyPropertyComments(context yqlib.Context, path []interface{}, comments []string) error { - assignmentOp := &yqlib.Operation{OperationType: assignOpType, Preferences: assignPreferences{}} - rhsCandidateNode := &yqlib.CandidateNode{ Tag: "!!str", Value: fmt.Sprintf("%v", path[len(path)-1]), HeadComment: dec.processComment(strings.Join(comments, "\n")), Kind: yqlib.ScalarNode, } - rhsCandidateNode.Tag = rhsCandidateNode.GuessTagFromCustomType() - - rhsOp := &yqlib.Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode} - - assignmentOpNode := &yqlib.ExpressionNode{ - Operation: assignmentOp, - LHS: createTraversalTree(path, traversePreferences{}, true), - RHS: &yqlib.ExpressionNode{Operation: rhsOp}, - } - - _, err := dec.d.GetMatchingNodes(context, assignmentOpNode) - return err + return dec.d.DeeplyAssignKey(context, path, rhsCandidateNode) } func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *properties.Properties, key string) error { @@ -84,7 +71,7 @@ func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *p } } - rhsNode := createStringScalarNode(value) + rhsNode := yqlib.CreateStringScalarNode(value) rhsNode.Tag = rhsNode.GuessTagFromCustomType() return dec.d.DeeplyAssign(context, path, rhsNode) diff --git a/pkg/properties/encoder_properties.go b/pkg/properties/encoder_properties.go index 379913d0..3a80905a 100644 --- a/pkg/properties/encoder_properties.go +++ b/pkg/properties/encoder_properties.go @@ -44,7 +44,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin } } else { - if err := writeString(writer, readline); err != nil { + if err := yqlib.WriteString(writer, readline); err != nil { return err } } @@ -52,7 +52,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin if errors.Is(errReading, io.EOF) { if readline != "" { // the last comment we read didn't have a newline, put one in - if err := writeString(writer, "\n"); err != nil { + if err := yqlib.WriteString(writer, "\n"); err != nil { return err } } @@ -65,10 +65,10 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin func (pe *propertiesEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error { if node.Kind == yqlib.ScalarNode { - return writeString(writer, node.Value+"\n") + return yqlib.WriteString(writer, node.Value+"\n") } - mapKeysToStrings(node) + node.ConvertKeysToStrings() p := properties.NewProperties() p.WriteSeparator = pe.prefs.KeyValueSeparator err := pe.doEncode(p, node, "", nil) @@ -80,14 +80,14 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) return err } -func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yqlib.CandidateNode, path string, keyNode *CandidateNode) error { +func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yqlib.CandidateNode, path string, keyNode *yqlib.CandidateNode) error { comments := "" if keyNode != nil { // include the key node comments if present - comments = headAndLineComment(keyNode) + comments = keyNode.CleanHeadAndLineComment() } - comments = comments + headAndLineComment(node) + comments = comments + node.CleanHeadAndLineComment() commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ") p.SetComments(path, strings.Split(commentsWithSpaces, "\n")) diff --git a/pkg/properties/encoder_properties_test.go b/pkg/properties/encoder_properties_test.go index f942c352..6f428920 100644 --- a/pkg/properties/encoder_properties_test.go +++ b/pkg/properties/encoder_properties_test.go @@ -1,4 +1,4 @@ -package yqlib +package properties import ( "bufio" diff --git a/pkg/properties/properties.go b/pkg/properties/properties.go index 69d82f34..b94df7a1 100644 --- a/pkg/properties/properties.go +++ b/pkg/properties/properties.go @@ -24,15 +24,16 @@ func (p *PropertiesPreferences) Copy() PropertiesPreferences { } } -var PropertiesFormat = &yqlib.Format{"props", []string{"p", "properties"}, +var PropertiesFormat = &yqlib.Format{"props", []string{"p", "properties"}, "properties", func() yqlib.Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) }, func() yqlib.Decoder { return NewPropertiesDecoder() }, + nil, } var propertyYqRules = []*yqlib.ParticipleYqRule{ - {"PropertiesDecode", `from_?props|@propsd`, decodeOp(PropertiesFormat), 0}, - {"PropsEncode", `to_?props|@props`, encodeWithIndent(PropertiesFormat, 2), 0}, - {"LoadProperties", `load_?props`, loadOp(NewPropertiesDecoder(), false), 0}, + {"PropertiesDecode", `from_?props|@propsd`, yqlib.CreateDecodeOpYqAction(PropertiesFormat), 0}, + {"PropsEncode", `to_?props|@props`, yqlib.CreateEncodeOpYqAction(PropertiesFormat, 2), 0}, + {"LoadProperties", `load_?props`, yqlib.CreateLoadOpYqAction(NewPropertiesDecoder()), 0}, } func RegisterPropertiesFormat() { diff --git a/pkg/xml/decoder_xml.go b/pkg/xml/decoder_xml.go index 71150d97..7ec0b2be 100644 --- a/pkg/xml/decoder_xml.go +++ b/pkg/xml/decoder_xml.go @@ -36,8 +36,8 @@ func (dec *xmlDecoder) Init(reader io.Reader) error { return nil } -func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib..CandidateNode, error) { - yamlNode := &yqlib..CandidateNode{Kind: SequenceNode, Tag: "!!seq"} +func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib.CandidateNode, error) { + yamlNode := &yqlib.CandidateNode{Kind: yqlib.SequenceNode, Tag: "!!seq"} for _, child := range nodes { yamlChild, err := dec.convertToYamlNode(child) if err != nil { @@ -64,14 +64,14 @@ func (dec *xmlDecoder) processComment(c string) string { return replacement } -func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib..CandidateNode, error) { - log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment) - yamlNode := &yqlib..CandidateNode{Kind: MappingNode, Tag: "!!map"} +func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) { + yqlib.GetLogger().Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment) + yamlNode := &yqlib.CandidateNode{Kind: yqlib.MappingNode, Tag: "!!map"} if len(n.Data) > 0 { - log.Debugf("creating content node for map: %v", dec.prefs.ContentName) + yqlib.GetLogger().Debugf("creating content node for map: %v", dec.prefs.ContentName) label := dec.prefs.ContentName - labelNode := createScalarNode(label, label) + labelNode := yqlib.CreateScalarNode(label, label) labelNode.HeadComment = dec.processComment(n.HeadComment) labelNode.LineComment = dec.processComment(n.LineComment) labelNode.FootComment = dec.processComment(n.FootComment) @@ -81,19 +81,19 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib..CandidateNode, error) { for i, keyValuePair := range n.Children { label := keyValuePair.K children := keyValuePair.V - labelNode := createScalarNode(label, label) - var valueNode *yqlib..CandidateNode + labelNode := yqlib.CreateScalarNode(label, label) + var valueNode *yqlib.CandidateNode var err error if i == 0 { - log.Debugf("head comment here") + yqlib.GetLogger().Debugf("head comment here") labelNode.HeadComment = dec.processComment(n.HeadComment) } - log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment) + yqlib.GetLogger().Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment) labelNode.FootComment = dec.processComment(keyValuePair.FootComment) - log.Debug("len of children in %v is %v", label, len(children)) + yqlib.GetLogger().Debug("len of children in %v is %v", label, len(children)) if len(children) > 1 { valueNode, err = dec.createSequence(children) if err != nil { @@ -106,7 +106,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib..CandidateNode, error) { if len(children[0].Children) == 0 && children[0].HeadComment != "" { if len(children[0].Data) > 0 { - log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment) + yqlib.GetLogger().Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment) labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n") children[0].HeadComment = "" } else { @@ -126,33 +126,33 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib..CandidateNode, error) { return yamlNode, nil } -func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib..CandidateNode { +func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib.CandidateNode { switch len(values) { case 0: - return createScalarNode(nil, "") + return yqlib.CreateScalarNode(nil, "") case 1: - return createScalarNode(values[0], values[0]) + return yqlib.CreateScalarNode(values[0], values[0]) default: - content := make([]*yqlib..CandidateNode, 0) + content := make([]*yqlib.CandidateNode, 0) for _, value := range values { - content = append(content, createScalarNode(value, value)) + content = append(content, yqlib.CreateScalarNode(value, value)) } - return &yqlib..CandidateNode{ - Kind: SequenceNode, + return &yqlib.CandidateNode{ + Kind: yqlib.SequenceNode, Tag: "!!seq", Content: content, } } } -func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib..CandidateNode, error) { +func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, error) { if len(n.Children) > 0 { return dec.createMap(n) } scalar := dec.createValueNodeFromData(n.Data) - log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment) + yqlib.GetLogger().Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment) scalar.HeadComment = dec.processComment(n.HeadComment) scalar.LineComment = dec.processComment(n.LineComment) if scalar.Tag == "!!seq" { @@ -165,7 +165,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib..CandidateNode, err return scalar, nil } -func (dec *xmlDecoder) Decode() (*yqlib..CandidateNode, error) { +func (dec *xmlDecoder) Decode() (*yqlib.CandidateNode, error) { if dec.finished { return nil, io.EOF } @@ -212,17 +212,17 @@ func (n *xmlNode) AddChild(s string, c *xmlNode) { if n.Children == nil { n.Children = make([]*xmlChildrenKv, 0) } - log.Debug("looking for %s", s) + yqlib.GetLogger().Debug("looking for %s", s) // see if we can find an existing entry to add to for _, childEntry := range n.Children { if childEntry.K == s { - log.Debug("found it, appending an entry%s", s) + yqlib.GetLogger().Debug("found it, appending an entry%s", s) childEntry.V = append(childEntry.V, c) - log.Debug("yay len of children in %v is %v", s, len(childEntry.V)) + yqlib.GetLogger().Debug("yay len of children in %v is %v", s, len(childEntry.V)) return } } - log.Debug("not there, making a new one %s", s) + yqlib.GetLogger().Debug("not there, making a new one %s", s) n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}}) } @@ -268,7 +268,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error { switch se := t.(type) { case xml.StartElement: - log.Debug("start element %v", se.Name.Local) + yqlib.GetLogger().Debug("start element %v", se.Name.Local) elem.state = "started" // Build new a new current element and link it to its parent elem = &element{ @@ -297,14 +297,14 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error { if len(newBit) > 0 { elem.n.Data = append(elem.n.Data, newBit) elem.state = "chardata" - log.Debug("chardata [%v] for %v", elem.n.Data, elem.label) + yqlib.GetLogger().Debug("chardata [%v] for %v", elem.n.Data, elem.label) } case xml.EndElement: if elem == nil { - log.Debug("no element, probably bad xml") + yqlib.GetLogger().Debug("no element, probably bad xml") continue } - log.Debug("end element %v", elem.label) + yqlib.GetLogger().Debug("end element %v", elem.label) elem.state = "finished" // And add it to its parent list if elem.parent != nil { @@ -320,10 +320,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error { applyFootComment(elem, commentStr) } else if elem.state == "chardata" { - log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr) + yqlib.GetLogger().Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr) elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ") } else { - log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr) + yqlib.GetLogger().Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr) elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ") } @@ -348,7 +348,7 @@ func applyFootComment(elem *element, commentStr string) { if len(elem.n.Children) > 0 { lastChildIndex := len(elem.n.Children) - 1 childKv := elem.n.Children[lastChildIndex] - log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr) + yqlib.GetLogger().Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr) // if it's an array of scalars, put the foot comment on the scalar itself if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 { nodeToUpdate := childKv.V[len(childKv.V)-1] @@ -357,7 +357,7 @@ func applyFootComment(elem *element, commentStr string) { childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ") } } else { - log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr) + yqlib.GetLogger().Debug("got a foot comment for %v: [%v]", elem.label, commentStr) elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ") } } diff --git a/pkg/xml/encoder_xml.go b/pkg/xml/encoder_xml.go index 47e9d5af..47c6b3ee 100644 --- a/pkg/xml/encoder_xml.go +++ b/pkg/xml/encoder_xml.go @@ -61,7 +61,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error { return err } if _, err := e.writer.Write([]byte("\n")); err != nil { - log.Warning("Unable to write newline, skipping: %w", err) + yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err) } } } @@ -102,7 +102,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error { } func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.CandidateNode) error { - err := e.encodeComment(encoder, headAndLineComment(node)) + err := e.encodeComment(encoder, node.CleanHeadAndLineComment()) if err != nil { return err } @@ -111,12 +111,12 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida value := node.Content[i+1] start := xml.StartElement{Name: xml.Name{Local: key.Value}} - log.Debugf("comments of key %v", key.Value) - err := e.encodeComment(encoder, headAndLineComment(key)) + yqlib.GetLogger().Debugf("comments of key %v", key.Value) + err := e.encodeComment(encoder, key.CleanHeadAndLineComment()) if err != nil { return err } - if headAndLineComment(key) != "" { + if key.CleanHeadAndLineComment() != "" { var newLine xml.CharData = []byte("\n") err = encoder.EncodeToken(newLine) if err != nil { @@ -133,7 +133,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida return err } if _, err := e.writer.Write([]byte("\n")); err != nil { - log.Warning("Unable to write newline, skipping: %w", err) + yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err) } } else if key.Value == e.prefs.DirectiveName { var directive xml.Directive = []byte(value.Value) @@ -141,23 +141,23 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida return err } if _, err := e.writer.Write([]byte("\n")); err != nil { - log.Warning("Unable to write newline, skipping: %w", err) + yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err) } } else { - log.Debugf("recursing") + yqlib.GetLogger().Debugf("recursing") err = e.doEncode(encoder, value, start) if err != nil { return err } } - err = e.encodeComment(encoder, footComment(key)) + err = e.encodeComment(encoder, key.CleanFootComment()) if err != nil { return err } } - return e.encodeComment(encoder, footComment(node)) + return e.encodeComment(encoder, node.CleanFootComment()) } func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error { @@ -165,7 +165,7 @@ func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yqlib.CandidateNode if err != nil { return err } - return e.encodeComment(encoder, headComment(node)) + return e.encodeComment(encoder, node.CleanHeadComment()) } func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error { @@ -173,16 +173,16 @@ func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yqlib.CandidateNode, if err != nil { return err } - return e.encodeComment(encoder, footComment(node)) + return e.encodeComment(encoder, node.CleanFootComment()) } func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error { switch node.Kind { - case MappingNode: + case yqlib.MappingNode: return e.encodeMap(encoder, node, start) - case SequenceNode: + case yqlib.SequenceNode: return e.encodeArray(encoder, node, start) - case ScalarNode: + case yqlib.ScalarNode: err := e.encodeStart(encoder, node, start) if err != nil { return err @@ -194,7 +194,7 @@ func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, s return err } - if err = e.encodeComment(encoder, lineComment(node)); err != nil { + if err = e.encodeComment(encoder, node.CleanLineComment()); err != nil { return err } @@ -209,13 +209,13 @@ var chompRegexp = regexp.MustCompile(`\n$`) func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error { if commentStr != "" { - log.Debugf("got comment [%v]", commentStr) + yqlib.GetLogger().Debugf("got comment [%v]", commentStr) // multi line string if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") { commentStr = chompRegexp.ReplaceAllString(commentStr, "") - log.Debugf("chompRegexp [%v]", commentStr) + yqlib.GetLogger().Debugf("chompRegexp [%v]", commentStr) commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2") - log.Debugf("processed multiline [%v]", commentStr) + yqlib.GetLogger().Debugf("processed multiline [%v]", commentStr) // if the first line is non blank, add a space if commentStr[0] != '\n' && commentStr[0] != ' ' { commentStr = " " + commentStr @@ -227,9 +227,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") { commentStr = commentStr + " " - log.Debugf("added suffix [%v]", commentStr) + yqlib.GetLogger().Debugf("added suffix [%v]", commentStr) } - log.Debugf("encoding comment [%v]", commentStr) + yqlib.GetLogger().Debugf("encoding comment [%v]", commentStr) var comment xml.Comment = []byte(commentStr) err := encoder.EncodeToken(comment) @@ -242,7 +242,7 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error { - if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil { + if err := e.encodeComment(encoder, node.CleanHeadAndLineComment()); err != nil { return err } @@ -252,7 +252,7 @@ func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode return err } } - return e.encodeComment(encoder, footComment(node)) + return e.encodeComment(encoder, node.CleanFootComment()) } func (e *xmlEncoder) isAttribute(name string) bool { @@ -263,7 +263,7 @@ func (e *xmlEncoder) isAttribute(name string) bool { } func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error { - log.Debug("its a map") + yqlib.GetLogger().Debug("its a map") //first find all the attributes and put them on the start token for i := 0; i < len(node.Content); i += 2 { @@ -271,7 +271,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, value := node.Content[i+1] if e.isAttribute(key.Value) { - if value.Kind == ScalarNode { + if value.Kind == yqlib.ScalarNode { attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1) start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value}) } else { @@ -290,7 +290,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, key := node.Content[i] value := node.Content[i+1] - err := e.encodeComment(encoder, headAndLineComment(key)) + err := e.encodeComment(encoder, key.CleanHeadAndLineComment()) if err != nil { return err } @@ -307,7 +307,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, } } else if key.Value == e.prefs.ContentName { // directly encode the contents - err = e.encodeComment(encoder, headAndLineComment(value)) + err = e.encodeComment(encoder, value.CleanHeadAndLineComment()) if err != nil { return err } @@ -316,7 +316,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, if err != nil { return err } - err = e.encodeComment(encoder, footComment(value)) + err = e.encodeComment(encoder, value.CleanFootComment()) if err != nil { return err } @@ -327,7 +327,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, return err } } - err = e.encodeComment(encoder, footComment(key)) + err = e.encodeComment(encoder, key.CleanFootComment()) if err != nil { return err } diff --git a/pkg/xml/xml.go b/pkg/xml/xml.go index 82d1e84e..87c71920 100644 --- a/pkg/xml/xml.go +++ b/pkg/xml/xml.go @@ -47,17 +47,22 @@ func (p *XmlPreferences) Copy() XmlPreferences { var ConfiguredXMLPreferences = NewDefaultXmlPreferences() -var XMLFormat = &yqlib.Format{"xml", []string{"x"}, +var XMLFormat = &yqlib.Format{"xml", []string{"x"}, "xml", func() yqlib.Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) }, func() yqlib.Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) }, + func(indent int) yqlib.Encoder { + prefs := ConfiguredXMLPreferences.Copy() + prefs.Indent = indent + return NewXMLEncoder(prefs) + }, } var xmlYqRules = []*yqlib.ParticipleYqRule{ - {"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLFormat), 0}, - {"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLFormat), 0}, - {"XMLEncode", `to_?xml`, encodeWithIndent(XMLFormat, 2), 0}, - {"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLFormat, 0), 0}, - {"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0}, + {"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, yqlib.CreateEncodeYqActionParsingIndent(XMLFormat), 0}, + {"XmlDecode", `from_?xml|@xmld`, yqlib.CreateDecodeOpYqAction(XMLFormat), 0}, + {"XMLEncode", `to_?xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 2), 0}, + {"XMLEncodeNoIndent", `@xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 0), 0}, + {"LoadXML", `load_?xml|xml_?load`, yqlib.CreateLoadOpYqAction(NewXMLDecoder(ConfiguredXMLPreferences)), 0}, } func RegisterXmlFormat() { diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index bdeddd20..c79fed62 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -63,7 +63,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string } if allDocuments.Len() == 0 { - candidateNode := createScalarNode(nil, "") + candidateNode := CreateScalarNode(nil, "") allDocuments.PushBack(candidateNode) } diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 88e73ea5..499aa907 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -27,14 +27,14 @@ const ( FlowStyle ) -func createStringScalarNode(stringValue string) *CandidateNode { +func CreateStringScalarNode(stringValue string) *CandidateNode { var node = &CandidateNode{Kind: ScalarNode} node.Value = stringValue node.Tag = "!!str" return node } -func createScalarNode(value interface{}, stringValue string) *CandidateNode { +func CreateScalarNode(value interface{}, stringValue string) *CandidateNode { var node = &CandidateNode{Kind: ScalarNode} node.Value = stringValue @@ -219,7 +219,7 @@ func (n *CandidateNode) AddChild(rawChild *CandidateNode) { value.Key.SetParent(n) } else { index := len(n.Content) - keyNode := createScalarNode(index, fmt.Sprintf("%v", index)) + keyNode := CreateScalarNode(index, fmt.Sprintf("%v", index)) keyNode.SetParent(n) value.Key = keyNode } @@ -428,3 +428,34 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP n.LineComment = other.LineComment } } + +func (n *CandidateNode) CleanHeadAndLineComment() string { + return n.CleanHeadComment() + n.CleanLineComment() +} + +func (n *CandidateNode) CleanHeadComment() string { + return strings.Replace(n.HeadComment, "#", "", 1) +} + +func (n *CandidateNode) CleanLineComment() string { + return strings.Replace(n.LineComment, "#", "", 1) +} + +func (n *CandidateNode) CleanFootComment() string { + return strings.Replace(n.FootComment, "#", "", 1) +} + +func (n *CandidateNode) ConvertKeysToStrings() { + + if n.Kind == MappingNode { + for index, child := range n.Content { + if index%2 == 0 { // its a map key + child.Tag = "!!str" + } + } + } + + for _, child := range n.Content { + child.ConvertKeysToStrings() + } +} diff --git a/pkg/yqlib/candidate_node_test.go b/pkg/yqlib/candidate_node_test.go index 2a09b48a..5dfa189d 100644 --- a/pkg/yqlib/candidate_node_test.go +++ b/pkg/yqlib/candidate_node_test.go @@ -64,13 +64,13 @@ func TestCandidateNodeChildWhenParentUpdated(t *testing.T) { test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index") } -type createScalarNodeScenario struct { +type CreateScalarNodeScenario struct { value interface{} stringValue string expectedTag string } -var createScalarScenarios = []createScalarNodeScenario{ +var createScalarScenarios = []CreateScalarNodeScenario{ { value: "mike", stringValue: "mike", @@ -100,20 +100,20 @@ var createScalarScenarios = []createScalarNodeScenario{ func TestCreateScalarNodeScenarios(t *testing.T) { for _, tt := range createScalarScenarios { - actual := createScalarNode(tt.value, tt.stringValue) + actual := CreateScalarNode(tt.value, tt.stringValue) test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue)) test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue)) } } func TestGetKeyForMapValue(t *testing.T) { - key := createStringScalarNode("yourKey") + key := CreateStringScalarNode("yourKey") n := CandidateNode{Key: key, Value: "meow", document: 3} test.AssertResult(t, "3 - yourKey", n.GetKey()) } func TestGetKeyForMapKey(t *testing.T) { - key := createStringScalarNode("yourKey") + key := CreateStringScalarNode("yourKey") key.IsMapKey = true key.document = 3 test.AssertResult(t, "key-yourKey-3 - ", key.GetKey()) @@ -125,7 +125,7 @@ func TestGetKeyForValue(t *testing.T) { } func TestGetParsedKeyForMapKey(t *testing.T) { - key := createStringScalarNode("yourKey") + key := CreateStringScalarNode("yourKey") key.IsMapKey = true key.document = 3 test.AssertResult(t, "yourKey", key.getParsedKey()) @@ -137,13 +137,13 @@ func TestGetParsedKeyForLooseValue(t *testing.T) { } func TestGetParsedKeyForMapValue(t *testing.T) { - key := createStringScalarNode("yourKey") + key := CreateStringScalarNode("yourKey") n := CandidateNode{Key: key, Value: "meow", document: 3} test.AssertResult(t, "yourKey", n.getParsedKey()) } func TestGetParsedKeyForArrayValue(t *testing.T) { - key := createScalarNode(4, "4") + key := CreateScalarNode(4, "4") n := CandidateNode{Key: key, Value: "meow", document: 3} test.AssertResult(t, 4, n.getParsedKey()) } diff --git a/pkg/yqlib/candidiate_node_json.go b/pkg/yqlib/candidiate_node_json.go index fe7d344e..6cf85fe2 100644 --- a/pkg/yqlib/candidiate_node_json.go +++ b/pkg/yqlib/candidiate_node_json.go @@ -96,7 +96,7 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error { if child == nil { // need to represent it as a null scalar - child = createScalarNode(nil, "null") + child = CreateScalarNode(nil, "null") } childKey := o.CreateChild() childKey.Kind = ScalarNode diff --git a/pkg/yqlib/data_tree_navigator.go b/pkg/yqlib/data_tree_navigator.go index 99bece43..1ffe7370 100644 --- a/pkg/yqlib/data_tree_navigator.go +++ b/pkg/yqlib/data_tree_navigator.go @@ -13,6 +13,7 @@ type DataTreeNavigator interface { GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) DeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error + DeeplyAssignKey(context Context, path []interface{}, rhsNode *CandidateNode) error } type dataTreeNavigator struct { @@ -23,6 +24,14 @@ func NewDataTreeNavigator() DataTreeNavigator { } func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error { + return d.deeplyAssign(context, path, rhsCandidateNode, false) +} + +func (d *dataTreeNavigator) DeeplyAssignKey(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error { + return d.deeplyAssign(context, path, rhsCandidateNode, true) +} + +func (d *dataTreeNavigator) deeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode, targetKey bool) error { assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}} @@ -41,7 +50,7 @@ func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rh assignmentOpNode := &ExpressionNode{ Operation: assignmentOp, - LHS: createTraversalTree(path, traversePreferences{}, false), + LHS: createTraversalTree(path, traversePreferences{}, targetKey), RHS: &ExpressionNode{Operation: rhsOp}, } diff --git a/pkg/yqlib/decoder_base64.go b/pkg/yqlib/decoder_base64.go index 50ed3507..00495f64 100644 --- a/pkg/yqlib/decoder_base64.go +++ b/pkg/yqlib/decoder_base64.go @@ -68,5 +68,5 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) { } } dec.readAnything = true - return createStringScalarNode(buf.String()), nil + return CreateStringScalarNode(buf.String()), nil } diff --git a/pkg/yqlib/decoder_csv_object.go b/pkg/yqlib/decoder_csv_object.go index 6273fcbb..d1665b9d 100644 --- a/pkg/yqlib/decoder_csv_object.go +++ b/pkg/yqlib/decoder_csv_object.go @@ -32,7 +32,7 @@ func (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode { // if we're not auto-parsing, then we wont put in parsed objects or arrays // but we still parse scalars if err != nil || (!dec.prefs.AutoParse && node.Kind != ScalarNode) { - return createScalarNode(content, content) + return CreateScalarNode(content, content) } return node } @@ -41,7 +41,7 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"} for i, header := range headerRow { - objectNode.AddKeyValueChild(createScalarNode(header, header), dec.convertToNode(contentRow[i])) + objectNode.AddKeyValueChild(CreateScalarNode(header, header), dec.convertToNode(contentRow[i])) } return objectNode } diff --git a/pkg/yqlib/decoder_toml.go b/pkg/yqlib/decoder_toml.go index d8a6264d..7e06e945 100644 --- a/pkg/yqlib/decoder_toml.go +++ b/pkg/yqlib/decoder_toml.go @@ -145,30 +145,30 @@ func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error) func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) - return createScalarNode(content, content), nil + return CreateScalarNode(content, content), nil } func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) - return createScalarNode(content == "true", content), nil + return CreateScalarNode(content == "true", content), nil } func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) _, num, err := parseInt64(content) - return createScalarNode(num, content), err + return CreateScalarNode(num, content), err } func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) val, err := parseDateTime(time.RFC3339, content) - return createScalarNode(val, content), err + return CreateScalarNode(val, content), err } func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) num, err := strconv.ParseFloat(content, 64) - return createScalarNode(num, content), err + return CreateScalarNode(num, content), err } func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) { diff --git a/pkg/yqlib/decoder_uri.go b/pkg/yqlib/decoder_uri.go index cd08d23c..321f0416 100644 --- a/pkg/yqlib/decoder_uri.go +++ b/pkg/yqlib/decoder_uri.go @@ -48,5 +48,5 @@ func (dec *uriDecoder) Decode() (*CandidateNode, error) { return nil, err } dec.readAnything = true - return createStringScalarNode(newValue), nil + return CreateStringScalarNode(newValue), nil } diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go index c17f9029..89a2cac1 100644 --- a/pkg/yqlib/decoder_yaml.go +++ b/pkg/yqlib/decoder_yaml.go @@ -151,7 +151,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) { } func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode { - node := createScalarNode(nil, "") + node := CreateScalarNode(nil, "") node.LeadingContent = dec.leadingContent return node } diff --git a/pkg/yqlib/encoder.go b/pkg/yqlib/encoder.go index ee63fd47..716d35b4 100644 --- a/pkg/yqlib/encoder.go +++ b/pkg/yqlib/encoder.go @@ -10,18 +10,3 @@ type Encoder interface { PrintLeadingContent(writer io.Writer, content string) error CanHandleAliases() bool } - -func mapKeysToStrings(node *CandidateNode) { - - if node.Kind == MappingNode { - for index, child := range node.Content { - if index%2 == 0 { // its a map key - child.Tag = "!!str" - } - } - } - - for _, child := range node.Content { - mapKeysToStrings(child) - } -} diff --git a/pkg/yqlib/encoder_csv.go b/pkg/yqlib/encoder_csv.go index 1fa4354c..5bd1249d 100644 --- a/pkg/yqlib/encoder_csv.go +++ b/pkg/yqlib/encoder_csv.go @@ -65,7 +65,7 @@ func (e *csvEncoder) createChildRow(child *CandidateNode, headers []*CandidateNo childRow := make([]*CandidateNode, 0) for _, header := range headers { keyIndex := findKeyInMap(child, header) - value := createScalarNode(nil, "") + value := CreateScalarNode(nil, "") if keyIndex != -1 { value = child.Content[keyIndex+1] } @@ -102,7 +102,7 @@ func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*CandidateNo func (e *csvEncoder) Encode(writer io.Writer, node *CandidateNode) error { if node.Kind == ScalarNode { - return writeString(writer, node.Value+"\n") + return WriteString(writer, node.Value+"\n") } csvWriter := csv.NewWriter(writer) diff --git a/pkg/yqlib/encoder_json.go b/pkg/yqlib/encoder_json.go index 54407cd0..b4b86d07 100644 --- a/pkg/yqlib/encoder_json.go +++ b/pkg/yqlib/encoder_json.go @@ -41,7 +41,7 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error { log.Debugf("kids %v", len(node.Content)) if node.Kind == ScalarNode && je.prefs.UnwrapScalar { - return writeString(writer, node.Value+"\n") + return WriteString(writer, node.Value+"\n") } destination := writer diff --git a/pkg/yqlib/encoder_lua.go b/pkg/yqlib/encoder_lua.go index 6fa85a57..8e75e76a 100644 --- a/pkg/yqlib/encoder_lua.go +++ b/pkg/yqlib/encoder_lua.go @@ -86,15 +86,15 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error case LiteralStyle, FoldedStyle, FlowStyle: for i := 0; i < 10; i++ { if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") { - err := writeString(writer, "["+strings.Repeat("=", i)+"[\n") + err := WriteString(writer, "["+strings.Repeat("=", i)+"[\n") if err != nil { return err } - err = writeString(writer, node.Value) + err = WriteString(writer, node.Value) if err != nil { return err } - return writeString(writer, "]"+strings.Repeat("=", i)+"]") + return WriteString(writer, "]"+strings.Repeat("=", i)+"]") } } case SingleQuotedStyle: @@ -102,22 +102,22 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error // fallthrough to regular ol' string } - return writeString(writer, quote+le.escape.Replace(node.Value)+quote) + return WriteString(writer, quote+le.escape.Replace(node.Value)+quote) } func (le *luaEncoder) writeIndent(writer io.Writer) error { if le.indentStr == "" { return nil } - err := writeString(writer, "\n") + err := WriteString(writer, "\n") if err != nil { return err } - return writeString(writer, strings.Repeat(le.indentStr, le.indent)) + return WriteString(writer, strings.Repeat(le.indentStr, le.indent)) } func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error { - err := writeString(writer, "{") + err := WriteString(writer, "{") if err != nil { return err } @@ -131,13 +131,13 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error { if err != nil { return err } - err = writeString(writer, ",") + err = WriteString(writer, ",") if err != nil { return err } if child.LineComment != "" { sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") - err = writeString(writer, " --"+sansPrefix) + err = WriteString(writer, " --"+sansPrefix) if err != nil { return err } @@ -150,7 +150,7 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error { return err } } - return writeString(writer, "}") + return WriteString(writer, "}") } func needsQuoting(s string) bool { @@ -181,7 +181,7 @@ func needsQuoting(s string) bool { func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bool) error { if !global { - err := writeString(writer, "{") + err := WriteString(writer, "{") if err != nil { return err } @@ -194,7 +194,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo if err != nil { return err } - err = writeString(writer, ";") + err = WriteString(writer, ";") if err != nil { return err } @@ -207,19 +207,19 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo } } if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) { - err := writeString(writer, child.Value+" = ") + err := WriteString(writer, child.Value+" = ") if err != nil { return err } } else { if global { // This only works in Lua 5.2+ - err := writeString(writer, "_ENV") + err := WriteString(writer, "_ENV") if err != nil { return err } } - err := writeString(writer, "[") + err := WriteString(writer, "[") if err != nil { return err } @@ -227,7 +227,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo if err != nil { return err } - err = writeString(writer, "] = ") + err = WriteString(writer, "] = ") if err != nil { return err } @@ -235,7 +235,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo } if child.LineComment != "" { sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") - err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix) + err := WriteString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix) if err != nil { return err } @@ -249,7 +249,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo } } if global { - return writeString(writer, "\n") + return WriteString(writer, "\n") } le.indent-- if len(node.Content) != 0 { @@ -258,7 +258,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo return err } } - return writeString(writer, "}") + return WriteString(writer, "}") } func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error { @@ -273,30 +273,30 @@ func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error { return le.encodeString(writer, node) case "!!null": // TODO reject invalid use as a table key - return writeString(writer, "nil") + return WriteString(writer, "nil") case "!!bool": // Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has // lower case - return writeString(writer, strings.ToLower(node.Value)) + return WriteString(writer, strings.ToLower(node.Value)) case "!!int": if strings.HasPrefix(node.Value, "0o") { _, octalValue, err := parseInt64(node.Value) if err != nil { return err } - return writeString(writer, fmt.Sprintf("%d", octalValue)) + return WriteString(writer, fmt.Sprintf("%d", octalValue)) } - return writeString(writer, strings.ToLower(node.Value)) + return WriteString(writer, strings.ToLower(node.Value)) case "!!float": switch strings.ToLower(node.Value) { case ".inf", "+.inf": - return writeString(writer, "(1/0)") + return WriteString(writer, "(1/0)") case "-.inf": - return writeString(writer, "(-1/0)") + return WriteString(writer, "(-1/0)") case ".nan": - return writeString(writer, "(0/0)") + return WriteString(writer, "(0/0)") default: - return writeString(writer, node.Value) + return WriteString(writer, node.Value) } default: return fmt.Errorf("Lua encoder NYI -- %s", node.Tag) @@ -307,7 +307,7 @@ func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error { } func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) error { - err := writeString(writer, le.docPrefix) + err := WriteString(writer, le.docPrefix) if err != nil { return err } @@ -315,7 +315,7 @@ func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) erro if err != nil { return err } - return writeString(writer, le.docSuffix) + return WriteString(writer, le.docSuffix) } func (le *luaEncoder) Encode(writer io.Writer, node *CandidateNode) error { diff --git a/pkg/yqlib/encoder_sh.go b/pkg/yqlib/encoder_sh.go index 1d993148..9cadef2b 100644 --- a/pkg/yqlib/encoder_sh.go +++ b/pkg/yqlib/encoder_sh.go @@ -34,7 +34,7 @@ func (e *shEncoder) Encode(writer io.Writer, node *CandidateNode) error { return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag) } - return writeString(writer, e.encode(node.Value)) + return WriteString(writer, e.encode(node.Value)) } // put any (shell-unsafe) characters into a single-quoted block, close the block lazily diff --git a/pkg/yqlib/encoder_shellvariables.go b/pkg/yqlib/encoder_shellvariables.go index d725dae3..6e868361 100644 --- a/pkg/yqlib/encoder_shellvariables.go +++ b/pkg/yqlib/encoder_shellvariables.go @@ -30,7 +30,7 @@ func (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) erro func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *CandidateNode) error { - mapKeysToStrings(node) + node.ConvertKeysToStrings() err := pe.doEncode(&writer, node, "") if err != nil { return err diff --git a/pkg/yqlib/encoder_toml.go b/pkg/yqlib/encoder_toml.go index f8ced30f..e7077556 100644 --- a/pkg/yqlib/encoder_toml.go +++ b/pkg/yqlib/encoder_toml.go @@ -14,7 +14,7 @@ func NewTomlEncoder() Encoder { func (te *tomlEncoder) Encode(writer io.Writer, node *CandidateNode) error { if node.Kind == ScalarNode { - return writeString(writer, node.Value+"\n") + return WriteString(writer, node.Value+"\n") } return fmt.Errorf("only scalars (e.g. strings, numbers, booleans) are supported for TOML output at the moment. Please use yaml output format (-oy) until the encoder has been fully implemented") } diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go index 5cce044c..769d70d7 100644 --- a/pkg/yqlib/encoder_yaml.go +++ b/pkg/yqlib/encoder_yaml.go @@ -26,7 +26,7 @@ func (ye *yamlEncoder) CanHandleAliases() bool { func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error { if ye.prefs.PrintDocSeparators { log.Debug("writing doc sep") - if err := writeString(writer, "---\n"); err != nil { + if err := WriteString(writer, "---\n"); err != nil { return err } } @@ -54,7 +54,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) { readline = "# " + readline } - if err := writeString(writer, readline); err != nil { + if err := WriteString(writer, readline); err != nil { return err } } @@ -62,7 +62,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err if errors.Is(errReading, io.EOF) { if readline != "" { // the last comment we read didn't have a newline, put one in - if err := writeString(writer, "\n"); err != nil { + if err := WriteString(writer, "\n"); err != nil { return err } } @@ -80,7 +80,7 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { if node.LeadingContent == "" || valueToPrint != "" { valueToPrint = valueToPrint + "\n" } - return writeString(writer, valueToPrint) + return WriteString(writer, valueToPrint) } destination := writer diff --git a/pkg/yqlib/expression_postfix.go b/pkg/yqlib/expression_postfix.go index 45d26859..fa7afa33 100644 --- a/pkg/yqlib/expression_postfix.go +++ b/pkg/yqlib/expression_postfix.go @@ -1,4 +1,4 @@ -package exp_parser +package yqlib import ( "errors" diff --git a/pkg/yqlib/format.go b/pkg/yqlib/format.go index 3f9a7d41..11b927af 100644 --- a/pkg/yqlib/format.go +++ b/pkg/yqlib/format.go @@ -6,63 +6,93 @@ import ( ) type EncoderFactoryFunction func() Encoder +type InDocumentEncoderFactoryFunction func(indent int) Encoder type DecoderFactoryFunction func() Decoder type Format struct { - FormalName string - Names []string - EncoderFactory EncoderFactoryFunction - DecoderFactory DecoderFactoryFunction + FormalName string + Names []string + DefaultExtension string + EncoderFactory EncoderFactoryFunction + DecoderFactory DecoderFactoryFunction + + /** + * Like the Encoder Factory, but for encoding content within the document itself. + * Should turn off colors and other settings to ensure the content comes out right. + * If this function is not configured, it will default to the EncoderFactory. + **/ + InDocumentEncoderFactory InDocumentEncoderFactoryFunction } -var YamlFormat = &Format{"yaml", []string{"y", "yml"}, +var YamlFormat = &Format{"yaml", []string{"y", "yml"}, "yml", func() Encoder { return NewYamlEncoder(ConfiguredYamlPreferences) }, func() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) }, + func(indent int) Encoder { + prefs := ConfiguredYamlPreferences.Copy() + prefs.Indent = indent + prefs.ColorsEnabled = false + return NewYamlEncoder(prefs) + }, } -var JSONFormat = &Format{"json", []string{"j"}, +var JSONFormat = &Format{"json", []string{"j"}, "json", func() Encoder { return NewJSONEncoder(ConfiguredJSONPreferences) }, func() Decoder { return NewJSONDecoder() }, + func(indent int) Encoder { + prefs := ConfiguredJSONPreferences.Copy() + prefs.Indent = indent + prefs.ColorsEnabled = false + prefs.UnwrapScalar = false + return NewJSONEncoder(prefs) + }, } -var CSVFormat = &Format{"csv", []string{"c"}, +var CSVFormat = &Format{"csv", []string{"c"}, "csv", func() Encoder { return NewCsvEncoder(ConfiguredCsvPreferences) }, func() Decoder { return NewCSVObjectDecoder(ConfiguredCsvPreferences) }, + nil, } -var TSVFormat = &Format{"tsv", []string{"t"}, +var TSVFormat = &Format{"tsv", []string{"t"}, "tsv", func() Encoder { return NewCsvEncoder(ConfiguredTsvPreferences) }, func() Decoder { return NewCSVObjectDecoder(ConfiguredTsvPreferences) }, + nil, } -var Base64Format = &Format{"base64", []string{}, +var Base64Format = &Format{"base64", []string{}, "txt", func() Encoder { return NewBase64Encoder() }, func() Decoder { return NewBase64Decoder() }, + nil, } -var UriFormat = &Format{"uri", []string{}, +var UriFormat = &Format{"uri", []string{}, "txt", func() Encoder { return NewUriEncoder() }, func() Decoder { return NewUriDecoder() }, + nil, } -var ShFormat = &Format{"", nil, +var ShFormat = &Format{"", nil, "sh", func() Encoder { return NewShEncoder() }, nil, -} - -var TomlFormat = &Format{"toml", []string{}, - func() Encoder { return NewTomlEncoder() }, - func() Decoder { return NewTomlDecoder() }, -} - -var ShellVariablesFormat = &Format{"shell", []string{"s", "sh"}, - func() Encoder { return NewShellVariablesEncoder() }, nil, } -var LuaFormat = &Format{"lua", []string{"l"}, +var TomlFormat = &Format{"toml", []string{}, "toml", + func() Encoder { return NewTomlEncoder() }, + func() Decoder { return NewTomlDecoder() }, + nil, +} + +var ShellVariablesFormat = &Format{"shell", []string{"s", "sh"}, "sh", + func() Encoder { return NewShellVariablesEncoder() }, + nil, + nil, +} + +var LuaFormat = &Format{"lua", []string{"l"}, "lua", func() Encoder { return NewLuaEncoder(ConfiguredLuaPreferences) }, func() Decoder { return NewLuaDecoder(ConfiguredLuaPreferences) }, + nil, } var Formats = []*Format{ @@ -94,7 +124,10 @@ func (f *Format) MatchesName(name string) bool { return false } -func (f *Format) GetConfiguredEncoder() Encoder { +func (f *Format) GetInDocumentEncoder(indent int) Encoder { + if f.InDocumentEncoderFactory != nil { + return f.InDocumentEncoderFactory(indent) + } return f.EncoderFactory() } diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 2025a576..85a4d4f6 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -1,4 +1,4 @@ -package exp_parser +package yqlib import ( "strconv" @@ -57,34 +57,34 @@ var participleYqRules = []*ParticipleYqRule{ {"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0}, - {"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlFormat), 0}, + {"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, CreateEncodeYqActionParsingIndent(YamlFormat), 0}, - {"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONFormat), 0}, + {"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, CreateEncodeYqActionParsingIndent(JSONFormat), 0}, - {"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlFormat), 0}, - {"YamlEncode", `to_?yaml|@yaml`, encodeWithIndent(YamlFormat, 2), 0}, + {"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, CreateDecodeOpYqAction(YamlFormat), 0}, + {"YamlEncode", `to_?yaml|@yaml`, CreateEncodeOpYqAction(YamlFormat, 2), 0}, - {"JSONEncode", `to_?json`, encodeWithIndent(JSONFormat, 2), 0}, - {"JSONEncodeNoIndent", `@json`, encodeWithIndent(JSONFormat, 0), 0}, + {"JSONEncode", `to_?json`, CreateEncodeOpYqAction(JSONFormat, 2), 0}, + {"JSONEncodeNoIndent", `@json`, CreateEncodeOpYqAction(JSONFormat, 0), 0}, - {"CSVDecode", `from_?csv|@csvd`, decodeOp(CSVFormat), 0}, - {"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVFormat, 0), 0}, + {"CSVDecode", `from_?csv|@csvd`, CreateDecodeOpYqAction(CSVFormat), 0}, + {"CSVEncode", `to_?csv|@csv`, CreateEncodeOpYqAction(CSVFormat, 0), 0}, - {"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVFormat), 0}, - {"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVFormat, 0), 0}, + {"TSVDecode", `from_?tsv|@tsvd`, CreateDecodeOpYqAction(TSVFormat), 0}, + {"TSVEncode", `to_?tsv|@tsv`, CreateEncodeOpYqAction(TSVFormat, 0), 0}, - {"Base64d", `@base64d`, decodeOp(Base64Format), 0}, - {"Base64", `@base64`, encodeWithIndent(Base64Format, 0), 0}, + {"Base64d", `@base64d`, CreateDecodeOpYqAction(Base64Format), 0}, + {"Base64", `@base64`, CreateEncodeOpYqAction(Base64Format, 0), 0}, - {"Urid", `@urid`, decodeOp(UriFormat), 0}, - {"Uri", `@uri`, encodeWithIndent(UriFormat, 0), 0}, - {"SH", `@sh`, encodeWithIndent(ShFormat, 0), 0}, + {"Urid", `@urid`, CreateDecodeOpYqAction(UriFormat), 0}, + {"Uri", `@uri`, CreateEncodeOpYqAction(UriFormat, 0), 0}, + {"SH", `@sh`, CreateEncodeOpYqAction(ShFormat, 0), 0}, - {"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0}, + {"LoadBase64", `load_?base64`, CreateLoadOpYqAction(NewBase64Decoder()), 0}, - {"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0}, + simpleOp(`load_?str|str_?load`, loadStringOpType), - {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences), false), 0}, + {"LoadYaml", `load`, CreateLoadOpYqAction(NewYamlDecoder(LoadYamlPreferences)), 0}, {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, @@ -518,7 +518,7 @@ func parentWithDefaultLevel() yqAction { } } -func encodeParseIndent(outputFormat *Format) yqAction { +func CreateEncodeYqActionParsingIndent(outputFormat *Format) yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value var indent, errParsingInt = extractNumberParameter(value) @@ -532,17 +532,17 @@ func encodeParseIndent(outputFormat *Format) yqAction { } } -func encodeWithIndent(outputFormat *Format, indent int) yqAction { +func CreateEncodeOpYqAction(outputFormat *Format, indent int) yqAction { prefs := encoderPreferences{format: outputFormat, indent: indent} return opTokenWithPrefs(encodeOpType, nil, prefs) } -func decodeOp(format *Format) yqAction { +func CreateDecodeOpYqAction(format *Format) yqAction { prefs := decoderPreferences{format: format} return opTokenWithPrefs(decodeOpType, nil, prefs) } -func loadOp(decoder Decoder) yqAction { +func CreateLoadOpYqAction(decoder Decoder) yqAction { prefs := loadPrefs{decoder} return opTokenWithPrefs(loadOpType, nil, prefs) } diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 23f6a1f8..3230ee49 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -98,7 +98,7 @@ func parseSnippet(value string) (*CandidateNode, error) { if result.Tag == "!!str" { // use the original string value, as // decoding drops new lines - return createScalarNode(value, value), nil + return CreateScalarNode(value, value), nil } result.Line = 0 result.Column = 0 @@ -161,22 +161,6 @@ func parseInt(numberString string) (int, error) { return int(parsed), err } -func headAndLineComment(node *CandidateNode) string { - return headComment(node) + lineComment(node) -} - -func headComment(node *CandidateNode) string { - return strings.Replace(node.HeadComment, "#", "", 1) -} - -func lineComment(node *CandidateNode) string { - return strings.Replace(node.LineComment, "#", "", 1) -} - -func footComment(node *CandidateNode) string { - return strings.Replace(node.FootComment, "#", "", 1) -} - // use for debugging only func NodesToString(collection *list.List) string { if !log.IsEnabledFor(logging.DEBUG) { diff --git a/pkg/yqlib/operation.go b/pkg/yqlib/operation.go index ea29e10d..6c24dc5a 100644 --- a/pkg/yqlib/operation.go +++ b/pkg/yqlib/operation.go @@ -26,7 +26,7 @@ var valueToStringFunc = func(p *Operation) string { func createValueOperation(value interface{}, stringValue string) *Operation { log.Debug("creating value op for string %v", stringValue) - var node = createScalarNode(value, stringValue) + var node = CreateScalarNode(value, stringValue) return &Operation{ OperationType: valueOpType, diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index 35336573..b0f1c4f1 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -9,33 +9,11 @@ import ( "strings" ) -func configureEncoder(format *Format, indent int) Encoder { - - switch format { - case JSONFormat: - prefs := ConfiguredJSONPreferences.Copy() - prefs.Indent = indent - prefs.ColorsEnabled = false - prefs.UnwrapScalar = false - return NewJSONEncoder(prefs) - case YamlFormat: - var prefs = ConfiguredYamlPreferences.Copy() - prefs.Indent = indent - prefs.ColorsEnabled = false - return NewYamlEncoder(prefs) - case XMLFormat: - var xmlPrefs = ConfiguredXMLPreferences.Copy() - xmlPrefs.Indent = indent - return NewXMLEncoder(xmlPrefs) - } - return format.EncoderFactory() -} - func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) { var output bytes.Buffer log.Debug("printing with indent: %v", prefs.indent) - encoder := configureEncoder(prefs.format, prefs.indent) + encoder := prefs.format.GetInDocumentEncoder(prefs.indent) if encoder == nil { return "", errors.New("no support for output format") } diff --git a/pkg/yqlib/operator_entries.go b/pkg/yqlib/operator_entries.go index b4ccb907..80af55a1 100644 --- a/pkg/yqlib/operator_entries.go +++ b/pkg/yqlib/operator_entries.go @@ -64,7 +64,7 @@ func toEntriesOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) func parseEntry(candidateNode *CandidateNode, position int) (*CandidateNode, *CandidateNode, error) { prefs := traversePreferences{DontAutoCreate: true} - keyResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("key"), prefs, false) + keyResults, err := traverseMap(Context{}, candidateNode, CreateStringScalarNode("key"), prefs, false) if err != nil { return nil, nil, err @@ -72,7 +72,7 @@ func parseEntry(candidateNode *CandidateNode, position int) (*CandidateNode, *Ca return nil, nil, fmt.Errorf("expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position) } - valueResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("value"), prefs, false) + valueResults, err := traverseMap(Context{}, candidateNode, CreateStringScalarNode("value"), prefs, false) if err != nil { return nil, nil, err diff --git a/pkg/yqlib/operator_strings.go b/pkg/yqlib/operator_strings.go index c90a78b9..8e52440a 100644 --- a/pkg/yqlib/operator_strings.go +++ b/pkg/yqlib/operator_strings.go @@ -98,7 +98,7 @@ func interpolate(d *dataTreeNavigator, context Context, str string) (string, err func stringInterpolationOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if !StringInterpolationEnabled { return context.SingleChildContext( - createScalarNode(expressionNode.Operation.StringValue, expressionNode.Operation.StringValue), + CreateScalarNode(expressionNode.Operation.StringValue, expressionNode.Operation.StringValue), ), nil } if context.MatchingNodes.Len() == 0 { @@ -106,7 +106,7 @@ func stringInterpolationOperator(d *dataTreeNavigator, context Context, expressi if err != nil { return Context{}, err } - node := createScalarNode(value, value) + node := CreateScalarNode(value, value) return context.SingleChildContext(node), nil } @@ -118,7 +118,7 @@ func stringInterpolationOperator(d *dataTreeNavigator, context Context, expressi if err != nil { return Context{}, err } - node := createScalarNode(value, value) + node := CreateScalarNode(value, value) results.PushBack(node) } @@ -261,29 +261,29 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN func addMatch(original []*CandidateNode, match string, offset int, name string) []*CandidateNode { newContent := append(original, - createScalarNode("string", "string")) + CreateScalarNode("string", "string")) if offset < 0 { // offset of -1 means there was no match, force a null value like jq newContent = append(newContent, - createScalarNode(nil, "null"), + CreateScalarNode(nil, "null"), ) } else { newContent = append(newContent, - createScalarNode(match, match), + CreateScalarNode(match, match), ) } newContent = append(newContent, - createScalarNode("offset", "offset"), - createScalarNode(offset, fmt.Sprintf("%v", offset)), - createScalarNode("length", "length"), - createScalarNode(len(match), fmt.Sprintf("%v", len(match)))) + CreateScalarNode("offset", "offset"), + CreateScalarNode(offset, fmt.Sprintf("%v", offset)), + CreateScalarNode("length", "length"), + CreateScalarNode(len(match), fmt.Sprintf("%v", len(match)))) if name != "" { newContent = append(newContent, - createScalarNode("name", "name"), - createScalarNode(name, name), + CreateScalarNode("name", "name"), + CreateScalarNode(name, name), ) } return newContent @@ -330,7 +330,7 @@ func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *Candida node := candidate.CreateReplacement(MappingNode, "!!map", "") node.AddChildren(addMatch(node.Content, match, allIndices[i][0], "")) - node.AddKeyValueChild(createScalarNode("captures", "captures"), capturesListNode) + node.AddKeyValueChild(CreateScalarNode("captures", "captures"), capturesListNode) results.PushBack(node) } @@ -353,15 +353,15 @@ func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *Candi _, submatches := matches[0], matches[1:] for j, submatch := range submatches { - keyNode := createScalarNode(subNames[j+1], subNames[j+1]) + keyNode := CreateScalarNode(subNames[j+1], subNames[j+1]) var valueNode *CandidateNode offset := allIndices[i][2+j*2] // offset of -1 means there was no match, force a null value like jq if offset < 0 { - valueNode = createScalarNode(nil, "null") + valueNode = CreateScalarNode(nil, "null") } else { - valueNode = createScalarNode(submatch, submatch) + valueNode = CreateScalarNode(submatch, submatch) } capturesNode.AddKeyValueChild(keyNode, valueNode) } diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index aef22ce7..8c90372a 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -54,7 +54,7 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation switch matchingNode.Kind { case MappingNode: log.Debug("its a map with %v entries", len(matchingNode.Content)/2) - return traverseMap(context, matchingNode, createStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false) + return traverseMap(context, matchingNode, CreateStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false) case SequenceNode: log.Debug("its a sequence of %v things!", len(matchingNode.Content)) @@ -149,7 +149,7 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT func traverseMapWithIndices(context Context, candidate *CandidateNode, indices []*CandidateNode, prefs traversePreferences) (*list.List, error) { if len(indices) == 0 { - return traverseMap(context, candidate, createStringScalarNode(""), prefs, true) + return traverseMap(context, candidate, CreateStringScalarNode(""), prefs, true) } var matchingNodeMap = list.New() @@ -196,7 +196,7 @@ func traverseArrayWithIndices(node *CandidateNode, indices []*CandidateNode, pre node.Style = 0 } - valueNode := createScalarNode(nil, "null") + valueNode := CreateScalarNode(nil, "null") node.AddChild(valueNode) contentLength = len(node.Content) } diff --git a/pkg/yqlib/printer_writer.go b/pkg/yqlib/printer_writer.go index b3e89466..114ef456 100644 --- a/pkg/yqlib/printer_writer.go +++ b/pkg/yqlib/printer_writer.go @@ -34,14 +34,7 @@ type multiPrintWriter struct { } func NewMultiPrinterWriter(expression *ExpressionNode, format *Format) PrinterWriter { - extension := "yml" - - switch format { - case JSONFormat: - extension = "json" - case PropertiesFormat: - extension = "properties" - } + extension := format.DefaultExtension return &multiPrintWriter{ nameExpression: expression, diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index 0c9ad6dd..9b5bcf9c 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -31,7 +31,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error if err != nil { return err } - candidateNode := createScalarNode(nil, "") + candidateNode := CreateScalarNode(nil, "") inputList := list.New() inputList.PushBack(candidateNode) diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 4424582a..a57f64dc 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -26,7 +26,7 @@ func readStream(filename string) (io.Reader, error) { } -func writeString(writer io.Writer, txt string) error { +func WriteString(writer io.Writer, txt string) error { _, errorWriting := writer.Write([]byte(txt)) return errorWriting } diff --git a/yq.go b/yq.go index d6394a8e..a90b4073 100644 --- a/yq.go +++ b/yq.go @@ -4,11 +4,13 @@ import ( "os" command "github.com/mikefarah/yq/v4/cmd" - "github.com/mikefarah/yq/v4/pkg/yqlib" + "github.com/mikefarah/yq/v4/pkg/properties" + "github.com/mikefarah/yq/v4/pkg/xml" ) func main() { - yqlib.RegisterPropertiesFormat() + properties.RegisterPropertiesFormat() + xml.RegisterXmlFormat() cmd := command.New()