diff --git a/cmd/constant.go b/cmd/constant.go index ef0c31db..a3bec40a 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -1,6 +1,5 @@ package cmd -var leadingContentPreProcessing = true var unwrapScalar = true var writeInplace = false diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index d6dc993f..fc761fdc 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -109,13 +109,13 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) { switch len(args) { case 0: if nullInput { - err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer, "") + err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer) } else { cmd.Println(cmd.UsageString()) return nil } default: - err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder) + err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder) } completedSuccessfully = err == nil diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index cc9ab672..3a352c18 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -123,13 +123,13 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) { switch len(args) { case 0: if nullInput { - err = streamEvaluator.EvaluateNew(processExpression(expression), printer, "") + err = streamEvaluator.EvaluateNew(processExpression(expression), printer) } else { cmd.Println(cmd.UsageString()) return nil } default: - err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder) + err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder) } completedSuccessfully = err == nil diff --git a/cmd/root.go b/cmd/root.go index 058dfd7d..24ef066d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,6 +59,11 @@ yq -P sample.json "naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " + "`+` please set that value explicityly with --xml-attribute-prefix.") } + + //copy preference form global setting + yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar + + yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators }, } @@ -97,7 +102,7 @@ yq -P sample.json rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors") rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact") rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.") - rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.") rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.") rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.") diff --git a/cmd/utils.go b/cmd/utils.go index 007ada69..0d303ff5 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -74,7 +74,7 @@ func configureDecoder() (yqlib.Decoder, error) { return yqlib.NewCSVObjectDecoder('\t'), nil } - return yqlib.NewYamlDecoder(), nil + return yqlib.NewYamlDecoder(yqlib.ConfiguredYamlPreferences), nil } func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) { @@ -105,7 +105,7 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder { case yqlib.TSVOutputFormat: return yqlib.NewCsvEncoder('\t') case yqlib.YamlOutputFormat: - return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar) + return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences) case yqlib.XMLOutputFormat: return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences) } diff --git a/examples/small.properties b/examples/small.properties index 7a75a2e5..fbb6df9f 100644 --- a/examples/small.properties +++ b/examples/small.properties @@ -1 +1,2 @@ +# great huh this.is = a properties file \ No newline at end of file diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index 066edd0d..4dbed3d1 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -8,7 +8,7 @@ import ( // A yaml expression evaluator that runs the expression once against all files/nodes in memory. type Evaluator interface { - EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error + EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error // EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) @@ -46,21 +46,16 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand return context.MatchingNodes, nil } -func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error { +func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error { fileIndex := 0 - firstFileLeadingContent := "" var allDocuments = list.New() for _, filename := range filenames { - reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing) + reader, err := readStream(filename) if err != nil { return err } - if fileIndex == 0 { - firstFileLeadingContent = leadingContent - } - fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder) if err != nil { return err @@ -75,11 +70,9 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string Filename: "", Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, FileIndex: 0, - LeadingContent: firstFileLeadingContent, + LeadingContent: "", } allDocuments.PushBack(candidateNode) - } else { - allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent } matches, err := e.EvaluateCandidateNodes(expression, allDocuments) diff --git a/pkg/yqlib/csv_test.go b/pkg/yqlib/csv_test.go index d58b43e2..4ab3306d 100644 --- a/pkg/yqlib/csv_test.go +++ b/pkg/yqlib/csv_test.go @@ -136,13 +136,13 @@ var csvScenarios = []formatScenario{ func testCSVScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "encode-csv": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description) case "encode-tsv": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description) case "decode-csv-object": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "decode-tsv-object": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "roundtrip-csv": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description) default: @@ -171,7 +171,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy } writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", - processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))), + processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))), ) } @@ -203,7 +203,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str } writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType, - processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))), + processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))), ) } diff --git a/pkg/yqlib/decoder_base64.go b/pkg/yqlib/decoder_base64.go index 6ec507cb..c4868fd2 100644 --- a/pkg/yqlib/decoder_base64.go +++ b/pkg/yqlib/decoder_base64.go @@ -19,21 +19,22 @@ func NewBase64Decoder() Decoder { return &base64Decoder{finished: false, encoding: *base64.StdEncoding} } -func (dec *base64Decoder) Init(reader io.Reader) { +func (dec *base64Decoder) Init(reader io.Reader) error { dec.reader = reader dec.readAnything = false dec.finished = false + return nil } -func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *base64Decoder) Decode() (*CandidateNode, error) { if dec.finished { - return io.EOF + return nil, io.EOF } base64Reader := base64.NewDecoder(&dec.encoding, dec.reader) buf := new(bytes.Buffer) if _, err := buf.ReadFrom(base64Reader); err != nil { - return err + return nil, err } if buf.Len() == 0 { dec.finished = true @@ -42,12 +43,15 @@ func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error { // otherwise if we've already read some bytes, and now we get // an empty string, then we are done. if dec.readAnything { - return io.EOF + return nil, io.EOF } } dec.readAnything = true - rootYamlNode.Kind = yaml.ScalarNode - rootYamlNode.Tag = "!!str" - rootYamlNode.Value = buf.String() - return nil + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: buf.String(), + }, + }, nil } diff --git a/pkg/yqlib/decoder_csv_object.go b/pkg/yqlib/decoder_csv_object.go index c2b8f164..e8e78023 100644 --- a/pkg/yqlib/decoder_csv_object.go +++ b/pkg/yqlib/decoder_csv_object.go @@ -19,12 +19,13 @@ func NewCSVObjectDecoder(separator rune) Decoder { return &csvObjectDecoder{separator: separator} } -func (dec *csvObjectDecoder) Init(reader io.Reader) { +func (dec *csvObjectDecoder) Init(reader io.Reader) error { cleanReader, enc := utfbom.Skip(reader) log.Debugf("Detected encoding: %s\n", enc) dec.reader = *csv.NewReader(cleanReader) dec.reader.Comma = dec.separator dec.finished = false + return nil } func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node { @@ -47,14 +48,14 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin return objectNode } -func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) { if dec.finished { - return io.EOF + return nil, io.EOF } headerRow, err := dec.reader.Read() log.Debugf(": headerRow%v", headerRow) if err != nil { - return err + return nil, err } rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} @@ -68,13 +69,13 @@ func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error { log.Debugf("Read next contentRow: %v, %v", contentRow, err) } if !errors.Is(err, io.EOF) { - return err + return nil, err } - log.Debugf("finished, contentRow%v", contentRow) - log.Debugf("err: %v", err) - - rootYamlNode.Kind = yaml.DocumentNode - rootYamlNode.Content = []*yaml.Node{rootArray} - return nil + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{rootArray}, + }, + }, nil } diff --git a/pkg/yqlib/decoder_json.go b/pkg/yqlib/decoder_json.go index e4594357..448550af 100644 --- a/pkg/yqlib/decoder_json.go +++ b/pkg/yqlib/decoder_json.go @@ -16,26 +16,31 @@ func NewJSONDecoder() Decoder { return &jsonDecoder{} } -func (dec *jsonDecoder) Init(reader io.Reader) { +func (dec *jsonDecoder) Init(reader io.Reader) error { dec.decoder = *json.NewDecoder(reader) + return nil } -func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *jsonDecoder) Decode() (*CandidateNode, error) { var dataBucket orderedMap log.Debug("going to decode") err := dec.decoder.Decode(&dataBucket) if err != nil { - return err + return nil, err } node, err := dec.convertToYamlNode(&dataBucket) if err != nil { - return err + return nil, err } - rootYamlNode.Kind = yaml.DocumentNode - rootYamlNode.Content = []*yaml.Node{node} - return nil + + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{node}, + }, + }, nil } func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) { diff --git a/pkg/yqlib/decoder_properties.go b/pkg/yqlib/decoder_properties.go index 0a56d4fe..ae1b04e4 100644 --- a/pkg/yqlib/decoder_properties.go +++ b/pkg/yqlib/decoder_properties.go @@ -20,9 +20,10 @@ func NewPropertiesDecoder() Decoder { return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false} } -func (dec *propertiesDecoder) Init(reader io.Reader) { +func (dec *propertiesDecoder) Init(reader io.Reader) error { dec.reader = reader dec.finished = false + return nil } func parsePropKey(key string) []interface{} { @@ -78,22 +79,22 @@ func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, c return err } -func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *propertiesDecoder) Decode() (*CandidateNode, error) { if dec.finished { - return io.EOF + return nil, io.EOF } buf := new(bytes.Buffer) if _, err := buf.ReadFrom(dec.reader); err != nil { - return err + return nil, err } if buf.Len() == 0 { dec.finished = true - return io.EOF + return nil, io.EOF } properties, err := properties.LoadString(buf.String()) if err != nil { - return err + return nil, err } properties.DisableExpansion = true @@ -109,14 +110,17 @@ func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error { for _, key := range properties.Keys() { if err := dec.applyProperty(properties, context, key); err != nil { - return err + return nil, err } } - - rootYamlNode.Kind = yaml.DocumentNode - rootYamlNode.Content = []*yaml.Node{rootMap.Node} dec.finished = true - return nil + + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{rootMap.Node}, + }, + }, nil } diff --git a/pkg/yqlib/decoder_test.go b/pkg/yqlib/decoder_test.go index 462eb4d1..ddd93ced 100644 --- a/pkg/yqlib/decoder_test.go +++ b/pkg/yqlib/decoder_test.go @@ -23,7 +23,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) s writer := bufio.NewWriter(&output) if decoder == nil { - decoder = NewYamlDecoder() + decoder = NewYamlDecoder(ConfiguredYamlPreferences) } inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder) diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go index 8a7e79f0..ced6c964 100644 --- a/pkg/yqlib/decoder_xml.go +++ b/pkg/yqlib/decoder_xml.go @@ -25,10 +25,11 @@ func NewXMLDecoder(prefs XmlPreferences) Decoder { } } -func (dec *xmlDecoder) Init(reader io.Reader) { +func (dec *xmlDecoder) Init(reader io.Reader) error { dec.reader = reader dec.readAnything = false dec.finished = false + return nil } func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) { @@ -118,32 +119,36 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) { return scalar, nil } -func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *xmlDecoder) Decode() (*CandidateNode, error) { if dec.finished { - return io.EOF + return nil, io.EOF } root := &xmlNode{} // cant use xj - it doesn't keep map order. err := dec.decodeXML(root) if err != nil { - return err + return nil, err } firstNode, err := dec.convertToYamlNode(root) if err != nil { - return err + return nil, err } else if firstNode.Tag == "!!null" { dec.finished = true if dec.readAnything { - return io.EOF + return nil, io.EOF } } dec.readAnything = true - rootYamlNode.Kind = yaml.DocumentNode - rootYamlNode.Content = []*yaml.Node{firstNode} dec.finished = true - return nil + + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{firstNode}, + }, + }, nil } type xmlNode struct { diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go index bf86a4e9..f18ac176 100644 --- a/pkg/yqlib/decoder_yaml.go +++ b/pkg/yqlib/decoder_yaml.go @@ -13,11 +13,11 @@ import ( type yamlDecoder struct { decoder yaml.Decoder // work around of various parsing issues by yaml.v3 with document headers - prefs yamlPreferences + prefs YamlPreferences leadingContent string } -func NewYamlDecoder(prefs yamlPreferences) Decoder { +func NewYamlDecoder(prefs YamlPreferences) Decoder { return &yamlDecoder{prefs: prefs} } @@ -57,7 +57,7 @@ func (dec *yamlDecoder) Init(reader io.Reader) error { readerToUse := reader leadingContent := "" var err error - if dec.leadingContentPreProcessing { + if dec.prefs.LeadingContentPreProcessing { readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader)) if err != nil { return err diff --git a/pkg/yqlib/doc/operators/comment-operators.md b/pkg/yqlib/doc/operators/comment-operators.md index e4dd23ed..9d1b9da2 100644 --- a/pkg/yqlib/doc/operators/comment-operators.md +++ b/pkg/yqlib/doc/operators/comment-operators.md @@ -246,6 +246,7 @@ Note the use of `...` to ensure key nodes are included. Given a sample.yml file of: ```yaml +# hi a: cat # comment # great b: # key comment @@ -263,6 +264,7 @@ b: ## Get line comment Given a sample.yml file of: ```yaml +# welcome! a: cat # meow # have a great day ``` diff --git a/pkg/yqlib/doc/usage/properties.md b/pkg/yqlib/doc/usage/properties.md index e7952fcf..e676ca1c 100644 --- a/pkg/yqlib/doc/usage/properties.md +++ b/pkg/yqlib/doc/usage/properties.md @@ -9,7 +9,7 @@ Note that empty arrays and maps are not encoded by default. Given a sample.yml file of: ```yaml -# block comments don't come through +# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -25,6 +25,7 @@ yq -o=props sample.yml ``` will output ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski @@ -38,7 +39,7 @@ Note that string values with blank characters in them are encapsulated with doub Given a sample.yml file of: ```yaml -# block comments don't come through +# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -54,6 +55,7 @@ yq -o=props --unwrapScalar=false sample.yml ``` will output ```properties +# block comments come through # comments on values appear person.name = "Mike Wazowski" @@ -65,7 +67,7 @@ person.food.0 = pizza ## Encode properties: no comments Given a sample.yml file of: ```yaml -# block comments don't come through +# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -91,7 +93,7 @@ Use a yq expression to set the empty maps and sequences to your desired value. Given a sample.yml file of: ```yaml -# block comments don't come through +# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -107,6 +109,7 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) = ``` will output ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski @@ -120,6 +123,7 @@ emptyMap = ## Decode properties Given a sample.properties file of: ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski @@ -145,6 +149,7 @@ person: ## Roundtrip Given a sample.properties file of: ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md index 714194c3..b5241765 100644 --- a/pkg/yqlib/doc/usage/xml.md +++ b/pkg/yqlib/doc/usage/xml.md @@ -402,7 +402,7 @@ yq -o=xml '.' sample.yml ``` will output ```xml - + val1 val2 diff --git a/pkg/yqlib/encoder_properties_test.go b/pkg/yqlib/encoder_properties_test.go index 2c2b7f58..d1ed5cf6 100644 --- a/pkg/yqlib/encoder_properties_test.go +++ b/pkg/yqlib/encoder_properties_test.go @@ -14,7 +14,7 @@ func yamlToProps(sampleYaml string, unwrapScalar bool) string { writer := bufio.NewWriter(&output) var propsEncoder = NewPropertiesEncoder(unwrapScalar) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_test.go b/pkg/yqlib/encoder_test.go index 98e652f6..7cc0b35d 100644 --- a/pkg/yqlib/encoder_test.go +++ b/pkg/yqlib/encoder_test.go @@ -14,7 +14,7 @@ func yamlToJSON(sampleYaml string, indent int) string { writer := bufio.NewWriter(&output) var jsonEncoder = NewJSONEncoder(indent, false) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go index 99351589..ebf7ece7 100644 --- a/pkg/yqlib/encoder_yaml.go +++ b/pkg/yqlib/encoder_yaml.go @@ -13,14 +13,14 @@ import ( type yamlEncoder struct { indent int colorise bool - prefs yamlPreferences + prefs YamlPreferences } -func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder { +func NewYamlEncoder(indent int, colorise bool, prefs YamlPreferences) Encoder { if indent < 0 { indent = 0 } - return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar} + return &yamlEncoder{indent, colorise, prefs} } func (ye *yamlEncoder) CanHandleAliases() bool { @@ -28,7 +28,7 @@ func (ye *yamlEncoder) CanHandleAliases() bool { } func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error { - if ye.printDocSeparators { + if ye.prefs.PrintDocSeparators { log.Debug("-- writing doc sep") if err := writeString(writer, "---\n"); err != nil { return err @@ -75,7 +75,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error { - if node.Kind == yaml.ScalarNode && ye.unwrapScalar { + if node.Kind == yaml.ScalarNode && ye.prefs.UnwrapScalar { return writeString(writer, node.Value+"\n") } diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go index f25751b8..63fdb3c1 100644 --- a/pkg/yqlib/json_test.go +++ b/pkg/yqlib/json_test.go @@ -262,11 +262,11 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, true, true)))) + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences)))) } func decodeJSON(t *testing.T, jsonString string) *CandidateNode { - docs, err := readDocumentWithLeadingContent(jsonString, "sample.json", 0) + docs, err := readDocument(jsonString, "sample.json", 0) if err != nil { t.Error(err) @@ -293,12 +293,12 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode { func testJSONScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "encode", "decode": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)), s.description) case "": var actual = resultToString(t, decodeJSON(t, s.input)) test.AssertResultWithContext(t, s.expected, actual, s.description) case "decode-ndjson": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "roundtrip-ndjson": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description) case "roundtrip-multi": @@ -385,7 +385,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) { } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)))) + writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)))) } func TestJSONScenarios(t *testing.T) { diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 66715f3a..ab73a475 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -84,7 +84,7 @@ var participleYqRules = []*participleYqRule{ {"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0}, - {"LoadYaml", `load`, loadOp(NewYamlDecoder(), false), 0}, + {"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0}, {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index f092cb33..461d15ea 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -21,14 +21,6 @@ func InitExpressionParser() { } } -type yamlPreferences struct { - LeadingContentPreProcessing bool - printDocSeparators bool - unwrapScalar bool -} - -var YamlPreferences = NewDefaultYamlPreferences() - var log = logging.MustGetLogger("yq-lib") var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""` @@ -256,14 +248,16 @@ func guessTagFromCustomType(node *yaml.Node) string { } func parseSnippet(value string) (*yaml.Node, error) { - decoder := NewYamlDecoder() - decoder.Init(strings.NewReader(value)) - var dataBucket yaml.Node - err := decoder.Decode(&dataBucket) - if len(dataBucket.Content) == 0 { + decoder := NewYamlDecoder(ConfiguredYamlPreferences) + err := decoder.Init(strings.NewReader(value)) + if err != nil { + return nil, err + } + parsedNode, err := decoder.Decode() + if len(parsedNode.Node.Content) == 0 { return nil, fmt.Errorf("bad data") } - return dataBucket.Content[0], err + return unwrapDoc(parsedNode.Node), err } func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool { diff --git a/pkg/yqlib/operator_comments.go b/pkg/yqlib/operator_comments.go index 60b3e068..5bbe2e59 100644 --- a/pkg/yqlib/operator_comments.go +++ b/pkg/yqlib/operator_comments.go @@ -83,6 +83,10 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode * log.Debugf("GetComments operator!") var results = list.New() + yamlPrefs := NewDefaultYamlPreferences() + yamlPrefs.PrintDocSeparators = false + yamlPrefs.UnwrapScalar = false + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) comment := "" @@ -92,7 +96,7 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode * var chompRegexp = regexp.MustCompile(`\n$`) var output bytes.Buffer var writer = bufio.NewWriter(&output) - var encoder = NewYamlEncoder(2, false, false, false) + var encoder = NewYamlEncoder(2, false, yamlPrefs) if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil { return Context{}, err } diff --git a/pkg/yqlib/operator_datetime.go b/pkg/yqlib/operator_datetime.go index d701fef5..da071fd3 100644 --- a/pkg/yqlib/operator_datetime.go +++ b/pkg/yqlib/operator_datetime.go @@ -4,7 +4,6 @@ import ( "container/list" "errors" "fmt" - "strings" "time" "gopkg.in/yaml.v3" @@ -54,7 +53,6 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { format, err := getStringParamter("format", d, context, expressionNode.RHS) layout := context.GetDateTimeLayout() - decoder := NewYamlDecoder() if err != nil { return Context{}, err @@ -69,19 +67,15 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err) } formattedTimeStr := parsedTime.Format(format) - decoder.Init(strings.NewReader(formattedTimeStr)) - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) - var node *yaml.Node + + node, errorReading := parseSnippet(formattedTimeStr) if errorReading != nil { - log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr) + log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading) node = &yaml.Node{ Kind: yaml.ScalarNode, Tag: "!!str", Value: formattedTimeStr, } - } else { - node = unwrapDoc(&dataBucket) } results.PushBack(candidate.CreateReplacement(node)) diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index 7a1c6a56..92e60369 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -21,7 +21,7 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder { case TSVOutputFormat: return NewCsvEncoder('\t') case YamlOutputFormat: - return NewYamlEncoder(indent, false, true, true) + return NewYamlEncoder(indent, false, ConfiguredYamlPreferences) case XMLOutputFormat: return NewXMLEncoder(indent, ConfiguredXMLPreferences) case Base64OutputFormat: @@ -102,7 +102,7 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre var decoder Decoder switch preferences.format { case YamlInputFormat: - decoder = NewYamlDecoder() + decoder = NewYamlDecoder(ConfiguredYamlPreferences) case XMLInputFormat: decoder = NewXMLDecoder(ConfiguredXMLPreferences) case Base64InputFormat: @@ -121,17 +121,19 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList()) - var dataBucket yaml.Node log.Debugf("got: [%v]", candidate.Node.Value) - decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value)) + err := decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value)) + if err != nil { + return Context{}, err + } - errorReading := decoder.Decode(&dataBucket) + decodedNode, errorReading := decoder.Decode() if errorReading != nil { return Context{}, errorReading } //first node is a doc - node := unwrapDoc(&dataBucket) + node := unwrapDoc(decodedNode.Node) results.PushBack(candidate.CreateReplacement(node)) } diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index a3256dea..79757895 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -40,21 +40,16 @@ func TestMain(m *testing.M) { } func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { - return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer)) + prefs := NewDefaultYamlPreferences() + prefs.PrintDocSeparators = printDocSeparators + prefs.UnwrapScalar = unwrapScalar + return NewPrinter(NewYamlEncoder(indent, colorsEnabled, prefs), NewSinglePrinterWriter(writer)) } -func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { - reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content))) - if err != nil { - return nil, err - } +func readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { + reader := bufio.NewReader(strings.NewReader(content)) - inputs, err := readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder()) - if err != nil { - return nil, err - } - inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent - return inputs, nil + return readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder(ConfiguredYamlPreferences)) } func testScenario(t *testing.T, s *expressionScenario) { @@ -67,7 +62,7 @@ func testScenario(t *testing.T, s *expressionScenario) { inputs := list.New() if s.document != "" { - inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0) + inputs, err = readDocument(s.document, "sample.yml", 0) if err != nil { t.Error(err, s.document, s.expression) @@ -75,7 +70,7 @@ func testScenario(t *testing.T, s *expressionScenario) { } if s.document2 != "" { - moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1) + moreInputs, err := readDocument(s.document2, "another.yml", 1) if err != nil { t.Error(err, s.document2, s.expression) return @@ -176,7 +171,7 @@ func formatYaml(yaml string, filename string) string { panic(err) } streamEvaluator := NewStreamEvaluator() - _, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "", NewYamlDecoder()) + _, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -322,13 +317,13 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt if s.document != "" { - inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0) + inputs, err = readDocument(formattedDoc, "sample.yml", 0) if err != nil { t.Error(err, s.document, s.expression) return } if s.document2 != "" { - moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1) + moreInputs, err := readDocument(formattedDoc2, "another.yml", 1) if err != nil { t.Error(err, s.document, s.expression) return diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index 721a19ad..88f86c77 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -38,7 +38,7 @@ func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -76,7 +76,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -118,7 +118,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -165,7 +165,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -215,7 +215,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -234,7 +234,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -263,7 +263,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -294,7 +294,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) { panic(err) } streamEvaluator := NewStreamEvaluator() - _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n", NewYamlDecoder()) + _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -316,7 +316,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) { // when outputing JSON. printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer)) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } diff --git a/pkg/yqlib/properties_test.go b/pkg/yqlib/properties_test.go index 3a59fcf3..b539270f 100644 --- a/pkg/yqlib/properties_test.go +++ b/pkg/yqlib/properties_test.go @@ -8,7 +8,7 @@ import ( "github.com/mikefarah/yq/v4/test" ) -const samplePropertiesYaml = `# block comments don't come through +const samplePropertiesYaml = `# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -18,7 +18,8 @@ emptyArray: [] emptyMap: [] ` -const expectedPropertiesUnwrapped = `# comments on values appear +const expectedPropertiesUnwrapped = `# block comments come through +# comments on values appear person.name = Mike Wazowski # comments on array values appear @@ -26,7 +27,8 @@ person.pets.0 = cat person.food.0 = pizza ` -const expectedPropertiesWrapped = `# comments on values appear +const expectedPropertiesWrapped = `# block comments come through +# comments on values appear person.name = "Mike Wazowski" # comments on array values appear @@ -55,7 +57,8 @@ person.pets.0 = cat person.food.0 = pizza ` -const expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear +const expectedPropertiesWithEmptyMapsAndArrays = `# block comments come through +# comments on values appear person.name = Mike Wazowski # comments on array values appear @@ -143,7 +146,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)))) } func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { @@ -168,7 +171,7 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)))) } func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) { @@ -193,7 +196,7 @@ func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, true, true)))) + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences)))) } func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) { @@ -245,11 +248,11 @@ func TestPropertyScenarios(t *testing.T) { for _, s := range propertyScenarios { switch s.scenarioType { case "": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description) case "decode": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "encode-wrapped": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description) diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index c430453c..8229e8a7 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -14,9 +14,9 @@ import ( // Uses less memory than loading all documents and running the expression once, but this cannot process // cross document expressions. type StreamEvaluator interface { - Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error) - EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error - EvaluateNew(expression string, printer Printer, leadingContent string) error + Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) + EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error + EvaluateNew(expression string, printer Printer) error } type streamEvaluator struct { @@ -28,17 +28,16 @@ func NewStreamEvaluator() StreamEvaluator { return &streamEvaluator{treeNavigator: NewDataTreeNavigator()} } -func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error { +func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error { node, err := ExpressionParser.ParseExpression(expression) if err != nil { return err } candidateNode := &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, - FileIndex: 0, - LeadingContent: leadingContent, + Document: 0, + Filename: "", + Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + FileIndex: 0, } inputList := list.New() inputList.PushBack(candidateNode) @@ -50,15 +49,13 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin return printer.PrintResults(result.MatchingNodes) } -func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error { +func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error { var totalProcessDocs uint node, err := ExpressionParser.ParseExpression(expression) if err != nil { return err } - var firstFileLeadingContent string - for _, filename := range filenames { reader, err := readStream(filename) @@ -78,7 +75,7 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p } if totalProcessDocs == 0 { - return s.EvaluateNew(expression, printer, firstFileLeadingContent) + return s.EvaluateNew(expression, printer) } return nil @@ -87,7 +84,10 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) { var currentIndex uint - decoder.Init(reader) + err := decoder.Init(reader) + if err != nil { + return 0, err + } for { candidateNode, errorReading := decoder.Decode() diff --git a/pkg/yqlib/string_evaluator.go b/pkg/yqlib/string_evaluator.go index 62bc253d..738b3066 100644 --- a/pkg/yqlib/string_evaluator.go +++ b/pkg/yqlib/string_evaluator.go @@ -6,12 +6,10 @@ import ( "errors" "fmt" "io" - - yaml "gopkg.in/yaml.v3" ) type StringEvaluator interface { - Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error) + Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) } type stringEvaluator struct { @@ -25,7 +23,7 @@ func NewStringEvaluator() StringEvaluator { } } -func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error) { +func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) { // Use bytes.Buffer for output of string out := new(bytes.Buffer) @@ -37,16 +35,18 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco return "", err } - reader, leadingContent, err := readString(input, leadingContentPreProcessing) + reader, err := readString(input) if err != nil { return "", err } var currentIndex uint - decoder.Init(reader) + err = decoder.Init(reader) + if err != nil { + return "", err + } for { - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) + candidateNode, errorReading := decoder.Decode() if errors.Is(errorReading, io.EOF) { s.fileIndex = s.fileIndex + 1 @@ -54,20 +54,9 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco } else if errorReading != nil { return "", fmt.Errorf("bad input '%v': %w", input, errorReading) } + candidateNode.Document = currentIndex + candidateNode.FileIndex = s.fileIndex - candidateNode := &CandidateNode{ - Document: currentIndex, - Node: &dataBucket, - FileIndex: s.fileIndex, - } - // move document comments into candidate node - // otherwise unwrap drops them. - candidateNode.TrailingContent = dataBucket.FootComment - dataBucket.FootComment = "" - - if currentIndex == 0 { - candidateNode.LeadingContent = leadingContent - } inputList := list.New() inputList.PushBack(candidateNode) diff --git a/pkg/yqlib/string_evaluator_test.go b/pkg/yqlib/string_evaluator_test.go index a359b3c3..e8e7ddf5 100644 --- a/pkg/yqlib/string_evaluator_test.go +++ b/pkg/yqlib/string_evaluator_test.go @@ -18,10 +18,10 @@ func TestStringEvaluator_Evaluate_Nominal(t *testing.T) { `---` + "\n" + ` - name: jq` + "\n" + ` description: Command-line JSON processor` + "\n" - encoder := NewYamlEncoder(2, true, true, true) - decoder := NewYamlDecoder() + encoder := NewYamlEncoder(2, true, ConfiguredYamlPreferences) + decoder := NewYamlDecoder(ConfiguredYamlPreferences) - result, err := NewStringEvaluator().Evaluate(expression, input, encoder, true, decoder) + result, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder) if err != nil { t.Error(err) } diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 55723af7..0d39046f 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -37,7 +37,10 @@ func writeString(writer io.Writer, txt string) error { } func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) { - decoder.Init(reader) + err := decoder.Init(reader) + if err != nil { + return nil, err + } inputList := list.New() var currentIndex uint diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index 9963c947..28d83341 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -414,19 +414,19 @@ var xmlScenarios = []formatScenario{ func testXMLScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "", "decode": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description) case "encode": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description) case "decode-keep-ns": prefs := NewDefaultXmlPreferences() prefs.KeepNamespace = true - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "decode-raw-token": prefs := NewDefaultXmlPreferences() prefs.UseRawToken = true - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "roundtrip-skip-directives": prefs := NewDefaultXmlPreferences() prefs.SkipDirectives = true @@ -480,7 +480,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression)) writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, true, true)))) + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)))) } func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) { @@ -549,7 +549,7 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n") writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, ConfiguredXMLPreferences)))) + writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)))) } func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) { diff --git a/pkg/yqlib/yaml.go b/pkg/yqlib/yaml.go new file mode 100644 index 00000000..068e1ead --- /dev/null +++ b/pkg/yqlib/yaml.go @@ -0,0 +1,17 @@ +package yqlib + +type YamlPreferences struct { + LeadingContentPreProcessing bool + PrintDocSeparators bool + UnwrapScalar bool +} + +func NewDefaultYamlPreferences() YamlPreferences { + return YamlPreferences{ + LeadingContentPreProcessing: true, + PrintDocSeparators: true, + UnwrapScalar: true, + } +} + +var ConfiguredYamlPreferences = NewDefaultYamlPreferences()