diff --git a/acceptance_tests/empty.sh b/acceptance_tests/empty.sh index 7ffb813f..63d5ca80 100755 --- a/acceptance_tests/empty.sh +++ b/acceptance_tests/empty.sh @@ -9,7 +9,7 @@ EOL testEmptyEval() { X=$(./yq e test.yml) - expected=$(cat test.yml) + expected="# comment" assertEquals 0 $? assertEquals "$expected" "$X" } diff --git a/cmd/constant.go b/cmd/constant.go index ef0c31db..a3bec40a 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -1,6 +1,5 @@ package cmd -var leadingContentPreProcessing = true var unwrapScalar = true var writeInplace = false diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index d6dc993f..63b68871 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -75,7 +75,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) { return err } - decoder, err := configureDecoder() + decoder, err := configureDecoder(true) if err != nil { return err } @@ -109,13 +109,13 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) { switch len(args) { case 0: if nullInput { - err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer, "") + err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer) } else { cmd.Println(cmd.UsageString()) return nil } default: - err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder) + err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder) } completedSuccessfully = err == nil diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index cc9ab672..53108165 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -97,7 +97,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) { printer := yqlib.NewPrinter(encoder, printerWriter) - decoder, err := configureDecoder() + decoder, err := configureDecoder(false) if err != nil { return err } @@ -123,13 +123,13 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) { switch len(args) { case 0: if nullInput { - err = streamEvaluator.EvaluateNew(processExpression(expression), printer, "") + err = streamEvaluator.EvaluateNew(processExpression(expression), printer) } else { cmd.Println(cmd.UsageString()) return nil } default: - err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder) + err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder) } completedSuccessfully = err == nil diff --git a/cmd/root.go b/cmd/root.go index 058dfd7d..24ef066d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,6 +59,11 @@ yq -P sample.json "naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " + "`+` please set that value explicityly with --xml-attribute-prefix.") } + + //copy preference form global setting + yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar + + yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators }, } @@ -97,7 +102,7 @@ yq -P sample.json rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors") rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact") rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.") - rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.") rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.") rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.") diff --git a/cmd/utils.go b/cmd/utils.go index 007ada69..c6f5b2ab 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -56,7 +56,7 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) { return expression, args, nil } -func configureDecoder() (yqlib.Decoder, error) { +func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat) if err != nil { return nil, err @@ -73,8 +73,9 @@ func configureDecoder() (yqlib.Decoder, error) { case yqlib.TSVObjectInputFormat: return yqlib.NewCSVObjectDecoder('\t'), nil } - - return yqlib.NewYamlDecoder(), nil + prefs := yqlib.ConfiguredYamlPreferences + prefs.EvaluateTogether = evaluateTogether + return yqlib.NewYamlDecoder(prefs), nil } func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) { @@ -105,7 +106,7 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder { case yqlib.TSVOutputFormat: return yqlib.NewCsvEncoder('\t') case yqlib.YamlOutputFormat: - return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar) + return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences) case yqlib.XMLOutputFormat: return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences) } diff --git a/examples/empty-no-comment.yaml b/examples/empty-no-comment.yaml new file mode 100644 index 00000000..e69de29b diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index 066edd0d..4dbed3d1 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -8,7 +8,7 @@ import ( // A yaml expression evaluator that runs the expression once against all files/nodes in memory. type Evaluator interface { - EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error + EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error // EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) @@ -46,21 +46,16 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand return context.MatchingNodes, nil } -func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error { +func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error { fileIndex := 0 - firstFileLeadingContent := "" var allDocuments = list.New() for _, filename := range filenames { - reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing) + reader, err := readStream(filename) if err != nil { return err } - if fileIndex == 0 { - firstFileLeadingContent = leadingContent - } - fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder) if err != nil { return err @@ -75,11 +70,9 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string Filename: "", Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, FileIndex: 0, - LeadingContent: firstFileLeadingContent, + LeadingContent: "", } allDocuments.PushBack(candidateNode) - } else { - allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent } matches, err := e.EvaluateCandidateNodes(expression, allDocuments) diff --git a/pkg/yqlib/csv_test.go b/pkg/yqlib/csv_test.go index d58b43e2..4ab3306d 100644 --- a/pkg/yqlib/csv_test.go +++ b/pkg/yqlib/csv_test.go @@ -136,13 +136,13 @@ var csvScenarios = []formatScenario{ func testCSVScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "encode-csv": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description) case "encode-tsv": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description) case "decode-csv-object": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "decode-tsv-object": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "roundtrip-csv": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description) default: @@ -171,7 +171,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy } writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", - processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))), + processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))), ) } @@ -203,7 +203,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str } writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType, - processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))), + processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))), ) } diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go index e8c63bf7..904b65c3 100644 --- a/pkg/yqlib/decoder.go +++ b/pkg/yqlib/decoder.go @@ -3,8 +3,6 @@ package yqlib import ( "fmt" "io" - - yaml "gopkg.in/yaml.v3" ) type InputFormat uint @@ -20,8 +18,8 @@ const ( ) type Decoder interface { - Init(reader io.Reader) - Decode(node *yaml.Node) error + Init(reader io.Reader) error + Decode() (*CandidateNode, error) } func InputFormatFromString(format string) (InputFormat, error) { diff --git a/pkg/yqlib/decoder_base64.go b/pkg/yqlib/decoder_base64.go index 6ec507cb..c4868fd2 100644 --- a/pkg/yqlib/decoder_base64.go +++ b/pkg/yqlib/decoder_base64.go @@ -19,21 +19,22 @@ func NewBase64Decoder() Decoder { return &base64Decoder{finished: false, encoding: *base64.StdEncoding} } -func (dec *base64Decoder) Init(reader io.Reader) { +func (dec *base64Decoder) Init(reader io.Reader) error { dec.reader = reader dec.readAnything = false dec.finished = false + return nil } -func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *base64Decoder) Decode() (*CandidateNode, error) { if dec.finished { - return io.EOF + return nil, io.EOF } base64Reader := base64.NewDecoder(&dec.encoding, dec.reader) buf := new(bytes.Buffer) if _, err := buf.ReadFrom(base64Reader); err != nil { - return err + return nil, err } if buf.Len() == 0 { dec.finished = true @@ -42,12 +43,15 @@ func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error { // otherwise if we've already read some bytes, and now we get // an empty string, then we are done. if dec.readAnything { - return io.EOF + return nil, io.EOF } } dec.readAnything = true - rootYamlNode.Kind = yaml.ScalarNode - rootYamlNode.Tag = "!!str" - rootYamlNode.Value = buf.String() - return nil + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: buf.String(), + }, + }, nil } diff --git a/pkg/yqlib/decoder_csv_object.go b/pkg/yqlib/decoder_csv_object.go index c2b8f164..e8e78023 100644 --- a/pkg/yqlib/decoder_csv_object.go +++ b/pkg/yqlib/decoder_csv_object.go @@ -19,12 +19,13 @@ func NewCSVObjectDecoder(separator rune) Decoder { return &csvObjectDecoder{separator: separator} } -func (dec *csvObjectDecoder) Init(reader io.Reader) { +func (dec *csvObjectDecoder) Init(reader io.Reader) error { cleanReader, enc := utfbom.Skip(reader) log.Debugf("Detected encoding: %s\n", enc) dec.reader = *csv.NewReader(cleanReader) dec.reader.Comma = dec.separator dec.finished = false + return nil } func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node { @@ -47,14 +48,14 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin return objectNode } -func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) { if dec.finished { - return io.EOF + return nil, io.EOF } headerRow, err := dec.reader.Read() log.Debugf(": headerRow%v", headerRow) if err != nil { - return err + return nil, err } rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} @@ -68,13 +69,13 @@ func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error { log.Debugf("Read next contentRow: %v, %v", contentRow, err) } if !errors.Is(err, io.EOF) { - return err + return nil, err } - log.Debugf("finished, contentRow%v", contentRow) - log.Debugf("err: %v", err) - - rootYamlNode.Kind = yaml.DocumentNode - rootYamlNode.Content = []*yaml.Node{rootArray} - return nil + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{rootArray}, + }, + }, nil } diff --git a/pkg/yqlib/decoder_json.go b/pkg/yqlib/decoder_json.go index e4594357..448550af 100644 --- a/pkg/yqlib/decoder_json.go +++ b/pkg/yqlib/decoder_json.go @@ -16,26 +16,31 @@ func NewJSONDecoder() Decoder { return &jsonDecoder{} } -func (dec *jsonDecoder) Init(reader io.Reader) { +func (dec *jsonDecoder) Init(reader io.Reader) error { dec.decoder = *json.NewDecoder(reader) + return nil } -func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *jsonDecoder) Decode() (*CandidateNode, error) { var dataBucket orderedMap log.Debug("going to decode") err := dec.decoder.Decode(&dataBucket) if err != nil { - return err + return nil, err } node, err := dec.convertToYamlNode(&dataBucket) if err != nil { - return err + return nil, err } - rootYamlNode.Kind = yaml.DocumentNode - rootYamlNode.Content = []*yaml.Node{node} - return nil + + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{node}, + }, + }, nil } func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) { diff --git a/pkg/yqlib/decoder_properties.go b/pkg/yqlib/decoder_properties.go index 0a56d4fe..f234f9c3 100644 --- a/pkg/yqlib/decoder_properties.go +++ b/pkg/yqlib/decoder_properties.go @@ -2,6 +2,7 @@ package yqlib import ( "bytes" + "fmt" "io" "strconv" "strings" @@ -20,9 +21,10 @@ func NewPropertiesDecoder() Decoder { return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false} } -func (dec *propertiesDecoder) Init(reader io.Reader) { +func (dec *propertiesDecoder) Init(reader io.Reader) error { dec.reader = reader dec.finished = false + return nil } func parsePropKey(key string) []interface{} { @@ -46,15 +48,49 @@ func (dec *propertiesDecoder) processComment(c string) string { return "# " + c } -func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, context Context, key string) error { +func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error { + assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}} + + rhsCandidateNode := &CandidateNode{ + Path: path, + Node: &yaml.Node{ + Tag: "!!str", + Value: fmt.Sprintf("%v", path[len(path)-1]), + HeadComment: dec.processComment(strings.Join(comments, "\n")), + Kind: yaml.ScalarNode, + }, + } + + rhsCandidateNode.Node.Tag = guessTagFromCustomType(rhsCandidateNode.Node) + + rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode} + + assignmentOpNode := &ExpressionNode{ + Operation: assignmentOp, + LHS: createTraversalTree(path, traversePreferences{}, true), + RHS: &ExpressionNode{Operation: rhsOp}, + } + + _, err := dec.d.GetMatchingNodes(context, assignmentOpNode) + return err +} + +func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error { value, _ := properties.Get(key) path := parsePropKey(key) + propertyComments := properties.GetComments(key) + if len(propertyComments) > 0 { + err := dec.applyPropertyComments(context, path, propertyComments) + if err != nil { + return nil + } + } + rhsNode := &yaml.Node{ - Value: value, - Tag: "!!str", - Kind: yaml.ScalarNode, - LineComment: dec.processComment(properties.GetComment(key)), + Value: value, + Tag: "!!str", + Kind: yaml.ScalarNode, } rhsNode.Tag = guessTagFromCustomType(rhsNode) @@ -78,22 +114,22 @@ func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, c return err } -func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *propertiesDecoder) Decode() (*CandidateNode, error) { if dec.finished { - return io.EOF + return nil, io.EOF } buf := new(bytes.Buffer) if _, err := buf.ReadFrom(dec.reader); err != nil { - return err + return nil, err } if buf.Len() == 0 { dec.finished = true - return io.EOF + return nil, io.EOF } properties, err := properties.LoadString(buf.String()) if err != nil { - return err + return nil, err } properties.DisableExpansion = true @@ -108,15 +144,18 @@ func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error { context = context.SingleChildContext(rootMap) for _, key := range properties.Keys() { - if err := dec.applyProperty(properties, context, key); err != nil { - return err + if err := dec.applyProperty(context, properties, key); err != nil { + return nil, err } } - - rootYamlNode.Kind = yaml.DocumentNode - rootYamlNode.Content = []*yaml.Node{rootMap.Node} dec.finished = true - return nil + + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{rootMap.Node}, + }, + }, nil } diff --git a/pkg/yqlib/decoder_test.go b/pkg/yqlib/decoder_test.go index 462eb4d1..ddd93ced 100644 --- a/pkg/yqlib/decoder_test.go +++ b/pkg/yqlib/decoder_test.go @@ -23,7 +23,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) s writer := bufio.NewWriter(&output) if decoder == nil { - decoder = NewYamlDecoder() + decoder = NewYamlDecoder(ConfiguredYamlPreferences) } inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder) diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go index 8a7e79f0..ced6c964 100644 --- a/pkg/yqlib/decoder_xml.go +++ b/pkg/yqlib/decoder_xml.go @@ -25,10 +25,11 @@ func NewXMLDecoder(prefs XmlPreferences) Decoder { } } -func (dec *xmlDecoder) Init(reader io.Reader) { +func (dec *xmlDecoder) Init(reader io.Reader) error { dec.reader = reader dec.readAnything = false dec.finished = false + return nil } func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) { @@ -118,32 +119,36 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) { return scalar, nil } -func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error { +func (dec *xmlDecoder) Decode() (*CandidateNode, error) { if dec.finished { - return io.EOF + return nil, io.EOF } root := &xmlNode{} // cant use xj - it doesn't keep map order. err := dec.decodeXML(root) if err != nil { - return err + return nil, err } firstNode, err := dec.convertToYamlNode(root) if err != nil { - return err + return nil, err } else if firstNode.Tag == "!!null" { dec.finished = true if dec.readAnything { - return io.EOF + return nil, io.EOF } } dec.readAnything = true - rootYamlNode.Kind = yaml.DocumentNode - rootYamlNode.Content = []*yaml.Node{firstNode} dec.finished = true - return nil + + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{firstNode}, + }, + }, nil } type xmlNode struct { diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go index 37c4ad31..207ec01e 100644 --- a/pkg/yqlib/decoder_yaml.go +++ b/pkg/yqlib/decoder_yaml.go @@ -1,23 +1,114 @@ package yqlib import ( + "bufio" + "errors" "io" + "regexp" + "strings" yaml "gopkg.in/yaml.v3" ) type yamlDecoder struct { decoder yaml.Decoder + // work around of various parsing issues by yaml.v3 with document headers + prefs YamlPreferences + leadingContent string + readAnything bool + firstFile bool } -func NewYamlDecoder() Decoder { - return &yamlDecoder{} +func NewYamlDecoder(prefs YamlPreferences) Decoder { + return &yamlDecoder{prefs: prefs, firstFile: true} } -func (dec *yamlDecoder) Init(reader io.Reader) { - dec.decoder = *yaml.NewDecoder(reader) +func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) { + var commentLineRegEx = regexp.MustCompile(`^\s*#`) + var sb strings.Builder + for { + peekBytes, err := reader.Peek(3) + if errors.Is(err, io.EOF) { + // EOF are handled else where.. + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } else if string(peekBytes) == "---" { + _, err := reader.ReadString('\n') + sb.WriteString("$yqDocSeperator$\n") + if errors.Is(err, io.EOF) { + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } + } else if commentLineRegEx.MatchString(string(peekBytes)) { + line, err := reader.ReadString('\n') + sb.WriteString(line) + if errors.Is(err, io.EOF) { + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } + } else { + return reader, sb.String(), nil + } + } } -func (dec *yamlDecoder) Decode(rootYamlNode *yaml.Node) error { - return dec.decoder.Decode(rootYamlNode) +func (dec *yamlDecoder) Init(reader io.Reader) error { + readerToUse := reader + leadingContent := "" + var err error + // if we 'evaluating together' - we only process the leading content + // of the first file - this ensures comments from subsequent files are + // merged together correctly. + if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) { + readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader)) + if err != nil { + return err + } + } + dec.leadingContent = leadingContent + dec.readAnything = false + dec.decoder = *yaml.NewDecoder(readerToUse) + dec.firstFile = false + return nil +} + +func (dec *yamlDecoder) Decode() (*CandidateNode, error) { + var dataBucket yaml.Node + + err := dec.decoder.Decode(&dataBucket) + if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything { + // force returning an empty node with a comment. + dec.readAnything = true + return dec.blankNodeWithComment(), nil + + } else if err != nil { + return nil, err + } + + candidateNode := &CandidateNode{ + Node: &dataBucket, + } + + if dec.leadingContent != "" { + candidateNode.LeadingContent = dec.leadingContent + dec.leadingContent = "" + } + // move document comments into candidate node + // otherwise unwrap drops them. + candidateNode.TrailingContent = dataBucket.FootComment + dataBucket.FootComment = "" + return candidateNode, nil +} + +func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode { + return &CandidateNode{ + Document: 0, + Filename: "", + Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + FileIndex: 0, + LeadingContent: dec.leadingContent, + } } diff --git a/pkg/yqlib/doc/operators/comment-operators.md b/pkg/yqlib/doc/operators/comment-operators.md index e4dd23ed..9d1b9da2 100644 --- a/pkg/yqlib/doc/operators/comment-operators.md +++ b/pkg/yqlib/doc/operators/comment-operators.md @@ -246,6 +246,7 @@ Note the use of `...` to ensure key nodes are included. Given a sample.yml file of: ```yaml +# hi a: cat # comment # great b: # key comment @@ -263,6 +264,7 @@ b: ## Get line comment Given a sample.yml file of: ```yaml +# welcome! a: cat # meow # have a great day ``` diff --git a/pkg/yqlib/doc/usage/properties.md b/pkg/yqlib/doc/usage/properties.md index e7952fcf..4e21e08b 100644 --- a/pkg/yqlib/doc/usage/properties.md +++ b/pkg/yqlib/doc/usage/properties.md @@ -9,7 +9,7 @@ Note that empty arrays and maps are not encoded by default. Given a sample.yml file of: ```yaml -# block comments don't come through +# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -25,6 +25,7 @@ yq -o=props sample.yml ``` will output ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski @@ -38,7 +39,7 @@ Note that string values with blank characters in them are encapsulated with doub Given a sample.yml file of: ```yaml -# block comments don't come through +# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -54,6 +55,7 @@ yq -o=props --unwrapScalar=false sample.yml ``` will output ```properties +# block comments come through # comments on values appear person.name = "Mike Wazowski" @@ -65,7 +67,7 @@ person.food.0 = pizza ## Encode properties: no comments Given a sample.yml file of: ```yaml -# block comments don't come through +# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -91,7 +93,7 @@ Use a yq expression to set the empty maps and sequences to your desired value. Given a sample.yml file of: ```yaml -# block comments don't come through +# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -107,6 +109,7 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) = ``` will output ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski @@ -120,6 +123,7 @@ emptyMap = ## Decode properties Given a sample.properties file of: ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski @@ -135,9 +139,12 @@ yq -p=props sample.properties will output ```yaml person: - name: Mike Wazowski # comments on values appear + # block comments come through + # comments on values appear + name: Mike Wazowski pets: - - cat # comments on array values appear + # comments on array values appear + - cat food: - pizza ``` @@ -145,6 +152,7 @@ person: ## Roundtrip Given a sample.properties file of: ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski @@ -159,6 +167,7 @@ yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties ``` will output ```properties +# block comments come through # comments on values appear person.name = Mike Wazowski diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md index 714194c3..a1d535b1 100644 --- a/pkg/yqlib/doc/usage/xml.md +++ b/pkg/yqlib/doc/usage/xml.md @@ -386,6 +386,7 @@ A best attempt is made to copy comments to xml. Given a sample.yml file of: ```yaml +# header comment # above_cat cat: # inline_cat # above_array @@ -402,7 +403,10 @@ yq -o=xml '.' sample.yml ``` will output ```xml - + val1 val2 diff --git a/pkg/yqlib/encoder_properties.go b/pkg/yqlib/encoder_properties.go index 745f1279..e62b3f16 100644 --- a/pkg/yqlib/encoder_properties.go +++ b/pkg/yqlib/encoder_properties.go @@ -65,7 +65,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error { mapKeysToStrings(node) p := properties.NewProperties() - err := pe.doEncode(p, node, "") + err := pe.doEncode(p, node, "", nil) if err != nil { return err } @@ -74,8 +74,17 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error { return err } -func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error { - p.SetComment(path, headAndLineComment(node)) +func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string, keyNode *yaml.Node) error { + + comments := "" + if keyNode != nil { + // include the key node comments if present + comments = headAndLineComment(keyNode) + } + comments = comments + headAndLineComment(node) + commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ") + p.SetComments(path, strings.Split(commentsWithSpaces, "\n")) + switch node.Kind { case yaml.ScalarNode: var nodeValue string @@ -87,13 +96,13 @@ func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, _, _, err := p.Set(path, nodeValue) return err case yaml.DocumentNode: - return pe.doEncode(p, node.Content[0], path) + return pe.doEncode(p, node.Content[0], path, node) case yaml.SequenceNode: return pe.encodeArray(p, node.Content, path) case yaml.MappingNode: return pe.encodeMap(p, node.Content, path) case yaml.AliasNode: - return pe.doEncode(p, node.Alias, path) + return pe.doEncode(p, node.Alias, path, nil) default: return fmt.Errorf("Unsupported node %v", node.Tag) } @@ -108,7 +117,7 @@ func (pe *propertiesEncoder) appendPath(path string, key interface{}) string { func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error { for index, child := range kids { - err := pe.doEncode(p, child, pe.appendPath(path, index)) + err := pe.doEncode(p, child, pe.appendPath(path, index), nil) if err != nil { return err } @@ -120,7 +129,7 @@ func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.No for index := 0; index < len(kids); index = index + 2 { key := kids[index] value := kids[index+1] - err := pe.doEncode(p, value, pe.appendPath(path, key.Value)) + err := pe.doEncode(p, value, pe.appendPath(path, key.Value), key) if err != nil { return err } diff --git a/pkg/yqlib/encoder_properties_test.go b/pkg/yqlib/encoder_properties_test.go index 2c2b7f58..d1ed5cf6 100644 --- a/pkg/yqlib/encoder_properties_test.go +++ b/pkg/yqlib/encoder_properties_test.go @@ -14,7 +14,7 @@ func yamlToProps(sampleYaml string, unwrapScalar bool) string { writer := bufio.NewWriter(&output) var propsEncoder = NewPropertiesEncoder(unwrapScalar) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_test.go b/pkg/yqlib/encoder_test.go index 98e652f6..7cc0b35d 100644 --- a/pkg/yqlib/encoder_test.go +++ b/pkg/yqlib/encoder_test.go @@ -14,7 +14,7 @@ func yamlToJSON(sampleYaml string, indent int) string { writer := bufio.NewWriter(&output) var jsonEncoder = NewJSONEncoder(indent, false) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go index 5d76194d..1450f495 100644 --- a/pkg/yqlib/encoder_xml.go +++ b/pkg/yqlib/encoder_xml.go @@ -4,24 +4,28 @@ import ( "encoding/xml" "fmt" "io" + "regexp" "strings" yaml "gopkg.in/yaml.v3" ) type xmlEncoder struct { - indentString string - writer io.Writer - prefs XmlPreferences + indentString string + writer io.Writer + prefs XmlPreferences + leadingContent string } +var commentPrefix = regexp.MustCompile(`(^|\n)\s*#`) + func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder { var indentString = "" for index := 0; index < indent; index++ { indentString = indentString + " " } - return &xmlEncoder{indentString, nil, prefs} + return &xmlEncoder{indentString, nil, prefs, ""} } func (e *xmlEncoder) CanHandleAliases() bool { @@ -33,6 +37,7 @@ func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error { } func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error { + e.leadingContent = commentPrefix.ReplaceAllString(content, "\n") return nil } @@ -42,6 +47,13 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error { e.writer = writer encoder.Indent("", e.indentString) + if e.leadingContent != "" { + err := e.encodeComment(encoder, e.leadingContent) + if err != nil { + return err + } + } + switch node.Kind { case yaml.MappingNode: err := e.encodeTopLevelMap(encoder, node) diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go index 39142c19..ebf7ece7 100644 --- a/pkg/yqlib/encoder_yaml.go +++ b/pkg/yqlib/encoder_yaml.go @@ -11,17 +11,16 @@ import ( ) type yamlEncoder struct { - indent int - colorise bool - printDocSeparators bool - unwrapScalar bool + indent int + colorise bool + prefs YamlPreferences } -func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder { +func NewYamlEncoder(indent int, colorise bool, prefs YamlPreferences) Encoder { if indent < 0 { indent = 0 } - return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar} + return &yamlEncoder{indent, colorise, prefs} } func (ye *yamlEncoder) CanHandleAliases() bool { @@ -29,7 +28,7 @@ func (ye *yamlEncoder) CanHandleAliases() bool { } func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error { - if ye.printDocSeparators { + if ye.prefs.PrintDocSeparators { log.Debug("-- writing doc sep") if err := writeString(writer, "---\n"); err != nil { return err @@ -76,7 +75,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error { - if node.Kind == yaml.ScalarNode && ye.unwrapScalar { + if node.Kind == yaml.ScalarNode && ye.prefs.UnwrapScalar { return writeString(writer, node.Value+"\n") } diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go index f25751b8..63fdb3c1 100644 --- a/pkg/yqlib/json_test.go +++ b/pkg/yqlib/json_test.go @@ -262,11 +262,11 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, true, true)))) + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences)))) } func decodeJSON(t *testing.T, jsonString string) *CandidateNode { - docs, err := readDocumentWithLeadingContent(jsonString, "sample.json", 0) + docs, err := readDocument(jsonString, "sample.json", 0) if err != nil { t.Error(err) @@ -293,12 +293,12 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode { func testJSONScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "encode", "decode": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)), s.description) case "": var actual = resultToString(t, decodeJSON(t, s.input)) test.AssertResultWithContext(t, s.expected, actual, s.description) case "decode-ndjson": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "roundtrip-ndjson": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description) case "roundtrip-multi": @@ -385,7 +385,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) { } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)))) + writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)))) } func TestJSONScenarios(t *testing.T) { diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 66715f3a..ab73a475 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -84,7 +84,7 @@ var participleYqRules = []*participleYqRule{ {"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0}, - {"LoadYaml", `load`, loadOp(NewYamlDecoder(), false), 0}, + {"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0}, {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 5494d72c..461d15ea 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -248,14 +248,16 @@ func guessTagFromCustomType(node *yaml.Node) string { } func parseSnippet(value string) (*yaml.Node, error) { - decoder := NewYamlDecoder() - decoder.Init(strings.NewReader(value)) - var dataBucket yaml.Node - err := decoder.Decode(&dataBucket) - if len(dataBucket.Content) == 0 { + decoder := NewYamlDecoder(ConfiguredYamlPreferences) + err := decoder.Init(strings.NewReader(value)) + if err != nil { + return nil, err + } + parsedNode, err := decoder.Decode() + if len(parsedNode.Node.Content) == 0 { return nil, fmt.Errorf("bad data") } - return dataBucket.Content[0], err + return unwrapDoc(parsedNode.Node), err } func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool { diff --git a/pkg/yqlib/operator_comments.go b/pkg/yqlib/operator_comments.go index 60b3e068..5bbe2e59 100644 --- a/pkg/yqlib/operator_comments.go +++ b/pkg/yqlib/operator_comments.go @@ -83,6 +83,10 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode * log.Debugf("GetComments operator!") var results = list.New() + yamlPrefs := NewDefaultYamlPreferences() + yamlPrefs.PrintDocSeparators = false + yamlPrefs.UnwrapScalar = false + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) comment := "" @@ -92,7 +96,7 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode * var chompRegexp = regexp.MustCompile(`\n$`) var output bytes.Buffer var writer = bufio.NewWriter(&output) - var encoder = NewYamlEncoder(2, false, false, false) + var encoder = NewYamlEncoder(2, false, yamlPrefs) if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil { return Context{}, err } diff --git a/pkg/yqlib/operator_datetime.go b/pkg/yqlib/operator_datetime.go index d701fef5..da071fd3 100644 --- a/pkg/yqlib/operator_datetime.go +++ b/pkg/yqlib/operator_datetime.go @@ -4,7 +4,6 @@ import ( "container/list" "errors" "fmt" - "strings" "time" "gopkg.in/yaml.v3" @@ -54,7 +53,6 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { format, err := getStringParamter("format", d, context, expressionNode.RHS) layout := context.GetDateTimeLayout() - decoder := NewYamlDecoder() if err != nil { return Context{}, err @@ -69,19 +67,15 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err) } formattedTimeStr := parsedTime.Format(format) - decoder.Init(strings.NewReader(formattedTimeStr)) - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) - var node *yaml.Node + + node, errorReading := parseSnippet(formattedTimeStr) if errorReading != nil { - log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr) + log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading) node = &yaml.Node{ Kind: yaml.ScalarNode, Tag: "!!str", Value: formattedTimeStr, } - } else { - node = unwrapDoc(&dataBucket) } results.PushBack(candidate.CreateReplacement(node)) diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index 7a1c6a56..92e60369 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -21,7 +21,7 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder { case TSVOutputFormat: return NewCsvEncoder('\t') case YamlOutputFormat: - return NewYamlEncoder(indent, false, true, true) + return NewYamlEncoder(indent, false, ConfiguredYamlPreferences) case XMLOutputFormat: return NewXMLEncoder(indent, ConfiguredXMLPreferences) case Base64OutputFormat: @@ -102,7 +102,7 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre var decoder Decoder switch preferences.format { case YamlInputFormat: - decoder = NewYamlDecoder() + decoder = NewYamlDecoder(ConfiguredYamlPreferences) case XMLInputFormat: decoder = NewXMLDecoder(ConfiguredXMLPreferences) case Base64InputFormat: @@ -121,17 +121,19 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList()) - var dataBucket yaml.Node log.Debugf("got: [%v]", candidate.Node.Value) - decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value)) + err := decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value)) + if err != nil { + return Context{}, err + } - errorReading := decoder.Decode(&dataBucket) + decodedNode, errorReading := decoder.Decode() if errorReading != nil { return Context{}, errorReading } //first node is a doc - node := unwrapDoc(&dataBucket) + node := unwrapDoc(decodedNode.Node) results.PushBack(candidate.CreateReplacement(node)) } diff --git a/pkg/yqlib/operator_load_test.go b/pkg/yqlib/operator_load_test.go index 619b9685..67a73e15 100644 --- a/pkg/yqlib/operator_load_test.go +++ b/pkg/yqlib/operator_load_test.go @@ -7,8 +7,16 @@ import ( var loadScenarios = []expressionScenario{ { skipDoc: true, - description: "Load empty file", + description: "Load empty file with a comment", expression: `load("../../examples/empty.yaml")`, + expected: []string{ + "D0, P[], (doc)::# comment\n\n", + }, + }, + { + skipDoc: true, + description: "Load empty file with no comment", + expression: `load("../../examples/empty-no-comment.yaml")`, expected: []string{ "D0, P[], (!!null)::\n", }, diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index a3256dea..79757895 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -40,21 +40,16 @@ func TestMain(m *testing.M) { } func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { - return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer)) + prefs := NewDefaultYamlPreferences() + prefs.PrintDocSeparators = printDocSeparators + prefs.UnwrapScalar = unwrapScalar + return NewPrinter(NewYamlEncoder(indent, colorsEnabled, prefs), NewSinglePrinterWriter(writer)) } -func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { - reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content))) - if err != nil { - return nil, err - } +func readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { + reader := bufio.NewReader(strings.NewReader(content)) - inputs, err := readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder()) - if err != nil { - return nil, err - } - inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent - return inputs, nil + return readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder(ConfiguredYamlPreferences)) } func testScenario(t *testing.T, s *expressionScenario) { @@ -67,7 +62,7 @@ func testScenario(t *testing.T, s *expressionScenario) { inputs := list.New() if s.document != "" { - inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0) + inputs, err = readDocument(s.document, "sample.yml", 0) if err != nil { t.Error(err, s.document, s.expression) @@ -75,7 +70,7 @@ func testScenario(t *testing.T, s *expressionScenario) { } if s.document2 != "" { - moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1) + moreInputs, err := readDocument(s.document2, "another.yml", 1) if err != nil { t.Error(err, s.document2, s.expression) return @@ -176,7 +171,7 @@ func formatYaml(yaml string, filename string) string { panic(err) } streamEvaluator := NewStreamEvaluator() - _, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "", NewYamlDecoder()) + _, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -322,13 +317,13 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt if s.document != "" { - inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0) + inputs, err = readDocument(formattedDoc, "sample.yml", 0) if err != nil { t.Error(err, s.document, s.expression) return } if s.document2 != "" { - moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1) + moreInputs, err := readDocument(formattedDoc2, "another.yml", 1) if err != nil { t.Error(err, s.document, s.expression) return diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 78b32c50..aef082d9 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -111,7 +111,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { mappedDoc := el.Value.(*CandidateNode) log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document) - + log.Debug("%v", NodeToString(mappedDoc)) writer, errorWriting := p.printerWriter.GetWriter(mappedDoc) if errorWriting != nil { return errorWriting diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index 721a19ad..88f86c77 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -38,7 +38,7 @@ func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -76,7 +76,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -118,7 +118,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -165,7 +165,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -215,7 +215,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -234,7 +234,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -263,7 +263,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -294,7 +294,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) { panic(err) } streamEvaluator := NewStreamEvaluator() - _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n", NewYamlDecoder()) + _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } @@ -316,7 +316,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) { // when outputing JSON. printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer)) - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder()) + inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } diff --git a/pkg/yqlib/properties_test.go b/pkg/yqlib/properties_test.go index 3a59fcf3..1d201f53 100644 --- a/pkg/yqlib/properties_test.go +++ b/pkg/yqlib/properties_test.go @@ -8,7 +8,49 @@ import ( "github.com/mikefarah/yq/v4/test" ) -const samplePropertiesYaml = `# block comments don't come through +const propertiesWithCommentsOnMap = `this.thing = hi hi +# important notes +# about this value +this.value = cool +` + +const expectedPropertiesWithCommentsOnMapProps = `this.thing = hi hi + +# important notes +# about this value +this.value = cool +` + +const expectedPropertiesWithCommentsOnMapYaml = `this: + thing: hi hi + # important notes + # about this value + value: cool +` + +const propertiesWithCommentInArray = ` +this.array.0 = cat +# important notes +# about dogs +this.array.1 = dog +` + +const expectedPropertiesWithCommentInArrayProps = `this.array.0 = cat + +# important notes +# about dogs +this.array.1 = dog +` + +const expectedPropertiesWithCommentInArrayYaml = `this: + array: + - cat + # important notes + # about dogs + - dog +` + +const samplePropertiesYaml = `# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: @@ -18,7 +60,8 @@ emptyArray: [] emptyMap: [] ` -const expectedPropertiesUnwrapped = `# comments on values appear +const expectedPropertiesUnwrapped = `# block comments come through +# comments on values appear person.name = Mike Wazowski # comments on array values appear @@ -26,7 +69,8 @@ person.pets.0 = cat person.food.0 = pizza ` -const expectedPropertiesWrapped = `# comments on values appear +const expectedPropertiesWrapped = `# block comments come through +# comments on values appear person.name = "Mike Wazowski" # comments on array values appear @@ -34,7 +78,8 @@ person.pets.0 = cat person.food.0 = pizza ` -const expectedUpdatedProperties = `# comments on values appear +const expectedUpdatedProperties = `# block comments come through +# comments on values appear person.name = Mike Wazowski # comments on array values appear @@ -43,9 +88,12 @@ person.food.0 = pizza ` const expectedDecodedYaml = `person: - name: Mike Wazowski # comments on values appear + # block comments come through + # comments on values appear + name: Mike Wazowski pets: - - cat # comments on array values appear + # comments on array values appear + - cat food: - pizza ` @@ -55,7 +103,8 @@ person.pets.0 = cat person.food.0 = pizza ` -const expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear +const expectedPropertiesWithEmptyMapsAndArrays = `# block comments come through +# comments on values appear person.name = Mike Wazowski # comments on array values appear @@ -112,6 +161,34 @@ var propertyScenarios = []formatScenario{ expected: expectedUpdatedProperties, scenarioType: "roundtrip", }, + { + skipDoc: true, + description: "comments on arrays roundtrip", + input: propertiesWithCommentInArray, + expected: expectedPropertiesWithCommentInArrayProps, + scenarioType: "roundtrip", + }, + { + skipDoc: true, + description: "comments on arrays decode", + input: propertiesWithCommentInArray, + expected: expectedPropertiesWithCommentInArrayYaml, + scenarioType: "decode", + }, + { + skipDoc: true, + description: "comments on map roundtrip", + input: propertiesWithCommentsOnMap, + expected: expectedPropertiesWithCommentsOnMapProps, + scenarioType: "roundtrip", + }, + { + skipDoc: true, + description: "comments on map decode", + input: propertiesWithCommentsOnMap, + expected: expectedPropertiesWithCommentsOnMapYaml, + scenarioType: "decode", + }, { description: "Empty doc", skipDoc: true, @@ -143,7 +220,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)))) } func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { @@ -168,7 +245,7 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)))) + writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)))) } func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) { @@ -193,7 +270,7 @@ func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, true, true)))) + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences)))) } func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) { @@ -245,11 +322,11 @@ func TestPropertyScenarios(t *testing.T) { for _, s := range propertyScenarios { switch s.scenarioType { case "": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description) case "decode": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "encode-wrapped": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description) diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index 55eb6623..ef78ca98 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -14,9 +14,9 @@ import ( // Uses less memory than loading all documents and running the expression once, but this cannot process // cross document expressions. type StreamEvaluator interface { - Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error) - EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error - EvaluateNew(expression string, printer Printer, leadingContent string) error + Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) + EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error + EvaluateNew(expression string, printer Printer) error } type streamEvaluator struct { @@ -28,17 +28,16 @@ func NewStreamEvaluator() StreamEvaluator { return &streamEvaluator{treeNavigator: NewDataTreeNavigator()} } -func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error { +func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error { node, err := ExpressionParser.ParseExpression(expression) if err != nil { return err } candidateNode := &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, - FileIndex: 0, - LeadingContent: leadingContent, + Document: 0, + Filename: "", + Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + FileIndex: 0, } inputList := list.New() inputList.PushBack(candidateNode) @@ -50,27 +49,20 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin return printer.PrintResults(result.MatchingNodes) } -func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error { +func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error { var totalProcessDocs uint node, err := ExpressionParser.ParseExpression(expression) if err != nil { return err } - var firstFileLeadingContent string - - for index, filename := range filenames { - reader, leadingContent, err := readStream(filename, leadingContentPreProcessing) - log.Debug("leadingContent: %v", leadingContent) - - if index == 0 { - firstFileLeadingContent = leadingContent - } + for _, filename := range filenames { + reader, err := readStream(filename) if err != nil { return err } - processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent, decoder) + processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder) if err != nil { return err } @@ -83,19 +75,22 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p } if totalProcessDocs == 0 { - return s.EvaluateNew(expression, printer, firstFileLeadingContent) + // problem is I've already slurped the leading content sadface + return s.EvaluateNew(expression, printer) } return nil } -func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error) { +func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) { var currentIndex uint - decoder.Init(reader) + err := decoder.Init(reader) + if err != nil { + return 0, err + } for { - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) + candidateNode, errorReading := decoder.Decode() if errors.Is(errorReading, io.EOF) { s.fileIndex = s.fileIndex + 1 @@ -103,21 +98,10 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr } else if errorReading != nil { return currentIndex, fmt.Errorf("bad file '%v': %w", filename, errorReading) } + candidateNode.Document = currentIndex + candidateNode.Filename = filename + candidateNode.FileIndex = s.fileIndex - candidateNode := &CandidateNode{ - Document: currentIndex, - Filename: filename, - Node: &dataBucket, - FileIndex: s.fileIndex, - } - // move document comments into candidate node - // otherwise unwrap drops them. - candidateNode.TrailingContent = dataBucket.FootComment - dataBucket.FootComment = "" - - if currentIndex == 0 { - candidateNode.LeadingContent = leadingContent - } inputList := list.New() inputList.PushBack(candidateNode) diff --git a/pkg/yqlib/string_evaluator.go b/pkg/yqlib/string_evaluator.go index 62bc253d..59d451ca 100644 --- a/pkg/yqlib/string_evaluator.go +++ b/pkg/yqlib/string_evaluator.go @@ -1,17 +1,17 @@ package yqlib import ( + "bufio" "bytes" "container/list" "errors" "fmt" "io" - - yaml "gopkg.in/yaml.v3" + "strings" ) type StringEvaluator interface { - Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error) + Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) } type stringEvaluator struct { @@ -25,7 +25,7 @@ func NewStringEvaluator() StringEvaluator { } } -func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error) { +func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) { // Use bytes.Buffer for output of string out := new(bytes.Buffer) @@ -37,16 +37,15 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco return "", err } - reader, leadingContent, err := readString(input, leadingContentPreProcessing) + reader := bufio.NewReader(strings.NewReader(input)) + + var currentIndex uint + err = decoder.Init(reader) if err != nil { return "", err } - - var currentIndex uint - decoder.Init(reader) for { - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) + candidateNode, errorReading := decoder.Decode() if errors.Is(errorReading, io.EOF) { s.fileIndex = s.fileIndex + 1 @@ -54,20 +53,9 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco } else if errorReading != nil { return "", fmt.Errorf("bad input '%v': %w", input, errorReading) } + candidateNode.Document = currentIndex + candidateNode.FileIndex = s.fileIndex - candidateNode := &CandidateNode{ - Document: currentIndex, - Node: &dataBucket, - FileIndex: s.fileIndex, - } - // move document comments into candidate node - // otherwise unwrap drops them. - candidateNode.TrailingContent = dataBucket.FootComment - dataBucket.FootComment = "" - - if currentIndex == 0 { - candidateNode.LeadingContent = leadingContent - } inputList := list.New() inputList.PushBack(candidateNode) diff --git a/pkg/yqlib/string_evaluator_test.go b/pkg/yqlib/string_evaluator_test.go index a359b3c3..e8e7ddf5 100644 --- a/pkg/yqlib/string_evaluator_test.go +++ b/pkg/yqlib/string_evaluator_test.go @@ -18,10 +18,10 @@ func TestStringEvaluator_Evaluate_Nominal(t *testing.T) { `---` + "\n" + ` - name: jq` + "\n" + ` description: Command-line JSON processor` + "\n" - encoder := NewYamlEncoder(2, true, true, true) - decoder := NewYamlDecoder() + encoder := NewYamlEncoder(2, true, ConfiguredYamlPreferences) + decoder := NewYamlDecoder(ConfiguredYamlPreferences) - result, err := NewStringEvaluator().Evaluate(expression, input, encoder, true, decoder) + result, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder) if err != nil { t.Error(err) } diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 27964d58..a1afff27 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -7,13 +7,9 @@ import ( "fmt" "io" "os" - "regexp" - "strings" - - yaml "gopkg.in/yaml.v3" ) -func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) { +func readStream(filename string) (io.Reader, error) { var reader *bufio.Reader if filename == "-" { reader = bufio.NewReader(os.Stdin) @@ -22,23 +18,12 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s // and ensuring that it's not possible to give a path to a file outside thar directory. file, err := os.Open(filename) // #nosec if err != nil { - return nil, "", err + return nil, err } reader = bufio.NewReader(file) } + return reader, nil - if !leadingContentPreProcessing { - return reader, "", nil - } - return processReadStream(reader) -} - -func readString(input string, leadingContentPreProcessing bool) (io.Reader, string, error) { - reader := bufio.NewReader(strings.NewReader(input)) - if !leadingContentPreProcessing { - return reader, "", nil - } - return processReadStream(reader) } func writeString(writer io.Writer, txt string) error { @@ -46,46 +31,16 @@ func writeString(writer io.Writer, txt string) error { return errorWriting } -func processReadStream(reader *bufio.Reader) (io.Reader, string, error) { - var commentLineRegEx = regexp.MustCompile(`^\s*#`) - var sb strings.Builder - for { - peekBytes, err := reader.Peek(3) - if errors.Is(err, io.EOF) { - // EOF are handled else where.. - return reader, sb.String(), nil - } else if err != nil { - return reader, sb.String(), err - } else if string(peekBytes) == "---" { - _, err := reader.ReadString('\n') - sb.WriteString("$yqDocSeperator$\n") - if errors.Is(err, io.EOF) { - return reader, sb.String(), nil - } else if err != nil { - return reader, sb.String(), err - } - } else if commentLineRegEx.MatchString(string(peekBytes)) { - line, err := reader.ReadString('\n') - sb.WriteString(line) - if errors.Is(err, io.EOF) { - return reader, sb.String(), nil - } else if err != nil { - return reader, sb.String(), err - } - } else { - return reader, sb.String(), nil - } - } -} - func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) { - decoder.Init(reader) + err := decoder.Init(reader) + if err != nil { + return nil, err + } inputList := list.New() var currentIndex uint for { - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) + candidateNode, errorReading := decoder.Decode() if errors.Is(errorReading, io.EOF) { switch reader := reader.(type) { @@ -96,18 +51,10 @@ func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Dec } else if errorReading != nil { return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) } - candidateNode := &CandidateNode{ - Document: currentIndex, - Filename: filename, - Node: &dataBucket, - FileIndex: fileIndex, - EvaluateTogether: true, - } - - //move document comments into candidate node - // otherwise unwrap drops them. - candidateNode.TrailingContent = dataBucket.FootComment - dataBucket.FootComment = "" + candidateNode.Document = currentIndex + candidateNode.Filename = filename + candidateNode.FileIndex = fileIndex + candidateNode.EvaluateTogether = true inputList.PushBack(candidateNode) diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index 9963c947..c2e804c0 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -137,7 +137,8 @@ in d before --> ` -const yamlWithComments = `# above_cat +const yamlWithComments = `# header comment +# above_cat cat: # inline_cat # above_array array: # inline_array @@ -147,7 +148,10 @@ cat: # inline_cat # below_cat ` -const expectedXMLWithComments = ` +const expectedXMLWithComments = ` val1 val2 @@ -414,19 +418,19 @@ var xmlScenarios = []formatScenario{ func testXMLScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "", "decode": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description) case "encode": - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description) case "decode-keep-ns": prefs := NewDefaultXmlPreferences() prefs.KeepNamespace = true - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "decode-raw-token": prefs := NewDefaultXmlPreferences() prefs.UseRawToken = true - test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description) + test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) case "roundtrip-skip-directives": prefs := NewDefaultXmlPreferences() prefs.SkipDirectives = true @@ -480,7 +484,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression)) writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, true, true)))) + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)))) } func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) { @@ -549,7 +553,7 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n") writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, ConfiguredXMLPreferences)))) + writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)))) } func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) { diff --git a/pkg/yqlib/yaml.go b/pkg/yqlib/yaml.go new file mode 100644 index 00000000..d7c8863a --- /dev/null +++ b/pkg/yqlib/yaml.go @@ -0,0 +1,19 @@ +package yqlib + +type YamlPreferences struct { + LeadingContentPreProcessing bool + PrintDocSeparators bool + UnwrapScalar bool + EvaluateTogether bool +} + +func NewDefaultYamlPreferences() YamlPreferences { + return YamlPreferences{ + LeadingContentPreProcessing: true, + PrintDocSeparators: true, + UnwrapScalar: true, + EvaluateTogether: false, + } +} + +var ConfiguredYamlPreferences = NewDefaultYamlPreferences()