From 1928e5103be46aaed7036fa65ee5f54986c4f2cc Mon Sep 17 00:00:00 2001 From: Mihail Vratchanski Date: Wed, 4 Jun 2025 10:30:55 +0300 Subject: [PATCH] Initial implementation --- cmd/root.go | 12 ++ pkg/yqlib/candidate_node_goccy_yaml.go | 82 ++++++++++ pkg/yqlib/decoder_goccy_yaml.go | 130 +++++++++++++-- pkg/yqlib/encoder_goccy_yaml.go | 126 +++++++++++++++ pkg/yqlib/format.go | 14 +- pkg/yqlib/goccy_yaml_test.go | 2 +- pkg/yqlib/lib.go | 11 ++ pkg/yqlib/parser_comparison_test.go | 210 +++++++++++++++++++++++++ pkg/yqlib/yaml.go | 3 + 9 files changed, 578 insertions(+), 12 deletions(-) create mode 100644 pkg/yqlib/encoder_goccy_yaml.go create mode 100644 pkg/yqlib/parser_comparison_test.go diff --git a/cmd/root.go b/cmd/root.go index b7360d3c..3d479576 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -94,6 +94,12 @@ yq -P -oy sample.json logging.SetBackend(backend) yqlib.InitExpressionParser() + // Handle legacy parser flag + useLegacyYamlParser, _ := cmd.Flags().GetBool("yaml-v3") + if useLegacyYamlParser { + yqlib.ConfiguredYamlPreferences.UseGoccyParser = false + } + return nil }, } @@ -197,6 +203,12 @@ yq -P -oy sample.json } rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.") + rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredYamlPreferences.UseGoccyParser, "yaml-goccy", true, "Use goccy/go-yaml parser (default)") + + // Add a flag to revert to the legacy parser + var useLegacyYamlParser bool + rootCmd.PersistentFlags().BoolVar(&useLegacyYamlParser, "yaml-v3", false, "Use legacy gopkg.in/yaml.v3 parser instead of goccy (for backwards compatibility)") + rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.") if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil { panic(err) diff --git a/pkg/yqlib/candidate_node_goccy_yaml.go b/pkg/yqlib/candidate_node_goccy_yaml.go index 3e861b83..8b9838dc 100644 --- a/pkg/yqlib/candidate_node_goccy_yaml.go +++ b/pkg/yqlib/candidate_node_goccy_yaml.go @@ -208,3 +208,85 @@ func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingVa return nil } + +func (o *CandidateNode) MarshalGoccyYAML() (interface{}, error) { + log.Debug("MarshalGoccyYAML to goccy: %v", o.Tag) + + switch o.Kind { + case AliasNode: + log.Debug("MarshalGoccyYAML - alias to goccy: %v", o.Tag) + // For goccy, we'll return the referenced value directly + // The goccy encoder will handle alias creation + if o.Alias != nil { + return o.Alias.MarshalGoccyYAML() + } + return o.Value, nil + + case ScalarNode: + log.Debug("MarshalGoccyYAML - scalar: %v", o.Value) + + // Handle different scalar types based on tag + switch o.Tag { + case "!!int": + if val, err := parseInt(o.Value); err == nil { + return val, nil + } + case "!!float": + if val, err := parseFloat(o.Value); err == nil { + return val, nil + } + case "!!bool": + if val, err := parseBool(o.Value); err == nil { + return val, nil + } + case "!!null": + return nil, nil + default: + // String or unknown type + return o.Value, nil + } + return o.Value, nil + + case MappingNode: + log.Debug("MarshalGoccyYAML - mapping: %v", NodeToString(o)) + result := make(map[string]interface{}) + + for i := 0; i < len(o.Content); i += 2 { + if i+1 >= len(o.Content) { + break + } + + keyNode := o.Content[i] + valueNode := o.Content[i+1] + + key := keyNode.Value + if key == "" { + key = NodeToString(keyNode) + } + + value, err := valueNode.MarshalGoccyYAML() + if err != nil { + return nil, err + } + + result[key] = value + } + return result, nil + + case SequenceNode: + log.Debug("MarshalGoccyYAML - sequence: %v", NodeToString(o)) + result := make([]interface{}, len(o.Content)) + + for i, childNode := range o.Content { + value, err := childNode.MarshalGoccyYAML() + if err != nil { + return nil, err + } + result[i] = value + } + return result, nil + } + + // Default case + return o.Value, nil +} diff --git a/pkg/yqlib/decoder_goccy_yaml.go b/pkg/yqlib/decoder_goccy_yaml.go index 6598e68d..f09c57e0 100644 --- a/pkg/yqlib/decoder_goccy_yaml.go +++ b/pkg/yqlib/decoder_goccy_yaml.go @@ -7,7 +7,12 @@ package yqlib import ( + "bufio" + "bytes" + "errors" "io" + "regexp" + "strings" yaml "github.com/goccy/go-yaml" "github.com/goccy/go-yaml/ast" @@ -16,34 +21,141 @@ import ( type goccyYamlDecoder struct { decoder yaml.Decoder cm yaml.CommentMap + + prefs YamlPreferences + + // work around of various parsing issues by handling document headers + leadingContent string + bufferRead bytes.Buffer + // anchor map persists over multiple documents for convenience. anchorMap map[string]*CandidateNode + + readAnything bool + firstFile bool + documentIndex uint } -func NewGoccyYAMLDecoder() Decoder { - return &goccyYamlDecoder{} +func NewGoccyYAMLDecoder(prefs YamlPreferences) Decoder { + return &goccyYamlDecoder{prefs: prefs, firstFile: true} +} + +func (dec *goccyYamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) { + var commentLineRegEx = regexp.MustCompile(`^\s*#`) + var yamlDirectiveLineRegEx = regexp.MustCompile(`^\s*%YA`) + var sb strings.Builder + for { + peekBytes, err := reader.Peek(4) + if errors.Is(err, io.EOF) { + // EOF are handled else where.. + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } else if string(peekBytes[0]) == "\n" { + _, err := reader.ReadString('\n') + sb.WriteString("\n") + if errors.Is(err, io.EOF) { + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } + } else if string(peekBytes) == "--- " { + _, err := reader.ReadString(' ') + sb.WriteString("$yqDocSeparator$\n") + if errors.Is(err, io.EOF) { + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } + } else if string(peekBytes) == "---\n" { + _, err := reader.ReadString('\n') + sb.WriteString("$yqDocSeparator$\n") + if errors.Is(err, io.EOF) { + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } + } else if commentLineRegEx.MatchString(string(peekBytes)) || yamlDirectiveLineRegEx.MatchString(string(peekBytes)) { + line, err := reader.ReadString('\n') + sb.WriteString(line) + if errors.Is(err, io.EOF) { + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } + } else { + return reader, sb.String(), nil + } + } } func (dec *goccyYamlDecoder) Init(reader io.Reader) error { + readerToUse := reader + leadingContent := "" + dec.bufferRead = bytes.Buffer{} + var err error + // if we 'evaluating together' - we only process the leading content + // of the first file - this ensures comments from subsequent files are + // merged together correctly. + if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) { + readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader)) + if err != nil { + return err + } + } else if !dec.prefs.LeadingContentPreProcessing { + // if we're not process the leading content + // keep a copy of what we've read. This is incase its a + // doc with only comments - the decoder will return nothing + // then we can read the comments from bufferRead + readerToUse = io.TeeReader(reader, &dec.bufferRead) + } + dec.leadingContent = leadingContent + dec.readAnything = false dec.cm = yaml.CommentMap{} - dec.decoder = *yaml.NewDecoder(reader, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap()) + dec.decoder = *yaml.NewDecoder(readerToUse, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap()) + dec.firstFile = false + dec.documentIndex = 0 dec.anchorMap = make(map[string]*CandidateNode) return nil } func (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) { + var astNode ast.Node + err := dec.decoder.Decode(&astNode) - var ast ast.Node - - err := dec.decoder.Decode(&ast) - if err != nil { + if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything { + // force returning an empty node with a comment. + dec.readAnything = true + return dec.blankNodeWithComment(), nil + } else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything { + // didn't find any yaml, + // check the tee buffer, maybe there were comments + dec.readAnything = true + dec.leadingContent = dec.bufferRead.String() + if dec.leadingContent != "" { + return dec.blankNodeWithComment(), nil + } + return nil, err + } else if err != nil { return nil, err } - candidateNode := &CandidateNode{} - if err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm, dec.anchorMap); err != nil { + candidateNode := &CandidateNode{document: dec.documentIndex} + if err := candidateNode.UnmarshalGoccyYAML(astNode, dec.cm, dec.anchorMap); err != nil { return nil, err } + if dec.leadingContent != "" { + candidateNode.LeadingContent = dec.leadingContent + dec.leadingContent = "" + } + dec.readAnything = true + dec.documentIndex++ return candidateNode, nil } + +func (dec *goccyYamlDecoder) blankNodeWithComment() *CandidateNode { + node := createScalarNode(nil, "") + node.LeadingContent = dec.leadingContent + return node +} diff --git a/pkg/yqlib/encoder_goccy_yaml.go b/pkg/yqlib/encoder_goccy_yaml.go new file mode 100644 index 00000000..cc26014d --- /dev/null +++ b/pkg/yqlib/encoder_goccy_yaml.go @@ -0,0 +1,126 @@ +package yqlib + +import ( + "bufio" + "bytes" + "errors" + "io" + "regexp" + "strings" + + "github.com/fatih/color" + yaml "github.com/goccy/go-yaml" +) + +type goccyYamlEncoder struct { + prefs YamlPreferences +} + +func NewGoccyYamlEncoder(prefs YamlPreferences) Encoder { + return &goccyYamlEncoder{prefs} +} + +func (ye *goccyYamlEncoder) CanHandleAliases() bool { + return true +} + +func (ye *goccyYamlEncoder) PrintDocumentSeparator(writer io.Writer) error { + if ye.prefs.PrintDocSeparators { + log.Debug("writing doc sep") + if err := writeString(writer, "---\n"); err != nil { + return err + } + } + return nil +} + +func (ye *goccyYamlEncoder) PrintLeadingContent(writer io.Writer, content string) error { + reader := bufio.NewReader(strings.NewReader(content)) + + var commentLineRegEx = regexp.MustCompile(`^\s*#`) + + for { + readline, errReading := reader.ReadString('\n') + if errReading != nil && !errors.Is(errReading, io.EOF) { + return errReading + } + if strings.Contains(readline, "$yqDocSeparator$") { + if err := ye.PrintDocumentSeparator(writer); err != nil { + return err + } + } else { + if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) { + readline = "# " + readline + } + if ye.prefs.ColorsEnabled && strings.TrimSpace(readline) != "" { + readline = format(color.FgHiBlack) + readline + format(color.Reset) + } + if err := writeString(writer, readline); err != nil { + return err + } + } + + if errors.Is(errReading, io.EOF) { + if readline != "" { + // the last comment we read didn't have a newline, put one in + if err := writeString(writer, "\n"); err != nil { + return err + } + } + break + } + } + + return nil +} + +func (ye *goccyYamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { + log.Debug("goccyYamlEncoder - going to print %v", NodeToString(node)) + if node.Kind == ScalarNode && ye.prefs.UnwrapScalar { + valueToPrint := node.Value + if node.LeadingContent == "" || valueToPrint != "" { + valueToPrint = valueToPrint + "\n" + } + return writeString(writer, valueToPrint) + } + + destination := writer + tempBuffer := bytes.NewBuffer(nil) + if ye.prefs.ColorsEnabled { + destination = tempBuffer + } + + // Create encoder with indent option + var encoder *yaml.Encoder + if ye.prefs.Indent > 0 { + encoder = yaml.NewEncoder(destination, yaml.Indent(ye.prefs.Indent)) + } else { + encoder = yaml.NewEncoder(destination) + } + + target, err := node.MarshalGoccyYAML() + if err != nil { + return err + } + + trailingContent := node.FootComment + if target != nil { + // Store and clear foot comment to handle it separately + if astNode, ok := target.(interface{ GetComment() interface{} }); ok { + _ = astNode // placeholder for potential future foot comment handling + } + } + + if err := encoder.Encode(target); err != nil { + return err + } + + if err := ye.PrintLeadingContent(destination, trailingContent); err != nil { + return err + } + + if ye.prefs.ColorsEnabled { + return colorizeAndPrint(tempBuffer.Bytes(), writer) + } + return nil +} diff --git a/pkg/yqlib/format.go b/pkg/yqlib/format.go index a74ce599..1a4fce25 100644 --- a/pkg/yqlib/format.go +++ b/pkg/yqlib/format.go @@ -18,8 +18,18 @@ type Format struct { } var YamlFormat = &Format{"yaml", []string{"y", "yml"}, - func() Encoder { return NewYamlEncoder(ConfiguredYamlPreferences) }, - func() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) }, + func() Encoder { + if ConfiguredYamlPreferences.UseGoccyParser { + return NewGoccyYamlEncoder(ConfiguredYamlPreferences) + } + return NewYamlEncoder(ConfiguredYamlPreferences) + }, + func() Decoder { + if ConfiguredYamlPreferences.UseGoccyParser { + return NewGoccyYAMLDecoder(ConfiguredYamlPreferences) + } + return NewYamlDecoder(ConfiguredYamlPreferences) + }, } var JSONFormat = &Format{"json", []string{"j"}, diff --git a/pkg/yqlib/goccy_yaml_test.go b/pkg/yqlib/goccy_yaml_test.go index 9c1158cb..8603d65b 100644 --- a/pkg/yqlib/goccy_yaml_test.go +++ b/pkg/yqlib/goccy_yaml_test.go @@ -272,7 +272,7 @@ var goccyYamlFormatScenarios = []formatScenario{ } func testGoccyYamlScenario(t *testing.T, s formatScenario) { - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) } func TestGoccyYmlFormatScenarios(t *testing.T) { diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 15561930..4460819a 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -175,6 +175,17 @@ func parseInt(numberString string) (int, error) { return int(parsed), err } +func parseFloat(numberString string) (float64, error) { + if strings.Contains(numberString, "_") { + numberString = strings.ReplaceAll(numberString, "_", "") + } + return strconv.ParseFloat(numberString, 64) +} + +func parseBool(boolString string) (bool, error) { + return strconv.ParseBool(boolString) +} + func headAndLineComment(node *CandidateNode) string { return headComment(node) + lineComment(node) } diff --git a/pkg/yqlib/parser_comparison_test.go b/pkg/yqlib/parser_comparison_test.go new file mode 100644 index 00000000..c96d65c1 --- /dev/null +++ b/pkg/yqlib/parser_comparison_test.go @@ -0,0 +1,210 @@ +package yqlib + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +var testYamlDocs = []string{ + `--- +name: John +age: 30 +address: + street: 123 Main St + city: Anytown + state: ST + zip: 12345 +hobbies: + - reading + - swimming + - cooking +`, + `--- +# This is a comment +version: "1.0" +database: + host: localhost + port: 5432 + name: myapp + credentials: + username: admin + password: secret123 +servers: + - name: web1 + ip: 192.168.1.10 + - name: web2 + ip: 192.168.1.11 +`, + `--- +# Complex YAML with various types +config: + enabled: true + timeout: 30 + ratio: 3.14159 + options: + debug: false + verbose: true + items: [] + metadata: null + multiline: | + This is a multiline + string with line breaks + and proper formatting +`, +} + +func TestYamlParsersStructuralEquivalence(t *testing.T) { + // Save original preferences + originalPrefs := ConfiguredYamlPreferences.Copy() + defer func() { + ConfiguredYamlPreferences = originalPrefs + }() + + for i, yamlDoc := range testYamlDocs { + t.Run(fmt.Sprintf("Document_%d", i), func(t *testing.T) { + // Test with yaml.v3 parser + ConfiguredYamlPreferences.UseGoccyParser = false + v3Node := parseYamlDocument(t, yamlDoc) + + // Test with goccy parser + ConfiguredYamlPreferences.UseGoccyParser = true + goccyNode := parseYamlDocument(t, yamlDoc) + + // Compare the parsed structures (not text output) + if !recursiveNodeEqual(v3Node, goccyNode) { + t.Errorf("Parsed structures differ between yaml.v3 and goccy for document %d", i) + t.Logf("yaml.v3 result: %s", NodeToString(v3Node)) + t.Logf("goccy result: %s", NodeToString(goccyNode)) + } + }) + } +} + +func parseYamlDocument(t *testing.T, yamlDoc string) *CandidateNode { + decoder := YamlFormat.DecoderFactory() + + inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder) + if err != nil { + t.Fatalf("Failed to decode YAML: %v", err) + } + + if inputs.Len() == 0 { + t.Fatal("No documents found") + } + + return inputs.Front().Value.(*CandidateNode) +} + +func TestYamlParsersEquivalence(t *testing.T) { + // Save original preferences + originalPrefs := ConfiguredYamlPreferences.Copy() + defer func() { + ConfiguredYamlPreferences = originalPrefs + }() + + // Note: This test documents the differences in output formatting between parsers + // These differences are acceptable as they don't affect semantic meaning + t.Log("Note: yaml.v3 and goccy may produce different key ordering, which is semantically equivalent") + + for i, yamlDoc := range testYamlDocs { + t.Run(fmt.Sprintf("Document_%d", i), func(t *testing.T) { + // Test with yaml.v3 parser + ConfiguredYamlPreferences.UseGoccyParser = false + v3Result := processYamlDocument(t, yamlDoc) + + // Test with goccy parser + ConfiguredYamlPreferences.UseGoccyParser = true + goccyResult := processYamlDocument(t, yamlDoc) + + // Log the differences for documentation purposes + if v3Result != goccyResult { + t.Logf("Output format differs (this is expected):") + t.Logf("yaml.v3 output:\n%s", v3Result) + t.Logf("goccy output:\n%s", goccyResult) + + // But both should be valid YAML that parses to the same structure + v3Reparse := parseYamlDocument(t, v3Result) + goccyReparse := parseYamlDocument(t, goccyResult) + + if !recursiveNodeEqual(v3Reparse, goccyReparse) { + t.Error("Reparsed structures differ - this indicates a semantic issue") + } + } + }) + } +} + +func processYamlDocument(t *testing.T, yamlDoc string) string { + decoder := YamlFormat.DecoderFactory() + encoder := YamlFormat.EncoderFactory() + + inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder) + if err != nil { + t.Fatalf("Failed to decode YAML: %v", err) + } + + if inputs.Len() == 0 { + t.Fatal("No documents found") + } + + node := inputs.Front().Value.(*CandidateNode) + + var output bytes.Buffer + err = encoder.Encode(&output, node) + if err != nil { + t.Fatalf("Failed to encode YAML: %v", err) + } + + return output.String() +} + +func TestGoccyParserBasicFunctionality(t *testing.T) { + // Save original preferences + originalPrefs := ConfiguredYamlPreferences.Copy() + defer func() { + ConfiguredYamlPreferences = originalPrefs + }() + + // Enable goccy parser + ConfiguredYamlPreferences.UseGoccyParser = true + + yamlDoc := `--- +test: + field1: value1 + field2: 42 + field3: true +` + + decoder := YamlFormat.DecoderFactory() + encoder := YamlFormat.EncoderFactory() + + inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder) + test.AssertResult(t, nil, err) + test.AssertResult(t, 1, inputs.Len()) + + node := inputs.Front().Value.(*CandidateNode) + + var output bytes.Buffer + err = encoder.Encode(&output, node) + test.AssertResult(t, nil, err) + + result := output.String() + + // Basic checks - the exact format might differ but structure should be preserved + if !strings.Contains(result, "test:") { + t.Error("Expected 'test:' in output") + } + if !strings.Contains(result, "field1: value1") { + t.Error("Expected 'field1: value1' in output") + } + if !strings.Contains(result, "field2: 42") { + t.Error("Expected 'field2: 42' in output") + } + if !strings.Contains(result, "field3: true") { + t.Error("Expected 'field3: true' in output") + } +} diff --git a/pkg/yqlib/yaml.go b/pkg/yqlib/yaml.go index 079bc749..99cf3d0d 100644 --- a/pkg/yqlib/yaml.go +++ b/pkg/yqlib/yaml.go @@ -7,6 +7,7 @@ type YamlPreferences struct { PrintDocSeparators bool UnwrapScalar bool EvaluateTogether bool + UseGoccyParser bool } func NewDefaultYamlPreferences() YamlPreferences { @@ -17,6 +18,7 @@ func NewDefaultYamlPreferences() YamlPreferences { PrintDocSeparators: true, UnwrapScalar: true, EvaluateTogether: false, + UseGoccyParser: true, // Default to goccy parser (actively maintained) } } @@ -28,6 +30,7 @@ func (p *YamlPreferences) Copy() YamlPreferences { PrintDocSeparators: p.PrintDocSeparators, UnwrapScalar: p.UnwrapScalar, EvaluateTogether: p.EvaluateTogether, + UseGoccyParser: p.UseGoccyParser, } }