Initial implementation

cmd/root.go

@@ -94,6 +94,12 @@ yq -P -oy sample.json
 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
 
+			// Handle legacy parser flag
+			useLegacyYamlParser, _ := cmd.Flags().GetBool("yaml-v3")
+			if useLegacyYamlParser {
+				yqlib.ConfiguredYamlPreferences.UseGoccyParser = false
+			}
+
 			return nil
 		},
 	}
@@ -197,6 +203,12 @@ yq -P -oy sample.json
 	}
 	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
 
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredYamlPreferences.UseGoccyParser, "yaml-goccy", true, "Use goccy/go-yaml parser (default)")
+
+	// Add a flag to revert to the legacy parser
+	var useLegacyYamlParser bool
+	rootCmd.PersistentFlags().BoolVar(&useLegacyYamlParser, "yaml-v3", false, "Use legacy gopkg.in/yaml.v3 parser instead of goccy (for backwards compatibility)")
+
 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.")
 	if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil {
 		panic(err)

pkg/yqlib/candidate_node_goccy_yaml.go

@@ -205,3 +205,85 @@ func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingVa
 
 	return nil
 }
+
+func (o *CandidateNode) MarshalGoccyYAML() (interface{}, error) {
+	log.Debug("MarshalGoccyYAML to goccy: %v", o.Tag)
+
+	switch o.Kind {
+	case AliasNode:
+		log.Debug("MarshalGoccyYAML - alias to goccy: %v", o.Tag)
+		// For goccy, we'll return the referenced value directly
+		// The goccy encoder will handle alias creation
+		if o.Alias != nil {
+			return o.Alias.MarshalGoccyYAML()
+		}
+		return o.Value, nil
+
+	case ScalarNode:
+		log.Debug("MarshalGoccyYAML - scalar: %v", o.Value)
+
+		// Handle different scalar types based on tag
+		switch o.Tag {
+		case "!!int":
+			if val, err := parseInt(o.Value); err == nil {
+				return val, nil
+			}
+		case "!!float":
+			if val, err := parseFloat(o.Value); err == nil {
+				return val, nil
+			}
+		case "!!bool":
+			if val, err := parseBool(o.Value); err == nil {
+				return val, nil
+			}
+		case "!!null":
+			return nil, nil
+		default:
+			// String or unknown type
+			return o.Value, nil
+		}
+		return o.Value, nil
+
+	case MappingNode:
+		log.Debug("MarshalGoccyYAML - mapping: %v", NodeToString(o))
+		result := make(map[string]interface{})
+
+		for i := 0; i < len(o.Content); i += 2 {
+			if i+1 >= len(o.Content) {
+				break
+			}
+
+			keyNode := o.Content[i]
+			valueNode := o.Content[i+1]
+
+			key := keyNode.Value
+			if key == "" {
+				key = NodeToString(keyNode)
+			}
+
+			value, err := valueNode.MarshalGoccyYAML()
+			if err != nil {
+				return nil, err
+			}
+
+			result[key] = value
+		}
+		return result, nil
+
+	case SequenceNode:
+		log.Debug("MarshalGoccyYAML - sequence: %v", NodeToString(o))
+		result := make([]interface{}, len(o.Content))
+
+		for i, childNode := range o.Content {
+			value, err := childNode.MarshalGoccyYAML()
+			if err != nil {
+				return nil, err
+			}
+			result[i] = value
+		}
+		return result, nil
+	}
+
+	// Default case
+	return o.Value, nil
+}

pkg/yqlib/decoder_goccy_yaml.go

@@ -7,7 +7,12 @@
 package yqlib
 
 import (
+	"bufio"
+	"bytes"
+	"errors"
 	"io"
+	"regexp"
+	"strings"
 
 	yaml "github.com/goccy/go-yaml"
 	"github.com/goccy/go-yaml/ast"
@@ -16,34 +21,141 @@ import (
 type goccyYamlDecoder struct {
 	decoder yaml.Decoder
 	cm      yaml.CommentMap
+
+	prefs YamlPreferences
+
+	// work around of various parsing issues by handling document headers
+	leadingContent string
+	bufferRead     bytes.Buffer
+
 	// anchor map persists over multiple documents for convenience.
 	anchorMap map[string]*CandidateNode
+
+	readAnything  bool
+	firstFile     bool
+	documentIndex uint
 }
 
-func NewGoccyYAMLDecoder() Decoder {
-	return &goccyYamlDecoder{}
+func NewGoccyYAMLDecoder(prefs YamlPreferences) Decoder {
+	return &goccyYamlDecoder{prefs: prefs, firstFile: true}
+}
+
+func (dec *goccyYamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+	var yamlDirectiveLineRegEx = regexp.MustCompile(`^\s*%YA`)
+	var sb strings.Builder
+	for {
+		peekBytes, err := reader.Peek(4)
+		if errors.Is(err, io.EOF) {
+			// EOF are handled else where..
+			return reader, sb.String(), nil
+		} else if err != nil {
+			return reader, sb.String(), err
+		} else if string(peekBytes[0]) == "\n" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("\n")
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if string(peekBytes) == "--- " {
+			_, err := reader.ReadString(' ')
+			sb.WriteString("$yqDocSeparator$\n")
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if string(peekBytes) == "---\n" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("$yqDocSeparator$\n")
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if commentLineRegEx.MatchString(string(peekBytes)) || yamlDirectiveLineRegEx.MatchString(string(peekBytes)) {
+			line, err := reader.ReadString('\n')
+			sb.WriteString(line)
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else {
+			return reader, sb.String(), nil
+		}
+	}
 }
 
 func (dec *goccyYamlDecoder) Init(reader io.Reader) error {
+	readerToUse := reader
+	leadingContent := ""
+	dec.bufferRead = bytes.Buffer{}
+	var err error
+	// if we 'evaluating together' - we only process the leading content
+	// of the first file - this ensures comments from subsequent files are
+	// merged together correctly.
+	if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) {
+		readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader))
+		if err != nil {
+			return err
+		}
+	} else if !dec.prefs.LeadingContentPreProcessing {
+		// if we're not process the leading content
+		// keep a copy of what we've read. This is incase its a
+		// doc with only comments - the decoder will return nothing
+		// then we can read the comments from bufferRead
+		readerToUse = io.TeeReader(reader, &dec.bufferRead)
+	}
+	dec.leadingContent = leadingContent
+	dec.readAnything = false
 	dec.cm = yaml.CommentMap{}
-	dec.decoder = *yaml.NewDecoder(reader, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap())
+	dec.decoder = *yaml.NewDecoder(readerToUse, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap())
+	dec.firstFile = false
+	dec.documentIndex = 0
 	dec.anchorMap = make(map[string]*CandidateNode)
 	return nil
 }
 
 func (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) {
+	var astNode ast.Node
+	err := dec.decoder.Decode(&astNode)
 
-	var ast ast.Node
-
-	err := dec.decoder.Decode(&ast)
-	if err != nil {
+	if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
+		// force returning an empty node with a comment.
+		dec.readAnything = true
+		return dec.blankNodeWithComment(), nil
+	} else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything {
+		// didn't find any yaml,
+		// check the tee buffer, maybe there were comments
+		dec.readAnything = true
+		dec.leadingContent = dec.bufferRead.String()
+		if dec.leadingContent != "" {
+			return dec.blankNodeWithComment(), nil
+		}
+		return nil, err
+	} else if err != nil {
 		return nil, err
 	}
 
-	candidateNode := &CandidateNode{}
-	if err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm, dec.anchorMap); err != nil {
+	candidateNode := &CandidateNode{document: dec.documentIndex}
+	if err := candidateNode.UnmarshalGoccyYAML(astNode, dec.cm, dec.anchorMap); err != nil {
 		return nil, err
 	}
 
+	if dec.leadingContent != "" {
+		candidateNode.LeadingContent = dec.leadingContent
+		dec.leadingContent = ""
+	}
+	dec.readAnything = true
+	dec.documentIndex++
 	return candidateNode, nil
 }
+
+func (dec *goccyYamlDecoder) blankNodeWithComment() *CandidateNode {
+	node := createScalarNode(nil, "")
+	node.LeadingContent = dec.leadingContent
+	return node
+}

pkg/yqlib/encoder_goccy_yaml.go

@@ -0,0 +1,126 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/fatih/color"
+	yaml "github.com/goccy/go-yaml"
+)
+
+type goccyYamlEncoder struct {
+	prefs YamlPreferences
+}
+
+func NewGoccyYamlEncoder(prefs YamlPreferences) Encoder {
+	return &goccyYamlEncoder{prefs}
+}
+
+func (ye *goccyYamlEncoder) CanHandleAliases() bool {
+	return true
+}
+
+func (ye *goccyYamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
+	if ye.prefs.PrintDocSeparators {
+		log.Debug("writing doc sep")
+		if err := writeString(writer, "---\n"); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ye *goccyYamlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+	reader := bufio.NewReader(strings.NewReader(content))
+
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+
+	for {
+		readline, errReading := reader.ReadString('\n')
+		if errReading != nil && !errors.Is(errReading, io.EOF) {
+			return errReading
+		}
+		if strings.Contains(readline, "$yqDocSeparator$") {
+			if err := ye.PrintDocumentSeparator(writer); err != nil {
+				return err
+			}
+		} else {
+			if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) {
+				readline = "# " + readline
+			}
+			if ye.prefs.ColorsEnabled && strings.TrimSpace(readline) != "" {
+				readline = format(color.FgHiBlack) + readline + format(color.Reset)
+			}
+			if err := writeString(writer, readline); err != nil {
+				return err
+			}
+		}
+
+		if errors.Is(errReading, io.EOF) {
+			if readline != "" {
+				// the last comment we read didn't have a newline, put one in
+				if err := writeString(writer, "\n"); err != nil {
+					return err
+				}
+			}
+			break
+		}
+	}
+
+	return nil
+}
+
+func (ye *goccyYamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
+	log.Debug("goccyYamlEncoder - going to print %v", NodeToString(node))
+	if node.Kind == ScalarNode && ye.prefs.UnwrapScalar {
+		valueToPrint := node.Value
+		if node.LeadingContent == "" || valueToPrint != "" {
+			valueToPrint = valueToPrint + "\n"
+		}
+		return writeString(writer, valueToPrint)
+	}
+
+	destination := writer
+	tempBuffer := bytes.NewBuffer(nil)
+	if ye.prefs.ColorsEnabled {
+		destination = tempBuffer
+	}
+
+	// Create encoder with indent option
+	var encoder *yaml.Encoder
+	if ye.prefs.Indent > 0 {
+		encoder = yaml.NewEncoder(destination, yaml.Indent(ye.prefs.Indent))
+	} else {
+		encoder = yaml.NewEncoder(destination)
+	}
+
+	target, err := node.MarshalGoccyYAML()
+	if err != nil {
+		return err
+	}
+
+	trailingContent := node.FootComment
+	if target != nil {
+		// Store and clear foot comment to handle it separately
+		if astNode, ok := target.(interface{ GetComment() interface{} }); ok {
+			_ = astNode // placeholder for potential future foot comment handling
+		}
+	}
+
+	if err := encoder.Encode(target); err != nil {
+		return err
+	}
+
+	if err := ye.PrintLeadingContent(destination, trailingContent); err != nil {
+		return err
+	}
+
+	if ye.prefs.ColorsEnabled {
+		return colorizeAndPrint(tempBuffer.Bytes(), writer)
+	}
+	return nil
+}

pkg/yqlib/format.go

@@ -18,8 +18,18 @@ type Format struct {
 }
 
 var YamlFormat = &Format{"yaml", []string{"y", "yml"},
-	func() Encoder { return NewYamlEncoder(ConfiguredYamlPreferences) },
-	func() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) },
+	func() Encoder {
+		if ConfiguredYamlPreferences.UseGoccyParser {
+			return NewGoccyYamlEncoder(ConfiguredYamlPreferences)
+		}
+		return NewYamlEncoder(ConfiguredYamlPreferences)
+	},
+	func() Decoder {
+		if ConfiguredYamlPreferences.UseGoccyParser {
+			return NewGoccyYAMLDecoder(ConfiguredYamlPreferences)
+		}
+		return NewYamlDecoder(ConfiguredYamlPreferences)
+	},
 }
 
 var JSONFormat = &Format{"json", []string{"j"},

pkg/yqlib/goccy_yaml_test.go

@@ -272,7 +272,7 @@ var goccyYamlFormatScenarios = []formatScenario{
 }
 
 func testGoccyYamlScenario(t *testing.T, s formatScenario) {
-	test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+	test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 }
 
 func TestGoccyYmlFormatScenarios(t *testing.T) {

pkg/yqlib/lib.go

@@ -175,6 +175,17 @@ func parseInt(numberString string) (int, error) {
 	return int(parsed), err
 }
 
+func parseFloat(numberString string) (float64, error) {
+	if strings.Contains(numberString, "_") {
+		numberString = strings.ReplaceAll(numberString, "_", "")
+	}
+	return strconv.ParseFloat(numberString, 64)
+}
+
+func parseBool(boolString string) (bool, error) {
+	return strconv.ParseBool(boolString)
+}
+
 func headAndLineComment(node *CandidateNode) string {
 	return headComment(node) + lineComment(node)
 }

pkg/yqlib/parser_comparison_test.go

@@ -0,0 +1,210 @@
+package yqlib
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+var testYamlDocs = []string{
+	`---
+name: John
+age: 30
+address:
+  street: 123 Main St
+  city: Anytown
+  state: ST
+  zip: 12345
+hobbies:
+  - reading
+  - swimming
+  - cooking
+`,
+	`---
+# This is a comment
+version: "1.0"
+database:
+  host: localhost
+  port: 5432
+  name: myapp
+  credentials:
+    username: admin
+    password: secret123
+servers:
+  - name: web1
+    ip: 192.168.1.10
+  - name: web2  
+    ip: 192.168.1.11
+`,
+	`---
+# Complex YAML with various types
+config:
+  enabled: true
+  timeout: 30
+  ratio: 3.14159
+  options:
+    debug: false
+    verbose: true
+  items: []
+  metadata: null
+  multiline: |
+    This is a multiline
+    string with line breaks
+    and proper formatting
+`,
+}
+
+func TestYamlParsersStructuralEquivalence(t *testing.T) {
+	// Save original preferences
+	originalPrefs := ConfiguredYamlPreferences.Copy()
+	defer func() {
+		ConfiguredYamlPreferences = originalPrefs
+	}()
+
+	for i, yamlDoc := range testYamlDocs {
+		t.Run(fmt.Sprintf("Document_%d", i), func(t *testing.T) {
+			// Test with yaml.v3 parser
+			ConfiguredYamlPreferences.UseGoccyParser = false
+			v3Node := parseYamlDocument(t, yamlDoc)
+
+			// Test with goccy parser
+			ConfiguredYamlPreferences.UseGoccyParser = true
+			goccyNode := parseYamlDocument(t, yamlDoc)
+
+			// Compare the parsed structures (not text output)
+			if !recursiveNodeEqual(v3Node, goccyNode) {
+				t.Errorf("Parsed structures differ between yaml.v3 and goccy for document %d", i)
+				t.Logf("yaml.v3 result: %s", NodeToString(v3Node))
+				t.Logf("goccy result: %s", NodeToString(goccyNode))
+			}
+		})
+	}
+}
+
+func parseYamlDocument(t *testing.T, yamlDoc string) *CandidateNode {
+	decoder := YamlFormat.DecoderFactory()
+
+	inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder)
+	if err != nil {
+		t.Fatalf("Failed to decode YAML: %v", err)
+	}
+
+	if inputs.Len() == 0 {
+		t.Fatal("No documents found")
+	}
+
+	return inputs.Front().Value.(*CandidateNode)
+}
+
+func TestYamlParsersEquivalence(t *testing.T) {
+	// Save original preferences
+	originalPrefs := ConfiguredYamlPreferences.Copy()
+	defer func() {
+		ConfiguredYamlPreferences = originalPrefs
+	}()
+
+	// Note: This test documents the differences in output formatting between parsers
+	// These differences are acceptable as they don't affect semantic meaning
+	t.Log("Note: yaml.v3 and goccy may produce different key ordering, which is semantically equivalent")
+
+	for i, yamlDoc := range testYamlDocs {
+		t.Run(fmt.Sprintf("Document_%d", i), func(t *testing.T) {
+			// Test with yaml.v3 parser
+			ConfiguredYamlPreferences.UseGoccyParser = false
+			v3Result := processYamlDocument(t, yamlDoc)
+
+			// Test with goccy parser
+			ConfiguredYamlPreferences.UseGoccyParser = true
+			goccyResult := processYamlDocument(t, yamlDoc)
+
+			// Log the differences for documentation purposes
+			if v3Result != goccyResult {
+				t.Logf("Output format differs (this is expected):")
+				t.Logf("yaml.v3 output:\n%s", v3Result)
+				t.Logf("goccy output:\n%s", goccyResult)
+
+				// But both should be valid YAML that parses to the same structure
+				v3Reparse := parseYamlDocument(t, v3Result)
+				goccyReparse := parseYamlDocument(t, goccyResult)
+
+				if !recursiveNodeEqual(v3Reparse, goccyReparse) {
+					t.Error("Reparsed structures differ - this indicates a semantic issue")
+				}
+			}
+		})
+	}
+}
+
+func processYamlDocument(t *testing.T, yamlDoc string) string {
+	decoder := YamlFormat.DecoderFactory()
+	encoder := YamlFormat.EncoderFactory()
+
+	inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder)
+	if err != nil {
+		t.Fatalf("Failed to decode YAML: %v", err)
+	}
+
+	if inputs.Len() == 0 {
+		t.Fatal("No documents found")
+	}
+
+	node := inputs.Front().Value.(*CandidateNode)
+
+	var output bytes.Buffer
+	err = encoder.Encode(&output, node)
+	if err != nil {
+		t.Fatalf("Failed to encode YAML: %v", err)
+	}
+
+	return output.String()
+}
+
+func TestGoccyParserBasicFunctionality(t *testing.T) {
+	// Save original preferences
+	originalPrefs := ConfiguredYamlPreferences.Copy()
+	defer func() {
+		ConfiguredYamlPreferences = originalPrefs
+	}()
+
+	// Enable goccy parser
+	ConfiguredYamlPreferences.UseGoccyParser = true
+
+	yamlDoc := `---
+test:
+  field1: value1
+  field2: 42
+  field3: true
+`
+
+	decoder := YamlFormat.DecoderFactory()
+	encoder := YamlFormat.EncoderFactory()
+
+	inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder)
+	test.AssertResult(t, nil, err)
+	test.AssertResult(t, 1, inputs.Len())
+
+	node := inputs.Front().Value.(*CandidateNode)
+
+	var output bytes.Buffer
+	err = encoder.Encode(&output, node)
+	test.AssertResult(t, nil, err)
+
+	result := output.String()
+
+	// Basic checks - the exact format might differ but structure should be preserved
+	if !strings.Contains(result, "test:") {
+		t.Error("Expected 'test:' in output")
+	}
+	if !strings.Contains(result, "field1: value1") {
+		t.Error("Expected 'field1: value1' in output")
+	}
+	if !strings.Contains(result, "field2: 42") {
+		t.Error("Expected 'field2: 42' in output")
+	}
+	if !strings.Contains(result, "field3: true") {
+		t.Error("Expected 'field3: true' in output")
+	}
+}

pkg/yqlib/yaml.go

@@ -7,6 +7,7 @@ type YamlPreferences struct {
 	PrintDocSeparators          bool
 	UnwrapScalar                bool
 	EvaluateTogether            bool
+	UseGoccyParser              bool
 }
 
 func NewDefaultYamlPreferences() YamlPreferences {
@@ -17,6 +18,7 @@ func NewDefaultYamlPreferences() YamlPreferences {
 		PrintDocSeparators:          true,
 		UnwrapScalar:                true,
 		EvaluateTogether:            false,
+		UseGoccyParser:              true, // Default to goccy parser (actively maintained)
 	}
 }
 
@@ -28,6 +30,7 @@ func (p *YamlPreferences) Copy() YamlPreferences {
 		PrintDocSeparators:          p.PrintDocSeparators,
 		UnwrapScalar:                p.UnwrapScalar,
 		EvaluateTogether:            p.EvaluateTogether,
+		UseGoccyParser:              p.UseGoccyParser,
 	}
 }