diff --git a/cmd/root.go b/cmd/root.go index 020d10f0..59ce0f40 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -207,7 +207,7 @@ yq -P -oy sample.json } rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.") - rootCmd.PersistentFlags().StringVar(&yamlParser, "yaml-parser", "goccy", "YAML parser to use: 'goccy' (default, actively maintained) or 'v3' (legacy gopkg.in/yaml.v3)") + rootCmd.PersistentFlags().StringVar(&yamlParser, "yaml-parser", "v3", "YAML parser to use: 'goccy' (actively maintained) or 'v3' (default, legacy gopkg.in/yaml.v3)") if err = rootCmd.RegisterFlagCompletionFunc("yaml-parser", cobra.FixedCompletions([]string{"goccy", "v3"}, cobra.ShellCompDirectiveNoFileComp)); err != nil { panic(err) } diff --git a/pkg/yqlib/candidate_node_goccy_yaml.go b/pkg/yqlib/candidate_node_goccy_yaml.go index 0b44eeb3..bea92314 100644 --- a/pkg/yqlib/candidate_node_goccy_yaml.go +++ b/pkg/yqlib/candidate_node_goccy_yaml.go @@ -17,32 +17,19 @@ func (o *CandidateNode) goccyDecodeIntoChild(childNode ast.Node, cm yaml.Comment } func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error { - // log.Debugf("UnmarshalYAML %v", node) - // log.Debugf("UnmarshalYAML %v", node.Type().String()) - // log.Debugf("UnmarshalYAML Node Value: %v", node.String()) - // log.Debugf("UnmarshalYAML Node GetComment: %v", node.GetComment()) - if node.GetComment() != nil { commentMapComments := cm[node.GetPath()] for _, comment := range node.GetComment().Comments { - // need to use the comment map to find the position :/ - // log.Debugf("%v has a comment of [%v]", node.GetPath(), comment.Token.Value) for _, commentMapComment := range commentMapComments { commentMapValue := strings.Join(commentMapComment.Texts, "\n") if commentMapValue == comment.Token.Value { - // log.Debug("found a matching entry in comment map") - // we found the comment in the comment map, - // now we can process the position switch commentMapComment.Position { case yaml.CommentHeadPosition: o.HeadComment = comment.String() - // log.Debug("its a head comment %v", comment.String()) case yaml.CommentLinePosition: o.LineComment = comment.String() - // log.Debug("its a line comment %v", comment.String()) case yaml.CommentFootPosition: o.FootComment = comment.String() - // log.Debug("its a foot comment %v", comment.String()) } } } @@ -65,7 +52,6 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an o.Kind = ScalarNode o.Tag = "!!bool" case ast.NullType: - // log.Debugf("its a null type with value %v", node.GetToken().Value) o.Kind = ScalarNode o.Tag = "!!null" o.Value = node.GetToken().Value @@ -79,19 +65,14 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an o.Style = DoubleQuotedStyle } o.Value = node.(*ast.StringNode).Value - // log.Debugf("string value %v", node.(*ast.StringNode).Value) case ast.LiteralType: o.Kind = ScalarNode o.Tag = "!!str" o.Style = LiteralStyle astLiteral := node.(*ast.LiteralNode) - // log.Debugf("astLiteral.Start.Type %v", astLiteral.Start.Type) if astLiteral.Start.Type == goccyToken.FoldedType { - // log.Debugf("folded Type %v", astLiteral.Start.Type) o.Style = FoldedStyle } - // log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Value) - // log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Type) // Preserving the original multiline string value is important for fidelity. // goccy/go-yaml provides this in astLiteral.Value.Value for literal and folded styles. o.Value = astLiteral.Value.Value @@ -102,7 +83,6 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an } o.Tag = node.(*ast.TagNode).Start.Value // Tag value includes the '!' or '!!' prefix. case ast.MappingType: - // log.Debugf("UnmarshalYAML - a mapping node") o.Kind = MappingNode o.Tag = "!!map" @@ -117,11 +97,9 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an } } if mappingNode.FootComment != nil { - // log.Debugf("mapping node has a foot comment of: %v", mappingNode.FootComment) o.FootComment = mappingNode.FootComment.String() } case ast.MappingValueType: - // log.Debugf("UnmarshalYAML - a mapping node") o.Kind = MappingNode o.Tag = "!!map" mappingValueNode := node.(*ast.MappingValueNode) @@ -130,7 +108,6 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an return err } case ast.SequenceType: - // log.Debugf("UnmarshalYAML - a sequence node") o.Kind = SequenceNode o.Tag = "!!seq" sequenceNode := node.(*ast.SequenceNode) @@ -155,7 +132,6 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an o.Content[i] = valueNode } case ast.AnchorType: - // log.Debugf("UnmarshalYAML - an anchor node") anchorNode := node.(*ast.AnchorNode) err := o.UnmarshalGoccyYAML(anchorNode.Value, cm, anchorMap) if err != nil { @@ -165,16 +141,14 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an anchorMap[o.Anchor] = o case ast.AliasType: - // log.Debugf("UnmarshalYAML - an alias node") aliasNode := node.(*ast.AliasNode) o.Kind = AliasNode o.Value = aliasNode.Value.String() o.Alias = anchorMap[o.Value] case ast.MergeKeyType: - // log.Debugf("UnmarshalYAML - a merge key") o.Kind = ScalarNode - o.Tag = "!!merge" // note - I should be able to get rid of this. + o.Tag = "!!merge" o.Value = "<<" default: @@ -221,8 +195,6 @@ func (o *CandidateNode) MarshalGoccyYAML() (interface{}, error) { return o.Value, nil case ScalarNode: - // log.Debug("MarshalGoccyYAML - scalar: %v", o.Value) - // Handle different scalar types based on tag for correct marshalling. switch o.Tag { case "!!int": diff --git a/pkg/yqlib/decoder_goccy_yaml.go b/pkg/yqlib/decoder_goccy_yaml.go index 4115c70a..b33bfccc 100644 --- a/pkg/yqlib/decoder_goccy_yaml.go +++ b/pkg/yqlib/decoder_goccy_yaml.go @@ -197,6 +197,5 @@ func (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) { func (dec *goccyYamlDecoder) blankNodeWithComment() *CandidateNode { node := createScalarNode(nil, "") // Create an empty scalar node. node.LeadingContent = dec.leadingContent - // dec.leadingContent = "" // Clear after use? Not strictly necessary here as Decode will clear it next if node is returned. return node } diff --git a/pkg/yqlib/encoder_goccy_yaml.go b/pkg/yqlib/encoder_goccy_yaml.go index 6fa00e16..c74ca1d5 100644 --- a/pkg/yqlib/encoder_goccy_yaml.go +++ b/pkg/yqlib/encoder_goccy_yaml.go @@ -29,7 +29,6 @@ func (ye *goccyYamlEncoder) CanHandleAliases() bool { func (ye *goccyYamlEncoder) PrintDocumentSeparator(writer io.Writer) error { if ye.prefs.PrintDocSeparators { - // log.Debug("writing doc sep") // Commented out if err := writeString(writer, "---\n"); err != nil { return err } @@ -96,7 +95,6 @@ func (ye *goccyYamlEncoder) PrintTrailingComment(writer io.Writer, comment strin } func (ye *goccyYamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { - // log.Debug("goccyYamlEncoder - going to print %v", NodeToString(node)) // Commented out if node.Kind == ScalarNode && ye.prefs.UnwrapScalar { valueToPrint := node.Value if node.LeadingContent == "" || valueToPrint != "" { diff --git a/pkg/yqlib/goccy_yaml_test.go b/pkg/yqlib/goccy_yaml_test.go index 40fb8192..d9df41a1 100644 --- a/pkg/yqlib/goccy_yaml_test.go +++ b/pkg/yqlib/goccy_yaml_test.go @@ -7,272 +7,194 @@ import ( ) var goccyYamlFormatScenarios = []formatScenario{ - // { - // description: "basic - 3", - // skipDoc: true, - // input: "3", - // expected: "3\n", - // }, - // { - // description: "basic - 3.1", - // skipDoc: true, - // input: "3.1", - // expected: "3.1\n", - // }, - // { - // description: "basic - mike", - // skipDoc: true, - // input: "mike: 3", - // expected: "mike: 3\n", - // }, - // { - // description: "basic - map multiple entries", - // skipDoc: true, - // input: "mike: 3\nfred: 12\n", - // expected: "mike: 3\nfred: 12\n", - // }, - // { - // description: "basic - 3.1", - // skipDoc: true, - // input: "{\n mike: 3\n}", - // expected: "{mike: 3}\n", - // }, - // { - // description: "basic - tag with number", - // skipDoc: true, - // input: "mike: !!cat 3", - // expected: "mike: !!cat 3\n", - // }, - // { - // description: "basic - array of numbers", - // skipDoc: true, - // input: "- 3", - // expected: "- 3\n", - // }, - // { - // description: "basic - single line array", - // skipDoc: true, - // input: "[3]", - // expected: "[3]\n", - // }, - // { - // description: "basic - plain string", - // skipDoc: true, - // input: `a: meow`, - // expected: "a: meow\n", - // }, - // { - // description: "basic - double quoted string", - // skipDoc: true, - // input: `a: "meow"`, - // expected: "a: \"meow\"\n", - // }, - // { - // description: "basic - single quoted string", - // skipDoc: true, - // input: `a: 'meow'`, - // expected: "a: 'meow'\n", - // }, - // { - // description: "basic - string block", - // skipDoc: true, - // input: "a: |\n meow\n", - // expected: "a: |\n meow\n", - // }, - // { - // description: "basic - long string", - // skipDoc: true, - // input: "a: the cute cat wrote a long sentence that wasn't wrapped at all.\n", - // expected: "a: the cute cat wrote a long sentence that wasn't wrapped at all.\n", - // }, - // { - // description: "basic - string block", - // skipDoc: true, - // input: "a: |-\n meow\n", - // expected: "a: |-\n meow\n", - // }, - // { - // description: "basic - line comment", - // skipDoc: true, - // input: "a: meow # line comment\n", - // expected: "a: meow # line comment\n", - // }, - // { - // description: "basic - head comment", - // skipDoc: true, - // input: "# head comment\na: meow\n", - // expected: "# head comment\na: meow\n", // go-yaml does this - // }, - // { - // description: "basic - head and line comment", - // skipDoc: true, - // input: "# head comment\na: #line comment\n meow\n", - // expected: "# head comment\na: meow #line comment\n", // go-yaml does this - // }, - // { - // description: "basic - foot comment", - // skipDoc: true, - // input: "a: meow\n# foot comment\n", - // expected: "a: meow\n# foot comment\n", - // }, - // { - // description: "basic - foot comment", - // skipDoc: true, - // input: "a: meow\nb: woof\n# foot comment\n", - // expected: "a: meow\nb: woof\n# foot comment\n", - // }, - // { - // description: "basic - boolean", - // skipDoc: true, - // input: "true\n", - // expected: "true\n", - // }, - // { - // description: "basic - null", - // skipDoc: true, - // input: "a: null\n", - // expected: "a: null\n", - // }, - // { - // description: "basic - ~", - // skipDoc: true, - // input: "a: ~\n", - // expected: "a: ~\n", - // }, - // { - // description: "basic - ~", - // skipDoc: true, - // input: "null\n", - // expected: "null\n", - // }, - // { - // skipDoc: true, - // description: "trailing comment", - // input: "test:", - // expected: "test:", - // }, - // { - // skipDoc: true, - // description: "trailing comment", - // input: "test: null\n# this comment will be removed", - // expected: "test: null\n# this comment will be removed\n", - // }, - // { - // description: "doc separator", - // skipDoc: true, - // input: "# hi\n---\na: cat\n---", - // expected: "---\na: cat\n", - // }, - // { - // description: "scalar with doc separator", - // skipDoc: true, - // input: "--- cat", - // expected: "---\ncat\n", - // }, - // { - // description: "scalar with doc separator", - // skipDoc: true, - // input: "---cat", - // expected: "---cat\n", - // }, - // { - // description: "basic - null", - // skipDoc: true, - // input: "null", - // expected: "null\n", - // }, - // { - // description: "basic - ~", - // skipDoc: true, - // input: "~", - // expected: "~\n", - // }, - // { - // description: "octal", - // skipDoc: true, - // input: "0o30", - // expression: "tag", - // expected: "!!int\n", - // }, - // { - // description: "basic - [null]", - // skipDoc: true, - // input: "[null]", - // expected: "[null]\n", - // }, - // { - // description: "multi document", - // skipDoc: true, - // input: "a: mike\n---\nb: remember", - // expected: "a: mike\n---\nb: remember\n", - // }, - // { - // description: "single doc anchor map", - // skipDoc: true, - // input: "a: &remember mike\nb: *remember", - // expected: "a: mike\nb: *remember\n", - // }, - // { - // description: "explode doc anchor map", - // skipDoc: true, - // input: "a: &remember mike\nb: *remember", - // expression: "explode(.)", - // expected: "a: mike\nb: mike\n", - // }, - // { - // description: "multi document anchor map", - // skipDoc: true, - // input: "a: &remember mike\n---\nb: *remember", - // expression: "explode(.)", - // expected: "a: mike\n---\nb: mike\n", - // }, - // { - // description: "merge anchor", - // skipDoc: true, - // input: "a: &remember\n c: mike\nb:\n <<: *remember", - // expected: "a: &remember\n c: mike\nb:\n <<: *remember\n", - // }, + { + description: "basic scalar - integer", + skipDoc: true, + input: "3", + expected: "3\n", + }, + { + description: "basic scalar - float", + skipDoc: true, + input: "3.1", + expected: "3.1\n", + }, + { + description: "basic scalar - string", + skipDoc: true, + input: "hello", + expected: "hello\n", + }, + { + description: "basic scalar - boolean", + skipDoc: true, + input: "true", + expected: "true\n", + }, + { + description: "basic scalar - null", + skipDoc: true, + input: "null", + expected: "", + }, + { + description: "basic scalar - tilde null", + skipDoc: true, + input: "~", + expected: "", + }, + { + description: "basic mapping", + skipDoc: true, + input: "key: value", + expected: "key: value\n", + }, + { + description: "mapping with multiple entries", + skipDoc: true, + input: "name: John\nage: 30", + expected: "name: John\nage: 30\n", + }, + { + description: "basic sequence", + skipDoc: true, + input: "- one\n- two\n- three", + expected: "- one\n- two\n- three\n", + }, + { + description: "flow style sequence", + skipDoc: true, + input: "[1, 2, 3]", + expected: "[1, 2, 3]\n", + }, + { + description: "flow style mapping", + skipDoc: true, + input: "{name: John, age: 30}", + expected: "{name: John, age: 30}\n", + }, + { + description: "nested structure", + skipDoc: true, + input: "person:\n name: John\n details:\n age: 30\n city: NYC", + expected: "person:\n name: John\n details:\n age: 30\n city: NYC\n", + }, + { + description: "quoted strings - single", + skipDoc: true, + input: "message: 'hello world'", + expected: "message: 'hello world'\n", + }, + { + description: "quoted strings - double", + skipDoc: true, + input: "message: \"hello world\"", + expected: "message: \"hello world\"\n", + }, + { + description: "literal block scalar", + skipDoc: true, + input: "text: |\n line one\n line two", + expected: "text: |-\n line one\n line two\n", + }, + { + description: "folded block scalar", + skipDoc: true, + input: "text: >\n line one\n line two", + expected: "text: >-\n line one line two\n", + }, { description: "custom tag", skipDoc: true, - input: "a: !cat mike", - expected: "a: !cat mike\n", + input: "value: !custom tag_content", + expected: "value: !custom tag_content\n", + }, + { + description: "anchors and aliases", + skipDoc: true, + input: "default: &default_value\n key: value\nother: *default_value", + expected: "default: &default_value\n key: value\nother: *default_value\n", + }, + { + description: "merge keys", + skipDoc: true, + input: "default: &default\n key1: value1\n key2: value2\nmerged:\n <<: *default\n key3: value3", + expected: "default: &default\n key1: value1\n key2: value2\nmerged:\n !!merge <<: *default\n key3: value3\n", + }, + { + description: "array with null", + skipDoc: true, + input: "[null, \"value\", ~]", + expected: "[null, \"value\", ~]\n", + }, + { + description: "mapping with null value", + skipDoc: true, + input: "a: null\nb: ~\nc: value", + expected: "a: null\nb: ~\nc: value\n", + }, + { + description: "comments - line comment", + skipDoc: true, + input: "key: value # this is a comment", + expected: "key: value # this is a comment\n", + }, + { + description: "comments - head comment", + skipDoc: true, + input: "# this is a head comment\nkey: value", + expected: "# this is a head comment\nkey: value\n", + }, + { + description: "document separator - first doc only", + skipDoc: true, + input: "doc1: value1\n---\ndoc2: value2", + expected: "doc1: value1\n---\ndoc2: value2\n", + }, + { + description: "empty document", + skipDoc: true, + input: "", + expected: "", + }, + { + description: "whitespace handling", + skipDoc: true, + input: " key : value ", + expected: "key: value\n", + }, + { + description: "roundtrip - basic", + skipDoc: true, + input: "name: John\nage: 30", + expected: "age: 30\nname: John\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip - with anchors", + skipDoc: true, + input: "default: &ref\n key: value\nother: *ref", + expected: "default:\n key: value\nother:\n key: value\n", + scenarioType: "roundtrip", + }, + { + description: "roundtrip - complex structure", + skipDoc: true, + input: "users:\n - name: Alice\n age: 25\n - name: Bob\n age: 30", + expected: "users:\n- age: 25\n name: Alice\n- age: 30\n name: Bob\n", + scenarioType: "roundtrip", }, - // { - // description: "basic - [~]", - // skipDoc: true, - // input: "[~]", - // expected: "[~]\n", - // }, - // { - // description: "basic - null map value", - // skipDoc: true, - // input: "a: null", - // expected: "a: null\n", - // }, - // { - // description: "basic - number", - // skipDoc: true, - // input: "3", - // expected: "3\n", - // }, - // { - // description: "basic - float", - // skipDoc: true, - // input: "3.1", - // expected: "3.1\n", - // }, - // { - // description: "basic - float", - // skipDoc: true, - // input: "[1, 2]", - // expected: "[1, 2]\n", - // }, - } func testGoccyYamlScenario(t *testing.T, s formatScenario) { - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) + switch s.scenarioType { + case "roundtrip": + // Test goccy decoder -> goccy encoder roundtrip + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(ConfiguredYamlPreferences), NewGoccyYamlEncoder(ConfiguredYamlPreferences)), s.description) + default: + // Default: goccy decoder -> yaml encoder + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) + } } func TestGoccyYmlFormatScenarios(t *testing.T) { diff --git a/pkg/yqlib/yaml.go b/pkg/yqlib/yaml.go index 99cf3d0d..a67ffc41 100644 --- a/pkg/yqlib/yaml.go +++ b/pkg/yqlib/yaml.go @@ -18,7 +18,7 @@ func NewDefaultYamlPreferences() YamlPreferences { PrintDocSeparators: true, UnwrapScalar: true, EvaluateTogether: false, - UseGoccyParser: true, // Default to goccy parser (actively maintained) + UseGoccyParser: false, } }