From 37fd85768ea7088a55ed9a9dce9e43caed253e6d Mon Sep 17 00:00:00 2001 From: Mihail Vratchanski Date: Sat, 7 Jun 2025 12:14:55 +0300 Subject: [PATCH] Remove parser comparison test, fix code after rebase --- .../operators/anchor-and-alias-operators.md | 4 +- pkg/yqlib/operators_test.go | 2 +- pkg/yqlib/parser_comparison_test.go | 452 ------------------ 3 files changed, 3 insertions(+), 455 deletions(-) delete mode 100644 pkg/yqlib/parser_comparison_test.go diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md index 7a138336..14b0bef2 100644 --- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md @@ -241,7 +241,7 @@ Given a sample.yml file of: ```yaml f: a: &a cat - "*a": b + *a: b ``` then ```bash @@ -251,7 +251,7 @@ will output ```yaml f: a: cat - "*a": b + cat: b ``` ## Explode with merge anchors diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 8a9030b0..ff557ac2 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -47,7 +47,7 @@ func TestMain(m *testing.M) { goccyTesting = os.Getenv("GOCCY") == "true" if goccyTesting { - testingDecoder = NewGoccyYAMLDecoder() + testingDecoder = NewGoccyYAMLDecoder(ConfiguredYamlPreferences) } Now = func() time.Time { diff --git a/pkg/yqlib/parser_comparison_test.go b/pkg/yqlib/parser_comparison_test.go deleted file mode 100644 index c19046a7..00000000 --- a/pkg/yqlib/parser_comparison_test.go +++ /dev/null @@ -1,452 +0,0 @@ -package yqlib - -import ( - "bytes" - "fmt" - "strings" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -var testYamlDocs = []string{ - `--- -name: John -age: 30 -address: - street: 123 Main St - city: town - state: ST - zip: 12345 -hobbies: - - reading - - swimming - - cooking -`, - `--- -# This is a comment -version: "1.0" -database: - host: localhost - port: 5432 - name: app - credentials: - username: admin - password: secret123 -servers: - - name: web1 - ip: 192.168.1.10 - - name: web2 - ip: 192.168.1.11 -`, - `--- -# Complex YAML with various types -config: - enabled: true - timeout: 30 - ratio: 3.14159 - options: - debug: false - verbose: true - items: [] - metadata: null - multiline: | - This is a multiline - string with line breaks - and proper formatting -`, - `--- -# Anchors and Aliases -defaults: &defaults - adapter: postgres - host: localhost - -development: - <<: *defaults - database: app_dev - -test: - <<: *defaults - database: app_test -`, - `--- -# Multiple documents -doc: 1 -key: value1 ---- -doc: 2 -key: value2 -`, - `--- -# Flow style and tags -flow_map: {item1: val1, item2: val2} -flow_seq: [alpha, bravo, charlie] -custom_tag: !myTag data -`, -} - -func TestYamlParsersStructuralEquivalence(t *testing.T) { - // Save original preferences - originalPrefs := ConfiguredYamlPreferences.Copy() - defer func() { - ConfiguredYamlPreferences = originalPrefs - }() - - for i, yamlDoc := range testYamlDocs { - t.Run(fmt.Sprintf("Document_%d", i), func(t *testing.T) { - // Test with yaml.v3 parser - ConfiguredYamlPreferences.UseGoccyParser = false - v3Nodes := parseYamlDocument(t, yamlDoc) - - // Test with goccy parser - ConfiguredYamlPreferences.UseGoccyParser = true - goccyNodes := parseYamlDocument(t, yamlDoc) - - if len(v3Nodes) != len(goccyNodes) { - t.Errorf("Number of parsed documents differ between yaml.v3 (%d) and goccy (%d) for document %d", len(v3Nodes), len(goccyNodes), i) - // Log nodes for debugging if lengths differ significantly or one is empty - if len(v3Nodes) < 5 && len(goccyNodes) < 5 { // Avoid excessive logging for large diffs - for idx, node := range v3Nodes { - t.Logf("yaml.v3 result[%d]: %s", idx, NodeToString(node)) - } - for idx, node := range goccyNodes { - t.Logf("goccy result[%d]: %s", idx, NodeToString(node)) - } - } - return // No point comparing nodes if doc counts differ - } - - // Compare the parsed structures (not text output) - for j := range v3Nodes { - v3Node := v3Nodes[j] - goccyNode := goccyNodes[j] - - // Use specialized comparison for documents with merge tags (Document 3) - var nodesEqual bool - if i == 3 { - // Document 3 contains merge tags which require specialized comparison - nodesEqual = recursiveNodeEqualWithMergeHandling(v3Node, goccyNode) - } else { - nodesEqual = recursiveNodeEqual(v3Node, goccyNode) - } - - if !nodesEqual { - t.Errorf("Parsed structures differ between yaml.v3 and goccy for document %d, sub-document %d", i, j) - t.Logf("yaml.v3 result: %s", NodeToString(v3Node)) - t.Logf("goccy result: %s", NodeToString(goccyNode)) - } - } - }) - } -} - -func parseYamlDocument(t *testing.T, yamlDoc string) []*CandidateNode { - decoder := YamlFormat.DecoderFactory() - - inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder) - if err != nil { - t.Fatalf("Failed to decode YAML: %v", err) - } - - if inputs.Len() == 0 { - t.Fatal("No documents found") - } - - var nodes []*CandidateNode - for el := inputs.Front(); el != nil; el = el.Next() { - nodes = append(nodes, el.Value.(*CandidateNode)) - } - return nodes -} - -func TestYamlParsersEquivalence(t *testing.T) { - // Save original preferences - originalPrefs := ConfiguredYamlPreferences.Copy() - defer func() { - ConfiguredYamlPreferences = originalPrefs - }() - - // Note: This test documents the differences in output formatting between parsers - // These differences are acceptable as they don't affect semantic meaning - t.Log("Note: yaml.v3 and goccy may produce different key ordering, which is semantically equivalent") - - for i, yamlDoc := range testYamlDocs { - t.Run(fmt.Sprintf("Document_%d", i), func(t *testing.T) { - // Test with yaml.v3 parser - ConfiguredYamlPreferences.UseGoccyParser = false - v3Result := processYamlDocument(t, yamlDoc) - - // Test with goccy parser - ConfiguredYamlPreferences.UseGoccyParser = true - goccyResult := processYamlDocument(t, yamlDoc) - - // Log the differences for documentation purposes - if v3Result != goccyResult { - t.Logf("Output format differs (this is expected):") - t.Logf("yaml.v3 output:\n%s", v3Result) - t.Logf("goccy output:\n%s", goccyResult) - - // But both should be valid YAML that parses to the same structure - v3ReparsedNodes := parseYamlDocument(t, v3Result) - goccyReparsedNodes := parseYamlDocument(t, goccyResult) - - if len(v3ReparsedNodes) != len(goccyReparsedNodes) { - t.Errorf("Number of reparsed documents differ between yaml.v3 (%d) and goccy (%d) for document %d", len(v3ReparsedNodes), len(goccyReparsedNodes), i) - // Log nodes for debugging if lengths differ significantly - if len(v3ReparsedNodes) < 5 && len(goccyReparsedNodes) < 5 { // Avoid excessive logging - for idx, node := range v3ReparsedNodes { - t.Logf("yaml.v3 reparsed result[%d]: %s", idx, NodeToString(node)) - } - for idx, node := range goccyReparsedNodes { - t.Logf("goccy reparsed result[%d]: %s", idx, NodeToString(node)) - } - } - } else { - for j := range v3ReparsedNodes { - v3ReparseNode := v3ReparsedNodes[j] - goccyReparseNode := goccyReparsedNodes[j] - if !recursiveNodeEqualWithMergeHandling(v3ReparseNode, goccyReparseNode) { - t.Errorf("Reparsed structures differ for document %d, sub-document %d - this indicates a semantic issue", i, j) - // Force logging of nodes involved in the direct failing comparison - t.Logf("Failed comparison details:") - t.Logf("--- yaml.v3 reparsed node ---\n%s\nNode Structure:\n%s", NodeToString(v3ReparseNode), GetNodeStructure(v3ReparseNode, 0)) - t.Logf("--- goccy reparsed node ---\n%s\nNode Structure:\n%s", NodeToString(goccyReparseNode), GetNodeStructure(goccyReparseNode, 0)) - } - } - } - } - }) - } -} - -func processYamlDocument(t *testing.T, yamlDoc string) string { - decoder := YamlFormat.DecoderFactory() - encoder := YamlFormat.EncoderFactory() - - inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder) - if err != nil { - t.Fatalf("Failed to decode YAML: %v", err) - } - - if inputs.Len() == 0 { - t.Fatal("No documents found") - } - - var allDocsOutput strings.Builder - for i, el := 0, inputs.Front(); el != nil; i, el = i+1, el.Next() { - node := el.Value.(*CandidateNode) - var singleDocOutput bytes.Buffer - err = encoder.Encode(&singleDocOutput, node) - if err != nil { - t.Fatalf("Failed to encode YAML document %d: %v", i, err) - } - if i > 0 { - allDocsOutput.WriteString("\n---\n") // Add separator for subsequent documents - } - allDocsOutput.Write(singleDocOutput.Bytes()) - } - - return allDocsOutput.String() -} - -func TestGoccyParserBasicFunctionality(t *testing.T) { - // Save original preferences - originalPrefs := ConfiguredYamlPreferences.Copy() - defer func() { - ConfiguredYamlPreferences = originalPrefs - }() - - // Enable goccy parser - ConfiguredYamlPreferences.UseGoccyParser = true - - yamlDoc := `--- -test: - field1: value1 - field2: 42 - field3: true -` - - decoder := YamlFormat.DecoderFactory() - encoder := YamlFormat.EncoderFactory() - - inputs, err := readDocuments(strings.NewReader(yamlDoc), "test.yml", 0, decoder) - test.AssertResult(t, nil, err) - test.AssertResult(t, 1, inputs.Len()) - - node := inputs.Front().Value.(*CandidateNode) - - var output bytes.Buffer - err = encoder.Encode(&output, node) - test.AssertResult(t, nil, err) - - result := output.String() - - // Basic checks - the exact format might differ but structure should be preserved - if !strings.Contains(result, "test:") { - t.Error("Expected 'test:' in output") - } - if !strings.Contains(result, "field1: value1") { - t.Error("Expected 'field1: value1' in output") - } - if !strings.Contains(result, "field2: 42") { - t.Error("Expected 'field2: 42' in output") - } - if !strings.Contains(result, "field3: true") { - t.Error("Expected 'field3: true' in output") - } -} - -// Add a helper function to get detailed node structure for logging -func GetNodeStructure(node *CandidateNode, depth int) string { - if node == nil { - return strings.Repeat(" ", depth) + "\n" - } - indent := strings.Repeat(" ", depth) - var sb strings.Builder - sb.WriteString(fmt.Sprintf("%sKind: %s, Tag: '%s', Value: '%s', Path: %s\n", indent, KindString(node.Kind), node.Tag, node.Value, node.GetNicePath())) - if node.Alias != nil { - sb.WriteString(fmt.Sprintf("%s AliasToTag: '%s', AliasToValue: '%s'\n", indent, node.Alias.Tag, node.Alias.Value)) - if node.Alias.Alias != nil { - sb.WriteString(fmt.Sprintf("%s PointsToTag: '%s', PointsToValue: '%s'\n", indent, node.Alias.Alias.Tag, node.Alias.Alias.Value)) - } - } - if len(node.Content) > 0 { - sb.WriteString(fmt.Sprintf("%s Content (%d items):\n", indent, len(node.Content))) - for i, child := range node.Content { - sb.WriteString(fmt.Sprintf("%s [%d]:\n", indent, i)) - sb.WriteString(GetNodeStructure(child, depth+2)) - } - } - return sb.String() -} - -// recursiveNodeEqualWithMergeHandling compares two nodes but handles the case where -// one parser uses alias nodes for merge tags while the other expands them directly -func recursiveNodeEqualWithMergeHandling(lhs *CandidateNode, rhs *CandidateNode) bool { - return recursiveNodeEqualWithMergeHandlingHelper(lhs, rhs, lhs, rhs) -} - -// recursiveNodeEqualWithMergeHandlingHelper includes root nodes for alias resolution -func recursiveNodeEqualWithMergeHandlingHelper(lhs *CandidateNode, rhs *CandidateNode, lhsRoot *CandidateNode, rhsRoot *CandidateNode) bool { - if recursiveNodeEqual(lhs, rhs) { - return true - } - - // Special handling for merge tag differences between parsers - if lhs != nil && rhs != nil && lhs.Kind == MappingNode && rhs.Kind == MappingNode { - return compareMappingNodesWithMergeHandling(lhs, rhs, lhsRoot, rhsRoot) - } - - // Handle sequence nodes - if lhs != nil && rhs != nil && lhs.Kind == SequenceNode && rhs.Kind == SequenceNode { - if len(lhs.Content) != len(rhs.Content) { - return false - } - for i := range lhs.Content { - if !recursiveNodeEqualWithMergeHandlingHelper(lhs.Content[i], rhs.Content[i], lhsRoot, rhsRoot) { - return false - } - } - return true - } - - return false -} - -// compareMappingNodesWithMergeHandling compares mapping nodes and handles merge tag differences -func compareMappingNodesWithMergeHandling(lhs *CandidateNode, rhs *CandidateNode, lhsRoot *CandidateNode, rhsRoot *CandidateNode) bool { - // Create maps of key-value pairs for comparison - lhsMap := make(map[string]*CandidateNode) - rhsMap := make(map[string]*CandidateNode) - - // Helper function to find alias target in the root mapping - findAliasTarget := func(rootNode *CandidateNode, aliasName string) *CandidateNode { - for i := 0; i < len(rootNode.Content); i += 2 { - if rootNode.Content[i].Value == aliasName && rootNode.Content[i+1].Kind == MappingNode { - return rootNode.Content[i+1] - } - } - return nil - } - - // Parse lhs content - for i := 0; i < len(lhs.Content); i += 2 { - key := lhs.Content[i].Value - value := lhs.Content[i+1] - - if key == "<<" && value.Kind == AliasNode { - // This is a merge tag with alias - find the target node - aliasNode := findAliasTarget(lhsRoot, value.Value) - if aliasNode != nil { - // Add each key-value pair from the alias - for k := 0; k < len(aliasNode.Content); k += 2 { - aliasKey := aliasNode.Content[k].Value - aliasValue := aliasNode.Content[k+1] - if _, exists := lhsMap[aliasKey]; !exists { - lhsMap[aliasKey] = aliasValue - } - } - } - } else if key == "<<" && value.Kind == MappingNode { - // This is a merge tag with expanded content - for j := 0; j < len(value.Content); j += 2 { - mergeKey := value.Content[j].Value - mergeValue := value.Content[j+1] - if _, exists := lhsMap[mergeKey]; !exists { - lhsMap[mergeKey] = mergeValue - } - } - } else { - lhsMap[key] = value - } - } - - // Parse rhs content - for i := 0; i < len(rhs.Content); i += 2 { - key := rhs.Content[i].Value - value := rhs.Content[i+1] - - if key == "<<" && value.Kind == AliasNode { - // This is a merge tag with alias - find the target node - aliasNode := findAliasTarget(rhsRoot, value.Value) - if aliasNode != nil { - // Add each key-value pair from the alias - for k := 0; k < len(aliasNode.Content); k += 2 { - aliasKey := aliasNode.Content[k].Value - aliasValue := aliasNode.Content[k+1] - if _, exists := rhsMap[aliasKey]; !exists { - rhsMap[aliasKey] = aliasValue - } - } - } - } else if key == "<<" && value.Kind == MappingNode { - // This is a merge tag with expanded content - for j := 0; j < len(value.Content); j += 2 { - mergeKey := value.Content[j].Value - mergeValue := value.Content[j+1] - if _, exists := rhsMap[mergeKey]; !exists { - rhsMap[mergeKey] = mergeValue - } - } - } else { - rhsMap[key] = value - } - } - - // Compare the flattened maps - if len(lhsMap) != len(rhsMap) { - return false - } - - for key, lhsValue := range lhsMap { - rhsValue, exists := rhsMap[key] - if !exists { - return false - } - if !recursiveNodeEqualWithMergeHandlingHelper(lhsValue, rhsValue, lhsRoot, rhsRoot) { - return false - } - } - - return true -}