fix(unique): ignore comments when comparing array elements

The unique operator computed the dedup key for non-scalar elements by
encoding them to YAML, which includes their head/line/foot comments. A
comment sitting between two otherwise equal objects attaches to one of
them, changing its encoding, so the duplicates were wrongly kept. Encode
a comment-free copy of the node for the key so uniqueness depends on the
data only.

Fixes #2491

pkg/yqlib/operator_unique.go

@@ -66,8 +66,32 @@ func getUniqueKeyValue(rhs Context) (string, error) {
 		keyCandidate := first.Value.(*CandidateNode)
 		keyValue = keyCandidate.Value
 		if keyCandidate.Kind != ScalarNode {
-			keyValue, err = encodeToString(keyCandidate, encoderPreferences{YamlFormat, 0})
+			// Encode a comment-free copy so that uniqueness is decided by the
+			// data only. Otherwise a comment attached to one of two otherwise
+			// equal nodes changes its encoding and they are wrongly kept as
+			// distinct (see #2491).
+			keyValue, err = encodeToString(withoutComments(keyCandidate), encoderPreferences{YamlFormat, 0})
 		}
 	}
 	return keyValue, err
 }
+
+// withoutComments returns a deep copy of the node with all head, line and foot
+// comments removed throughout the tree, so comments do not affect equality.
+func withoutComments(node *CandidateNode) *CandidateNode {
+	clone := node.Copy()
+	clearComments(clone)
+	return clone
+}
+
+func clearComments(node *CandidateNode) {
+	node.HeadComment = ""
+	node.LineComment = ""
+	node.FootComment = ""
+	if node.Key != nil {
+		clearComments(node.Key)
+	}
+	for _, child := range node.Content {
+		clearComments(child)
+	}
+}

pkg/yqlib/operator_unique_test.go

@@ -58,6 +58,15 @@ var uniqueOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\n",
 		},
 	},
+	{
+		description: "Unique array of objects with a comment between equal items",
+		skipDoc:     true,
+		document:    "- id: 1001\n# Comment\n- id: 1001\n",
+		expression:  `unique`,
+		expected: []string{
+			"D0, P[], (!!seq)::- id: 1001\n",
+		},
+	},
 	{
 		description: "Unique array of arrays",
 		document:    `[[cat,dog], [cat, sheep], [cat,dog]]`,