wip

pkg/yqlib/all_at_once_evaluator.go

@@ -2,8 +2,6 @@ package yqlib
 
 import (
 	"container/list"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
@@ -11,7 +9,7 @@ type Evaluator interface {
 	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
-	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
+	EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error)
 
 	// EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes
 	EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error)
@@ -26,10 +24,10 @@ func NewAllAtOnceEvaluator() Evaluator {
 	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()}
 }
 
-func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) {
+func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) {
 	inputCandidates := list.New()
 	for _, node := range nodes {
-		inputCandidates.PushBack(&CandidateNode{Node: node})
+		inputCandidates.PushBack(node)
 	}
 	return e.EvaluateCandidateNodes(expression, inputCandidates)
 }
@@ -68,7 +66,8 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		candidateNode := &CandidateNode{
 			Document:       0,
 			Filename:       "",
-			Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+			Kind:           DocumentNode,
+			Content:        []*CandidateNode{createScalarNode(nil, "")},
 			FileIndex:      0,
 			LeadingContent: "",
 		}

pkg/yqlib/candidate_node.go

@@ -4,15 +4,72 @@ import (
 	"container/list"
 	"fmt"
 	"strings"
-
-	"github.com/jinzhu/copier"
-	yaml "gopkg.in/yaml.v3"
 )
 
+type Kind uint32
+
+const (
+	DocumentNode Kind = 1 << iota
+	SequenceNode
+	MappingNode
+	ScalarNode
+	AliasNode
+)
+
+type Style uint32
+
+const (
+	TaggedStyle Style = 1 << iota
+	DoubleQuotedStyle
+	SingleQuotedStyle
+	LiteralStyle
+	FoldedStyle
+	FlowStyle
+)
+
+func createIntegerScalarNode(num int) *CandidateNode {
+	return &CandidateNode{
+		Kind:  ScalarNode,
+		Tag:   "!!int",
+		Value: fmt.Sprintf("%v", num),
+	}
+}
+
+func createScalarNode(value interface{}, stringValue string) *CandidateNode {
+	var node = &CandidateNode{Kind: ScalarNode}
+	node.Value = stringValue
+
+	switch value.(type) {
+	case float32, float64:
+		node.Tag = "!!float"
+	case int, int64, int32:
+		node.Tag = "!!int"
+	case bool:
+		node.Tag = "!!bool"
+	case string:
+		node.Tag = "!!str"
+	case nil:
+		node.Tag = "!!null"
+	}
+	return node
+}
+
 type CandidateNode struct {
-	Node   *yaml.Node     // the actual node
+	Kind  Kind
+	Style Style
+
+	Tag     string
+	Value   string
+	Anchor  string
+	Alias   *CandidateNode
+	Content []*CandidateNode
+
+	HeadComment string
+	LineComment string
+	FootComment string
+
 	Parent *CandidateNode // parent node
-	Key    *yaml.Node     // node key, if this is a value from a map (or index in an array)
+	Key    *CandidateNode // node key, if this is a value from a map (or index in an array)
 
 	LeadingContent  string
 	TrailingContent string
@@ -20,6 +77,10 @@ type CandidateNode struct {
 	Path     []interface{} /// the path we took to get to this node
 	Document uint          // the document index of this node
 	Filename string
+
+	Line   int
+	Column int
+
 	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
 	// (e.g. top level cross document merge). This property does not propagate to child nodes.
@@ -35,8 +96,15 @@ func (n *CandidateNode) GetKey() string {
 	return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
 }
 
+func (n *CandidateNode) unwrapDocument() *CandidateNode {
+	if n.Kind == DocumentNode {
+		return n.Content[0]
+	}
+	return n
+}
+
 func (n *CandidateNode) GetNiceTag() string {
-	return unwrapDoc(n.Node).Tag
+	return n.unwrapDocument().Tag
 }
 
 func (n *CandidateNode) GetNicePath() string {
@@ -56,13 +124,30 @@ func (n *CandidateNode) AsList() *list.List {
 	return elMap
 }
 
-func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *CandidateNode {
+func (n *CandidateNode) guessTagFromCustomType() string {
+	if strings.HasPrefix(n.Tag, "!!") {
+		return n.Tag
+	} else if n.Value == "" {
+		log.Debug("guessTagFromCustomType: node has no value to guess the type with")
+		return n.Tag
+	}
+	dataBucket, errorReading := parseSnippet(n.Value)
+
+	if errorReading != nil {
+		log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
+		return n.Tag
+	}
+	guessedTag := unwrapDoc(dataBucket).Tag
+	log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag)
+	return guessedTag
+}
+
+func (n *CandidateNode) CreateChildInMap(key *CandidateNode) *CandidateNode {
 	var value interface{}
 	if key != nil {
 		value = key.Value
 	}
 	return &CandidateNode{
-		Node:      node,
 		Path:      n.createChildPath(value),
 		Parent:    n,
 		Key:       key,
@@ -72,21 +157,19 @@ func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *Candi
 	}
 }
 
-func (n *CandidateNode) CreateChildInArray(index int, node *yaml.Node) *CandidateNode {
+func (n *CandidateNode) CreateChildInArray(index int) *CandidateNode {
 	return &CandidateNode{
-		Node:      node,
 		Path:      n.createChildPath(index),
 		Parent:    n,
-		Key:       &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", index), Tag: "!!int"},
+		Key:       createIntegerScalarNode(index),
 		Document:  n.Document,
 		Filename:  n.Filename,
 		FileIndex: n.FileIndex,
 	}
 }
 
-func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
+func (n *CandidateNode) CreateReplacement() *CandidateNode {
 	return &CandidateNode{
-		Node:      node,
 		Path:      n.createChildPath(nil),
 		Parent:    n.Parent,
 		Key:       n.Key,
@@ -97,12 +180,12 @@ func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
 	}
 }
 
-func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode {
+// func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode {
-	replacement := n.CreateReplacement(node)
+// 	replacement := n.CreateReplacement(node)
-	replacement.LeadingContent = n.LeadingContent
+// 	replacement.LeadingContent = n.LeadingContent
-	replacement.TrailingContent = n.TrailingContent
+// 	replacement.TrailingContent = n.TrailingContent
-	return replacement
+// 	return replacement
-}
+// }
 
 func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
 	if path == nil {
@@ -118,29 +201,64 @@ func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
 	return newPath
 }
 
-func (n *CandidateNode) Copy() (*CandidateNode, error) {
+func (n *CandidateNode) CopyChildren() []*CandidateNode {
-	clone := &CandidateNode{}
+	clonedKids := make([]*CandidateNode, len(n.Content))
-	err := copier.Copy(clone, n)
+	for i, child := range n.Content {
-	if err != nil {
+		clonedKids[i] = child.Copy()
-		return nil, err
+	}
+	return clonedKids
+}
+
+func (n *CandidateNode) Copy() *CandidateNode {
+	return &CandidateNode{
+		Kind:  n.Kind,
+		Style: n.Style,
+
+		Tag:    n.Tag,
+		Value:  n.Value,
+		Anchor: n.Anchor,
+
+		// ok not to clone this,
+		// as its a reference to somewhere else.
+		Alias:   n.Alias,
+		Content: n.CopyChildren(),
+
+		HeadComment: n.HeadComment,
+		LineComment: n.LineComment,
+		FootComment: n.FootComment,
+
+		Parent: n.Parent,
+		Key:    n.Key.Copy(),
+
+		LeadingContent:  n.LeadingContent,
+		TrailingContent: n.TrailingContent,
+
+		Path:     n.Path,
+		Document: n.Document,
+		Filename: n.Filename,
+
+		Line:   n.Line,
+		Column: n.Column,
+
+		FileIndex:        n.FileIndex,
+		EvaluateTogether: n.EvaluateTogether,
+		IsMapKey:         n.IsMapKey,
 	}
-	clone.Node = deepClone(n.Node)
-	return clone, nil
 }
 
 // updates this candidate from the given candidate node
 func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) {
 
 	// if this is an empty map or empty array, use the style of other node.
-	if (n.Node.Kind != yaml.ScalarNode && len(n.Node.Content) == 0) ||
+	if (n.Kind != ScalarNode && len(n.Content) == 0) ||
 		// if the tag has changed (e.g. from str to bool)
-		(guessTagFromCustomType(n.Node) != guessTagFromCustomType(other.Node)) {
+		(n.guessTagFromCustomType() != other.guessTagFromCustomType()) {
-		n.Node.Style = other.Node.Style
+		n.Style = other.Style
 	}
 
-	n.Node.Content = deepCloneContent(other.Node.Content)
+	n.Content = other.CopyChildren()
-	n.Node.Kind = other.Node.Kind
+	n.Kind = other.Kind
-	n.Node.Value = other.Node.Value
+	n.Value = other.Value
 
 	n.UpdateAttributesFrom(other, prefs)
 
@@ -148,42 +266,42 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences
 
 func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
 	log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey())
-	if n.Node.Kind != other.Node.Kind {
+	if n.Kind != other.Kind {
 		// clear out the contents when switching to a different type
 		// e.g. map to array
-		n.Node.Content = make([]*yaml.Node, 0)
+		n.Content = make([]*CandidateNode, 0)
-		n.Node.Value = ""
+		n.Value = ""
 	}
-	n.Node.Kind = other.Node.Kind
+	n.Kind = other.Kind
 
 	// don't clobber custom tags...
-	if prefs.ClobberCustomTags || strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" {
+	if prefs.ClobberCustomTags || strings.HasPrefix(n.Tag, "!!") || n.Tag == "" {
-		n.Node.Tag = other.Node.Tag
+		n.Tag = other.Tag
 	}
 
-	n.Node.Alias = other.Node.Alias
+	n.Alias = other.Alias
 
 	if !prefs.DontOverWriteAnchor {
-		n.Node.Anchor = other.Node.Anchor
+		n.Anchor = other.Anchor
 	}
 
 	// merge will pickup the style of the new thing
 	// when autocreating nodes
 
-	if n.Node.Style == 0 {
+	if n.Style == 0 {
-		n.Node.Style = other.Node.Style
+		n.Style = other.Style
 	}
 
-	if other.Node.FootComment != "" {
+	if other.FootComment != "" {
-		n.Node.FootComment = other.Node.FootComment
+		n.FootComment = other.FootComment
 	}
 	if other.TrailingContent != "" {
 		n.TrailingContent = other.TrailingContent
 	}
-	if other.Node.HeadComment != "" {
+	if other.HeadComment != "" {
-		n.Node.HeadComment = other.Node.HeadComment
+		n.HeadComment = other.HeadComment
 	}
-	if other.Node.LineComment != "" {
+	if other.LineComment != "" {
-		n.Node.LineComment = other.Node.LineComment
+		n.LineComment = other.LineComment
 	}
 }

pkg/yqlib/context.go

@@ -83,11 +83,7 @@ func (n *Context) DeepClone() Context {
 	// copier doesn't do lists properly for some reason
 	clone.MatchingNodes = list.New()
 	for el := n.MatchingNodes.Front(); el != nil; el = el.Next() {
-		clonedNode, err := el.Value.(*CandidateNode).Copy()
+		clonedNode := el.Value.(*CandidateNode).Copy()
-		if err != nil {
-			log.Error("Error cloning context :(")
-			panic(err)
-		}
 		clone.MatchingNodes.PushBack(clonedNode)
 	}
 

pkg/yqlib/decoder_yaml.go

@@ -96,6 +96,80 @@ func (dec *yamlDecoder) Init(reader io.Reader) error {
 	return nil
 }
 
+func (dec *yamlDecoder) convertKind(oKind yaml.Kind) Kind {
+	switch oKind {
+	case yaml.DocumentNode:
+		return DocumentNode
+	case yaml.SequenceNode:
+		return SequenceNode
+	case yaml.MappingNode:
+		return MappingNode
+	case yaml.ScalarNode:
+		return ScalarNode
+	case yaml.AliasNode:
+		return AliasNode
+	}
+	return ScalarNode
+}
+
+func (dec *yamlDecoder) convertStyle(oStyle yaml.Style) Style {
+	switch oStyle {
+	case yaml.TaggedStyle:
+		return TaggedStyle
+	case yaml.DoubleQuotedStyle:
+		return DoubleQuotedStyle
+	case yaml.SingleQuotedStyle:
+		return SingleQuotedStyle
+	case yaml.LiteralStyle:
+		return LiteralStyle
+	case yaml.FoldedStyle:
+		return FoldedStyle
+	case yaml.FlowStyle:
+		return FlowStyle
+	}
+	return 0
+}
+
+func (dec *yamlDecoder) ConvertToCandidateNode(yamlNode *yaml.Node) *CandidateNode {
+	kids := make([]*CandidateNode, len(yamlNode.Content))
+	for i, v := range yamlNode.Content {
+		kids[i] = dec.ConvertToCandidateNode(v)
+	}
+
+	return &CandidateNode{
+		Kind:  dec.convertKind(yamlNode.Kind),
+		Style: dec.convertStyle(yamlNode.Style),
+
+		Tag:     yamlNode.Tag,
+		Value:   yamlNode.Value,
+		Anchor:  yamlNode.Anchor,
+		Alias:   dec.ConvertToCandidateNode(yamlNode.Alias),
+		Content: kids,
+
+		HeadComment: yamlNode.HeadComment,
+		LineComment: yamlNode.LineComment,
+		FootComment: yamlNode.FootComment,
+
+		// Parent: yamlNode.Parent,
+		// Key:    yamlNode.Key,
+
+		// LeadingContent:  yamlNode.LeadingContent,
+		// TrailingContent: yamlNode.TrailingContent,
+
+		// Path:     yamlNode.Path,
+		// Document: yamlNode.Document,
+		// Filename: yamlNode.Filename,
+
+		Line:   yamlNode.Line,
+		Column: yamlNode.Column,
+
+		// FileIndex:        yamlNode.FileIndex,
+		// EvaluateTogether: yamlNode.EvaluateTogether,
+		// IsMapKey:         yamlNode.IsMapKey,
+	}
+
+}
+
 func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 	var dataBucket yaml.Node
 	err := dec.decoder.Decode(&dataBucket)
@@ -116,9 +190,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 		return nil, err
 	}
 
-	candidateNode := &CandidateNode{
+	candidateNode := dec.ConvertToCandidateNode(&dataBucket)
-		Node: &dataBucket,
-	}
 
 	if dec.leadingContent != "" {
 		candidateNode.LeadingContent = dec.leadingContent
@@ -136,7 +208,8 @@ func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
 	return &CandidateNode{
 		Document:       0,
 		Filename:       "",
-		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+		Kind:           DocumentNode,
+		Content:        []*CandidateNode{createScalarNode(nil, "")},
 		FileIndex:      0,
 		LeadingContent: dec.leadingContent,
 	}

pkg/yqlib/lib.go

@@ -389,25 +389,6 @@ func createStringScalarNode(stringValue string) *yaml.Node {
 	return node
 }
 
-func createScalarNode(value interface{}, stringValue string) *yaml.Node {
-	var node = &yaml.Node{Kind: yaml.ScalarNode}
-	node.Value = stringValue
-
-	switch value.(type) {
-	case float32, float64:
-		node.Tag = "!!float"
-	case int, int64, int32:
-		node.Tag = "!!int"
-	case bool:
-		node.Tag = "!!bool"
-	case string:
-		node.Tag = "!!str"
-	case nil:
-		node.Tag = "!!null"
-	}
-	return node
-}
-
 func headAndLineComment(node *yaml.Node) string {
 	return headComment(node) + lineComment(node)
 }

pkg/yqlib/stream_evaluator.go

@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 // A yaml expression evaluator that runs the expression multiple times for each given yaml document.
@@ -36,7 +34,8 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	candidateNode := &CandidateNode{
 		Document:  0,
 		Filename:  "",
-		Node:      &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+		Kind:      DocumentNode,
+		Content:   []*CandidateNode{createScalarNode(nil, "")},
 		FileIndex: 0,
 	}
 	inputList := list.New()