From 6a891e52391033fce1250d5077beb93e8036a9c5 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 3 Apr 2023 16:16:07 +1000 Subject: [PATCH] wip --- pkg/yqlib/all_at_once_evaluator.go | 11 +- pkg/yqlib/candidate_node.go | 220 ++++++++++++++++++++++------- pkg/yqlib/context.go | 6 +- pkg/yqlib/decoder_yaml.go | 81 ++++++++++- pkg/yqlib/lib.go | 19 --- pkg/yqlib/stream_evaluator.go | 5 +- 6 files changed, 254 insertions(+), 88 deletions(-) diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index 4dbed3d1..1ff3fb4b 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -2,8 +2,6 @@ package yqlib import ( "container/list" - - yaml "gopkg.in/yaml.v3" ) // A yaml expression evaluator that runs the expression once against all files/nodes in memory. @@ -11,7 +9,7 @@ type Evaluator interface { EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error // EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes - EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) + EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) // EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error) @@ -26,10 +24,10 @@ func NewAllAtOnceEvaluator() Evaluator { return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()} } -func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) { +func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) { inputCandidates := list.New() for _, node := range nodes { - inputCandidates.PushBack(&CandidateNode{Node: node}) + inputCandidates.PushBack(node) } return e.EvaluateCandidateNodes(expression, inputCandidates) } @@ -68,7 +66,8 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string candidateNode := &CandidateNode{ Document: 0, Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + Kind: DocumentNode, + Content: []*CandidateNode{createScalarNode(nil, "")}, FileIndex: 0, LeadingContent: "", } diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 61675c8a..004e687d 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -4,22 +4,83 @@ import ( "container/list" "fmt" "strings" - - "github.com/jinzhu/copier" - yaml "gopkg.in/yaml.v3" ) +type Kind uint32 + +const ( + DocumentNode Kind = 1 << iota + SequenceNode + MappingNode + ScalarNode + AliasNode +) + +type Style uint32 + +const ( + TaggedStyle Style = 1 << iota + DoubleQuotedStyle + SingleQuotedStyle + LiteralStyle + FoldedStyle + FlowStyle +) + +func createIntegerScalarNode(num int) *CandidateNode { + return &CandidateNode{ + Kind: ScalarNode, + Tag: "!!int", + Value: fmt.Sprintf("%v", num), + } +} + +func createScalarNode(value interface{}, stringValue string) *CandidateNode { + var node = &CandidateNode{Kind: ScalarNode} + node.Value = stringValue + + switch value.(type) { + case float32, float64: + node.Tag = "!!float" + case int, int64, int32: + node.Tag = "!!int" + case bool: + node.Tag = "!!bool" + case string: + node.Tag = "!!str" + case nil: + node.Tag = "!!null" + } + return node +} + type CandidateNode struct { - Node *yaml.Node // the actual node + Kind Kind + Style Style + + Tag string + Value string + Anchor string + Alias *CandidateNode + Content []*CandidateNode + + HeadComment string + LineComment string + FootComment string + Parent *CandidateNode // parent node - Key *yaml.Node // node key, if this is a value from a map (or index in an array) + Key *CandidateNode // node key, if this is a value from a map (or index in an array) LeadingContent string TrailingContent string - Path []interface{} /// the path we took to get to this node - Document uint // the document index of this node - Filename string + Path []interface{} /// the path we took to get to this node + Document uint // the document index of this node + Filename string + + Line int + Column int + FileIndex int // when performing op against all nodes given, this will treat all the nodes as one // (e.g. top level cross document merge). This property does not propagate to child nodes. @@ -35,8 +96,15 @@ func (n *CandidateNode) GetKey() string { return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path) } +func (n *CandidateNode) unwrapDocument() *CandidateNode { + if n.Kind == DocumentNode { + return n.Content[0] + } + return n +} + func (n *CandidateNode) GetNiceTag() string { - return unwrapDoc(n.Node).Tag + return n.unwrapDocument().Tag } func (n *CandidateNode) GetNicePath() string { @@ -56,13 +124,30 @@ func (n *CandidateNode) AsList() *list.List { return elMap } -func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *CandidateNode { +func (n *CandidateNode) guessTagFromCustomType() string { + if strings.HasPrefix(n.Tag, "!!") { + return n.Tag + } else if n.Value == "" { + log.Debug("guessTagFromCustomType: node has no value to guess the type with") + return n.Tag + } + dataBucket, errorReading := parseSnippet(n.Value) + + if errorReading != nil { + log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading) + return n.Tag + } + guessedTag := unwrapDoc(dataBucket).Tag + log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag) + return guessedTag +} + +func (n *CandidateNode) CreateChildInMap(key *CandidateNode) *CandidateNode { var value interface{} if key != nil { value = key.Value } return &CandidateNode{ - Node: node, Path: n.createChildPath(value), Parent: n, Key: key, @@ -72,21 +157,19 @@ func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *Candi } } -func (n *CandidateNode) CreateChildInArray(index int, node *yaml.Node) *CandidateNode { +func (n *CandidateNode) CreateChildInArray(index int) *CandidateNode { return &CandidateNode{ - Node: node, Path: n.createChildPath(index), Parent: n, - Key: &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", index), Tag: "!!int"}, + Key: createIntegerScalarNode(index), Document: n.Document, Filename: n.Filename, FileIndex: n.FileIndex, } } -func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode { +func (n *CandidateNode) CreateReplacement() *CandidateNode { return &CandidateNode{ - Node: node, Path: n.createChildPath(nil), Parent: n.Parent, Key: n.Key, @@ -97,12 +180,12 @@ func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode { } } -func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode { - replacement := n.CreateReplacement(node) - replacement.LeadingContent = n.LeadingContent - replacement.TrailingContent = n.TrailingContent - return replacement -} +// func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode { +// replacement := n.CreateReplacement(node) +// replacement.LeadingContent = n.LeadingContent +// replacement.TrailingContent = n.TrailingContent +// return replacement +// } func (n *CandidateNode) createChildPath(path interface{}) []interface{} { if path == nil { @@ -118,29 +201,64 @@ func (n *CandidateNode) createChildPath(path interface{}) []interface{} { return newPath } -func (n *CandidateNode) Copy() (*CandidateNode, error) { - clone := &CandidateNode{} - err := copier.Copy(clone, n) - if err != nil { - return nil, err +func (n *CandidateNode) CopyChildren() []*CandidateNode { + clonedKids := make([]*CandidateNode, len(n.Content)) + for i, child := range n.Content { + clonedKids[i] = child.Copy() + } + return clonedKids +} + +func (n *CandidateNode) Copy() *CandidateNode { + return &CandidateNode{ + Kind: n.Kind, + Style: n.Style, + + Tag: n.Tag, + Value: n.Value, + Anchor: n.Anchor, + + // ok not to clone this, + // as its a reference to somewhere else. + Alias: n.Alias, + Content: n.CopyChildren(), + + HeadComment: n.HeadComment, + LineComment: n.LineComment, + FootComment: n.FootComment, + + Parent: n.Parent, + Key: n.Key.Copy(), + + LeadingContent: n.LeadingContent, + TrailingContent: n.TrailingContent, + + Path: n.Path, + Document: n.Document, + Filename: n.Filename, + + Line: n.Line, + Column: n.Column, + + FileIndex: n.FileIndex, + EvaluateTogether: n.EvaluateTogether, + IsMapKey: n.IsMapKey, } - clone.Node = deepClone(n.Node) - return clone, nil } // updates this candidate from the given candidate node func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) { // if this is an empty map or empty array, use the style of other node. - if (n.Node.Kind != yaml.ScalarNode && len(n.Node.Content) == 0) || + if (n.Kind != ScalarNode && len(n.Content) == 0) || // if the tag has changed (e.g. from str to bool) - (guessTagFromCustomType(n.Node) != guessTagFromCustomType(other.Node)) { - n.Node.Style = other.Node.Style + (n.guessTagFromCustomType() != other.guessTagFromCustomType()) { + n.Style = other.Style } - n.Node.Content = deepCloneContent(other.Node.Content) - n.Node.Kind = other.Node.Kind - n.Node.Value = other.Node.Value + n.Content = other.CopyChildren() + n.Kind = other.Kind + n.Value = other.Value n.UpdateAttributesFrom(other, prefs) @@ -148,42 +266,42 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) { log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey()) - if n.Node.Kind != other.Node.Kind { + if n.Kind != other.Kind { // clear out the contents when switching to a different type // e.g. map to array - n.Node.Content = make([]*yaml.Node, 0) - n.Node.Value = "" + n.Content = make([]*CandidateNode, 0) + n.Value = "" } - n.Node.Kind = other.Node.Kind + n.Kind = other.Kind // don't clobber custom tags... - if prefs.ClobberCustomTags || strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" { - n.Node.Tag = other.Node.Tag + if prefs.ClobberCustomTags || strings.HasPrefix(n.Tag, "!!") || n.Tag == "" { + n.Tag = other.Tag } - n.Node.Alias = other.Node.Alias + n.Alias = other.Alias if !prefs.DontOverWriteAnchor { - n.Node.Anchor = other.Node.Anchor + n.Anchor = other.Anchor } // merge will pickup the style of the new thing // when autocreating nodes - if n.Node.Style == 0 { - n.Node.Style = other.Node.Style + if n.Style == 0 { + n.Style = other.Style } - if other.Node.FootComment != "" { - n.Node.FootComment = other.Node.FootComment + if other.FootComment != "" { + n.FootComment = other.FootComment } if other.TrailingContent != "" { n.TrailingContent = other.TrailingContent } - if other.Node.HeadComment != "" { - n.Node.HeadComment = other.Node.HeadComment + if other.HeadComment != "" { + n.HeadComment = other.HeadComment } - if other.Node.LineComment != "" { - n.Node.LineComment = other.Node.LineComment + if other.LineComment != "" { + n.LineComment = other.LineComment } } diff --git a/pkg/yqlib/context.go b/pkg/yqlib/context.go index 17cddb3c..d2dc656b 100644 --- a/pkg/yqlib/context.go +++ b/pkg/yqlib/context.go @@ -83,11 +83,7 @@ func (n *Context) DeepClone() Context { // copier doesn't do lists properly for some reason clone.MatchingNodes = list.New() for el := n.MatchingNodes.Front(); el != nil; el = el.Next() { - clonedNode, err := el.Value.(*CandidateNode).Copy() - if err != nil { - log.Error("Error cloning context :(") - panic(err) - } + clonedNode := el.Value.(*CandidateNode).Copy() clone.MatchingNodes.PushBack(clonedNode) } diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go index a87233bc..f84444a3 100644 --- a/pkg/yqlib/decoder_yaml.go +++ b/pkg/yqlib/decoder_yaml.go @@ -96,6 +96,80 @@ func (dec *yamlDecoder) Init(reader io.Reader) error { return nil } +func (dec *yamlDecoder) convertKind(oKind yaml.Kind) Kind { + switch oKind { + case yaml.DocumentNode: + return DocumentNode + case yaml.SequenceNode: + return SequenceNode + case yaml.MappingNode: + return MappingNode + case yaml.ScalarNode: + return ScalarNode + case yaml.AliasNode: + return AliasNode + } + return ScalarNode +} + +func (dec *yamlDecoder) convertStyle(oStyle yaml.Style) Style { + switch oStyle { + case yaml.TaggedStyle: + return TaggedStyle + case yaml.DoubleQuotedStyle: + return DoubleQuotedStyle + case yaml.SingleQuotedStyle: + return SingleQuotedStyle + case yaml.LiteralStyle: + return LiteralStyle + case yaml.FoldedStyle: + return FoldedStyle + case yaml.FlowStyle: + return FlowStyle + } + return 0 +} + +func (dec *yamlDecoder) ConvertToCandidateNode(yamlNode *yaml.Node) *CandidateNode { + kids := make([]*CandidateNode, len(yamlNode.Content)) + for i, v := range yamlNode.Content { + kids[i] = dec.ConvertToCandidateNode(v) + } + + return &CandidateNode{ + Kind: dec.convertKind(yamlNode.Kind), + Style: dec.convertStyle(yamlNode.Style), + + Tag: yamlNode.Tag, + Value: yamlNode.Value, + Anchor: yamlNode.Anchor, + Alias: dec.ConvertToCandidateNode(yamlNode.Alias), + Content: kids, + + HeadComment: yamlNode.HeadComment, + LineComment: yamlNode.LineComment, + FootComment: yamlNode.FootComment, + + // Parent: yamlNode.Parent, + // Key: yamlNode.Key, + + // LeadingContent: yamlNode.LeadingContent, + // TrailingContent: yamlNode.TrailingContent, + + // Path: yamlNode.Path, + // Document: yamlNode.Document, + // Filename: yamlNode.Filename, + + Line: yamlNode.Line, + Column: yamlNode.Column, + + // FileIndex: yamlNode.FileIndex, + // EvaluateTogether: yamlNode.EvaluateTogether, + // IsMapKey: yamlNode.IsMapKey, + } + +} + func (dec *yamlDecoder) Decode() (*CandidateNode, error) { var dataBucket yaml.Node err := dec.decoder.Decode(&dataBucket) @@ -116,9 +190,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) { return nil, err } - candidateNode := &CandidateNode{ - Node: &dataBucket, - } + candidateNode := dec.ConvertToCandidateNode(&dataBucket) if dec.leadingContent != "" { candidateNode.LeadingContent = dec.leadingContent @@ -136,7 +208,8 @@ func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode { return &CandidateNode{ Document: 0, Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + Kind: DocumentNode, + Content: []*CandidateNode{createScalarNode(nil, "")}, FileIndex: 0, LeadingContent: dec.leadingContent, } diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 5fc503ae..05ce2bd2 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -389,25 +389,6 @@ func createStringScalarNode(stringValue string) *yaml.Node { return node } -func createScalarNode(value interface{}, stringValue string) *yaml.Node { - var node = &yaml.Node{Kind: yaml.ScalarNode} - node.Value = stringValue - - switch value.(type) { - case float32, float64: - node.Tag = "!!float" - case int, int64, int32: - node.Tag = "!!int" - case bool: - node.Tag = "!!bool" - case string: - node.Tag = "!!str" - case nil: - node.Tag = "!!null" - } - return node -} - func headAndLineComment(node *yaml.Node) string { return headComment(node) + lineComment(node) } diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index ef78ca98..dad06273 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -6,8 +6,6 @@ import ( "fmt" "io" "os" - - yaml "gopkg.in/yaml.v3" ) // A yaml expression evaluator that runs the expression multiple times for each given yaml document. @@ -36,7 +34,8 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error candidateNode := &CandidateNode{ Document: 0, Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + Kind: DocumentNode, + Content: []*CandidateNode{createScalarNode(nil, "")}, FileIndex: 0, } inputList := list.New()