diff --git a/cmd/constant.go b/cmd/constant.go index f9bad317..45400f76 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -11,19 +11,18 @@ var unwrapScalar = true var customStyle = "" var anchorName = "" var makeAlias = false -var stripComments = false var writeInplace = false var writeScript = "" var sourceYamlFile = "" var outputToJSON = false var exitStatus = false -var prettyPrint = false var explodeAnchors = false var forceColor = false var forceNoColor = false var colorsEnabled = false var defaultValue = "" var indent = 2 +var printDocSeparators = true var overwriteFlag = false var autoCreateFlag = true var arrayMergeStrategyFlag = "update" diff --git a/cmd/root.go b/cmd/root.go index 9ab13b0c..17602f65 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/mikefarah/yq/v4/pkg/yqlib/treeops" + "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" logging "gopkg.in/op/go-logging.v1" ) @@ -40,7 +40,7 @@ func New() *cobra.Command { // } cmd.SilenceUsage = true - var treeCreator = treeops.NewPathTreeCreator() + var treeCreator = yqlib.NewPathTreeCreator() expression := "" if len(args) > 0 { @@ -53,13 +53,13 @@ func New() *cobra.Command { } if outputToJSON { - explodeOp := treeops.Operation{OperationType: treeops.Explode} - explodeNode := treeops.PathTreeNode{Operation: &explodeOp} - pipeOp := treeops.Operation{OperationType: treeops.Pipe} - pathNode = &treeops.PathTreeNode{Operation: &pipeOp, Lhs: pathNode, Rhs: &explodeNode} + explodeOp := yqlib.Operation{OperationType: yqlib.Explode} + explodeNode := yqlib.PathTreeNode{Operation: &explodeOp} + pipeOp := yqlib.Operation{OperationType: yqlib.Pipe} + pathNode = &yqlib.PathTreeNode{Operation: &pipeOp, Lhs: pathNode, Rhs: &explodeNode} } - matchingNodes, err := evaluate("-", pathNode) + matchingNodes, err := yqlib.Evaluate("-", pathNode) if err != nil { return err } @@ -71,7 +71,14 @@ func New() *cobra.Command { out := cmd.OutOrStdout() - return printResults(matchingNodes, out) + fileInfo, _ := os.Stdout.Stat() + + if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { + colorsEnabled = true + } + printer := yqlib.NewPrinter(outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators) + + return printer.PrintResults(matchingNodes, out) }, PersistentPreRun: func(cmd *cobra.Command, args []string) { cmd.SetOut(cmd.OutOrStdout()) @@ -92,8 +99,7 @@ func New() *cobra.Command { } rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode") - rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. By default it prints a json document in one line, use the prettyPrint flag to print a formatted doc.") - rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print") + rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. Set indent to 0 to print json in one line.") rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output") rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit") diff --git a/cmd/utils.go b/cmd/utils.go deleted file mode 100644 index debd25fa..00000000 --- a/cmd/utils.go +++ /dev/null @@ -1,232 +0,0 @@ -package cmd - -import ( - "bufio" - "container/list" - "errors" - "io" - "os" - - "github.com/mikefarah/yq/v4/pkg/yqlib" - "github.com/mikefarah/yq/v4/pkg/yqlib/treeops" - yaml "gopkg.in/yaml.v3" -) - -func readStream(filename string) (*yaml.Decoder, error) { - if filename == "" { - return nil, errors.New("Must provide filename") - } - - var stream io.Reader - if filename == "-" { - stream = bufio.NewReader(os.Stdin) - } else { - file, err := os.Open(filename) // nolint gosec - if err != nil { - return nil, err - } - defer safelyCloseFile(file) - stream = file - } - return yaml.NewDecoder(stream), nil -} - -func evaluate(filename string, node *treeops.PathTreeNode) (*list.List, error) { - - var treeNavigator = treeops.NewDataTreeNavigator(treeops.NavigationPrefs{}) - - var matchingNodes = list.New() - - var currentIndex uint = 0 - var decoder, err = readStream(filename) - if err != nil { - return nil, err - } - - for { - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) - - if errorReading == io.EOF { - return matchingNodes, nil - } else if errorReading != nil { - return nil, errorReading - } - candidateNode := &treeops.CandidateNode{ - Document: currentIndex, - Filename: filename, - Node: &dataBucket, - } - inputList := list.New() - inputList.PushBack(candidateNode) - - newMatches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node) - if errorParsing != nil { - return nil, errorParsing - } - matchingNodes.PushBackList(newMatches) - currentIndex = currentIndex + 1 - } - - return matchingNodes, nil -} - -func printNode(node *yaml.Node, writer io.Writer) error { - var encoder yqlib.Encoder - if node.Kind == yaml.ScalarNode && unwrapScalar && !outputToJSON { - return writeString(writer, node.Value+"\n") - } - if outputToJSON { - encoder = yqlib.NewJsonEncoder(writer, prettyPrint, indent) - } else { - encoder = yqlib.NewYamlEncoder(writer, indent, colorsEnabled) - } - return encoder.Encode(node) -} - -func removeComments(matchingNodes *list.List) { - for el := matchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*treeops.CandidateNode) - removeCommentOfNode(candidate.Node) - } -} - -func removeCommentOfNode(node *yaml.Node) { - node.HeadComment = "" - node.LineComment = "" - node.FootComment = "" - - for _, child := range node.Content { - removeCommentOfNode(child) - } -} - -func setStyle(matchingNodes *list.List, style yaml.Style) { - for el := matchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*treeops.CandidateNode) - updateStyleOfNode(candidate.Node, style) - } -} - -func updateStyleOfNode(node *yaml.Node, style yaml.Style) { - node.Style = style - - for _, child := range node.Content { - updateStyleOfNode(child, style) - } -} - -func writeString(writer io.Writer, txt string) error { - _, errorWriting := writer.Write([]byte(txt)) - return errorWriting -} - -func printResults(matchingNodes *list.List, writer io.Writer) error { - if prettyPrint { - setStyle(matchingNodes, 0) - } - - if stripComments { - removeComments(matchingNodes) - } - - fileInfo, _ := os.Stdout.Stat() - - if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { - colorsEnabled = true - } - - bufferedWriter := bufio.NewWriter(writer) - defer safelyFlush(bufferedWriter) - - if matchingNodes.Len() == 0 { - log.Debug("no matching results, nothing to print") - if defaultValue != "" { - return writeString(bufferedWriter, defaultValue) - } - return nil - } - var errorWriting error - - for el := matchingNodes.Front(); el != nil; el = el.Next() { - mappedDoc := el.Value.(*treeops.CandidateNode) - - switch printMode { - case "p": - errorWriting = writeString(bufferedWriter, mappedDoc.PathStackToString()+"\n") - if errorWriting != nil { - return errorWriting - } - case "pv", "vp": - // put it into a node and print that. - var parentNode = yaml.Node{Kind: yaml.MappingNode} - parentNode.Content = make([]*yaml.Node, 2) - parentNode.Content[0] = &yaml.Node{Kind: yaml.ScalarNode, Value: mappedDoc.PathStackToString()} - if mappedDoc.Node.Kind == yaml.DocumentNode { - parentNode.Content[1] = mappedDoc.Node.Content[0] - } else { - parentNode.Content[1] = mappedDoc.Node - } - if err := printNode(&parentNode, bufferedWriter); err != nil { - return err - } - default: - if err := printNode(mappedDoc.Node, bufferedWriter); err != nil { - return err - } - } - } - - return nil -} - -func safelyRenameFile(from string, to string) { - if renameError := os.Rename(from, to); renameError != nil { - log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to) - log.Debug(renameError.Error()) - // can't do this rename when running in docker to a file targeted in a mounted volume, - // so gracefully degrade to copying the entire contents. - if copyError := copyFileContents(from, to); copyError != nil { - log.Errorf("Failed copying from %v to %v", from, to) - log.Error(copyError.Error()) - } else { - removeErr := os.Remove(from) - if removeErr != nil { - log.Errorf("failed removing original file: %s", from) - } - } - } -} - -// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) // nolint gosec - if err != nil { - return err - } - defer safelyCloseFile(in) - out, err := os.Create(dst) - if err != nil { - return err - } - defer safelyCloseFile(out) - if _, err = io.Copy(out, in); err != nil { - return err - } - return out.Sync() -} - -func safelyFlush(writer *bufio.Writer) { - if err := writer.Flush(); err != nil { - log.Error("Error flushing writer!") - log.Error(err.Error()) - } - -} -func safelyCloseFile(file *os.File) { - err := file.Close() - if err != nil { - log.Error("Error closing file!") - log.Error(err.Error()) - } -} diff --git a/pkg/yqlib/treeops/candidate_node.go b/pkg/yqlib/candidate_node.go similarity index 99% rename from pkg/yqlib/treeops/candidate_node.go rename to pkg/yqlib/candidate_node.go index 61843a99..1f9daad4 100644 --- a/pkg/yqlib/treeops/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "fmt" diff --git a/pkg/yqlib/treeops/data_tree_navigator.go b/pkg/yqlib/data_tree_navigator.go similarity index 98% rename from pkg/yqlib/treeops/data_tree_navigator.go rename to pkg/yqlib/data_tree_navigator.go index 8d541253..78ff4410 100644 --- a/pkg/yqlib/treeops/data_tree_navigator.go +++ b/pkg/yqlib/data_tree_navigator.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "fmt" diff --git a/pkg/yqlib/treeops/data_tree_navigator_test.go b/pkg/yqlib/data_tree_navigator_test.go similarity index 99% rename from pkg/yqlib/treeops/data_tree_navigator_test.go rename to pkg/yqlib/data_tree_navigator_test.go index e82eac70..571a93e1 100644 --- a/pkg/yqlib/treeops/data_tree_navigator_test.go +++ b/pkg/yqlib/data_tree_navigator_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/doc/Equal Operator.md b/pkg/yqlib/doc/Equal Operator.md new file mode 100644 index 00000000..c8c94687 --- /dev/null +++ b/pkg/yqlib/doc/Equal Operator.md @@ -0,0 +1,64 @@ +# Equal Operator +## Examples +### Example 0 +sample.yml: +```yaml +[cat,goat,dog] +``` +Expression +```bash +yq '.[] | (. == "*at")' < sample.yml +``` +Result +```yaml +true +true +false +``` +### Example 1 +sample.yml: +```yaml +[3, 4, 5] +``` +Expression +```bash +yq '.[] | (. == 4)' < sample.yml +``` +Result +```yaml +false +true +false +``` +### Example 2 +sample.yml: +```yaml +a: { cat: {b: apple, c: whatever}, pat: {b: banana} } +``` +Expression +```bash +yq '.a | (.[].b == "apple")' < sample.yml +``` +Result +```yaml +true +false +``` +### Example 3 +Expression +```bash +yq 'null == null' < sample.yml +``` +Result +```yaml +true +``` +### Example 4 +Expression +```bash +yq 'null == ~' < sample.yml +``` +Result +```yaml +true +``` diff --git a/pkg/yqlib/encoder.go b/pkg/yqlib/encoder.go index e9ab9587..11bad1e8 100644 --- a/pkg/yqlib/encoder.go +++ b/pkg/yqlib/encoder.go @@ -74,16 +74,14 @@ func mapKeysToStrings(node *yaml.Node) { } } -func NewJsonEncoder(destination io.Writer, prettyPrint bool, indent int) Encoder { +func NewJsonEncoder(destination io.Writer, indent int) Encoder { var encoder = json.NewEncoder(destination) var indentString = "" for index := 0; index < indent; index++ { indentString = indentString + " " } - if prettyPrint { - encoder.SetIndent("", indentString) - } + encoder.SetIndent("", indentString) return &jsonEncoder{encoder} } diff --git a/pkg/yqlib/treeops/lib.go b/pkg/yqlib/lib.go similarity index 98% rename from pkg/yqlib/treeops/lib.go rename to pkg/yqlib/lib.go index c9ba0b0c..f229c136 100644 --- a/pkg/yqlib/treeops/lib.go +++ b/pkg/yqlib/lib.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "bytes" @@ -9,8 +9,6 @@ import ( "gopkg.in/yaml.v3" ) -var log = logging.MustGetLogger("yq-treeops") - type OperationType struct { Type string NumArgs uint // number of arguments to the op @@ -25,6 +23,9 @@ type OperationType struct { // - mergeAppend (merges and appends arrays) // - mergeEmpty (sets only if the document is empty, do I do that now?) // - updateTag - not recursive +// - select by tag (tag==) +// - get tag (tag) +// - select by style (style==) // - compare ?? // - validate ?? // - exists diff --git a/pkg/yqlib/treeops/matchKeyString.go b/pkg/yqlib/matchKeyString.go similarity index 97% rename from pkg/yqlib/treeops/matchKeyString.go rename to pkg/yqlib/matchKeyString.go index 34714fd5..0ccb4f91 100644 --- a/pkg/yqlib/treeops/matchKeyString.go +++ b/pkg/yqlib/matchKeyString.go @@ -1,4 +1,4 @@ -package treeops +package yqlib func Match(name string, pattern string) (matched bool) { if pattern == "" { diff --git a/pkg/yqlib/treeops/operator_assign_update.go b/pkg/yqlib/operator_assign_update.go similarity index 98% rename from pkg/yqlib/treeops/operator_assign_update.go rename to pkg/yqlib/operator_assign_update.go index e124b7be..8381b479 100644 --- a/pkg/yqlib/treeops/operator_assign_update.go +++ b/pkg/yqlib/operator_assign_update.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import "container/list" diff --git a/pkg/yqlib/treeops/operator_assign_update_test.go b/pkg/yqlib/operator_assign_update_test.go similarity index 99% rename from pkg/yqlib/treeops/operator_assign_update_test.go rename to pkg/yqlib/operator_assign_update_test.go index bbd8e59b..e1b5870a 100644 --- a/pkg/yqlib/treeops/operator_assign_update_test.go +++ b/pkg/yqlib/operator_assign_update_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_booleans.go b/pkg/yqlib/operator_booleans.go similarity index 99% rename from pkg/yqlib/treeops/operator_booleans.go rename to pkg/yqlib/operator_booleans.go index 26b7b46b..28e2fbac 100644 --- a/pkg/yqlib/treeops/operator_booleans.go +++ b/pkg/yqlib/operator_booleans.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_booleans_test.go b/pkg/yqlib/operator_booleans_test.go similarity index 97% rename from pkg/yqlib/treeops/operator_booleans_test.go rename to pkg/yqlib/operator_booleans_test.go index f3013fbd..61437926 100644 --- a/pkg/yqlib/treeops/operator_booleans_test.go +++ b/pkg/yqlib/operator_booleans_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_collect.go b/pkg/yqlib/operator_collect.go similarity index 98% rename from pkg/yqlib/treeops/operator_collect.go rename to pkg/yqlib/operator_collect.go index c4e1f7c0..9eda72ec 100644 --- a/pkg/yqlib/treeops/operator_collect.go +++ b/pkg/yqlib/operator_collect.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_collect_object.go b/pkg/yqlib/operator_collect_object.go similarity index 98% rename from pkg/yqlib/treeops/operator_collect_object.go rename to pkg/yqlib/operator_collect_object.go index eea0db8b..4845aee7 100644 --- a/pkg/yqlib/treeops/operator_collect_object.go +++ b/pkg/yqlib/operator_collect_object.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_collect_object_test.go b/pkg/yqlib/operator_collect_object_test.go similarity index 98% rename from pkg/yqlib/treeops/operator_collect_object_test.go rename to pkg/yqlib/operator_collect_object_test.go index a573a05f..22c99577 100644 --- a/pkg/yqlib/treeops/operator_collect_object_test.go +++ b/pkg/yqlib/operator_collect_object_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_collect_test.go b/pkg/yqlib/operator_collect_test.go similarity index 98% rename from pkg/yqlib/treeops/operator_collect_test.go rename to pkg/yqlib/operator_collect_test.go index f2de5a05..a6018183 100644 --- a/pkg/yqlib/treeops/operator_collect_test.go +++ b/pkg/yqlib/operator_collect_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_create_map.go b/pkg/yqlib/operator_create_map.go similarity index 98% rename from pkg/yqlib/treeops/operator_create_map.go rename to pkg/yqlib/operator_create_map.go index 5abfdbd8..6c4d9949 100644 --- a/pkg/yqlib/treeops/operator_create_map.go +++ b/pkg/yqlib/operator_create_map.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_create_map_test.go b/pkg/yqlib/operator_create_map_test.go similarity index 98% rename from pkg/yqlib/treeops/operator_create_map_test.go rename to pkg/yqlib/operator_create_map_test.go index dee6e360..24d25ad6 100644 --- a/pkg/yqlib/treeops/operator_create_map_test.go +++ b/pkg/yqlib/operator_create_map_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_delete.go b/pkg/yqlib/operator_delete.go similarity index 99% rename from pkg/yqlib/treeops/operator_delete.go rename to pkg/yqlib/operator_delete.go index c45a46b3..362207d2 100644 --- a/pkg/yqlib/treeops/operator_delete.go +++ b/pkg/yqlib/operator_delete.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_equals.go b/pkg/yqlib/operator_equals.go similarity index 97% rename from pkg/yqlib/treeops/operator_equals.go rename to pkg/yqlib/operator_equals.go index 63bf6d71..8f663f37 100644 --- a/pkg/yqlib/treeops/operator_equals.go +++ b/pkg/yqlib/operator_equals.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_equals_test.go b/pkg/yqlib/operator_equals_test.go similarity index 92% rename from pkg/yqlib/treeops/operator_equals_test.go rename to pkg/yqlib/operator_equals_test.go index 02e4479d..6db4199d 100644 --- a/pkg/yqlib/treeops/operator_equals_test.go +++ b/pkg/yqlib/operator_equals_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" @@ -49,4 +49,5 @@ func TestEqualOperatorScenarios(t *testing.T) { for _, tt := range equalsOperatorScenarios { testScenario(t, &tt) } + documentScenarios(t, "Equal Operator", equalsOperatorScenarios) } diff --git a/pkg/yqlib/treeops/operator_explode.go b/pkg/yqlib/operator_explode.go similarity index 99% rename from pkg/yqlib/treeops/operator_explode.go rename to pkg/yqlib/operator_explode.go index de3a73bd..1d67c484 100644 --- a/pkg/yqlib/treeops/operator_explode.go +++ b/pkg/yqlib/operator_explode.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_explode_test.go b/pkg/yqlib/operator_explode_test.go similarity index 98% rename from pkg/yqlib/treeops/operator_explode_test.go rename to pkg/yqlib/operator_explode_test.go index 8117e142..7318b9a7 100644 --- a/pkg/yqlib/treeops/operator_explode_test.go +++ b/pkg/yqlib/operator_explode_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_multilpy.go b/pkg/yqlib/operator_multilpy.go similarity index 99% rename from pkg/yqlib/treeops/operator_multilpy.go rename to pkg/yqlib/operator_multilpy.go index 3aa2feb1..b97a4364 100644 --- a/pkg/yqlib/treeops/operator_multilpy.go +++ b/pkg/yqlib/operator_multilpy.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "fmt" diff --git a/pkg/yqlib/treeops/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go similarity index 99% rename from pkg/yqlib/treeops/operator_multiply_test.go rename to pkg/yqlib/operator_multiply_test.go index fbcd7f90..4b8dc283 100644 --- a/pkg/yqlib/treeops/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_not.go b/pkg/yqlib/operator_not.go similarity index 97% rename from pkg/yqlib/treeops/operator_not.go rename to pkg/yqlib/operator_not.go index 0497bac3..3ae7e970 100644 --- a/pkg/yqlib/treeops/operator_not.go +++ b/pkg/yqlib/operator_not.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import "container/list" diff --git a/pkg/yqlib/treeops/operator_not_test.go b/pkg/yqlib/operator_not_test.go similarity index 98% rename from pkg/yqlib/treeops/operator_not_test.go rename to pkg/yqlib/operator_not_test.go index f1e22535..1c441644 100644 --- a/pkg/yqlib/treeops/operator_not_test.go +++ b/pkg/yqlib/operator_not_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_recursive_descent.go b/pkg/yqlib/operator_recursive_descent.go similarity index 98% rename from pkg/yqlib/treeops/operator_recursive_descent.go rename to pkg/yqlib/operator_recursive_descent.go index 5a364db6..78dd67cc 100644 --- a/pkg/yqlib/treeops/operator_recursive_descent.go +++ b/pkg/yqlib/operator_recursive_descent.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_recursive_descent_test.go b/pkg/yqlib/operator_recursive_descent_test.go similarity index 99% rename from pkg/yqlib/treeops/operator_recursive_descent_test.go rename to pkg/yqlib/operator_recursive_descent_test.go index 7ced6738..e1a41e7d 100644 --- a/pkg/yqlib/treeops/operator_recursive_descent_test.go +++ b/pkg/yqlib/operator_recursive_descent_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_select.go b/pkg/yqlib/operator_select.go similarity index 97% rename from pkg/yqlib/treeops/operator_select.go rename to pkg/yqlib/operator_select.go index f0e57f2b..72ba06a1 100644 --- a/pkg/yqlib/treeops/operator_select.go +++ b/pkg/yqlib/operator_select.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operator_select_test.go b/pkg/yqlib/operator_select_test.go similarity index 98% rename from pkg/yqlib/treeops/operator_select_test.go rename to pkg/yqlib/operator_select_test.go index 3a2dae1e..2af3c989 100644 --- a/pkg/yqlib/treeops/operator_select_test.go +++ b/pkg/yqlib/operator_select_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_self.go b/pkg/yqlib/operator_self.go similarity index 90% rename from pkg/yqlib/treeops/operator_self.go rename to pkg/yqlib/operator_self.go index bd98d32d..58bf2aa1 100644 --- a/pkg/yqlib/treeops/operator_self.go +++ b/pkg/yqlib/operator_self.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import "container/list" diff --git a/pkg/yqlib/treeops/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go similarity index 86% rename from pkg/yqlib/treeops/operator_traverse_path.go rename to pkg/yqlib/operator_traverse_path.go index 3b12e9d3..08626f1b 100644 --- a/pkg/yqlib/treeops/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "fmt" @@ -186,26 +186,32 @@ func traverseArray(candidate *CandidateNode, operation *Operation) ([]*Candidate } - index := operation.Value.(int64) - indexToUse := index - contentLength := int64(len(candidate.Node.Content)) - for contentLength <= index { - candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}) - contentLength = int64(len(candidate.Node.Content)) - } + switch operation.Value.(type) { + case int64: + index := operation.Value.(int64) + indexToUse := index + contentLength := int64(len(candidate.Node.Content)) + for contentLength <= index { + candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}) + contentLength = int64(len(candidate.Node.Content)) + } - if indexToUse < 0 { - indexToUse = contentLength + indexToUse - } + if indexToUse < 0 { + indexToUse = contentLength + indexToUse + } - if indexToUse < 0 { - return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength) - } + if indexToUse < 0 { + return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength) + } - return []*CandidateNode{&CandidateNode{ - Node: candidate.Node.Content[indexToUse], - Document: candidate.Document, - Path: append(candidate.Path, index), - }}, nil + return []*CandidateNode{&CandidateNode{ + Node: candidate.Node.Content[indexToUse], + Document: candidate.Document, + Path: append(candidate.Path, index), + }}, nil + default: + log.Debug("argument not an int (%v), no array matches", operation.Value) + return nil, nil + } } diff --git a/pkg/yqlib/treeops/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go similarity index 91% rename from pkg/yqlib/treeops/operator_traverse_path_test.go rename to pkg/yqlib/operator_traverse_path_test.go index c5bbd94d..9006ea4c 100644 --- a/pkg/yqlib/treeops/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" @@ -126,6 +126,26 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[b c], (!!str)::frog\n", }, }, + { + document: `[1,2,3]`, + expression: `.b`, + expected: []string{}, + }, + { + document: `[1,2,3]`, + expression: `[0]`, + expected: []string{ + "D0, P[0], (!!int)::1\n", + }, + }, + { + description: `Maps can have numbers as keys, so this default to a non-exisiting key behaviour.`, + document: `{a: b}`, + expression: `[0]`, + expected: []string{ + "D0, P[0], (!!null)::null\n", + }, + }, { document: mergeDocSample, expression: `.foobar`, diff --git a/pkg/yqlib/treeops/operator_union.go b/pkg/yqlib/operator_union.go similarity index 96% rename from pkg/yqlib/treeops/operator_union.go rename to pkg/yqlib/operator_union.go index 9c50d434..f436fa0e 100644 --- a/pkg/yqlib/treeops/operator_union.go +++ b/pkg/yqlib/operator_union.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import "container/list" diff --git a/pkg/yqlib/treeops/operator_union_test.go b/pkg/yqlib/operator_union_test.go similarity index 97% rename from pkg/yqlib/treeops/operator_union_test.go rename to pkg/yqlib/operator_union_test.go index ff80f1d6..d28e4a98 100644 --- a/pkg/yqlib/treeops/operator_union_test.go +++ b/pkg/yqlib/operator_union_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operator_value.go b/pkg/yqlib/operator_value.go similarity index 94% rename from pkg/yqlib/treeops/operator_value.go rename to pkg/yqlib/operator_value.go index 0769a212..94650050 100644 --- a/pkg/yqlib/treeops/operator_value.go +++ b/pkg/yqlib/operator_value.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import "container/list" diff --git a/pkg/yqlib/treeops/operator_value_test.go b/pkg/yqlib/operator_value_test.go similarity index 98% rename from pkg/yqlib/treeops/operator_value_test.go rename to pkg/yqlib/operator_value_test.go index 43408f76..0642600a 100644 --- a/pkg/yqlib/treeops/operator_value_test.go +++ b/pkg/yqlib/operator_value_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/operators.go b/pkg/yqlib/operators.go similarity index 99% rename from pkg/yqlib/treeops/operators.go rename to pkg/yqlib/operators.go index b035b1f6..b1dfc1b7 100644 --- a/pkg/yqlib/treeops/operators.go +++ b/pkg/yqlib/operators.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go new file mode 100644 index 00000000..c9745b16 --- /dev/null +++ b/pkg/yqlib/operators_test.go @@ -0,0 +1,85 @@ +package yqlib + +import ( + "bufio" + "bytes" + "fmt" + "os" + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +type expressionScenario struct { + description string + document string + expression string + expected []string +} + +func testScenario(t *testing.T, s *expressionScenario) { + + nodes := readDoc(t, s.document) + path, errPath := treeCreator.ParsePath(s.expression) + if errPath != nil { + t.Error(errPath) + return + } + results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + + if errNav != nil { + t.Error(errNav) + return + } + test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document)) +} + +func documentScenarios(t *testing.T, title string, scenarios []expressionScenario) { + f, err := os.Create(fmt.Sprintf("doc/%v.md", title)) + + if err != nil { + panic(err) + } + defer f.Close() + w := bufio.NewWriter(f) + w.WriteString(fmt.Sprintf("# %v\n", title)) + w.WriteString(fmt.Sprintf("## Examples\n")) + + printer := NewPrinter(false, true, false, 2, true) + + for index, s := range scenarios { + if s.description != "" { + w.WriteString(fmt.Sprintf("### %v\n", s.description)) + } else { + w.WriteString(fmt.Sprintf("### Example %v\n", index)) + } + if s.document != "" { + w.WriteString(fmt.Sprintf("sample.yml:\n")) + w.WriteString(fmt.Sprintf("```yaml\n%v\n```\n", s.document)) + } + if s.expression != "" { + w.WriteString(fmt.Sprintf("Expression\n")) + w.WriteString(fmt.Sprintf("```bash\nyq '%v' < sample.yml\n```\n", s.expression)) + } + + w.WriteString(fmt.Sprintf("Result\n")) + + nodes := readDoc(t, s.document) + path, errPath := treeCreator.ParsePath(s.expression) + if errPath != nil { + t.Error(errPath) + return + } + var output bytes.Buffer + results, err := treeNavigator.GetMatchingNodes(nodes, path) + printer.PrintResults(results, bufio.NewWriter(&output)) + + w.WriteString(fmt.Sprintf("```yaml\n%v```\n", output.String())) + + if err != nil { + panic(err) + } + + } + w.Flush() +} diff --git a/pkg/yqlib/treeops/operatory_style.go b/pkg/yqlib/operatory_style.go similarity index 99% rename from pkg/yqlib/treeops/operatory_style.go rename to pkg/yqlib/operatory_style.go index 68209463..a0d4aab0 100644 --- a/pkg/yqlib/treeops/operatory_style.go +++ b/pkg/yqlib/operatory_style.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "container/list" diff --git a/pkg/yqlib/treeops/operatory_style_test.go b/pkg/yqlib/operatory_style_test.go similarity index 98% rename from pkg/yqlib/treeops/operatory_style_test.go rename to pkg/yqlib/operatory_style_test.go index 89e26498..b2aa40bc 100644 --- a/pkg/yqlib/treeops/operatory_style_test.go +++ b/pkg/yqlib/operatory_style_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "testing" diff --git a/pkg/yqlib/treeops/path_parse_test.go b/pkg/yqlib/path_parse_test.go similarity index 99% rename from pkg/yqlib/treeops/path_parse_test.go rename to pkg/yqlib/path_parse_test.go index 1142f4bf..cc2e2e3b 100644 --- a/pkg/yqlib/treeops/path_parse_test.go +++ b/pkg/yqlib/path_parse_test.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "fmt" diff --git a/pkg/yqlib/treeops/path_postfix.go b/pkg/yqlib/path_postfix.go similarity index 99% rename from pkg/yqlib/treeops/path_postfix.go rename to pkg/yqlib/path_postfix.go index 147e74d1..992a48ee 100644 --- a/pkg/yqlib/treeops/path_postfix.go +++ b/pkg/yqlib/path_postfix.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "errors" diff --git a/pkg/yqlib/treeops/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go similarity index 99% rename from pkg/yqlib/treeops/path_tokeniser.go rename to pkg/yqlib/path_tokeniser.go index 189b50f9..4c5b44a0 100644 --- a/pkg/yqlib/treeops/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import ( "fmt" diff --git a/pkg/yqlib/treeops/path_tree.go b/pkg/yqlib/path_tree.go similarity index 99% rename from pkg/yqlib/treeops/path_tree.go rename to pkg/yqlib/path_tree.go index bee15ff8..aca5b386 100644 --- a/pkg/yqlib/treeops/path_tree.go +++ b/pkg/yqlib/path_tree.go @@ -1,4 +1,4 @@ -package treeops +package yqlib import "fmt" diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go new file mode 100644 index 00000000..55dcec6c --- /dev/null +++ b/pkg/yqlib/printer.go @@ -0,0 +1,72 @@ +package yqlib + +import ( + "bufio" + "container/list" + "io" + + "gopkg.in/yaml.v3" +) + +type Printer interface { + PrintResults(matchingNodes *list.List, writer io.Writer) error +} + +type resultsPrinter struct { + outputToJSON bool + unwrapScalar bool + colorsEnabled bool + indent int + printDocSeparators bool +} + +func NewPrinter(outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { + return &resultsPrinter{outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators} +} + +func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error { + var encoder Encoder + if node.Kind == yaml.ScalarNode && p.unwrapScalar && !p.outputToJSON { + return p.writeString(writer, node.Value+"\n") + } + if p.outputToJSON { + encoder = NewJsonEncoder(writer, p.indent) + } else { + encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled) + } + return encoder.Encode(node) +} + +func (p *resultsPrinter) writeString(writer io.Writer, txt string) error { + _, errorWriting := writer.Write([]byte(txt)) + return errorWriting +} + +func (p *resultsPrinter) PrintResults(matchingNodes *list.List, writer io.Writer) error { + + bufferedWriter := bufio.NewWriter(writer) + defer safelyFlush(bufferedWriter) + + if matchingNodes.Len() == 0 { + log.Debug("no matching results, nothing to print") + return nil + } + + var previousDocIndex uint = 0 + + for el := matchingNodes.Front(); el != nil; el = el.Next() { + mappedDoc := el.Value.(*CandidateNode) + + if previousDocIndex != mappedDoc.Document && p.printDocSeparators { + p.writeString(bufferedWriter, "---\n") + } + + if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil { + return err + } + + previousDocIndex = mappedDoc.Document + } + + return nil +} diff --git a/pkg/yqlib/treeops/operators_test.go b/pkg/yqlib/treeops/operators_test.go deleted file mode 100644 index 57a4c4b5..00000000 --- a/pkg/yqlib/treeops/operators_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package treeops - -import ( - "fmt" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -type expressionScenario struct { - document string - expression string - expected []string -} - -func testScenario(t *testing.T, s *expressionScenario) { - - nodes := readDoc(t, s.document) - path, errPath := treeCreator.ParsePath(s.expression) - if errPath != nil { - t.Error(errPath) - return - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - return - } - test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document)) -} diff --git a/pkg/yqlib/treeops/sample.yaml b/pkg/yqlib/treeops/sample.yaml deleted file mode 100644 index f151d6f8..00000000 --- a/pkg/yqlib/treeops/sample.yaml +++ /dev/null @@ -1 +0,0 @@ -{name: Mike, pets: [cat, dog]} \ No newline at end of file diff --git a/pkg/yqlib/treeops/temp.json b/pkg/yqlib/treeops/temp.json deleted file mode 100644 index f5cf07d9..00000000 --- a/pkg/yqlib/treeops/temp.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "Mike", - "pets": [ - "cat", - "dog" - ] -} diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go new file mode 100644 index 00000000..73408d44 --- /dev/null +++ b/pkg/yqlib/utils.go @@ -0,0 +1,122 @@ +package yqlib + +import ( + "bufio" + "container/list" + "errors" + "io" + "os" + + yaml "gopkg.in/yaml.v3" +) + +func readStream(filename string) (*yaml.Decoder, error) { + if filename == "" { + return nil, errors.New("Must provide filename") + } + + var stream io.Reader + if filename == "-" { + stream = bufio.NewReader(os.Stdin) + } else { + file, err := os.Open(filename) // nolint gosec + if err != nil { + return nil, err + } + defer safelyCloseFile(file) + stream = file + } + return yaml.NewDecoder(stream), nil +} + +// put this in lib +func Evaluate(filename string, node *PathTreeNode) (*list.List, error) { + + var treeNavigator = NewDataTreeNavigator(NavigationPrefs{}) + + var matchingNodes = list.New() + + var currentIndex uint = 0 + var decoder, err = readStream(filename) + if err != nil { + return nil, err + } + + for { + var dataBucket yaml.Node + errorReading := decoder.Decode(&dataBucket) + + if errorReading == io.EOF { + return matchingNodes, nil + } else if errorReading != nil { + return nil, errorReading + } + candidateNode := &CandidateNode{ + Document: currentIndex, + Filename: filename, + Node: &dataBucket, + } + inputList := list.New() + inputList.PushBack(candidateNode) + + newMatches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node) + if errorParsing != nil { + return nil, errorParsing + } + matchingNodes.PushBackList(newMatches) + currentIndex = currentIndex + 1 + } + + return matchingNodes, nil +} + +func safelyRenameFile(from string, to string) { + if renameError := os.Rename(from, to); renameError != nil { + log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to) + log.Debug(renameError.Error()) + // can't do this rename when running in docker to a file targeted in a mounted volume, + // so gracefully degrade to copying the entire contents. + if copyError := copyFileContents(from, to); copyError != nil { + log.Errorf("Failed copying from %v to %v", from, to) + log.Error(copyError.Error()) + } else { + removeErr := os.Remove(from) + if removeErr != nil { + log.Errorf("failed removing original file: %s", from) + } + } + } +} + +// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) // nolint gosec + if err != nil { + return err + } + defer safelyCloseFile(in) + out, err := os.Create(dst) + if err != nil { + return err + } + defer safelyCloseFile(out) + if _, err = io.Copy(out, in); err != nil { + return err + } + return out.Sync() +} + +func safelyFlush(writer *bufio.Writer) { + if err := writer.Flush(); err != nil { + log.Error("Error flushing writer!") + log.Error(err.Error()) + } + +} +func safelyCloseFile(file *os.File) { + err := file.Close() + if err != nil { + log.Error("Error closing file!") + log.Error(err.Error()) + } +}