suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-11-12 05:38:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed for most cases, except strip comments

Browse Source
This commit is contained in:
Mike Farah 2021-07-20 09:18:40 +10:00
parent 38dd4175fb
commit bbebebe30c
12 changed files with 199 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
acceptance_tests/basic.sh
View File

@ -40,4 +40,33 @@ testBasicExitStatus() {
assertEquals 1 "$?"
}
testBasicExtractFieldWithSeperator() {
cat >test.yml <<EOL
---
name: chart-name
version: 1.2.3
EOL
X=$(./yq e '.name' test.yml)
assertEquals "chart-name" "$X"
}
testBasicExtractMultipleFieldWithSeperator() {
cat >test.yml <<EOL
---
name: chart-name
version: 1.2.3
---
name: thing
version: 1.2.3
EOL
read -r -d '' expected << EOM
chart-name
---
thing
EOM
X=$(./yq e '.name' test.yml)
assertEquals "$expected" "$X"
}
source ./scripts/shunit2

110
acceptance_tests/leading-seperator.sh
View File

@ -229,6 +229,116 @@ EOM
assertEquals "$expected" "$X"
}
testLeadingSeperatorMultiDocEvalCommentsStripComments() {
cat >test.yml <<EOL
---
# hi peeps
# cool
a: test
---
# this is another doc
# great
b: sane
EOL
# it will be hard to remove that top level separator
read -r -d '' expected << EOM
a: test
---
b: sane
EOM
X=$(./yq e --header-preprocess=false '... comments=""' test.yml)
assertEquals "$expected" "$X"
}
testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
cat >test.yml <<EOL
---
# hi peeps
# cool
a: test
---
# this is another doc
# great
b: sane
EOL
# it will be hard to remove that top level separator
read -r -d '' expected << EOM
---
# hi peeps
# cool
a: test
# this is another doc
# great
b: sane
EOM
X=$(./yq e '.' --no-doc test.yml)
assertEquals "$expected" "$X"
}
testLeadingSeperatorMultiDocEvalJsonFlag() {
cat >test.yml <<EOL
---
# hi peeps
# cool
a: test
EOL
cat >test2.yml <<EOL
---
# this is another doc
# great
b: sane
EOL
read -r -d '' expected << EOM
{
"a": "test"
}
{
"b": "sane"
}
EOM
X=$(./yq e '.' -j test.yml test2.yml)
assertEquals "$expected" "$X"
}
testLeadingSeperatorMultiDocEvalAllJsonFlag() {
cat >test.yml <<EOL
---
# hi peeps
# cool
a: test
EOL
cat >test2.yml <<EOL
---
# this is another doc
# great
b: sane
EOL
read -r -d '' expected << EOM
{
"a": "test"
}
{
"b": "sane"
}
EOM
X=$(./yq ea '.' -j test.yml test2.yml)
assertEquals "$expected" "$X"
}
testLeadingSeperatorMultiDocEvalAll() {
read -r -d '' expected << EOM
---

1
cmd/constant.go
View File

@ -1,5 +1,6 @@
package cmd
var leadingContentPreProcessing = true
var unwrapScalar = true
var writeInplace = false

6
cmd/evaluate_all_command.go
View File

@ -112,7 +112,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
switch len(args) {
case 0:
if pipingStdIn {
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
} else {
cmd.Println(cmd.UsageString())
return nil
@ -121,10 +121,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
if nullInput {
err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "")
} else {
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
}
default:
err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
}
completedSuccessfully = err == nil

6
cmd/evalute_sequence_command.go
View File

@ -125,7 +125,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
switch len(args) {
case 0:
if pipingStdIn {
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
} else {
cmd.Println(cmd.UsageString())
return nil
@ -134,10 +134,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
if nullInput {
err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "")
} else {
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
}
default:
err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
}
completedSuccessfully = err == nil

1
cmd/root.go
View File

@ -55,6 +55,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. You will want this off if you want to remove leading comments.")
rootCmd.AddCommand(
createEvaluateSequenceCommand(),
createEvaluateAllCommand(),

8
examples/data1.yaml
View File

@ -1,2 +1,8 @@
---
{"a":{"b":1}}
# hi peeps
# cool
a: test
---
# this is another doc
# great
b: sane

6
pkg/yqlib/all_at_once_evaluator.go
View File

@ -8,7 +8,7 @@ import (
// A yaml expression evaluator that runs the expression once against all files/nodes in memory.
type Evaluator interface {
EvaluateFiles(expression string, filenames []string, printer Printer) error
EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@ -46,13 +46,13 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
return context.MatchingNodes, nil
}
func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
fileIndex := 0
firstFileLeadingContent := ""
var allDocuments *list.List = list.New()
for _, filename := range filenames {
reader, leadingContent, err := readStream(filename)
reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
if err != nil {
return err
}

11
pkg/yqlib/printer.go
View File

@ -128,9 +128,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s
}
if !printedLead {
// we want to print this after the seperator logic
if err := p.writeString(bufferedWriter, leadingContent); err != nil {
return err
// dont print leading comments and seperator if:
// - we are print json; or
// - we are printing an unwrapped scalar node
if !p.outputToJSON && (mappedDoc.Node.Kind != yaml.ScalarNode || !p.unwrapScalar) {
// we want to print this after the seperator logic
if err := p.writeString(bufferedWriter, leadingContent); err != nil {
return err
}
}
printedLead = true
}

27
pkg/yqlib/printer_test.go
View File

@ -276,6 +276,31 @@ a: coconut
test.AssertResult(t, expected, output.String())
}
func TestPrinterScalarWithLeadingCont(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, false, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".a")
if err != nil {
panic(err)
}
streamEvaluator := NewStreamEvaluator()
_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n")
if err != nil {
panic(err)
}
writer.Flush()
expected := `banana
---
apple
---
coconut
`
test.AssertResult(t, expected, output.String())
}
func TestPrinterMultipleDocsJson(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
@ -288,7 +313,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
panic(err)
}
err = printer.PrintResults(inputs, "")
err = printer.PrintResults(inputs, "# ignore this")
if err != nil {
panic(err)
}

6
pkg/yqlib/stream_evaluator.go
View File

@ -13,7 +13,7 @@ import (
// cross document expressions.
type StreamEvaluator interface {
Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error)
EvaluateFiles(expression string, filenames []string, printer Printer) error
EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
EvaluateNew(expression string, printer Printer, leadingContent string) error
}
@ -48,7 +48,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
return printer.PrintResults(result.MatchingNodes, leadingContent)
}
func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
var totalProcessDocs uint = 0
node, err := s.treeCreator.ParseExpression(expression)
if err != nil {
@ -58,7 +58,7 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
var firstFileLeadingContent string
for index, filename := range filenames {
reader, leadingContent, err := readStream(filename)
reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
if index == 0 {
firstFileLeadingContent = leadingContent

6
pkg/yqlib/utils.go
View File

@ -11,7 +11,7 @@ import (
yaml "gopkg.in/yaml.v3"
)
func readStream(filename string) (io.Reader, string, error) {
func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) {
var commentLineRegEx = regexp.MustCompile(`^\s*#`)
var reader *bufio.Reader
if filename == "-" {
@ -27,6 +27,10 @@ func readStream(filename string) (io.Reader, string, error) {
}
var sb strings.Builder
if !leadingContentPreProcessing {
return reader, "", nil
}
for {
peekBytes, err := reader.Peek(3)